1 #!/usr/bin/python -B 2 3 # Copyright 2017 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Generates the timezone data files used by Android.""" 18 19 import glob 20 import os 21 import re 22 import subprocess 23 import sys 24 import tarfile 25 import tempfile 26 27 sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP')) 28 import i18nutil 29 import icuutil 30 import tzdatautil 31 32 33 # Calculate the paths that are referred to by multiple functions. 34 android_build_top = i18nutil.GetAndroidRootOrDie() 35 timezone_dir = os.path.realpath('%s/system/timezone' % android_build_top) 36 i18nutil.CheckDirExists(timezone_dir, 'system/timezone') 37 38 android_host_out = i18nutil.GetAndroidHostOutOrDie() 39 40 zone_compactor_dir = os.path.realpath('%s/system/timezone/zone_compactor' % android_build_top) 41 i18nutil.CheckDirExists(timezone_dir, 'system/timezone/zone_zompactor') 42 43 timezone_input_tools_dir = os.path.realpath('%s/input_tools' % timezone_dir) 44 timezone_input_data_dir = os.path.realpath('%s/input_data' % timezone_dir) 45 46 timezone_output_data_dir = '%s/output_data' % timezone_dir 47 i18nutil.CheckDirExists(timezone_output_data_dir, 'output_data') 48 49 tmp_dir = tempfile.mkdtemp('-tzdata') 50 51 def GenerateZicInputFile(extracted_iana_data_dir): 52 # Android APIs assume DST means "summer time" so we follow the rearguard format 53 # introduced in 2018e. 54 zic_input_file_name = 'rearguard.zi' 55 56 # 'NDATA=' is used to remove unnecessary rules files. 57 subprocess.check_call(['make', '-C', extracted_iana_data_dir, 'NDATA=', zic_input_file_name]) 58 59 zic_input_file = '%s/%s' % (extracted_iana_data_dir, zic_input_file_name) 60 if not os.path.exists(zic_input_file): 61 print 'Could not find %s' % zic_input_file 62 sys.exit(1) 63 return zic_input_file 64 65 66 def WriteSetupFile(zic_input_file): 67 """Writes the list of zones that ZoneCompactor should process.""" 68 links = [] 69 zones = [] 70 for line in open(zic_input_file): 71 fields = line.split() 72 if fields: 73 if fields[0] == 'Link': 74 links.append('%s %s %s' % (fields[0], fields[1], fields[2])) 75 zones.append(fields[2]) 76 elif fields[0] == 'Zone': 77 zones.append(fields[1]) 78 zones.sort() 79 80 zone_compactor_setup_file = '%s/setup' % tmp_dir 81 setup = open(zone_compactor_setup_file, 'w') 82 for link in sorted(set(links)): 83 setup.write('%s\n' % link) 84 for zone in sorted(set(zones)): 85 setup.write('%s\n' % zone) 86 setup.close() 87 return zone_compactor_setup_file 88 89 90 def BuildIcuData(iana_data_tar_file): 91 icu_build_dir = '%s/icu' % tmp_dir 92 93 icuutil.PrepareIcuBuild(icu_build_dir) 94 icuutil.MakeTzDataFiles(icu_build_dir, iana_data_tar_file) 95 96 # Create ICU system image files. 97 icuutil.MakeAndCopyIcuDataFiles(icu_build_dir) 98 99 # Create the ICU overlay time zone file. 100 icu_overlay_dat_file = '%s/icu_overlay/icu_tzdata.dat' % timezone_output_data_dir 101 icuutil.MakeAndCopyOverlayTzIcuData(icu_build_dir, icu_overlay_dat_file) 102 103 104 def GetIanaVersion(iana_tar_file): 105 iana_tar_filename = os.path.basename(iana_tar_file) 106 iana_version = re.search('tz(?:data|code)(.+)\\.tar\\.gz', iana_tar_filename).group(1) 107 return iana_version 108 109 110 def ExtractTarFile(tar_file, dir): 111 print 'Extracting %s...' % tar_file 112 if not os.path.exists(dir): 113 os.mkdir(dir) 114 tar = tarfile.open(tar_file, 'r') 115 tar.extractall(dir) 116 117 118 def BuildZic(iana_tools_dir): 119 iana_zic_code_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'code') 120 iana_zic_code_version = GetIanaVersion(iana_zic_code_tar_file) 121 iana_zic_data_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'data') 122 iana_zic_data_version = GetIanaVersion(iana_zic_data_tar_file) 123 124 print 'Found IANA zic release %s/%s in %s/%s ...' \ 125 % (iana_zic_code_version, iana_zic_data_version, iana_zic_code_tar_file, iana_zic_data_tar_file) 126 127 zic_build_dir = '%s/zic' % tmp_dir 128 ExtractTarFile(iana_zic_code_tar_file, zic_build_dir) 129 ExtractTarFile(iana_zic_data_tar_file, zic_build_dir) 130 131 # zic 132 print 'Building zic...' 133 # VERSION_DEPS= is to stop the build process looking for files that might not 134 # be present across different versions. 135 subprocess.check_call(['make', '-C', zic_build_dir, 'zic']) 136 137 zic_binary_file = '%s/zic' % zic_build_dir 138 if not os.path.exists(zic_binary_file): 139 print 'Could not find %s' % zic_binary_file 140 sys.exit(1) 141 return zic_binary_file 142 143 144 def BuildTzdata(zic_binary_file, extracted_iana_data_dir, iana_data_version): 145 print 'Generating zic input file...' 146 zic_input_file = GenerateZicInputFile(extracted_iana_data_dir) 147 148 print 'Calling zic...' 149 zic_output_dir = '%s/data' % tmp_dir 150 os.mkdir(zic_output_dir) 151 zic_cmd = [zic_binary_file, '-d', zic_output_dir, zic_input_file] 152 subprocess.check_call(zic_cmd) 153 154 # ZoneCompactor 155 zone_compactor_setup_file = WriteSetupFile(zic_input_file) 156 157 print 'Calling ZoneCompactor to update tzdata to %s...' % iana_data_version 158 subprocess.check_call(['make', '-C', android_build_top, '-j30', 'zone_compactor']) 159 160 # Create args for ZoneCompactor 161 zone_tab_file = '%s/zone.tab' % extracted_iana_data_dir 162 jar_file = '%s/framework/zone_compactor.jar' % android_host_out 163 header_string = 'tzdata%s' % iana_data_version 164 165 print 'Executing ZoneCompactor...' 166 iana_output_data_dir = '%s/iana' % timezone_output_data_dir 167 subprocess.check_call(['java', '-jar', jar_file, 168 zone_compactor_setup_file, zic_output_dir, zone_tab_file, 169 iana_output_data_dir, header_string]) 170 171 172 def BuildTzlookup(iana_data_dir): 173 countryzones_source_file = '%s/android/countryzones.txt' % timezone_input_data_dir 174 tzlookup_dest_file = '%s/android/tzlookup.xml' % timezone_output_data_dir 175 176 print 'Calling TzLookupGenerator to create tzlookup.xml...' 177 subprocess.check_call(['make', '-C', android_build_top, '-j30', 'tzlookup_generator']) 178 179 jar_file = '%s/framework/tzlookup_generator.jar' % android_host_out 180 zone_tab_file = '%s/zone.tab' % iana_data_dir 181 subprocess.check_call(['java', '-jar', jar_file, 182 countryzones_source_file, zone_tab_file, tzlookup_dest_file]) 183 184 185 def CreateDistroFiles(iana_data_version, output_dir): 186 create_distro_script = '%s/distro/tools/create-distro.py' % timezone_dir 187 188 tzdata_file = '%s/iana/tzdata' % timezone_output_data_dir 189 icu_file = '%s/icu_overlay/icu_tzdata.dat' % timezone_output_data_dir 190 tzlookup_file = '%s/android/tzlookup.xml' % timezone_output_data_dir 191 192 distro_file_pattern = '%s/*.zip' % output_dir 193 existing_distro_files = glob.glob(distro_file_pattern) 194 195 distro_file_metadata_pattern = '%s/*.txt' % output_dir 196 existing_distro_metadata_files = glob.glob(distro_file_metadata_pattern) 197 existing_files = existing_distro_files + existing_distro_metadata_files 198 199 print 'Removing %s' % existing_files 200 for existing_file in existing_files: 201 os.remove(existing_file) 202 203 subprocess.check_call([create_distro_script, 204 '-iana_version', iana_data_version, 205 '-tzdata', tzdata_file, 206 '-icu', icu_file, 207 '-tzlookup', tzlookup_file, 208 '-output', output_dir]) 209 210 def UpdateTestFiles(): 211 testing_data_dir = '%s/testing/data' % timezone_dir 212 update_test_files_script = '%s/create-test-data.sh' % testing_data_dir 213 subprocess.check_call([update_test_files_script], cwd=testing_data_dir) 214 215 216 # Run with no arguments from any directory, with no special setup required. 217 # See http://www.iana.org/time-zones/ for more about the source of this data. 218 def main(): 219 print 'Source data file structure: %s' % timezone_input_data_dir 220 print 'Source tools file structure: %s' % timezone_input_tools_dir 221 print 'Output data file structure: %s' % timezone_output_data_dir 222 223 iana_input_data_dir = '%s/iana' % timezone_input_data_dir 224 iana_data_tar_file = tzdatautil.GetIanaTarFile(iana_input_data_dir, 'data') 225 iana_data_version = GetIanaVersion(iana_data_tar_file) 226 print 'IANA time zone data release %s in %s ...' % (iana_data_version, iana_data_tar_file) 227 228 icu_dir = icuutil.icuDir() 229 print 'Found icu in %s ...' % icu_dir 230 231 BuildIcuData(iana_data_tar_file) 232 233 iana_tools_dir = '%s/iana' % timezone_input_tools_dir 234 zic_binary_file = BuildZic(iana_tools_dir) 235 236 iana_data_dir = '%s/iana_data' % tmp_dir 237 ExtractTarFile(iana_data_tar_file, iana_data_dir) 238 BuildTzdata(zic_binary_file, iana_data_dir, iana_data_version) 239 BuildTzlookup(iana_data_dir) 240 241 # Create a distro file from the output from prior stages. 242 distro_output_dir = '%s/distro' % timezone_output_data_dir 243 CreateDistroFiles(iana_data_version, distro_output_dir) 244 245 # Update test versions of distro files too. 246 UpdateTestFiles() 247 248 print 'Look in %s and %s for new files' % (timezone_output_data_dir, icu_dir) 249 sys.exit(0) 250 251 252 if __name__ == '__main__': 253 main() 254
