1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2010 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 # Generate ICU dat files for locale relevant resources. 18 # 19 # Usage: 20 # icu_dat_generator.py icu-version [-v] [-h] 21 # 22 # Sample usage: 23 # $ANDROID_BUILD_TOP/external/icu4c/stubdata$ ./icu_dat_generator.py 4.4 --verbose 24 # 25 # Add new dat file: 26 # 1. Add icudtxxl-<datname>.txt to $ANDROID_BUILD_TOP/external/icu4c/stubdata. 27 # Check the exemplar file under 28 # $ANDROID_BUILD_TOP/external/icu4c/stubdata/icudt42l-us.dat. 29 # 2. Add an entry to main() --> datlist[] 30 # 3. Run this script to generate dat files. 31 # 32 # For ICU upgrade 33 # We cannot get CLDR version from dat file unless calling ICU function. 34 # If there is a CLDR version change, please modify "global CLDR_VERSION". 35 36 import getopt 37 import os.path 38 import shutil 39 import subprocess 40 import sys 41 42 # Return 0 if the version_string contains non-digit characters. 43 def GetIcuVersion(version_string): 44 list = version_string.split(".") 45 version = "" 46 for number in list: 47 if (number.isdigit()): 48 version += number 49 else: 50 return -1 51 return version 52 53 54 def PrintHelp(): 55 print "Usage:" 56 print "icu_dat_generator.py icu-version [-v|--verbose] [-h|--help]" 57 print "Example:" 58 print "$ANDROID_BUILD_TOP/external/icu4c/stubdata$ ./icu_dat_generator.py 4.4" 59 60 61 def InvokeIcuTool(tool, working_dir, args): 62 command_list = [os.path.join(GetIcuPrebuiltDir(), tool)] 63 command_list.extend(args) 64 65 if VERBOSE: 66 command = "[%s] %s" % (working_dir, " ".join(command_list)) 67 print command 68 69 ret = subprocess.call(command_list, cwd = working_dir) 70 if ret != 0: 71 sys.exit(command_list[0:]) 72 73 74 def GetIcuPrebuiltDir(): 75 return os.path.join(os.environ.get("ANDROID_EABI_TOOLCHAIN"), "..", "..", 76 "..", "icu-" + ICU_VERSION) 77 78 79 def ExtractAllResourceToTempDir(): 80 # copy icudtxxl-all.dat to icudtxxl.dat 81 source_dat = os.path.join(ANDROID_ROOT, "external", "icu4c", "stubdata", 82 ICUDATA + "-all.dat") 83 dest_dat = os.path.join(ANDROID_ROOT, "external", "icu4c", "stubdata", 84 ICUDATA_DAT) 85 shutil.copyfile(source_dat, dest_dat) 86 InvokeIcuTool("icupkg", None, [dest_dat, "-x", "*", "-d", TMP_DAT_PATH]) 87 88 89 def MakeDat(icu_dat_path, dat_name): 90 # Get the resource list. e.g. icudt42l-us.txt, icudt42l-default.txt. 91 dat_list_file_path = os.path.join(icu_dat_path, ICUDATA + "-" + dat_name + 92 ".txt") 93 if not os.path.isfile(dat_list_file_path): 94 print "%s not present for resource list." % dat_list_file_path 95 return 96 GenResIndex(dat_list_file_path) 97 CopyAndroidCnvFiles(icu_dat_path) 98 os.chdir(TMP_DAT_PATH) 99 # Run command such as "icupkg -tl -s icudt42l -a icudt42l-us.txt 100 # new icudt42l.dat" 101 InvokeIcuTool("icupkg", None, ["-tl", "-s", TMP_DAT_PATH, "-a", dat_list_file_path, "new", 102 ICUDATA_DAT]) 103 104 105 def WriteIndex(path, list, cldr_version = None): 106 res_index = "res_index.txt" 107 empty_value = " {\"\"}\n" # key-value pair for all locale entries 108 109 f = open(path, "w") 110 f.write("res_index:table(nofallback) {\n") 111 if cldr_version: 112 f.write(" CLDRVersion { %s }\n" % cldr_version) 113 f.write(" InstalledLocales {\n") 114 for item in list: 115 f.write(item + empty_value) 116 117 f.write(" }\n") 118 f.write("}\n") 119 f.close() 120 121 122 def ShowMissing(whats, locales, data_dir_name, dat_file): 123 if not len(whats): 124 return 125 dat_file = os.path.basename(dat_file) 126 for missing in locales.difference(whats): 127 p = os.path.join(ANDROID_ROOT, "external", "icu4c", "data", data_dir_name, 128 missing + ".txt") 129 if os.path.exists(p): 130 print "warning: %s exists but isn't included in %s" % (p, dat_file) 131 132 133 # Open dat file such as icudt42l-us.txt. 134 # Go through the list and generate res_index.txt for locales, brkitr, 135 # coll and rbnf. 136 def GenResIndex(dat_list_file_path): 137 res_index = "res_index.txt" 138 139 locales = set() 140 brkitrs = set() 141 colls = set() 142 currs = set() 143 langs = set() 144 regions = set() 145 zones = set() 146 rbnfs = set() 147 148 for line in open(dat_list_file_path, "r"): 149 if line.find("root.") >= 0: 150 continue 151 if line.find("res_index") >= 0: 152 continue 153 if line.find("_.res") >= 0: 154 continue; 155 if line.find("brkitr/") >= 0: 156 end = line.find(".res") 157 if end > 0: 158 brkitrs.add(line[line.find("/")+1:end]) 159 elif line.find("coll/") >= 0: 160 end = line.find(".res") 161 if end > 0: 162 colls.add(line[line.find("/")+1:end]) 163 elif line.find("curr/") >= 0: 164 end = line.find(".res") 165 if end > 0: 166 currs.add(line[line.find("/")+1:end]) 167 elif line.find("lang/") >= 0: 168 end = line.find(".res") 169 if end > 0: 170 langs.add(line[line.find("/")+1:end]) 171 elif line.find("region/") >= 0: 172 end = line.find(".res") 173 if end > 0: 174 regions.add(line[line.find("/")+1:end]) 175 elif line.find("zone/") >= 0: 176 end = line.find(".res") 177 if end > 0: 178 zones.add(line[line.find("/")+1:end]) 179 elif line.find("rbnf/") >= 0: 180 end = line.find(".res") 181 if end > 0: 182 rbnfs.add(line[line.find("/")+1:end]) 183 elif line.find(".res") >= 0: 184 # We need to determine the resource is locale resource or misc resource. 185 # To determine the locale resource, we assume max script length is 3. 186 end = line.find(".res") 187 if end <= 3 or (line.find("_") <= 3 and line.find("_") > 0): 188 locales.add(line[:end]) 189 190 ShowMissing(brkitrs, locales, "brkitr", dat_list_file_path) 191 ShowMissing(colls, locales, "coll", dat_list_file_path) 192 ShowMissing(currs, locales, "curr", dat_list_file_path) 193 ShowMissing(langs, locales, "lang", dat_list_file_path) 194 ShowMissing(regions, locales, "region", dat_list_file_path) 195 ShowMissing(zones, locales, "zone", dat_list_file_path) 196 ShowMissing(rbnfs, locales, "rbnf", dat_list_file_path) 197 198 WriteIndex(os.path.join(TMP_DAT_PATH, res_index), locales, CLDR_VERSION) 199 WriteIndex(os.path.join(TMP_DAT_PATH, "brkitr", res_index), brkitrs) 200 WriteIndex(os.path.join(TMP_DAT_PATH, "coll", res_index), colls) 201 WriteIndex(os.path.join(TMP_DAT_PATH, "curr", res_index), currs) 202 WriteIndex(os.path.join(TMP_DAT_PATH, "lang", res_index), langs) 203 WriteIndex(os.path.join(TMP_DAT_PATH, "region", res_index), regions) 204 WriteIndex(os.path.join(TMP_DAT_PATH, "zone", res_index), zones) 205 WriteIndex(os.path.join(TMP_DAT_PATH, "rbnf", res_index), rbnfs) 206 207 # Call genrb to generate new res_index.res. 208 InvokeIcuTool("genrb", TMP_DAT_PATH, [res_index]) 209 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, "brkitr"), [res_index]) 210 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, "coll"), [res_index]) 211 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, "curr"), [res_index]) 212 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, "lang"), [res_index]) 213 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, "region"), [res_index]) 214 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, "zone"), [res_index]) 215 if len(rbnfs): 216 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, "rbnf"), [res_index]) 217 218 219 def CopyAndroidCnvFiles(icu_dat_path): 220 android_specific_cnv = ["gsm-03.38-2000.cnv", 221 "iso-8859_16-2001.cnv", 222 "docomo-shift_jis-2007.cnv", 223 "kddi-jisx-208-2007.cnv", 224 "kddi-shift_jis-2007.cnv", 225 "softbank-jisx-208-2007.cnv", 226 "softbank-shift_jis-2007.cnv"] 227 for cnv_file in android_specific_cnv: 228 source_path = os.path.join(icu_dat_path, "cnv", cnv_file) 229 dest_path = os.path.join(TMP_DAT_PATH, cnv_file) 230 shutil.copyfile(source_path, dest_path) 231 if VERBOSE: 232 print "copy " + source_path + " " + dest_path 233 234 235 def main(): 236 global ANDROID_ROOT # Android project home directory 237 global ICU_VERSION # ICU version number 238 global ICUDATA # e.g. "icudt42l" 239 global ICUDATA_DAT # e.g. "icudt42l.dat" 240 global CLDR_VERSION # CLDR version. The value can be vary upon ICU release. 241 global TMP_DAT_PATH # temp directory to store all resource files and 242 # intermediate dat files. 243 global HELP 244 global VERBOSE 245 246 argc = len(sys.argv) 247 if argc < 2: 248 print "You must provide icu version number." 249 print "Example: ./icu_dat_generator.py 4.4" 250 sys.exit(1) 251 ICU_VERSION = sys.argv[1] 252 version = GetIcuVersion(ICU_VERSION) 253 if (version == -1): 254 print sys.argv[1] + " is not a valid icu version number!" 255 sys.exit(1) 256 ICUDATA = "icudt" + version + "l" 257 CLDR_VERSION = "1.8" 258 ANDROID_ROOT = os.environ.get("ANDROID_BUILD_TOP") 259 if not ANDROID_ROOT: 260 print "$ANDROID_BUILD_TOP not set! Run 'env_setup.sh'." 261 sys.exit(1) 262 ICUDATA_DAT = ICUDATA + ".dat" 263 HELP = False 264 VERBOSE = False 265 266 try: 267 opts, args = getopt.getopt(sys.argv[2:], 'hv', ['help', 'verbose']) 268 except getopt.error: 269 print "Invalid option" 270 PrintHelp() 271 sys.exit(1) 272 for opt, arg in opts: 273 if opt in ('-h', '--help'): 274 PrintHelp() 275 sys.exit(1) 276 elif opt in ('-v', '--verbose'): 277 VERBOSE = True 278 279 280 # Check for requiered source files. 281 icu_dat_path = os.path.join(ANDROID_ROOT, "external", "icu4c", "stubdata") 282 full_data_filename = os.path.join(icu_dat_path, ICUDATA + "-all.dat") 283 if not os.path.isfile(full_data_filename): 284 print "%s not present." % full_data_filename 285 sys.exit(1) 286 287 # Create a temporary working directory. 288 TMP_DAT_PATH = os.path.join(ANDROID_ROOT, "external", "icu4c", "tmp") 289 if os.path.exists(TMP_DAT_PATH): 290 shutil.rmtree(TMP_DAT_PATH) 291 os.mkdir(TMP_DAT_PATH) 292 293 # Extract resource files from icudtxxl-all.dat to TMP_DAT_PATH. 294 ExtractAllResourceToTempDir() 295 296 datlist = ["us", "us-euro", "default", "us-japan", "zh", "medium", "large"] 297 for dat_subtag in datlist: 298 MakeDat(icu_dat_path, dat_subtag) 299 # Copy icudtxxl.dat to stubdata directory with corresponding subtag. 300 shutil.copyfile(os.path.join(TMP_DAT_PATH, ICUDATA_DAT), 301 os.path.join(icu_dat_path, ICUDATA + "-" + dat_subtag + ".dat")) 302 print "Generate ICU data:" + os.path.join(icu_dat_path, ICUDATA + "-" + dat_subtag + ".dat") 303 304 # Cleanup temporary working directory and icudtxxl.dat 305 shutil.rmtree(TMP_DAT_PATH) 306 os.remove(os.path.join(icu_dat_path, ICUDATA_DAT)) 307 308 if __name__ == "__main__": 309 main() 310
