1 /* 2 ****************************************************************************** 3 * Copyright (C) 2004-2013, International Business Machines Corporation and * 4 * others. All Rights Reserved. * 5 ****************************************************************************** 6 */ 7 package org.unicode.cldr.tool; 8 9 import java.io.BufferedReader; 10 import java.io.File; 11 import java.io.IOException; 12 import java.io.PrintWriter; 13 import java.util.ArrayList; 14 import java.util.Arrays; 15 import java.util.Collection; 16 import java.util.Comparator; 17 import java.util.HashMap; 18 import java.util.HashSet; 19 import java.util.Iterator; 20 import java.util.LinkedHashMap; 21 import java.util.List; 22 import java.util.Map; 23 import java.util.Map.Entry; 24 import java.util.Set; 25 import java.util.TreeMap; 26 import java.util.TreeSet; 27 import java.util.regex.Matcher; 28 import java.util.regex.Pattern; 29 30 import org.unicode.cldr.draft.FileUtilities; 31 import org.unicode.cldr.util.CLDRConfig; 32 import org.unicode.cldr.util.CLDRFile; 33 import org.unicode.cldr.util.CLDRPaths; 34 import org.unicode.cldr.util.CldrUtility; 35 import org.unicode.cldr.util.Factory; 36 import org.unicode.cldr.util.Iso639Data; 37 import org.unicode.cldr.util.IsoCurrencyParser; 38 import org.unicode.cldr.util.IsoCurrencyParser.Data; 39 import org.unicode.cldr.util.IsoRegionData; 40 import org.unicode.cldr.util.Level; 41 import org.unicode.cldr.util.Log; 42 import org.unicode.cldr.util.Organization; 43 import org.unicode.cldr.util.Pair; 44 import org.unicode.cldr.util.PatternCache; 45 import org.unicode.cldr.util.StandardCodes; 46 import org.unicode.cldr.util.SupplementalDataInfo; 47 import org.unicode.cldr.util.Tabber; 48 import org.unicode.cldr.util.TimezoneFormatter; 49 import org.unicode.cldr.util.UnicodeSetPrettyPrinter; 50 import org.unicode.cldr.util.XPathParts; 51 import org.unicode.cldr.util.props.ICUPropertyFactory; 52 53 import com.ibm.icu.dev.util.CollectionUtilities; 54 import com.ibm.icu.dev.util.UnicodeMap; 55 import com.ibm.icu.dev.util.UnicodeMapIterator; 56 import com.ibm.icu.impl.Relation; 57 import com.ibm.icu.impl.Row; 58 import com.ibm.icu.impl.Row.R2; 59 import com.ibm.icu.impl.Row.R3; 60 import com.ibm.icu.text.Collator; 61 import com.ibm.icu.text.NumberFormat; 62 import com.ibm.icu.text.RuleBasedCollator; 63 import com.ibm.icu.text.Transform; 64 import com.ibm.icu.text.UnicodeSet; 65 import com.ibm.icu.util.ULocale; 66 67 /** 68 * Simple program to count the amount of data in CLDR. Internal Use. 69 */ 70 public class CountItems { 71 72 private static final Collator ROOT_PRIMARY_COLLATOR = Collator.getInstance(ULocale.ROOT) 73 .setStrength2(Collator.PRIMARY); 74 75 static final String needsTranslationString = "America/Buenos_Aires " // America/Rio_Branco 76 + " America/Manaus America/Belem " 77 + " America/Campo_Grande America/Sao_Paulo " 78 + " Australia/Perth Australia/Darwin Australia/Brisbane Australia/Adelaide Australia/Sydney Australia/Hobart " 79 + " America/Vancouver America/Edmonton America/Regina America/Winnipeg America/Toronto America/Halifax America/St_Johns " 80 + " Asia/Jakarta " 81 + " America/Tijuana America/Hermosillo America/Chihuahua America/Mexico_City " 82 + " Europe/Moscow Europe/Kaliningrad Europe/Moscow Asia/Yekaterinburg Asia/Novosibirsk Asia/Yakutsk Asia/Vladivostok" 83 + " Pacific/Honolulu America/Indiana/Indianapolis America/Anchorage " 84 + " America/Los_Angeles America/Phoenix America/Denver America/Chicago America/Indianapolis" 85 + " America/New_York"; 86 87 static final Map<String, String> country_map = CollectionUtilities.asMap(new String[][] { 88 { "AQ", "http://www.worldtimezone.com/time-antarctica24.php" }, 89 { "AR", "http://www.worldtimezone.com/time-south-america24.php" }, 90 { "AU", "http://www.worldtimezone.com/time-australia24.php" }, 91 { "BR", "http://www.worldtimezone.com/time-south-america24.php" }, 92 { "CA", "http://www.worldtimezone.com/time-canada24.php" }, 93 { "CD", "http://www.worldtimezone.com/time-africa24.php" }, 94 { "CL", "http://www.worldtimezone.com/time-south-america24.php" }, 95 { "CN", "http://www.worldtimezone.com/time-cis24.php" }, 96 { "EC", "http://www.worldtimezone.com/time-south-america24.php" }, 97 { "ES", "http://www.worldtimezone.com/time-europe24.php" }, 98 { "FM", "http://www.worldtimezone.com/time-oceania24.php" }, 99 { "GL", "http://www.worldtimezone.com/index24.php" }, 100 { "ID", "http://www.worldtimezone.com/time-asia24.php" }, 101 { "KI", "http://www.worldtimezone.com/time-oceania24.php" }, 102 { "KZ", "http://www.worldtimezone.com/time-cis24.php" }, 103 { "MH", "http://www.worldtimezone.com/time-oceania24.php" }, 104 { "MN", "http://www.worldtimezone.com/time-cis24.php" }, 105 { "MX", "http://www.worldtimezone.com/index24.php" }, 106 { "MY", "http://www.worldtimezone.com/time-asia24.php" }, 107 { "NZ", "http://www.worldtimezone.com/time-oceania24.php" }, 108 { "PF", "http://www.worldtimezone.com/time-oceania24.php" }, 109 { "PT", "http://www.worldtimezone.com/time-europe24.php" }, 110 { "RU", "http://www.worldtimezone.com/time-russia24.php" }, 111 { "SJ", "http://www.worldtimezone.com/index24.php" }, 112 { "UA", "http://www.worldtimezone.com/time-cis24.php" }, 113 { "UM", "http://www.worldtimezone.com/time-oceania24.php" }, 114 { "US", "http://www.worldtimezone.com/time-usa24.php" }, 115 { "UZ", "http://www.worldtimezone.com/time-cis24.php" }, }); 116 117 /** 118 * Count the data. 119 * 120 * @throws IOException 121 */ 122 public static void main(String[] args) throws Exception { 123 double deltaTime = System.currentTimeMillis(); 124 try { 125 String methodName = System.getProperty("method"); 126 if (methodName != null) { 127 CldrUtility.callMethod(methodName, CountItems.class); 128 } else { 129 ShowZoneEquivalences.getZoneEquivalences(); 130 } 131 } finally { 132 deltaTime = System.currentTimeMillis() - deltaTime; 133 System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds"); 134 System.out.println("Done"); 135 } 136 } 137 138 static void subheader(PrintWriter out, Tabber tabber) { 139 // out.println("<tr><td colspan='6' class='gap'> </td></tr>"); 140 out.println(tabber.process("Cnty" + "\t" + "Grp" + "\t" + "ZoneID" + "\t" 141 + "Formatted ID" + "\t" + "MaxOffset" + "\t" + "MinOffset")); 142 } 143 144 /** 145 * 146 */ 147 private static void getPatternBlocks() { 148 UnicodeSet patterns = new UnicodeSet("[:pattern_syntax:]"); 149 UnicodeSet unassigned = new UnicodeSet("[:unassigned:]"); 150 UnicodeSet punassigned = new UnicodeSet(patterns).retainAll(unassigned); 151 UnicodeMap<String> blocks = ICUPropertyFactory.make().getProperty("block") 152 .getUnicodeMap(); 153 blocks.setMissing("<Reserved-Block>"); 154 // blocks.composeWith(new UnicodeMap().putAll(new UnicodeSet(patterns).retainAll(unassigned),"<reserved>"), 155 // new UnicodeMap.Composer() { 156 // public Object compose(int codePoint, Object a, Object b) { 157 // if (a == null) { 158 // return b; 159 // } 160 // if (b == null) { 161 // return a; 162 // } 163 // return a.toString() + " " + b.toString(); 164 // }}); 165 for (UnicodeMapIterator<String> it = new UnicodeMapIterator<String>(blocks); it 166 .nextRange();) { 167 UnicodeSet range = new UnicodeSet(it.codepoint, it.codepointEnd); 168 boolean hasPat = range.containsSome(patterns); 169 String prefix = !hasPat ? "Not-Syntax" 170 : !range.containsSome(unassigned) ? "Closed" : !range 171 .containsSome(punassigned) ? "Closed2" : "Open"; 172 173 boolean show = (prefix.equals("Open") || prefix.equals("Closed2")); 174 175 if (show) 176 System.out.println(); 177 System.out.println(prefix + "\t" + range + "\t" + it.value); 178 if (show) { 179 System.out.println(new UnicodeMap<String>().putAll(unassigned, "<reserved>") 180 .putAll(punassigned, "<reserved-for-syntax>").setMissing( 181 "<assigned>") 182 .putAll(range.complement(), null)); 183 } 184 } 185 } 186 187 /** 188 * @throws IOException 189 * 190 */ 191 private static void showExemplars() throws IOException { 192 PrintWriter out = FileUtilities.openUTF8Writer(CLDRPaths.GEN_DIRECTORY, "fixed_exemplars.txt"); 193 Factory cldrFactory = Factory.make(CLDRPaths.MAIN_DIRECTORY, ".*"); 194 Set<String> locales = cldrFactory.getAvailable(); 195 for (Iterator<String> it = locales.iterator(); it.hasNext();) { 196 System.out.print('.'); 197 String locale = it.next(); 198 CLDRFile cldrfile = cldrFactory.make(locale, false); 199 String v = cldrfile 200 .getStringValue("//ldml/characters/exemplarCharacters"); 201 if (v == null) 202 continue; 203 UnicodeSet exemplars = new UnicodeSet(v); 204 if (exemplars.size() != 0 && exemplars.size() < 500) { 205 Collator col = Collator.getInstance(new ULocale(locale)); 206 Collator spaceCol = Collator.getInstance(new ULocale(locale)); 207 spaceCol.setStrength(Collator.PRIMARY); 208 out.println(locale + ":\t\u200E" + v + '\u200E'); 209 // String fixedFull = CollectionUtilities.prettyPrint(exemplars, col, false); 210 // System.out.println(" =>\t" + fixedFull); 211 // verifyEquality(exemplars, new UnicodeSet(fixedFull)); 212 String fixed = new UnicodeSetPrettyPrinter() 213 .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT)) 214 .setSpaceComparator(spaceCol != null ? spaceCol : ROOT_PRIMARY_COLLATOR) 215 .setCompressRanges(true) 216 .format(exemplars); 217 out.println(" =>\t\u200E" + fixed + '\u200E'); 218 219 verifyEquality(exemplars, new UnicodeSet(fixed)); 220 out.flush(); 221 } 222 } 223 out.close(); 224 } 225 226 /** 227 * 228 */ 229 private static void verifyEquality(UnicodeSet exemplars, UnicodeSet others) { 230 if (others.equals(exemplars)) 231 return; 232 System.out.println("FAIL\ta-b\t" 233 + new UnicodeSet(exemplars).removeAll(others)); 234 System.out.println("\tb-a\t" + new UnicodeSet(others).removeAll(exemplars)); 235 } 236 237 /** 238 * 239 */ 240 public static void generateSupplementalCurrencyItems() { 241 IsoCurrencyParser isoCurrencyParser = IsoCurrencyParser.getInstance(); 242 Relation<String, Data> codeList = isoCurrencyParser.getCodeList(); 243 Map<String, String> numericTocurrencyCode = new TreeMap<String, String>(); 244 StringBuffer list = new StringBuffer(); 245 246 for (Iterator<String> it = codeList.keySet().iterator(); it.hasNext();) { 247 String currencyCode = it.next(); 248 int numericCode = -1; 249 Set<Data> dataSet = codeList.getAll(currencyCode); 250 boolean first = true; 251 for (Data data : dataSet) { 252 if (first) { 253 first = false; 254 } 255 numericCode = data.getNumericCode(); 256 } 257 258 String strNumCode = "" + numericCode; 259 String otherCode = numericTocurrencyCode.get(strNumCode); 260 if (otherCode != null) { 261 System.out.println("Warning: duplicate code " + otherCode + 262 "for " + numericCode); 263 } 264 numericTocurrencyCode.put(strNumCode, currencyCode); 265 if (list.length() != 0) 266 list.append(" "); 267 String currencyLine = "<currencyCodes type=" + "\"" + currencyCode + 268 "\"" + " numeric=" + "\"" + numericCode + "\"/>"; 269 list.append(currencyLine); 270 System.out.println(currencyLine); 271 272 } 273 System.out.println(); 274 275 } 276 277 /** 278 * 279 */ 280 public static void generateCurrencyItems() { 281 IsoCurrencyParser isoCurrencyParser = IsoCurrencyParser.getInstance(); 282 Relation<String, Data> codeList = isoCurrencyParser.getCodeList(); 283 StringBuffer list = new StringBuffer(); 284 for (Iterator<String> it = codeList.keySet().iterator(); it.hasNext();) { 285 // String lastField = (String) it.next(); 286 // String zone = (String) fullMap.get(lastField); 287 String currencyCode = it.next(); 288 Set<Data> dataSet = codeList.getAll(currencyCode); 289 boolean first = true; 290 for (Data data : dataSet) { 291 if (first) { 292 System.out.print(currencyCode); 293 first = false; 294 } 295 System.out.println("\t" + data); 296 } 297 298 if (list.length() != 0) 299 list.append(" "); 300 list.append(currencyCode); 301 302 } 303 System.out.println(); 304 String sep = CldrUtility.LINE_SEPARATOR + "\t\t\t\t"; 305 // "((?:[-+_A-Za-z0-9]+[/])+[A-Za-z0-9])[-+_A-Za-z0-9]*" 306 String broken = CldrUtility.breakLines(list.toString(), sep, PatternCache.get( 307 "([A-Z])[A-Z][A-Z]").matcher(""), 80); 308 assert (list.toString().equals(broken.replace(sep, " "))); 309 //System.out.println("\t\t\t<variable id=\"$currency\" type=\"choice\">" 310 // + broken + CldrUtility.LINE_SEPARATOR + "\t\t\t</variable>"); 311 Set<String> isoTextFileCodes = StandardCodes.make().getAvailableCodes( 312 "currency"); 313 Set<String> temp = new TreeSet<String>(codeList.keySet()); 314 temp.removeAll(isoTextFileCodes); 315 if (temp.size() != 0) { 316 throw new IllegalArgumentException("Missing from ISO4217.txt file: " + temp); 317 } 318 } 319 320 public static void genSupplementalZoneData() throws IOException { 321 genSupplementalZoneData(false); 322 } 323 324 public static void genSupplementalZoneData(boolean skipUnaliased) 325 throws IOException { 326 RuleBasedCollator col = (RuleBasedCollator) Collator.getInstance(); 327 col.setNumericCollation(true); 328 StandardCodes sc = StandardCodes.make(); 329 Map<String, String> zone_country = sc.getZoneToCounty(); 330 Map<String, Set<String>> country_zone = sc.getCountryToZoneSet(); 331 Factory cldrFactory = Factory.make(CLDRPaths.MAIN_DIRECTORY, ".*"); 332 CLDRFile english = cldrFactory.make("en", true); 333 334 writeZonePrettyPath(col, zone_country, english); 335 writeMetazonePrettyPath(); 336 337 Map<String, String> old_new = sc.getZoneLinkold_new(); 338 Map<String, Set<String>> new_old = new TreeMap<String, Set<String>>(); 339 340 for (Iterator<String> it = old_new.keySet().iterator(); it.hasNext();) { 341 String old = it.next(); 342 String newOne = old_new.get(old); 343 Set<String> oldSet = new_old.get(newOne); 344 if (oldSet == null) 345 new_old.put(newOne, oldSet = new TreeSet<String>()); 346 oldSet.add(old); 347 } 348 Map<String, String> fullMap = new TreeMap<String, String>(col); 349 for (Iterator<String> it = zone_country.keySet().iterator(); it.hasNext();) { 350 String zone = it.next(); 351 String defaultName = TimezoneFormatter.getFallbackName(zone); 352 Object already = fullMap.get(defaultName); 353 if (already != null) 354 System.out.println("CONFLICT: " + already + ", " + zone); 355 fullMap.put(defaultName, zone); 356 } 357 // fullSet.addAll(zone_country.keySet()); 358 // fullSet.addAll(new_old.keySet()); 359 360 System.out 361 .println("<!-- Generated by org.unicode.cldr.tool.CountItems -->"); 362 System.out.println("<supplementalData>"); 363 System.out.println("\t<timezoneData>"); 364 System.out.println(); 365 366 Set<String> multizone = new TreeSet<String>(); 367 for (Iterator<String> it = country_zone.keySet().iterator(); it.hasNext();) { 368 String country = it.next(); 369 Set<String> zones = country_zone.get(country); 370 if (zones != null && zones.size() != 1) 371 multizone.add(country); 372 } 373 374 System.out.println("\t\t<zoneFormatting multizone=\"" 375 + toString(multizone, " ") + "\"" + " tzidVersion=\"" 376 + sc.getZoneVersion() + "\"" + ">"); 377 378 Set<String> orderedSet = new TreeSet<String>(col); 379 orderedSet.addAll(zone_country.keySet()); 380 orderedSet.addAll(sc.getDeprecatedZoneIDs()); 381 StringBuffer tzid = new StringBuffer(); 382 383 for (Iterator<String> it = orderedSet.iterator(); it.hasNext();) { 384 // String lastField = (String) it.next(); 385 // String zone = (String) fullMap.get(lastField); 386 String zone = it.next(); 387 if (tzid.length() != 0) 388 tzid.append(' '); 389 tzid.append(zone); 390 391 String country = zone_country.get(zone); 392 if (country == null) 393 continue; // skip deprecated 394 395 Set<String> aliases = new_old.get(zone); 396 if (aliases != null) { 397 aliases = new TreeSet<String>(aliases); 398 aliases.remove(zone); 399 } 400 if (skipUnaliased) 401 if (aliases == null || aliases.size() == 0) 402 continue; 403 404 System.out.println("\t\t\t<zoneItem" 405 + " type=\"" 406 + zone 407 + "\"" 408 + " territory=\"" 409 + country 410 + "\"" 411 + (aliases != null && aliases.size() > 0 ? " aliases=\"" 412 + toString(aliases, " ") + "\"" : "") 413 + "/>"); 414 } 415 416 System.out.println("\t\t</zoneFormatting>"); 417 System.out.println(); 418 System.out.println("\t</timezoneData>"); 419 System.out.println("</supplementalData>"); 420 System.out.println(); 421 String sep = CldrUtility.LINE_SEPARATOR + "\t\t\t\t"; 422 // "((?:[-+_A-Za-z0-9]+[/])+[A-Za-z0-9])[-+_A-Za-z0-9]*" 423 String broken = CldrUtility.breakLines(tzid, sep, PatternCache.get( 424 "((?:[-+_A-Za-z0-9]+[/])+[-+_A-Za-z0-9])[-+_A-Za-z0-9]*").matcher(""), 425 80); 426 assert (tzid.toString().equals(broken.replace(sep, " "))); 427 System.out.println("\t\t\t<variable id=\"$tzid\" type=\"choice\">" + broken 428 + CldrUtility.LINE_SEPARATOR + "\t\t\t</variable>"); 429 } 430 431 public static void writeMetazonePrettyPath() { 432 CLDRConfig testInfo = ToolConfig.getToolInstance(); 433 Map<String, Map<String, String>> map = testInfo.getSupplementalDataInfo().getMetazoneToRegionToZone(); 434 Map zoneToCountry = testInfo.getStandardCodes().getZoneToCounty(); 435 Set<Pair<String, String>> results = new TreeSet<Pair<String, String>>(); 436 Map<String, String> countryToContinent = getCountryToContinent(testInfo.getSupplementalDataInfo(), 437 testInfo.getEnglish()); 438 439 for (String metazone : map.keySet()) { 440 Map<String, String> regionToZone = map.get(metazone); 441 String zone = regionToZone.get("001"); 442 if (zone == null) { 443 throw new IllegalArgumentException("Missing 001 for metazone " + metazone); 444 } 445 String continent = zone.split("/")[0]; 446 447 final Object country = zoneToCountry.get(zone); 448 results.add(new Pair<String, String>(continent + "\t" + country + "\t" + countryToContinent.get(country) 449 + "\t" + metazone, metazone)); 450 } 451 for (Pair<String, String> line : results) { 452 System.out.println("'" + line.getSecond() + "'\t>\t'\t" + line.getFirst() + "\t'"); 453 } 454 } 455 456 private static Map<String, String> getCountryToContinent(SupplementalDataInfo supplementalDataInfo, CLDRFile english) { 457 Relation<String, String> countryToContinent = Relation.of(new TreeMap<String, Set<String>>(), TreeSet.class); 458 Set<String> continents = new HashSet<String>(Arrays.asList("002", "019", "142", "150", "009")); 459 // note: we don't need more than 3 levels 460 for (String continent : continents) { 461 final Set<String> subcontinents = supplementalDataInfo.getContained(continent); 462 countryToContinent.putAll(subcontinents, continent); 463 for (String subcontinent : subcontinents) { 464 if (subcontinent.equals("EU")) continue; 465 final Set<String> countries = supplementalDataInfo.getContained(subcontinent); 466 countryToContinent.putAll(countries, continent); 467 } 468 } 469 // convert to map 470 Map<String, String> results = new TreeMap<String, String>(); 471 for (String item : countryToContinent.keySet()) { 472 final Set<String> containees = countryToContinent.getAll(item); 473 if (containees.size() != 1) { 474 throw new IllegalArgumentException(item + "\t" + containees); 475 } 476 results.put(item, english.getName(CLDRFile.TERRITORY_NAME, containees.iterator().next())); 477 } 478 return results; 479 } 480 481 private static void writeZonePrettyPath(RuleBasedCollator col, Map<String, String> zone_country, 482 CLDRFile english) throws IOException { 483 System.out.println("Writing zonePrettyPath"); 484 Set<String> masked = new HashSet<String>(); 485 Map<String, String> zoneNew_Old = new TreeMap<String, String>(col); 486 String lastZone = "XXX"; 487 for (String zone : new TreeSet<String>(zone_country.keySet())) { 488 String[] parts = zone.split("/"); 489 String newPrefix = zone_country.get(zone); // english.getName("tzid", zone_country.get(zone), 490 // false).replace(' ', '_'); 491 if (newPrefix.equals("001")) { 492 newPrefix = "ZZ"; 493 } 494 parts[0] = newPrefix; 495 String newName; 496 if (parts.length > 2) { 497 System.out.println("\tMultifield: " + zone); 498 if (parts.length == 3 && parts[1].equals("Argentina")) { 499 newName = parts[0] + "/" + parts[1]; 500 } else { 501 newName = CldrUtility.join(parts, "/"); 502 } 503 } else { 504 newName = CldrUtility.join(parts, "/"); 505 } 506 zoneNew_Old.put(newName, zone); 507 if (zone.startsWith(lastZone)) { 508 masked.add(zone); // find "masked items" and do them first. 509 } else { 510 lastZone = zone; 511 } 512 } 513 514 Log.setLog(CLDRPaths.GEN_DIRECTORY + "/supplemental/prettyPathZone.txt"); 515 String lastCountry = ""; 516 for (int i = 0; i < 2; ++i) { 517 Set<String> orderedList = zoneNew_Old.keySet(); 518 if (i == 0) { 519 Log 520 .println("# Short IDs for zone names: country code + last part of TZID"); 521 Log 522 .println("# First are items that would be masked, and are moved forwards and sorted in reverse order"); 523 Log.println(); 524 //Comparator c; 525 Set<String> temp = new TreeSet<String>(new ReverseComparator<Object>(col)); 526 temp.addAll(orderedList); 527 orderedList = temp; 528 } else { 529 Log.println(); 530 Log.println("# Normal items, sorted by country code"); 531 Log.println(); 532 } 533 534 // do masked items first 535 536 for (String newName : orderedList) { 537 String oldName = zoneNew_Old.get(newName); 538 if (masked.contains(oldName) != (i == 0)) { 539 continue; 540 } 541 String newCountry = newName.split("/")[0]; 542 if (!newCountry.equals(lastCountry)) { 543 Log.println("# " + newCountry + "\t" 544 + english.getName("territory", newCountry)); 545 lastCountry = newCountry; 546 } 547 Log.println("\t'" + oldName + "'\t>\t'" + newName + "';"); 548 } 549 } 550 Log.close(); 551 System.out.println("Done Writing zonePrettyPath"); 552 } 553 554 public static class ReverseComparator<T> implements Comparator<T> { 555 Comparator<T> other; 556 557 public ReverseComparator(Comparator<T> other) { 558 this.other = other; 559 } 560 561 public int compare(T o1, T o2) { 562 return other.compare(o2, o1); 563 } 564 } 565 566 public static void getSubtagVariables2() throws IOException { 567 Log.setLogNoBOM(CLDRPaths.GEN_DIRECTORY + "/supplemental", "supplementalMetadata.xml"); 568 BufferedReader oldFile = FileUtilities.openUTF8Reader(CLDRPaths.SUPPLEMENTAL_DIRECTORY, "supplementalMetadata.xml"); 569 CldrUtility.copyUpTo(oldFile, PatternCache.get("\\s*<!-- start of data generated with CountItems.*"), 570 Log.getLog(), true); 571 572 Map<String, String> variableSubstitutions = getVariables(VariableType.partial); 573 for (Entry<String, String> type : variableSubstitutions.entrySet()) { 574 Log.println(type.getValue()); 575 } 576 577 // String sep = CldrUtility.LINE_SEPARATOR + "\t\t\t"; 578 // String broken = CldrUtility.breakLines(CldrUtility.join(defaultLocaleContent," "), sep, 579 // PatternCache.get("(\\S)\\S*").matcher(""), 80); 580 // 581 // Log.println("\t\t<defaultContent locales=\"" + broken + "\""); 582 // Log.println("\t\t/>"); 583 584 // Log.println("</supplementalData>"); 585 CldrUtility.copyUpTo(oldFile, PatternCache.get("\\s<!-- end of data generated by CountItems.*"), null, true); 586 CldrUtility.copyUpTo(oldFile, null, Log.getLog(), true); 587 588 Log.close(); 589 oldFile.close(); 590 } 591 592 static final SupplementalDataInfo supplementalData = SupplementalDataInfo 593 .getInstance(CLDRPaths.SUPPLEMENTAL_DIRECTORY); 594 static final StandardCodes sc = StandardCodes.make(); 595 596 public static void getSubtagVariables() { 597 // This section no longer necessary, as it has been replaced by the new attributeValueValidity.xml 598 // 599 // System.out.println("Validity variables"); 600 // System.out.println("Cut/paste into supplementalMetadata.xml under the line"); 601 // System.out.println("<!-- start of data generated with CountItems tool ..."); 602 // Map<String, String> variableSubstitutions = getVariables(VariableType.partial); 603 604 // for (Entry<String, String> type : variableSubstitutions.entrySet()) { 605 // System.out.println(type.getValue()); 606 // } 607 // System.out.println("<!-- end of Validity variables generated with CountItems tool ..."); 608 // System.out.println(); 609 System.out.println("Language aliases"); 610 System.out.println("Cut/paste into supplementalMetadata.xml under the line"); 611 System.out.println("<!-- start of data generated with CountItems tool ..."); 612 613 Map<String, Map<String, String>> languageReplacement = StandardCodes.getLStreg().get("language"); 614 Map<String, Map<String, R2<List<String>, String>>> localeAliasInfo = supplementalData.getLocaleAliasInfo(); 615 Map<String, R2<List<String>, String>> languageAliasInfo = localeAliasInfo.get("language"); 616 617 Set<String> available = Iso639Data.getAvailable(); 618 // <languageAlias type="aju" replacement="jrb"/> <!-- Moroccan Judeo-Arabic Judeo-Arabic --> 619 Set<String> bad3letter = new HashSet<String>(); 620 for (String lang : available) { 621 if (lang.length() != 2) continue; 622 String target = lang; 623 Map<String, String> lstregData = languageReplacement.get(lang); 624 if (lstregData == null) { 625 throw new IllegalArgumentException("illegal language code"); 626 } else { 627 String replacement = lstregData.get("Preferred-Value"); 628 if (replacement != null) { 629 target = replacement; 630 } 631 } 632 String alpha3 = Iso639Data.toAlpha3(lang); 633 bad3letter.add(alpha3); 634 String targetAliased; 635 if (languageAliasInfo.containsKey(target)) { 636 targetAliased = CollectionUtilities.join(languageAliasInfo.get(target).get0(), " "); 637 } else { 638 targetAliased = target; 639 } 640 System.out.println("\t\t\t<languageAlias type=\"" + alpha3 + "\" replacement=\"" + targetAliased 641 + "\" reason=\"overlong\"/> <!-- " + 642 Iso639Data.getNames(target) + " -->"); 643 } 644 System.out.println("\t\t\t<!-- Bibliographic -->"); 645 TreeMap<String, String> sorted = new TreeMap<>(); 646 for (String hasBiblio : Iso639Data.hasBiblio3()) { 647 String biblio = Iso639Data.toBiblio3(hasBiblio); 648 sorted.put(biblio, hasBiblio); 649 } 650 for (Entry<String, String> entry : sorted.entrySet()) { 651 String biblio = entry.getKey(); 652 String hasBiblio = entry.getValue(); 653 System.out.println("\t\t\t<languageAlias type=\"" + biblio + "\" replacement=\"" + hasBiblio 654 + "\" reason=\"bibliographic\"/> <!-- " + 655 Iso639Data.getNames(hasBiblio) + " -->"); 656 } 657 System.out.println("<!-- end of Language alises generated with CountItems tool ..."); 658 659 Set<String> encompassed = Iso639Data.getEncompassed(); 660 Set<String> macros = Iso639Data.getMacros(); 661 Map<String, String> encompassed_macro = new HashMap<String, String>(); 662 for (Entry<String, R2<List<String>, String>> typeAndData : languageAliasInfo.entrySet()) { 663 String type = typeAndData.getKey(); 664 R2<List<String>, String> data = typeAndData.getValue(); 665 List<String> replacements = data.get0(); 666 if (!encompassed.contains(type)) continue; 667 if (replacements == null || replacements.size() != 1) continue; 668 String replacement = replacements.get(0); 669 if (macros.contains(replacement)) { 670 // we have a match, encompassed => replacement 671 encompassed_macro.put(type, replacement); 672 } 673 } 674 Set<String> missing = new TreeSet<String>(); 675 missing.addAll(macros); 676 missing.remove("no"); 677 missing.remove("sh"); 678 679 missing.removeAll(encompassed_macro.values()); 680 if (missing.size() != 0) { 681 for (String missingMacro : missing) { 682 System.err.println("ERROR: Missing <languageAlias type=\"" + "???" + "\" replacement=\"" + missingMacro 683 + "\"/> <!-- ??? => " + 684 Iso639Data.getNames(missingMacro) + " -->"); 685 System.out.println("\tOptions for ???:"); 686 for (String enc : Iso639Data.getEncompassedForMacro(missingMacro)) { 687 System.out.println("\t" + enc + "\t// " + Iso639Data.getNames(enc)); 688 } 689 } 690 } 691 // verify that every macro language has a encompassed mapping to it 692 // and remember those codes 693 694 // verify that nobody contains a bad code 695 696 for (Entry<String, R2<List<String>, String>> typeAndData : languageAliasInfo.entrySet()) { 697 String type = typeAndData.getKey(); 698 List<String> replacements = typeAndData.getValue().get0(); 699 if (replacements == null) continue; 700 for (String replacement : replacements) { 701 if (bad3letter.contains(replacement)) { 702 System.err.println("ERROR: Replacement(s) for type=\"" + type + 703 "\" contains " + replacement + ", which should be: " + Iso639Data.fromAlpha3(replacement)); 704 } 705 } 706 } 707 708 // get the bad ISO codes 709 710 Factory cldrFactory = Factory.make(CLDRPaths.MAIN_DIRECTORY, ".*"); 711 CLDRFile english = cldrFactory.make("en", true); 712 713 Set<String> territories = new TreeSet<String>(); 714 Relation<String, String> containers = supplementalData.getTerritoryToContained(); 715 for (String region : sc.getAvailableCodes("territory")) { 716 if (containers.containsKey(region)) continue; 717 territories.add(region); 718 } 719 System.out.println(); 720 System.out.println("Territory aliases"); 721 System.out.println("Cut/paste into supplementalMetadata.xml under the line"); 722 System.out.println("<!-- start of data generated with CountItems tool ..."); 723 //final Map<String, R2<List<String>, String>> territoryAliasInfo = localeAliasInfo.get("territory"); 724 725 addRegions(english, territories, "alpha3", "EA,EU,IC".split(","), new Transform<String, String>() { 726 public String transform(String region) { 727 return IsoRegionData.get_alpha3(region); 728 } 729 }); 730 addRegions(english, territories, "numeric", "AC,CP,DG,EA,EU,IC,TA".split(","), new Transform<String, String>() { 731 public String transform(String region) { 732 return IsoRegionData.getNumeric(region); 733 } 734 }); 735 System.out.println("<!-- end of Territory alises generated with CountItems tool ..."); 736 System.out.println(); 737 System.out.println("Deprecated codes check (informational)"); 738 // check that all deprecated codes are in fact deprecated 739 Map<String, Map<String, Map<String, String>>> fullData = StandardCodes.getLStreg(); 740 741 checkCodes("language", sc, localeAliasInfo, fullData); 742 checkCodes("script", sc, localeAliasInfo, fullData); 743 checkCodes("territory", sc, localeAliasInfo, fullData); 744 System.out.println("End of Deprecated codes check..."); 745 746 // generate mapping equivalences 747 // { "aar", "aar", "aa" }, // Afar 748 // b, t, bcp47 749 System.out.println(); 750 System.out.println("Mapping equivalences - (Informational only...)"); 751 System.out.println("{ bib , tech , bcp47 }"); 752 753 Set<R3<String, String, String>> rows = new TreeSet<R3<String, String, String>>(); 754 for (String lang : Iso639Data.getAvailable()) { 755 String bib = Iso639Data.toBiblio3(lang); 756 String tech = Iso639Data.toAlpha3(lang); 757 R3<String, String, String> row = Row.of(tech, bib, lang); 758 rows.add(row); 759 } 760 for (R3<String, String, String> row : rows) { 761 String tech = row.get0(); 762 String bib = row.get1(); 763 String lang = row.get2(); 764 String name = Iso639Data.getNames(lang).iterator().next(); // english.getName(lang); 765 if ((bib != null && !lang.equals(bib)) || (tech != null && !lang.equals(tech))) { 766 System.out.println(" { \"" + bib + "\", \"" + tech + "\", \"" + lang + "\" }, // " + name); 767 } 768 } 769 System.out.println("End of Mapping equivalences..."); 770 771 // generate the codeMappings 772 // <codeMappings> 773 // <territoryCodes type="CS" numeric="891" alpha3="SCG" fips10="YI"/> 774 775 System.out.println(); 776 System.out.println("Code Mappings"); 777 System.out.println("Cut/paste into supplementaData.xml under the line"); 778 System.out.println("<!-- start of data generated with CountItems tool ..."); 779 List<String> warnings = new ArrayList<String>(); 780 territories.add("QO"); 781 territories.add("EU"); 782 // territories.add("MF"); 783 //Map<String, R2<List<String>, String>> territoryAliases = supplementalData.getLocaleAliasInfo().get("territory"); 784 Relation<String, String> numeric2region = Relation.of(new HashMap<String, Set<String>>(), TreeSet.class); 785 Relation<String, String> alpha32region = Relation.of(new HashMap<String, Set<String>>(), TreeSet.class); 786 for (String region : territories) { 787 String numeric = IsoRegionData.getNumeric(region); 788 String alpha3 = IsoRegionData.get_alpha3(region); 789 numeric2region.put(numeric, region); 790 alpha32region.put(alpha3, region); 791 } 792 793 System.out.println(" <codeMappings>"); 794 795 for (String region : territories) { 796 String numeric = IsoRegionData.getNumeric(region); 797 String alpha3 = IsoRegionData.get_alpha3(region); 798 String fips10 = IsoRegionData.get_fips10(region); 799 System.out.println(" <territoryCodes" 800 + " type=\"" + region + "\"" 801 + (numeric == null ? "" : " numeric=\"" + numeric + "\"") 802 + (alpha3 == null ? "" : " alpha3=\"" + alpha3 + "\"") 803 + (fips10 == null || fips10.equals(region) ? "" : " fips10=\"" + fips10 + "\"") 804 + "/>"); 805 } 806 System.out.println(" </codeMappings>"); 807 System.out.println("<!-- end of Code Mappings generated with CountItems tool ..."); 808 System.out.println(CollectionUtilities.join(warnings, CldrUtility.LINE_SEPARATOR)); 809 } 810 811 enum VariableType { 812 full, partial 813 } 814 815 public static Map<String, String> getVariables(VariableType variableType) { 816 String sep = CldrUtility.LINE_SEPARATOR + "\t\t\t\t"; 817 Map<String, String> variableSubstitutions = new LinkedHashMap<String, String>(); 818 for (String type : new String[] { "grandfathered", "territory", "script", "variant" }) { 819 Set<String> i; 820 i = (variableType == VariableType.full || type.equals("grandfathered")) ? sc.getAvailableCodes(type) : sc.getGoodAvailableCodes(type); 821 addVariable(variableSubstitutions, type, i, sep); 822 } 823 824 Relation<String, String> bcp47Keys = supplementalData.getBcp47Keys(); 825 Relation<R2<String, String>, String> aliases = supplementalData.getBcp47Aliases(); 826 for (String key : bcp47Keys.keySet()) { 827 Set<String> keyAliases = aliases.getAll(Row.of(key, "")); 828 Set<String> rawsubtypes = bcp47Keys.getAll(key); 829 TreeSet<String> subtypes = new TreeSet<String>(); 830 for (String subtype : rawsubtypes) { 831 Set<String> keySubtypeAliases = aliases.getAll(Row.of(key, subtype)); 832 if (keySubtypeAliases != null) { 833 subtypes.addAll(keySubtypeAliases); 834 } 835 } 836 subtypes.addAll(rawsubtypes); 837 String alias = (keyAliases == null ? key : keyAliases.iterator().next()) + "_XXX"; 838 addVariable(variableSubstitutions, alias, subtypes, sep); 839 } 840 return variableSubstitutions; 841 } 842 843 private static final Pattern BreakerPattern = PatternCache.get("([-_A-Za-z0-9])[-/+_A-Za-z0-9]*"); 844 845 private static void addVariable(Map<String, String> variableSubstitutions, String type, Set<String> sinput, 846 String sep) { 847 TreeSet<String> s = new TreeSet<String>(ROOT_PRIMARY_COLLATOR); 848 s.addAll(sinput); 849 850 StringBuffer b = new StringBuffer(); 851 for (String code : s) { 852 if (b.length() != 0) 853 b.append(' '); 854 b.append(code); 855 } 856 // "((?:[-+_A-Za-z0-9]+[/])+[A-Za-z0-9])[-+_A-Za-z0-9]*" 857 String broken = CldrUtility.breakLines(b, sep, BreakerPattern.matcher(""), 80); 858 assert (b.toString().equals(broken.replace(sep, " "))); 859 variableSubstitutions.put(type, "\t\t\t<variable id=\"$" + type 860 + "\" type=\"choice\">" + broken + CldrUtility.LINE_SEPARATOR + "\t\t\t</variable>"); 861 } 862 863 private static void checkCodes(String type, StandardCodes sc, 864 Map<String, Map<String, R2<List<String>, String>>> localeAliasInfo, 865 Map<String, Map<String, Map<String, String>>> fullData) { 866 Map<String, Map<String, String>> typeData = fullData.get("territory".equals(type) ? "region" : type); 867 Map<String, R2<List<String>, String>> aliasInfo = localeAliasInfo.get(type); 868 for (String code : sc.getAvailableCodes(type)) { 869 Map<String, String> subdata = typeData.get(code); 870 String deprecated = subdata.get("Deprecated"); 871 if (deprecated == null) continue; 872 String replacement = subdata.get("Preferred-Value"); 873 R2<List<String>, String> supplementalReplacements = aliasInfo.get(code); 874 if (supplementalReplacements == null) { 875 System.out.println("Deprecated in LSTR, but not in supplementalData: " + type + "\t" + code + "\t" 876 + replacement); 877 } 878 } 879 } 880 881 private static void addRegions(CLDRFile english, Set<String> availableCodes, String codeType, String[] exceptions, 882 Transform<String, String> trans) { 883 Set<String> missingRegions = new TreeSet<String>(); 884 Set<String> exceptionSet = new HashSet<String>(Arrays.asList(exceptions)); 885 List<String> duplicateDestroyer = new ArrayList<String>(); 886 for (String region : availableCodes) { 887 888 if (exceptionSet.contains(region)) continue; 889 String alpha3 = trans.transform(region); 890 if (alpha3 == null) { 891 missingRegions.add(region); 892 continue; 893 } 894 Map<String, R2<List<String>, String>> territoryAliasInfo = supplementalData.getLocaleAliasInfo().get("territory"); 895 String result; 896 if (territoryAliasInfo.containsKey(region)) { 897 result = CollectionUtilities.join(territoryAliasInfo.get(region).get0(), " "); 898 } else { 899 result = region; 900 } 901 String name = english.getName(CLDRFile.TERRITORY_NAME, result); 902 if (!(duplicateDestroyer.contains(alpha3 + result + name))) { 903 duplicateDestroyer.add(alpha3 + result + name); 904 System.out.println("\t\t\t<territoryAlias type=\"" + alpha3 + "\" replacement=\"" + result 905 + "\" reason=\"overlong\"/> <!-- " + name + " -->"); 906 } 907 } 908 for (String region : missingRegions) { 909 String name = english.getName(CLDRFile.TERRITORY_NAME, region); 910 System.err.println("ERROR: Missing " + codeType + " code for " + region + "\t" + name); 911 } 912 } 913 914 /** 915 * 916 */ 917 private static String toString(Collection aliases, String separator) { 918 StringBuffer result = new StringBuffer(); 919 boolean first = true; 920 for (Iterator<Object> it = aliases.iterator(); it.hasNext();) { 921 Object item = it.next(); 922 if (first) 923 first = false; 924 else 925 result.append(separator); 926 result.append(item); 927 } 928 return result.toString(); 929 } 930 931 public static void showZoneInfo() throws IOException { 932 StandardCodes sc = StandardCodes.make(); 933 Map<String, String> m = sc.getZoneLinkold_new(); 934 int i = 0; 935 System.out.println("/* Generated by org.unicode.cldr.tool.CountItems */"); 936 System.out.println(); 937 i = 0; 938 System.out.println("/* zoneID, canonical zoneID */"); 939 for (Iterator<String> it = m.keySet().iterator(); it.hasNext();) { 940 String old = it.next(); 941 String newOne = m.get(old); 942 System.out.println("{\"" + old + "\", \"" + newOne + "\"},"); 943 ++i; 944 } 945 System.out.println("/* Total: " + i + " */"); 946 947 System.out.println(); 948 i = 0; 949 System.out.println("/* All canonical zoneIDs */"); 950 for (Iterator<String> it = sc.getZoneData().keySet().iterator(); it.hasNext();) { 951 String old = it.next(); 952 System.out.println("\"" + old + "\","); 953 ++i; 954 } 955 System.out.println("/* Total: " + i + " */"); 956 957 Factory mainCldrFactory = Factory.make(CLDRPaths.COMMON_DIRECTORY + "main" 958 + File.separator, ".*"); 959 CLDRFile desiredLocaleFile = mainCldrFactory.make("root", true); 960 String temp = desiredLocaleFile 961 .getFullXPath("//ldml/dates/timeZoneNames/singleCountries"); 962 String singleCountriesList = new XPathParts(null, null).set(temp) 963 .findAttributes("singleCountries").get("list"); 964 Set<String> singleCountriesSet = new TreeSet<String>(CldrUtility.splitList(singleCountriesList, 965 ' ')); 966 967 Map<String, String> zone_countries = StandardCodes.make().getZoneToCounty(); 968 Map<String, Set<String>> countries_zoneSet = StandardCodes.make().getCountryToZoneSet(); 969 System.out.println(); 970 i = 0; 971 System.out.println("/* zoneID, country, isSingle */"); 972 for (Iterator<String> it = zone_countries.keySet().iterator(); it.hasNext();) { 973 String old = it.next(); 974 String newOne = zone_countries.get(old); 975 Set<String> s = countries_zoneSet.get(newOne); 976 String isSingle = (s != null && s.size() == 1 || singleCountriesSet 977 .contains(old)) ? "T" : "F"; 978 System.out.println("{\"" + old + "\", \"" + newOne + "\", \"" + isSingle 979 + "\"},"); 980 ++i; 981 } 982 System.out.println("/* Total: " + i + " */"); 983 984 if (true) 985 return; 986 987 Factory cldrFactory = Factory.make(CLDRPaths.MAIN_DIRECTORY, ".*"); 988 Map<Organization, Map<String, Level>> platform_locale_status = StandardCodes.make().getLocaleTypes(); 989 Map<String, Level> onlyLocales = platform_locale_status.get(Organization.ibm); 990 Set<String> locales = onlyLocales.keySet(); 991 CLDRFile english = cldrFactory.make("en", true); 992 for (Iterator<String> it = locales.iterator(); it.hasNext();) { 993 String locale = it.next(); 994 System.out.println(locale + "\t" + english.getName(locale) + "\t" 995 + onlyLocales.get(locale)); 996 } 997 } 998 999 static final NumberFormat decimal = NumberFormat.getNumberInstance(); 1000 static { 1001 decimal.setGroupingUsed(true); 1002 } 1003 1004 public static void countItems() { 1005 // CLDRKey.main(new String[]{"-mde.*"}); 1006 String dir = CldrUtility.getProperty("source", CLDRPaths.MAIN_DIRECTORY); 1007 Factory cldrFactory = Factory.make(dir, ".*"); 1008 countItems(cldrFactory, false); 1009 } 1010 1011 /** 1012 * @param cldrFactory 1013 * @param resolved 1014 */ 1015 private static int countItems(Factory cldrFactory, boolean resolved) { 1016 int count = 0; 1017 int resolvedCount = 0; 1018 Set<String> locales = cldrFactory.getAvailable(); 1019 Set<String> keys = new HashSet<String>(); 1020 Set<String> values = new HashSet<String>(); 1021 Set<String> fullpaths = new HashSet<String>(); 1022 Matcher alt = CLDRFile.ALT_PROPOSED_PATTERN.matcher(""); 1023 1024 Set<String> temp = new HashSet<String>(); 1025 for (Iterator<String> it = locales.iterator(); it.hasNext();) { 1026 String locale = it.next(); 1027 if (CLDRFile.isSupplementalName(locale)) 1028 continue; 1029 CLDRFile item = cldrFactory.make(locale, false); 1030 1031 temp.clear(); 1032 for (Iterator<String> it2 = item.iterator(); it2.hasNext();) { 1033 String path = it2.next(); 1034 if (alt.reset(path).matches()) { 1035 continue; 1036 } 1037 temp.add(path); 1038 keys.add(path); 1039 values.add(item.getStringValue(path)); 1040 fullpaths.add(item.getFullXPath(path)); 1041 } 1042 int current = temp.size(); 1043 1044 CLDRFile itemResolved = cldrFactory.make(locale, true); 1045 temp.clear(); 1046 CollectionUtilities.addAll(itemResolved.iterator(), temp); 1047 int resolvedCurrent = temp.size(); 1048 1049 System.out.println(locale + "\tPlain:\t" + current + "\tResolved:\t" 1050 + resolvedCurrent + "\tUnique Paths:\t" + keys.size() 1051 + "\tUnique Values:\t" + values.size() + "\tUnique Full Paths:\t" 1052 + fullpaths.size()); 1053 count += current; 1054 resolvedCount += resolvedCurrent; 1055 } 1056 System.out.println("Total Items\t" + decimal.format(count)); 1057 System.out 1058 .println("Total Resolved Items\t" + decimal.format(resolvedCount)); 1059 System.out.println("Unique Paths\t" + decimal.format(keys.size())); 1060 System.out.println("Unique Values\t" + decimal.format(values.size())); 1061 System.out 1062 .println("Unique Full Paths\t" + decimal.format(fullpaths.size())); 1063 return count; 1064 } 1065 1066 } 1067
