1 package org.unicode.cldr.test; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.PrintWriter; 6 import java.util.ArrayList; 7 import java.util.Arrays; 8 import java.util.Collection; 9 import java.util.Collections; 10 import java.util.Comparator; 11 import java.util.EnumSet; 12 import java.util.HashMap; 13 import java.util.HashSet; 14 import java.util.Iterator; 15 import java.util.List; 16 import java.util.Map; 17 import java.util.Set; 18 import java.util.TreeMap; 19 import java.util.TreeSet; 20 import java.util.regex.Matcher; 21 22 import org.unicode.cldr.draft.FileUtilities; 23 import org.unicode.cldr.test.CheckCLDR.CheckStatus; 24 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 25 import org.unicode.cldr.test.CheckCLDR.CompoundCheckCLDR; 26 import org.unicode.cldr.test.CheckCLDR.FormatDemo; 27 import org.unicode.cldr.test.CheckCLDR.Options; 28 import org.unicode.cldr.test.CheckCLDR.Phase; 29 import org.unicode.cldr.test.CheckCLDR.SimpleDemo; 30 import org.unicode.cldr.test.ExampleGenerator.ExampleContext; 31 import org.unicode.cldr.test.ExampleGenerator.ExampleType; 32 import org.unicode.cldr.tool.Option; 33 import org.unicode.cldr.tool.Option.Params; 34 import org.unicode.cldr.tool.ShowData; 35 import org.unicode.cldr.tool.TablePrinter; 36 import org.unicode.cldr.util.CLDRConfig; 37 import org.unicode.cldr.util.CLDRConfig.Environment; 38 import org.unicode.cldr.util.CLDRFile; 39 import org.unicode.cldr.util.CLDRFile.Status; 40 import org.unicode.cldr.util.CLDRPaths; 41 import org.unicode.cldr.util.CLDRTool; 42 import org.unicode.cldr.util.CldrUtility; 43 import org.unicode.cldr.util.Counter; 44 import org.unicode.cldr.util.CoverageInfo; 45 import org.unicode.cldr.util.Factory; 46 import org.unicode.cldr.util.LanguageTagParser; 47 import org.unicode.cldr.util.Level; 48 import org.unicode.cldr.util.LocaleIDParser; 49 import org.unicode.cldr.util.Organization; 50 import org.unicode.cldr.util.Pair; 51 import org.unicode.cldr.util.PathDescription; 52 import org.unicode.cldr.util.PathHeader; 53 import org.unicode.cldr.util.PatternCache; 54 import org.unicode.cldr.util.SimpleFactory; 55 import org.unicode.cldr.util.StandardCodes; 56 import org.unicode.cldr.util.StringId; 57 import org.unicode.cldr.util.SupplementalDataInfo; 58 import org.unicode.cldr.util.UnicodeSetPrettyPrinter; 59 import org.unicode.cldr.util.VoteResolver; 60 import org.unicode.cldr.util.VoteResolver.CandidateInfo; 61 import org.unicode.cldr.util.VoteResolver.UnknownVoterException; 62 import org.unicode.cldr.util.XMLSource; 63 64 import com.ibm.icu.dev.tool.UOption; 65 import com.ibm.icu.dev.util.ElapsedTimer; 66 import com.ibm.icu.impl.Relation; 67 import com.ibm.icu.impl.Row; 68 import com.ibm.icu.lang.UCharacter; 69 import com.ibm.icu.text.Collator; 70 import com.ibm.icu.text.UnicodeSet; 71 import com.ibm.icu.util.ULocale; 72 73 /** 74 * Console test for CheckCLDR. <br> 75 * Some common source directories: 76 * 77 * <pre> 78 * -s C:/cvsdata/unicode/cldr/incoming/vetted/main 79 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 80 * -s C:/cvsdata/unicode/cldr/incoming/proposed/main 81 * -s C:/cvsdata/unicode/cldr/testdata/main 82 * </pre> 83 * 84 * @author markdavis 85 * 86 */ 87 @CLDRTool(alias = "check", 88 description = "Run CheckCLDR against CLDR data") 89 public class ConsoleCheckCLDR { 90 public static boolean showStackTrace = false; 91 public static boolean errorsOnly = false; 92 static boolean SHOW_LOCALE = true; 93 static boolean SHOW_EXAMPLES = false; 94 // static PrettyPath prettyPathMaker = new PrettyPath(); 95 96 private static final int HELP1 = 0, 97 HELP2 = 1, 98 COVERAGE = 2, 99 EXAMPLES = 3, 100 FILE_FILTER = 4, 101 TEST_FILTER = 5, 102 DATE_FORMATS = 6, 103 ORGANIZATION = 7, 104 SHOWALL = 8, 105 PATH_FILTER = 9, 106 ERRORS_ONLY = 10, 107 CHECK_ON_SUBMIT = 11, 108 NO_ALIASES = 12, 109 SOURCE_DIRECTORY = 13, 110 USER = 14, 111 PHASE = 15, 112 GENERATE_HTML = 16, 113 VOTE_RESOLVE = 17, 114 ID_VIEW = 18, 115 SUBTYPE_FILTER = 19, 116 SOURCE_ALL = 20, 117 BAILEY = 21 118 // VOTE_RESOLVE2 = 21 119 ; 120 121 static final String SOURCE_DIRS = CLDRPaths.MAIN_DIRECTORY + "," + CLDRPaths.ANNOTATIONS_DIRECTORY + CLDRPaths.SEED_DIRECTORY; 122 123 enum MyOptions { 124 coverage(new Params().setHelp("Set the coverage: eg -c comprehensive") 125 .setMatch("comprehensive|modern|moderate|basic")), // UOption.REQUIRES_ARG 126 examples(new Params().setHelp("Turn on examples (actually a summary of the demo)") 127 .setFlag('x')), //, 'x', UOption.NO_ARG), 128 file_filter(new Params().setHelp("Pick the locales (files) to check: arg is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)") 129 .setDefault(".*").setMatch(".*")), //, 'f', UOption.REQUIRES_ARG).setDefault(".*"), 130 test_filter(new Params() 131 .setHelp("Filter the Checks: arg is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)") 132 .setDefault(".*").setMatch(".*")), //, 't', UOption.REQUIRES_ARG).setDefault(".*"), 133 date_formats(new Params().setHelp("Turn on special date format checks")), //, 'd', UOption.NO_ARG), 134 organization(new Params().setHelp("Organization: ibm, google, ....; Uses Locales.txt for to filter locales and set coverage levels") 135 .setDefault(".*").setMatch(".*")), //, 'o', UOption.REQUIRES_ARG), 136 showall(new Params().setHelp("Show all paths, including aliased").setFlag('a')), //, 'a', UOption.NO_ARG), 137 path_filter(new Params().setHelp("Pick the paths to check, eg -p.*languages.*") 138 .setDefault(".*").setMatch(".*")), //, 'p', UOption.REQUIRES_ARG).setDefault(".*"), 139 errors_only(new Params().setHelp("Show errors only (with -ef, only final processing errors)")), //, 'e', UOption.NO_ARG), 140 check_on_submit(new Params().setHelp("") 141 .setFlag('k')), //, 'k', UOption.NO_ARG), 142 noaliases(new Params().setHelp("No aliases")), //, 'n', UOption.NO_ARG), 143 source_directory(new Params().setHelp("Fully qualified source directories. (Conflicts with -S.)") 144 .setDefault(SOURCE_DIRS).setMatch(".*")), //, 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 145 user(new Params().setHelp("User, eg -uu148") 146 .setMatch(".*")), //, 'u', UOption.REQUIRES_ARG), 147 phase(new Params().setHelp("?") 148 .setMatch(Phase.class).setFlag('z')), //, 'z', UOption.REQUIRES_ARG), 149 generate_html(new Params().setHelp("Generate HTML-style chart in directory.") 150 .setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/").setMatch(".*")), //, 'g', UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 151 vote_resolution(new Params().setHelp("")), //, 'v', UOption.NO_ARG), 152 id_view(new Params().setHelp("")), //, 'i', UOption.NO_ARG), 153 subtype_filter(new Params().setHelp("error/warning subtype filter, eg unexpectedOrderOfEraYear") 154 .setDefault(".*").setMatch(".*").setFlag('y')), //, 'y', UOption.REQUIRES_ARG), 155 source_all(new Params().setHelp( 156 "Partially qualified directories. Standard subdirectories added if not specified (/main, /annotations, /subdivisions). (Conflicts with -s.)") 157 .setMatch(".*").setFlag('S').setDefault("common,seed,exemplars")), //, 'S', <changed>), 158 bailey(new Params().setHelp("check bailey values (" + CldrUtility.INHERITANCE_MARKER + ")")), //, 'b', UOption.NO_ARG) 159 exemplarError(new Params().setFlag('E').setHelp("include to force strict Exemplar check")); 160 161 // BOILERPLATE TO COPY 162 final Option option; 163 164 private MyOptions(Params params) { 165 option = new Option(this, params); 166 } 167 168 private static Option.Options myOptions = new Option.Options(); 169 static { 170 for (MyOptions option : MyOptions.values()) { 171 myOptions.add(option, option.option); 172 } 173 } 174 175 private static Set<String> parse(String[] args, boolean showArguments) { 176 return myOptions.parse(MyOptions.values()[0], args, true); 177 } 178 } 179 180 private static final UOption[] options = { 181 UOption.HELP_H(), 182 UOption.HELP_QUESTION_MARK(), 183 UOption.create("coverage", 'c', UOption.REQUIRES_ARG), 184 UOption.create("examples", 'x', UOption.NO_ARG), 185 UOption.create("file_filter", 'f', UOption.REQUIRES_ARG).setDefault(".*"), 186 UOption.create("test_filter", 't', UOption.REQUIRES_ARG).setDefault(".*"), 187 UOption.create("date_formats", 'd', UOption.NO_ARG), 188 UOption.create("organization", 'o', UOption.REQUIRES_ARG), 189 UOption.create("showall", 'a', UOption.NO_ARG), 190 UOption.create("path_filter", 'p', UOption.REQUIRES_ARG).setDefault(".*"), 191 UOption.create("errors_only", 'e', UOption.NO_ARG), 192 UOption.create("check-on-submit", 'k', UOption.NO_ARG), 193 UOption.create("noaliases", 'n', UOption.NO_ARG), 194 UOption.create("source_directory", 's', UOption.REQUIRES_ARG).setDefault(SOURCE_DIRS), 195 UOption.create("user", 'u', UOption.REQUIRES_ARG), 196 UOption.create("phase", 'z', UOption.REQUIRES_ARG), 197 UOption.create("generate_html", 'g', UOption.OPTIONAL_ARG).setDefault(CLDRPaths.CHART_DIRECTORY + "/errors/"), 198 UOption.create("vote resolution", 'v', UOption.NO_ARG), 199 UOption.create("id view", 'i', UOption.NO_ARG), 200 UOption.create("subtype_filter", 'y', UOption.REQUIRES_ARG), 201 UOption.create("source_all", 'S', UOption.OPTIONAL_ARG).setDefault("common,seed,exemplars"), 202 UOption.create("bailey", 'b', UOption.NO_ARG), 203 UOption.create("exemplarError", 'E', UOption.NO_ARG) 204 // UOption.create("vote resolution2", 'w', UOption.OPTIONAL_ARG).setDefault(Utility.BASE_DIRECTORY + 205 // "incoming/vetted/main/votes/"), 206 }; 207 208 private static final Comparator<String> baseFirstCollator = new Comparator<String>() { 209 LanguageTagParser languageTagParser1 = new LanguageTagParser(); 210 LanguageTagParser languageTagParser2 = new LanguageTagParser(); 211 212 public int compare(String o1, String o2) { 213 String ls1 = languageTagParser1.set(o1).getLanguageScript(); 214 String ls2 = languageTagParser2.set(o2).getLanguageScript(); 215 int result = ls1.compareTo(ls2); 216 if (result != 0) return result; 217 return o1.compareTo(o2); 218 } 219 }; 220 private static final boolean PATH_IN_COUNT = false; 221 222 /* 223 * TODO: unused? Should be used? 224 */ 225 private static String[] HelpMessage = { 226 "-h \t This message", 227 "-s \t Source directory, default = " + SOURCE_DIRS, 228 "-S common,seed\t Use common AND seed directories. ( Set CLDR_DIR, don't use this with -s. )\n", 229 "-fxxx \t Pick the locales (files) to check: xxx is a regular expression, eg -f fr, or -f fr.*, or -f (fr|en-.*)", 230 "-pxxx \t Pick the paths to check, eg -p(.*languages.*)", 231 "-cxxx \t Set the coverage: eg -c comprehensive or -c modern or -c moderate or -c basic", 232 "-txxx \t Filter the Checks: xxx is a regular expression, eg -t.*number.*. To check all BUT a given test, use the style -t ((?!.*CheckZones).*)", 233 "-oxxx \t Organization: ibm, google, ....; filters locales and uses Locales.txt for coverage tests", 234 "-x \t Turn on examples (actually a summary of the demo).", 235 "-d \t Turn on special date format checks", 236 "-a \t Show all paths", 237 "-e \t Show errors only (with -ef, only final processing errors)", 238 "-n \t No aliases", 239 "-u \t User, eg -uu148", 240 "-y \t error/warning subtype filter, eg unexpectedOrderOfEraYear", 241 "-b \t check bailey values (" + CldrUtility.INHERITANCE_MARKER + ")", 242 }; 243 244 static Counter<ErrorType> subtotalCount = new Counter<ErrorType>(true); // new ErrorCount(); 245 static Counter<ErrorType> totalCount = new Counter<ErrorType>(true); 246 247 /** 248 * This will be the test framework way of using these tests. It is preliminary for now. 249 * The Survey Tool will call setDisplayInformation, and getCheckAll. 250 * For each cldrfile, it will set the cldrFile. 251 * Then on each path in the file it will call check. 252 * Right now it doesn't work with resolved files, so just use unresolved ones. 253 * 254 * @param args 255 * @throws IOException 256 */ 257 public static void main(String[] args) throws IOException { 258 MyOptions.parse(args, true); 259 ElapsedTimer totalTimer = new ElapsedTimer(); 260 //CldrUtility.showOptions(args); 261 UOption.parseArgs(args, options); 262 // if (options[HELP1].doesOccur || options[HELP2].doesOccur) { 263 // for (int i = 0; i < HelpMessage.length; ++i) { 264 // System.out.println(HelpMessage[i]); 265 // } 266 // return; 267 // } 268 String factoryFilter = options[FILE_FILTER].value; 269 if (factoryFilter.equals("key")) { 270 factoryFilter = "(en|ru|nl|fr|de|it|pl|es|tr|th|ja|zh|ko|ar|bg|sr|uk|ca|hr|cs|da|fil|fi|hu|id|lv|lt|nb|pt|ro|sk|sl|sv|vi|el|he|fa|hi|am|af|et|is|ms|sw|zu|bn|mr|ta|eu|gl|ur|gu|kn|ml|te|zh_Hant|pt_PT|en_GB)"; 271 } 272 String checkFilter = options[TEST_FILTER].value; 273 String subtypeFilterString = options[SUBTYPE_FILTER].value; 274 EnumSet<Subtype> subtypeFilter = null; 275 if (subtypeFilterString != null) { 276 subtypeFilter = EnumSet.noneOf(Subtype.class); 277 Matcher m = PatternCache.get(subtypeFilterString).matcher(""); 278 for (Subtype value : Subtype.values()) { 279 if (m.reset(value.toString()).find() || m.reset(value.name()).find()) { 280 subtypeFilter.add(value); 281 } 282 } 283 if (subtypeFilter.size() == 0) { 284 System.err.println("No subtype match for " + subtypeFilterString); 285 return; 286 } 287 } 288 289 errorsOnly = options[ERRORS_ONLY].doesOccur; 290 // if ("f".equals(options[ERRORS_ONLY].value)) { 291 // CheckCLDR.finalErrorType = CheckStatus.warningType; 292 // } 293 294 SHOW_EXAMPLES = options[EXAMPLES].doesOccur; 295 boolean showAll = options[SHOWALL].doesOccur; 296 boolean checkFlexibleDates = options[DATE_FORMATS].doesOccur; 297 String pathFilterString = options[PATH_FILTER].value; 298 Matcher pathFilter = null; 299 if (!pathFilterString.equals(".*")) { 300 pathFilter = PatternCache.get(pathFilterString).matcher(""); 301 } 302 boolean checkOnSubmit = options[CHECK_ON_SUBMIT].doesOccur; 303 boolean noaliases = options[NO_ALIASES].doesOccur; 304 305 Level coverageLevel = null; 306 String coverageLevelInput = options[COVERAGE].value; 307 if (coverageLevelInput != null) { 308 coverageLevel = Level.get(coverageLevelInput); 309 if (coverageLevel == Level.UNDETERMINED) { 310 throw new IllegalArgumentException("-c" + coverageLevelInput + "\t is invalid: must be one of: " 311 + "basic,moderate,..."); 312 } 313 } 314 315 Organization organization = options[ORGANIZATION].value == null ? null : Organization.fromString(options[ORGANIZATION].value); 316 if (organization != null) { 317 Set<Organization> organizations = StandardCodes.make().getLocaleCoverageOrganizations(); 318 if (!organizations.contains(organization)) { 319 throw new IllegalArgumentException("-o" + organization + "\t is invalid: must be one of: " 320 + organizations); 321 } 322 } 323 final CLDRConfig cldrConf = CLDRConfig.getInstance(); 324 // set the envronment to UNITTEST as suggested 325 cldrConf.setEnvironment(Environment.UNITTEST); 326 // get the Phase from CLDRConfig object 327 final Phase phase; 328 // Phase phase = Phase.BUILD; 329 if (options[PHASE].doesOccur) { 330 String phaseVal = options[PHASE].value; 331 try { 332 // no null check for argument; if it is is null, Phase.forString would return the one from CLDRConfig 333 phase = Phase.forString(phaseVal); 334 } catch (IllegalArgumentException e) { 335 StringBuilder sb = new StringBuilder("Incorrect Phase value"); 336 if (phaseVal != null && !phaseVal.isEmpty()) { 337 sb.append(" '"); 338 sb.append(phaseVal); 339 sb.append("'"); 340 } 341 sb.append(": should be one of "); 342 for (Phase curPhase : Phase.values()) { 343 // implicitly does a toString; 344 sb.append(curPhase); 345 sb.append(", "); 346 } 347 int lastIdx = sb.lastIndexOf(","); 348 // remove the last comma, if it occurs 349 if (lastIdx > -1) { 350 String tmpBuf = sb.substring(0, lastIdx); 351 sb.setLength(0); 352 sb.append(tmpBuf); 353 } 354 sb.append("."); 355 // TODO: Reporting should be similar to an error (wrong parameter...), and not actually an Exception 356 throw new IllegalArgumentException(sb.toString(), e); 357 } 358 } else { 359 phase = cldrConf.getPhase(); 360 } 361 362 boolean baileyTest = options[BAILEY].doesOccur; 363 364 File sourceDirectories[] = null; 365 366 if (MyOptions.source_all.option.doesOccur()) { 367 if (MyOptions.source_directory.option.doesOccur()) { 368 throw new IllegalArgumentException("Don't use -s and -S together."); 369 } 370 sourceDirectories = cldrConf.addStandardSubdirectories(cldrConf.getCLDRDataDirectories(MyOptions.source_all.option.getValue())); 371 } else { 372 String[] sdirs = options[SOURCE_DIRECTORY].value.split(",\\s*"); 373 sourceDirectories = new File[sdirs.length]; 374 for (int i = 0; i < sdirs.length; ++i) { 375 sourceDirectories[i] = new File(CldrUtility.checkValidDirectory(sdirs[i], 376 "Fix with -s. Use -h for help.")); 377 } 378 } 379 380 if (options[GENERATE_HTML].doesOccur) { 381 coverageLevel = Level.MODERN; // reset 382 ErrorFile.generated_html_directory = options[GENERATE_HTML].value; 383 ErrorFile.generated_html_count = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "count.txt"); 384 // try { 385 // ErrorFile.voteFactory = CLDRFile.Factory.make(sourceDirectory + "../../proposed/main/", ".*"); 386 // } catch (RuntimeException e) { 387 // ErrorFile.voteFactory = null; 388 // } 389 // PrintWriter cssFile = FileUtilities.openUTF8Writer(generated_html_directory, "index.css"); 390 // Utility; 391 } 392 393 idView = options[ID_VIEW].doesOccur; 394 395 if (options[VOTE_RESOLVE].doesOccur) { 396 resolveVotesDirectory = CldrUtility.checkValidFile(CLDRPaths.BASE_DIRECTORY + "incoming/vetted/votes/", 397 true, null); 398 VoteResolver.setVoterToInfo(CldrUtility.checkValidFile(CLDRPaths.BASE_DIRECTORY 399 + "incoming/vetted/usersa/usersa.xml", false, null)); 400 voteResolver = new VoteResolver<String>(); 401 } 402 403 // check stuff 404 // Comparator cc = StandardCodes.make().getTZIDComparator(); 405 // System.out.println(cc.compare("Antarctica/Rothera", "America/Cordoba")); 406 // System.out.println(cc.compare("Antarctica/Rothera", "America/Indianapolis")); 407 408 String user = options[USER].value; 409 410 System.out.println("Source directories:\n"); 411 for (File f : sourceDirectories) { 412 System.out.println(" " + f.getPath() + "\t(" 413 + f.getCanonicalPath() + ")"); 414 } 415 // System.out.println("factoryFilter: " + factoryFilter); 416 // System.out.println("test filter: " + checkFilter); 417 // System.out.println("organization: " + organization); 418 // System.out.println("show examples: " + SHOW_EXAMPLES); 419 // System.out.println("phase: " + phase); 420 // System.out.println("path filter: " + pathFilterString); 421 // System.out.println("coverage level: " + coverageLevel); 422 // System.out.println("checking dates: " + checkFlexibleDates); 423 // System.out.println("only check-on-submit: " + checkOnSubmit); 424 // System.out.println("show all: " + showAll); 425 // System.out.println("errors only?: " + errorsOnly); 426 // System.out.println("generate error counts: " + ErrorFile.generated_html_directory); 427 // // System.out.println("vote directory: " + (ErrorFile.voteFactory == null ? null : 428 // // ErrorFile.voteFactory.getSourceDirectory())); 429 // System.out.println("resolve votes: " + resolveVotesDirectory); 430 // System.out.println("id view: " + idView); 431 // System.out.println("subtype filter: " + subtypeFilter); 432 433 // set up the test 434 Factory cldrFactory = SimpleFactory.make(sourceDirectories, factoryFilter) 435 .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 436 CompoundCheckCLDR checkCldr = CheckCLDR.getCheckAll(cldrFactory, checkFilter); 437 if (checkCldr.getFilteredTestList().size() == 0) { 438 throw new IllegalArgumentException("The filter doesn't match any tests."); 439 } 440 System.out.println("filtered tests: " + checkCldr.getFilteredTests()); 441 Factory backCldrFactory = Factory.make(CLDRPaths.MAIN_DIRECTORY, factoryFilter) 442 .setSupplementalDirectory(new File(CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 443 english = backCldrFactory.make("en", true); 444 445 checkCldr.setDisplayInformation(english); 446 checkCldr.setEnglishFile(english); 447 setExampleGenerator(new ExampleGenerator(english, english, CLDRPaths.SUPPLEMENTAL_DIRECTORY)); 448 PathShower pathShower = new PathShower(); 449 450 // call on the files 451 Set<String> locales = new TreeSet<String>(baseFirstCollator); 452 locales.addAll(cldrFactory.getAvailable()); 453 454 List<CheckStatus> result = new ArrayList<CheckStatus>(); 455 Set<PathHeader> paths = new TreeSet<PathHeader>(); // CLDRFile.ldmlComparator); 456 Map m = new TreeMap(); 457 // double testNumber = 0; 458 Map<String, String> options = new HashMap<String, String>(); 459 FlexibleDateFromCLDR fset = new FlexibleDateFromCLDR(); 460 Set<String> englishPaths = null; 461 462 Set<String> fatalErrors = new TreeSet<String>(); 463 464 showHeaderLine(); 465 466 supplementalDataInfo = SupplementalDataInfo.getInstance(CLDRPaths.SUPPLEMENTAL_DIRECTORY); 467 468 LocaleIDParser localeIDParser = new LocaleIDParser(); 469 String lastBaseLanguage = ""; 470 PathHeader.Factory pathHeaderFactory = PathHeader.getFactory(english); 471 472 final List<String> specialPurposeLocales = new ArrayList<String>(Arrays.asList("en_US_POSIX", "en_ZZ", "und", "und_ZZ")); 473 for (String localeID : locales) { 474 if (CLDRFile.isSupplementalName(localeID)) continue; 475 if (supplementalDataInfo.getDefaultContentLocales().contains(localeID)) { 476 System.out.println("# Skipping default content locale: " + localeID); 477 continue; 478 } 479 480 // We don't really need to check the POSIX locale, as it is a special purpose locale 481 if (specialPurposeLocales.contains(localeID)) { 482 System.out.println("# Skipping special purpose locale: " + localeID); 483 continue; 484 } 485 486 boolean isLanguageLocale = localeID.equals(localeIDParser.set(localeID).getLanguageScript()); 487 options.clear(); 488 489 if (MyOptions.exemplarError.option.doesOccur()) { 490 options.put(Options.Option.exemplarErrors.toString(), "true"); 491 } 492 493 // if the organization is set, skip any locale that doesn't have a value in Locales.txt 494 Level level = coverageLevel; 495 if (level == null) { 496 level = Level.BASIC; 497 } 498 if (organization != null) { 499 Map<String, Level> locale_status = StandardCodes.make().getLocaleToLevel(organization); 500 if (locale_status == null) continue; 501 level = locale_status.get(localeID); 502 if (level == null) continue; 503 if (level.compareTo(Level.BASIC) <= 0) continue; 504 } else if (!isLanguageLocale) { 505 // otherwise, skip all language locales 506 options.put(Options.Option.CheckCoverage_skip.getKey(), "true"); 507 } 508 509 // if (coverageLevel != null) options.put("CoverageLevel.requiredLevel", coverageLevel.toString()); 510 if (organization != null) options.put(Options.Option.CoverageLevel_localeType.getKey(), organization.toString()); 511 options.put(Options.Option.phase.getKey(), phase.toString()); 512 //options.put(Options.Option.SHOW_TIMES.getKey(), "true"); 513 514 if (SHOW_LOCALE) System.out.println(); 515 516 // options.put("CheckCoverage.requiredLevel","comprehensive"); 517 518 CLDRFile file; 519 CLDRFile englishFile = english; 520 CLDRFile parent = null; 521 522 ElapsedTimer timer = new ElapsedTimer(); 523 try { 524 file = cldrFactory.make(localeID, true); 525 if (ErrorFile.voteFactory != null) { 526 ErrorFile.voteFile = ErrorFile.voteFactory.make(localeID, true); 527 } 528 final String parentID = LocaleIDParser.getParent(localeID); 529 if (parentID != null) { 530 parent = cldrFactory.make(parentID, true); 531 } 532 //englishFile = cldrFactory.make("en", true); 533 } catch (RuntimeException e) { 534 fatalErrors.add(localeID); 535 System.out.println("FATAL ERROR: " + localeID); 536 e.printStackTrace(System.out); 537 continue; 538 } 539 540 // generate HTML if asked for 541 if (ErrorFile.generated_html_directory != null) { 542 String baseLanguage = localeIDParser.set(localeID).getLanguageScript(); 543 544 if (!baseLanguage.equals(lastBaseLanguage)) { 545 lastBaseLanguage = baseLanguage; 546 ErrorFile.openErrorFile(localeID, baseLanguage); 547 } 548 549 } 550 551 if (user != null) { 552 file = new CLDRFile.TestUser(file, user, isLanguageLocale); 553 if (parent != null) { 554 parent = new CLDRFile.TestUser(parent, user, isLanguageLocale); 555 } 556 } 557 checkCldr.setCldrFileToCheck(file, options, result); 558 559 subtotalCount.clear(); 560 561 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext();) { 562 CheckStatus status = it3.next(); 563 String statusString = status.toString(); // com.ibm.icu.impl.Utility.escape( 564 CheckStatus.Type statusType = status.getType(); 565 566 if (errorsOnly) { 567 if (!statusType.equals(CheckStatus.errorType)) continue; 568 } 569 570 if (subtypeFilter != null) { 571 if (!subtypeFilter.contains(status.getSubtype())) { 572 continue; 573 } 574 } 575 576 if (checkOnSubmit) { 577 if (!status.isCheckOnSubmit() || !statusType.equals(CheckStatus.errorType)) continue; 578 } 579 showValue(file, null, localeID, null, null, null, null, statusString, status.getSubtype(), null); 580 // showSummary(checkCldr, localeID, level, statusString); 581 } 582 paths.clear(); 583 // CollectionUtilities.addAll(file.iterator(pathFilter), paths); 584 CoverageInfo covInfo = cldrConf.getCoverageInfo(); 585 for (String path : file.fullIterable()) { 586 if (pathFilter != null && !pathFilter.reset(path).find()) { 587 continue; 588 } 589 if (coverageLevel != null) { 590 Level currentLevel = covInfo.getCoverageLevel(path, localeID); 591 if (currentLevel.compareTo(coverageLevel) > 0) { 592 continue; 593 } 594 } 595 paths.add(pathHeaderFactory.fromPath(path)); 596 } 597 // addPrettyPaths(file, pathFilter, prettyPathMaker, noaliases, false, paths); 598 // addPrettyPaths(file, file.getExtraPaths(), pathFilter, prettyPathMaker, noaliases, false, paths); 599 600 // also add the English paths 601 // CollectionUtilities.addAll(checkCldr.getDisplayInformation().iterator(pathFilter), paths); 602 // initialize the first time in. 603 if (englishPaths == null) { 604 englishPaths = new HashSet<String>(); 605 final CLDRFile displayFile = CheckCLDR.getDisplayInformation(); 606 addPrettyPaths(displayFile, pathFilter, pathHeaderFactory, noaliases, true, englishPaths); 607 addPrettyPaths(displayFile, displayFile.getExtraPaths(), pathFilter, pathHeaderFactory, noaliases, 608 true, englishPaths); 609 englishPaths = Collections.unmodifiableSet(englishPaths); // for robustness 610 } 611 // paths.addAll(englishPaths); 612 613 UnicodeSet missingExemplars = new UnicodeSet(); 614 UnicodeSet missingCurrencyExemplars = new UnicodeSet(); 615 if (checkFlexibleDates) { 616 fset.set(file); 617 } 618 pathShower.set(localeID); 619 620 // only create if we are going to use 621 ExampleGenerator exampleGenerator = SHOW_EXAMPLES ? new ExampleGenerator(file, englishFile, 622 CLDRPaths.DEFAULT_SUPPLEMENTAL_DIRECTORY) : null; 623 ExampleContext exampleContext = new ExampleContext(); 624 625 // Status pathStatus = new Status(); 626 int pathCount = 0; 627 Status otherPath = new Status(); 628 629 for (PathHeader pathHeader : paths) { 630 pathCount++; 631 String path = pathHeader.getOriginalPath(); 632 String prettyPath = pathHeader.toString().replace('\t', '|').replace(' ', '_'); 633 // String prettyPath = it2.next(); 634 // String path = prettyPathMaker.getOriginal(prettyPath); 635 // if (path == null) { 636 // prettyPathMaker.getOriginal(prettyPath); 637 // } 638 639 if (!showAll && !file.isWinningPath(path)) { 640 continue; 641 } 642 if (!isLanguageLocale && !baileyTest) { 643 final String sourceLocaleID = file.getSourceLocaleID(path, otherPath); 644 if (!localeID.equals(sourceLocaleID)) { 645 continue; 646 } 647 // also skip aliases 648 if (!path.equals(otherPath.pathWhereFound)) { 649 continue; 650 } 651 } 652 653 if (path.contains("@alt")) { 654 if (path.contains("proposed")) continue; 655 } 656 String value = file.getStringValue(path); 657 if (baileyTest) { 658 value = CldrUtility.INHERITANCE_MARKER; 659 } 660 String fullPath = file.getFullXPath(path); 661 662 String example = ""; 663 664 if (SHOW_EXAMPLES) { 665 example = ExampleGenerator.simplify(exampleGenerator.getExampleHtml(path, value, exampleContext, 666 ExampleType.NATIVE)); 667 showExamples(checkCldr, prettyPath, localeID, exampleGenerator, path, value, fullPath, example, 668 exampleContext); 669 // continue; // don't show problems 670 } 671 672 if (checkFlexibleDates) { 673 fset.checkFlexibles(path, value, fullPath); 674 } 675 676 if (path.contains("duration-century")) { 677 int debug = 0; 678 } 679 680 int limit = 1; 681 for (int jj = 0; jj < limit; ++jj) { 682 if (jj == 0) { 683 checkCldr.check(path, fullPath, value, new Options(options), result); 684 } else { 685 checkCldr.getExamples(path, fullPath, value, new Options(options), result); 686 } 687 688 boolean showedOne = false; 689 for (Iterator<CheckStatus> it3 = result.iterator(); it3.hasNext();) { 690 CheckStatus status = it3.next(); 691 String statusString = status.toString(); // com.ibm.icu.impl.Utility.escape( 692 CheckStatus.Type statusType = status.getType(); 693 if (errorsOnly && !statusType.equals(CheckStatus.errorType)) continue; 694 695 if (subtypeFilter != null) { 696 if (!subtypeFilter.contains(status.getSubtype())) { 697 continue; 698 } 699 } 700 if (checkOnSubmit) { 701 if (!status.isCheckOnSubmit() || !statusType.equals(status.errorType)) continue; 702 } 703 // pathShower.showHeader(path, value); 704 705 // System.out.print("Locale:\t" + getLocaleAndName(localeID) + "\t"); 706 if (statusType.equals(CheckStatus.demoType)) { 707 SimpleDemo d = status.getDemo(); 708 if (d != null && d instanceof FormatDemo) { 709 FormatDemo fd = (FormatDemo) d; 710 m.clear(); 711 // m.put("pattern", fd.getPattern()); 712 // m.put("input", fd.getRandomInput()); 713 if (d.processPost(m)) System.out.println("\tDemo:\t" + fd.getPlainText(m)); 714 } 715 continue; 716 } 717 showValue(file, prettyPath, localeID, example, path, value, fullPath, statusString, 718 status.getSubtype(), exampleContext); 719 showedOne = true; 720 721 Object[] parameters = status.getParameters(); 722 if (parameters != null) { 723 if (parameters.length >= 1 && status.getCause().getClass() == CheckForExemplars.class) { 724 try { 725 UnicodeSet set = new UnicodeSet(parameters[0].toString()); 726 if (status.getMessage().contains("currency")) { 727 missingCurrencyExemplars.addAll(set); 728 } else { 729 missingExemplars.addAll(set); 730 } 731 } catch (RuntimeException e) { 732 } // skip if not parseable as set 733 } 734 for (int i = 0; i < parameters.length; ++i) { 735 if (showStackTrace && parameters[i] instanceof Throwable) { 736 ((Throwable) parameters[i]).printStackTrace(); 737 } 738 } 739 } 740 // survey tool will use: if (status.hasHTMLMessage()) 741 // System.out.println(status.getHTMLMessage()); 742 } 743 if (!showedOne && phase != Phase.FINAL_TESTING) { 744 // if (fullPath != null && draftStatusMatcher.reset(fullPath).find() && 745 // localeID.equals(sourceLocaleID) && path.equals(otherPath.pathWhereFound)) { 746 // final String draftStatus = draftStatusMatcher.group(1); 747 // // see if value is same as parents, then skip 748 // String parentValue = parent == null ? null : parent.getStringValue(path); 749 // if (parentValue == null || !parentValue.equals(value)) { 750 // showValue(file, prettyPath, localeID, example, path, value, fullPath, draftStatus, 751 // Subtype.none, exampleContext); 752 // showedOne = true; 753 // } 754 // } 755 if (!showedOne && showAll) { 756 showValue(file, prettyPath, localeID, example, path, value, fullPath, "ok", Subtype.none, 757 exampleContext); 758 showedOne = true; 759 // pathShower.showHeader(path, value); 760 } 761 } 762 763 } 764 } 765 766 if (resolveVotesDirectory != null) { 767 LocaleVotingData.resolveErrors(localeID); 768 } 769 770 showSummary(checkCldr, localeID, level, "Items (including inherited):\t" + pathCount); 771 if (missingExemplars.size() != 0) { 772 missingExemplars.removeAll(new UnicodeSet("[[:Uppercase:]-[]]")); // remove uppercase #4670 773 if (missingExemplars.size() != 0) { 774 Collator col = Collator.getInstance(new ULocale(localeID)); 775 showSummary(checkCldr, localeID, level, "Total missing from general exemplars:\t" + new UnicodeSetPrettyPrinter() 776 .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT)) 777 .setSpaceComparator(col != null ? col : Collator.getInstance(ULocale.ROOT) 778 .setStrength2(Collator.PRIMARY)) 779 .setCompressRanges(true) 780 .format(missingExemplars)); 781 } 782 } 783 if (missingCurrencyExemplars.size() != 0) { 784 Collator col = Collator.getInstance(new ULocale(localeID)); 785 showSummary(checkCldr, localeID, level, "Total missing from currency exemplars:\t" 786 + new UnicodeSetPrettyPrinter() 787 .setOrdering(col != null ? col : Collator.getInstance(ULocale.ROOT)) 788 .setSpaceComparator(col != null ? col : Collator.getInstance(ULocale.ROOT) 789 .setStrength2(Collator.PRIMARY)) 790 .setCompressRanges(true) 791 .format(missingCurrencyExemplars)); 792 } 793 for (ErrorType type : subtotalCount.keySet()) { 794 showSummary(checkCldr, localeID, level, "Subtotal " + type + ":\t" + subtotalCount.getCount(type)); 795 } 796 if (checkFlexibleDates) { 797 fset.showFlexibles(); 798 } 799 if (SHOW_EXAMPLES) { 800 // ldml/dates/timeZoneNames/zone[@type="America/Argentina/San_Juan"]/exemplarCity 801 for (String zone : StandardCodes.make().getGoodAvailableCodes("tzid")) { 802 String path = "//ldml/dates/timeZoneNames/zone[@type=\"" + zone + "\"]/exemplarCity"; 803 // String prettyPath = prettyPathMaker.getPrettyPath(path, false); 804 PathHeader pathHeader = pathHeaderFactory.fromPath(path); 805 String prettyPath = pathHeader.toString().replace('\t', '|').replace(' ', '_'); 806 if (pathFilter != null && !pathFilter.reset(path).matches()) continue; 807 String fullPath = file.getStringValue(path); 808 if (fullPath != null) continue; 809 String example = ExampleGenerator.simplify(exampleGenerator.getExampleHtml(path, null, 810 exampleContext, ExampleType.NATIVE)); 811 showExamples(checkCldr, prettyPath, localeID, exampleGenerator, path, null, fullPath, example, 812 exampleContext); 813 } 814 } 815 System.out.println("# Elapsed time: " + timer); 816 System.out.flush(); 817 } 818 819 if (ErrorFile.errorFileWriter != null) { 820 ErrorFile.closeErrorFile(); 821 } 822 823 if (ErrorFile.generated_html_directory != null) { 824 ErrorFile.writeErrorCountsText(); 825 ErrorFile.writeErrorFileIndex(); 826 } 827 System.out.println(); 828 for (ErrorType type : totalCount.keySet()) { 829 System.out.println("# Total " + type + ":\t" + totalCount.getCount(type)); 830 } 831 832 System.out.println(); 833 System.out.println("# Total elapsed time: " + totalTimer); 834 if (fatalErrors.size() != 0) { 835 System.out.println("# FATAL ERRORS:"); 836 } 837 long errorCount = totalCount.getCount(ErrorType.error) + fatalErrors.size(); 838 if (errorCount != 0) { 839 // System.exit((int) errorCount); // cast is safe; we'll never have that many errors 840 System.out.println(); 841 System.out.println("<< FAILURE - Error count is " + errorCount + " . >>"); 842 System.exit(-1); 843 } else { 844 System.out.println(); 845 System.out.println("<< SUCCESS - No errors found. >>"); 846 } 847 checkCldr.handleFinish(); 848 } 849 850 static class LocaleVotingData { 851 private int disputedCount = 0; 852 Counter<Organization> missingOrganizationCounter = new Counter<Organization>(true); 853 Counter<Organization> goodOrganizationCounter = new Counter<Organization>(true); 854 Counter<Organization> conflictedOrganizations = new Counter<Organization>(true); 855 Counter<VoteResolver.Status> winningStatusCounter = new Counter<VoteResolver.Status>(true); 856 857 static Map<String, LocaleVotingData> localeToErrors = new HashMap<String, LocaleVotingData>(); 858 private static Map<Integer, String> idToPath; 859 860 public static void resolveErrors(String locale) { 861 localeToErrors.put(locale, new LocaleVotingData(locale)); 862 } 863 864 public LocaleVotingData(String locale) { 865 866 Map<Organization, VoteResolver.Level> orgToMaxVote = VoteResolver.getOrganizationToMaxVote(locale); 867 868 Map<Integer, Map<Integer, CandidateInfo>> info = VoteResolver 869 .getBaseToAlternateToInfo(resolveVotesDirectory + locale + ".xml"); 870 871 Map<String, Integer> valueToItem = new HashMap<String, Integer>(); 872 873 for (int basePath : info.keySet()) { 874 final Map<Integer, CandidateInfo> itemInfo = info.get(basePath); 875 876 // find the last release status and value 877 voteResolver.clear(); 878 valueToItem.clear(); 879 880 for (int item : itemInfo.keySet()) { 881 String itemValue = getValue(item); 882 valueToItem.put(itemValue, item); 883 884 CandidateInfo candidateInfo = itemInfo.get(item); 885 if (candidateInfo.oldStatus != null) { 886 voteResolver.setLastRelease(itemValue, candidateInfo.oldStatus); 887 } 888 voteResolver.add(itemValue); 889 for (int voter : candidateInfo.voters) { 890 try { 891 voteResolver.add(itemValue, voter); 892 } catch (UnknownVoterException e) { 893 // skip 894 } 895 } 896 } 897 898 EnumSet<Organization> basePathConflictedOrganizations = voteResolver.getConflictedOrganizations(); 899 conflictedOrganizations.addAll(basePathConflictedOrganizations, 1); 900 901 VoteResolver.Status winningStatus = voteResolver.getWinningStatus(); 902 String winningValue = voteResolver.getWinningValue(); 903 904 winningStatusCounter.add(winningStatus, 1); 905 906 if (winningStatus == VoteResolver.Status.approved) { 907 continue; 908 } 909 910 CandidateInfo candidateInfo = itemInfo.get(valueToItem.get(winningValue)); 911 Map<Organization, VoteResolver.Level> orgToMaxVoteHere = VoteResolver 912 .getOrganizationToMaxVote(candidateInfo.voters); 913 914 // if the winning item is less than contributed, record the organizations that haven't given their 915 // maximum vote to the winning item. 916 if (winningStatus.compareTo(VoteResolver.Status.contributed) < 0) { 917 // showPaths(basePath, itemInfo); 918 for (Organization org : orgToMaxVote.keySet()) { 919 VoteResolver.Level maxVote = orgToMaxVote.get(org); 920 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 921 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 922 missingOrganizationCounter.add(org, 1); 923 } 924 } 925 if (voteResolver.isDisputed()) { 926 disputedCount++; 927 String path = getIdToPath(basePath); 928 ErrorFile.addDataToErrorFile(locale, path, null, ErrorType.disputed, Subtype.none); 929 } 930 } else { 931 for (Organization org : orgToMaxVote.keySet()) { 932 VoteResolver.Level maxVote = orgToMaxVote.get(org); 933 VoteResolver.Level maxVoteHere = orgToMaxVoteHere.get(org); 934 if (maxVoteHere == null || maxVoteHere.compareTo(maxVote) < 0) { 935 } else { 936 goodOrganizationCounter.add(org, 1); 937 } 938 } 939 } 940 } 941 System.out.println(getLocaleAndName(locale) + "\tEnabled Organizations:\t" + orgToMaxVote); 942 if (disputedCount != 0) { 943 System.out.println(getLocaleAndName(locale) + "\tDisputed Items:\t" + disputedCount); 944 } 945 946 if (missingOrganizationCounter.size() > 0) { 947 System.out.println(getLocaleAndName(locale) + "\tMIA organizations:\t" + missingOrganizationCounter); 948 System.out 949 .println(getLocaleAndName(locale) + "\tConflicted organizations:\t" + conflictedOrganizations); 950 System.out.println(getLocaleAndName(locale) + "\tCool organizations!:\t" + goodOrganizationCounter); 951 } 952 System.out.println(getLocaleAndName(locale) + "\tOptimal Status:\t" + winningStatusCounter); 953 } 954 955 private static String getIdToPath(int basePath) { 956 if (idToPath == null) { 957 idToPath = VoteResolver.getIdToPath(resolveVotesDirectory + "xpathTable.xml"); 958 } 959 return idToPath.get(basePath); 960 } 961 962 public static LocaleVotingData get(String locale) { 963 return localeToErrors.get(locale); 964 } 965 966 int getDisputedCount() { 967 return disputedCount; 968 } 969 970 String getConflictedHTML() { 971 String result = conflictedOrganizations.toString(); 972 if (result.length() == 0) { 973 return ""; 974 } 975 result = result.substring(1, result.length() - 1); 976 result = result.replace(", ", "<br>"); 977 return result; 978 } 979 } 980 981 private static String getValue(int item) { 982 return String.valueOf(item); 983 } 984 985 static Matcher draftStatusMatcher = PatternCache.get("\\[@draft=\"(provisional|unconfirmed)\"]").matcher(""); 986 987 enum ErrorType { 988 ok, error, disputed, warning, core, posix, minimal, basic, moderate, modern, comprehensive, optional, contributed, provisional, unconfirmed, unknown; 989 static EnumSet<ErrorType> unapproved = EnumSet.range(ErrorType.contributed, ErrorType.unconfirmed); 990 static EnumSet<ErrorType> coverage = EnumSet.range(ErrorType.posix, ErrorType.optional); 991 static EnumSet<ErrorType> showInSummary = EnumSet.of( 992 ErrorType.error, ErrorType.warning, ErrorType.posix, ErrorType.minimal, ErrorType.basic); 993 994 static ErrorType fromStatusString(String statusString) { 995 ErrorType shortStatus = statusString.equals("ok") ? ErrorType.ok 996 : statusString.startsWith("Error") ? ErrorType.error 997 : statusString.equals("disputed") ? ErrorType.disputed 998 : statusString.startsWith("Warning") ? ErrorType.warning 999 : statusString.equals("contributed") ? ErrorType.contributed 1000 : statusString.equals("provisional") ? ErrorType.provisional 1001 : statusString.equals("unconfirmed") ? ErrorType.unconfirmed 1002 : ErrorType.unknown; 1003 if (shortStatus == ErrorType.unknown) { 1004 throw new IllegalArgumentException("Unknown error type: " + statusString); 1005 } else if (shortStatus == ErrorType.warning) { 1006 if (coverageMatcher.reset(statusString).find()) { 1007 shortStatus = ErrorType.valueOf(coverageMatcher.group(1)); 1008 } 1009 } 1010 return shortStatus; 1011 } 1012 }; 1013 1014 /* 1015 * static class ErrorCount implements Comparable<ErrorCount> { 1016 * private Counter<ErrorType> counter = new Counter<ErrorType>(); 1017 * 1018 * public int compareTo(ErrorCount o) { 1019 * // we don't really need a good comparison - aren't going to be sorting 1020 * return total() < o.total() ? -1 : total() > o.total() ? 1 : 0; 1021 * } 1022 * public long total() { 1023 * return counter.getTotal(); 1024 * } 1025 * public void clear() { 1026 * counter.clear(); 1027 * } 1028 * public Set<ErrorType> keySet() { 1029 * return counter.getKeysetSortedByKey(); 1030 * } 1031 * public long getCount(ErrorType input) { 1032 * return counter.getCount(input); 1033 * } 1034 * public void increment(ErrorType errorType) { 1035 * counter.add(errorType, 1); 1036 * } 1037 * } 1038 */ 1039 1040 static class ErrorFile { 1041 1042 private static final boolean SHOW_VOTING_INFO = false; 1043 public static CLDRFile voteFile; 1044 public static Factory voteFactory; 1045 1046 private static void openErrorFile(String localeID, String baseLanguage) throws IOException { 1047 htmlOpenedFileLocale = localeID; 1048 if (ErrorFile.errorFileWriter != null) { 1049 ErrorFile.closeErrorFile(); 1050 } 1051 ErrorFile.errorFileWriter = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, baseLanguage + ".html"); 1052 ErrorFile.errorFileTable = new TablePrinter(); 1053 errorFileCounter.clear(); 1054 ErrorFile.errorFileTable.setCaption("Problem Details") 1055 .addColumn("Problem").setCellAttributes("align=\"left\" class=\"{0}\"").setSortPriority(0) 1056 .setSpanRows(true) 1057 .setBreakSpans(true).setRepeatHeader(true).setHeaderCell(true) 1058 .addColumn("Subtype").setCellAttributes("align=\"left\" class=\"{1}\"").setSortPriority(1) 1059 .setSpanRows(true) 1060 .setBreakSpans(true).setRepeatHeader(true).setHeaderCell(true) 1061 .addColumn("Locale").setCellAttributes("class=\"{1}\"") 1062 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}\">{0}</a>").setSortPriority(2) 1063 .setSpanRows(true).setBreakSpans(true)//.setRepeatDivider(true) 1064 .addColumn("Name").setCellAttributes("class=\"{1}\"").setSpanRows(true) 1065 .setBreakSpans(true) 1066 // .addColumn("HIDDEN").setSortPriority(2).setHidden(true) 1067 .addColumn("Section").setCellAttributes("class=\"{1}\"").setSortPriority(3) 1068 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={3}&x={0}\">{0}</a>") 1069 .setSpanRows(true) 1070 .addColumn("Count").setCellAttributes("class=\"{1}\" align=\"right\""); 1071 // showLineHeaders(generated_html_table); 1072 // "<a href='http://unicode.org/cldr/apps/survey?_=" + locale + "'>" + locale + "</a>"; 1073 1074 ErrorFile.htmlOpenedFileLanguage = baseLanguage; 1075 showIndexHead("", localeID, ErrorFile.errorFileWriter); 1076 } 1077 1078 static TablePrinter errorFileTable = new TablePrinter(); 1079 static Counter<Row.R4<String, String, ErrorType, Subtype>> errorFileCounter = new Counter<Row.R4<String, String, ErrorType, Subtype>>( 1080 true); 1081 1082 private static void addDataToErrorFile(String localeID, String path, String value, ErrorType shortStatus, 1083 Subtype subType) { 1084 String section = path == null 1085 ? null 1086 : org.unicode.cldr.util.PathUtilities.xpathToMenu(path); 1087 if (section == null) { 1088 section = "general"; 1089 } 1090 if (voteFile != null) { 1091 String fullVotePath = voteFile.getFullXPath(path); 1092 String users = ""; 1093 } 1094 errorFileCounter.add( 1095 new Row.R4<String, String, ErrorType, Subtype>(localeID, section, shortStatus, subType), 1); 1096 ErrorFile.sectionToProblemsToLocaleToCount.add( 1097 new Row.R4<String, ErrorType, Subtype, String>(section, shortStatus, subType, localeID), 1); 1098 } 1099 1100 private static void closeErrorFile() { 1101 Set<String> locales = new TreeSet<String>(); 1102 for (Row.R4<String, String, ErrorType, Subtype> item : errorFileCounter.keySet()) { 1103 String localeID = item.get0(); 1104 locales.add(localeID); 1105 String section = item.get1(); 1106 ErrorType shortStatus = item.get2(); 1107 Subtype subtype = item.get3(); 1108 // final String prettyPath = path == null ? "general" : prettyPathMaker.getPrettyPath(path, true); 1109 // final String outputForm = path == null ? "general" : prettyPathMaker.getOutputForm(prettyPath); 1110 errorFileTable.addRow() 1111 .addCell(shortStatus) 1112 .addCell(subtype) 1113 .addCell(localeID) 1114 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1115 // .addCell(prettyPath) // menuPath == null ? "" : "<a href='" + link + "'>" + menuPath + "</a>" 1116 .addCell(section) // menuPath == null ? "" : "<a href='" + link + "'>" + menuPath + "</a>" 1117 .addCell(errorFileCounter.getCount(item)) 1118 // .addCell(ConsoleCheckCLDR.safeForHtml(path == null ? null : 1119 // ConsoleCheckCLDR.getEnglishPathValue(path))) 1120 // .addCell(ConsoleCheckCLDR.safeForHtml(value)) 1121 .finishRow(); 1122 } 1123 1124 if (SHOW_VOTING_INFO) { 1125 TablePrinter data = new TablePrinter().setCaption("Voting Information") 1126 .addColumn("Locale").setHeaderCell(true) 1127 .addColumn("Name").setHeaderCell(true) 1128 .addColumn("Organization") 1129 .addColumn("Missing") 1130 .addColumn("Conflicted") 1131 // .addColumn("Good") 1132 ; 1133 for (String localeID : locales) { 1134 // now the voting info 1135 LocaleVotingData localeVotingData = LocaleVotingData.localeToErrors.get(localeID); 1136 if (localeVotingData != null) { 1137 // find all the orgs with data 1138 EnumSet<Organization> orgs = EnumSet.noneOf(Organization.class); 1139 orgs.addAll(localeVotingData.missingOrganizationCounter.keySet()); 1140 orgs.addAll(localeVotingData.conflictedOrganizations.keySet()); 1141 orgs.addAll(localeVotingData.goodOrganizationCounter.keySet()); 1142 for (Organization org : orgs) { 1143 data.addRow() 1144 .addCell(ConsoleCheckCLDR.getLinkedLocale(localeID)) 1145 .addCell(ConsoleCheckCLDR.getLocaleName(localeID)) 1146 .addCell(org) 1147 .addCell(localeVotingData.missingOrganizationCounter.getCount(org)) 1148 .addCell(localeVotingData.conflictedOrganizations.getCount(org)) 1149 // .addCell(localeVotingData.goodOrganizationCounter.getCount(org)) 1150 .finishRow(); 1151 } 1152 } 1153 } 1154 ErrorFile.errorFileWriter.println(data.toTable()); 1155 ErrorFile.errorFileWriter.println("<p></p>"); 1156 } 1157 1158 // generated_html.println("<table border='1' style='border-collapse: collapse' bordercolor='#CCCCFF'>"); 1159 // Locale Group Error Warning Missing Votes: Contributed Missing Votes: Provisional Missing Votes: 1160 // Unconfirmed Missing Coverage: Posix Missing Coverage: Minimal Missing Coverage: Basic Missing Coverage: 1161 // Moderate Missing Coverage: Modern 1162 ErrorFile.errorFileWriter.println(ErrorFile.errorFileTable.toTable()); 1163 ErrorFile.errorFileWriter.println(ShowData.dateFooter()); 1164 ErrorFile.errorFileWriter.println(CldrUtility.ANALYTICS); 1165 ErrorFile.errorFileWriter.println("</body></html>"); 1166 ErrorFile.errorFileWriter.close(); 1167 ErrorFile.errorFileTable = null; 1168 } 1169 1170 // ================ Index File =================== 1171 1172 static void showErrorFileIndex(PrintWriter generated_html_index) { 1173 1174 // get organizations 1175 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1176 1177 TablePrinter indexTablePrinter = new TablePrinter().setCaption("Problem Summary") 1178 .setTableAttributes("border='1' style='border-collapse: collapse' bordercolor='blue'") 1179 .addColumn("BASE").setHidden(true)//.setRepeatDivider(true) 1180 .addColumn("Locale").setCellPattern("<a name=\"{0}\" href=\"{1}.html\">{0}</a>") // link to base, anchor 1181 // with full 1182 .addColumn("Name"); 1183 if (SHOW_VOTING_INFO) { 1184 indexTablePrinter.addColumn("Summary") 1185 .addColumn("Missing"); 1186 } 1187 for (Organization org : orgToLocales.keySet()) { 1188 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1189 } 1190 indexTablePrinter 1191 .addColumn("Disputed").setHeaderAttributes("class='disputed'").setCellAttributes("class='disputed'") 1192 .addColumn("Conflicted").setHeaderAttributes("class='conflicted'") 1193 .setCellAttributes("class='conflicted'"); 1194 1195 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1196 String columnTitle = UCharacter.toTitleCase(type.toString(), null); 1197 final boolean coverage = ConsoleCheckCLDR.ErrorType.coverage.contains(type); 1198 if (coverage) { 1199 columnTitle = "MC: " + columnTitle; 1200 } else if (ConsoleCheckCLDR.ErrorType.unapproved.contains(type)) { 1201 columnTitle = "MV: " + columnTitle; 1202 } 1203 indexTablePrinter.addColumn(columnTitle).setHeaderAttributes("class='" + type + "'") 1204 .setCellAttributes("class='" + type + "'"); 1205 } 1206 1207 // now fill in the data 1208 LanguageTagParser ltp = new LanguageTagParser(); 1209 for (String key : ErrorFile.errorFileIndexData.keySet()) { 1210 Pair<String, Counter<ErrorType>> pair = ErrorFile.errorFileIndexData.get(key); 1211 String htmlOpenedFileLanguage = pair.getFirst(); 1212 Counter<ErrorType> counts = pair.getSecond(); 1213 LocaleVotingData votingData = LocaleVotingData.get(htmlOpenedFileLanguage); 1214 if (counts.getTotal() == 0) { 1215 continue; 1216 } 1217 final String baseLanguage = ltp.set(htmlOpenedFileLanguage).getLanguage(); 1218 indexTablePrinter.addRow() 1219 .addCell(baseLanguage) 1220 .addCell(htmlOpenedFileLanguage) 1221 .addCell(ConsoleCheckCLDR.getLocaleName(htmlOpenedFileLanguage)); 1222 if (SHOW_VOTING_INFO) { 1223 indexTablePrinter.addCell(votingData == null ? "" : votingData.winningStatusCounter.toString()) 1224 .addCell(votingData == null ? "" : votingData.missingOrganizationCounter.toString()); 1225 } 1226 for (Organization org : orgToLocales.keySet()) { 1227 indexTablePrinter.addCell(orgToLocales.getAll(org).contains(htmlOpenedFileLanguage) ? org.toString() 1228 .substring(0, 2) : ""); 1229 } 1230 indexTablePrinter 1231 .addCell(votingData == null ? "" : formatSkippingZero(votingData.getDisputedCount())) 1232 .addCell(votingData == null ? "" : votingData.getConflictedHTML()); 1233 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1234 indexTablePrinter.addCell(formatSkippingZero(counts.getCount(type))); 1235 } 1236 indexTablePrinter.finishRow(); 1237 } 1238 generated_html_index.println(indexTablePrinter.toTable()); 1239 generated_html_index.println(ShowData.dateFooter()); 1240 generated_html_index.println(CldrUtility.ANALYTICS); 1241 generated_html_index.println("</body></html>"); 1242 } 1243 1244 static Relation<Organization, String> orgToLocales; 1245 1246 private static Relation<Organization, String> getOrgToLocales() { 1247 if (orgToLocales == null) { 1248 orgToLocales = Relation.of(new TreeMap<Organization, Set<String>>(), TreeSet.class); 1249 StandardCodes sc = StandardCodes.make(); 1250 for (Organization org : sc.getLocaleCoverageOrganizations()) { 1251 for (String locale : sc.getLocaleCoverageLocales(org)) { 1252 Level x = sc.getLocaleCoverageLevel(org, locale); 1253 if (x.compareTo(Level.BASIC) > 0) { 1254 orgToLocales.put(org, locale); 1255 } 1256 } 1257 } 1258 } 1259 return orgToLocales; 1260 } 1261 1262 static void showSections() throws IOException { 1263 Relation<Organization, String> orgToLocales = getOrgToLocales(); 1264 TablePrinter indexTablePrinter = new TablePrinter().setCaption("Problem Summary") 1265 .setTableAttributes("border='1' style='border-collapse: collapse' bordercolor='blue'") 1266 .addColumn("Section").setSpanRows(true).setBreakSpans(true)//.setRepeatDivider(true) 1267 .addColumn("Problems").setCellAttributes("style=\"text-align:left\" class=\"{2}\"").setSpanRows(true) 1268 .addColumn("Subtype").setCellAttributes("style=\"text-align:left\" class=\"{2}\"").setSpanRows(true) 1269 .addColumn("Locale").setCellAttributes("class=\"{2}\"") 1270 .addColumn("Code").setCellAttributes("class=\"{2}\"") 1271 .setCellPattern("<a href=\"http://unicode.org/cldr/apps/survey?_={0}&x={1}\">{0}</a>") // TODO: use CLDRConfig.urls() 1272 .addColumn("Count").setCellAttributes("class=\"{2}\""); 1273 for (Organization org : orgToLocales.keySet()) { 1274 indexTablePrinter.addColumn(org.toString().substring(0, 2)); 1275 } 1276 1277 for (Row.R4<String, ErrorType, Subtype, String> sectionAndProblemsAndLocale : ErrorFile.sectionToProblemsToLocaleToCount 1278 .getKeysetSortedByKey()) { 1279 final ErrorType problem = sectionAndProblemsAndLocale.get1(); 1280 final Subtype subtype = sectionAndProblemsAndLocale.get2(); 1281 if (!ConsoleCheckCLDR.ErrorType.showInSummary.contains(problem)) { 1282 continue; 1283 } 1284 final String locale = sectionAndProblemsAndLocale.get3(); 1285 if (problem != ErrorType.error && problem != ErrorType.disputed && !orgToLocales.containsValue(locale)) { 1286 continue; 1287 } 1288 long count = ErrorFile.sectionToProblemsToLocaleToCount.getCount(sectionAndProblemsAndLocale); 1289 final String section = sectionAndProblemsAndLocale.get0(); 1290 indexTablePrinter.addRow() 1291 .addCell(section) 1292 .addCell(problem) 1293 .addCell(subtype) 1294 .addCell(ConsoleCheckCLDR.getLocaleName(locale)) 1295 .addCell(locale) 1296 .addCell(count); 1297 for (Organization org : orgToLocales.keySet()) { 1298 indexTablePrinter.addCell(orgToLocales.getAll(org).contains(locale) ? org.toString().substring(0, 2) : ""); 1299 } 1300 indexTablePrinter.finishRow(); 1301 } 1302 PrintWriter generated_html_index = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "sections.html"); 1303 ConsoleCheckCLDR.ErrorFile.showIndexHead("Error Report Index by Section", "", generated_html_index); 1304 generated_html_index.println(indexTablePrinter.toTable()); 1305 generated_html_index.println(ShowData.dateFooter()); 1306 generated_html_index.println(CldrUtility.ANALYTICS); 1307 generated_html_index.println("</body></html>"); 1308 generated_html_index.close(); 1309 } 1310 1311 static String formatSkippingZero(long count) { 1312 if (count == 0) { 1313 return ""; 1314 } 1315 return String.valueOf(count); 1316 } 1317 1318 static void showIndexHead(String title, String localeID, PrintWriter generated_html_index) { 1319 final boolean notLocaleSpecific = localeID.length() == 0; 1320 if ((!notLocaleSpecific)) { 1321 title = "Errors in " + ConsoleCheckCLDR.getNameAndLocale(localeID, false); 1322 } 1323 generated_html_index 1324 .println("<html>" + 1325 "<head><meta http-equiv='Content-Type' content='text/html; charset=utf-8'>" 1326 + CldrUtility.LINE_SEPARATOR 1327 + 1328 "<title>" 1329 + title 1330 + "</title>" 1331 + CldrUtility.LINE_SEPARATOR 1332 + 1333 "<link rel='stylesheet' href='errors.css' type='text/css'>" 1334 + CldrUtility.LINE_SEPARATOR 1335 + 1336 "<base target='_blank'>" 1337 + CldrUtility.LINE_SEPARATOR 1338 + 1339 "</head><body>" 1340 + CldrUtility.LINE_SEPARATOR 1341 + 1342 "<h1>" 1343 + title 1344 + "</h1>" 1345 + CldrUtility.LINE_SEPARATOR 1346 + 1347 "<p>" 1348 + 1349 "<a href='index.html" 1350 + (notLocaleSpecific ? "" : "#" + localeID) 1351 + "'>Index</a>" 1352 + 1353 " | " 1354 + 1355 "<a href='sections.html" 1356 + (notLocaleSpecific ? "" : "#" + localeID) 1357 + "'>Index by Section</a>" 1358 + 1359 " | " 1360 + 1361 "<a href='http://unicode.org/cldr/data/docs/survey/vetting.html'><b style='background-color: yellow;'><i>Help: How to Vet</i></b></a>" 1362 + 1363 "</p>" 1364 + 1365 "<p>The following errors have been detected in the locale" 1366 + 1367 (notLocaleSpecific 1368 ? "s. " + org.unicode.cldr.test.HelpMessages.getChartMessages("error_index_header") 1369 : " " + ConsoleCheckCLDR.getNameAndLocale(localeID, false) + ". " 1370 + ErrorFile.ERROR_CHART_HEADER)); 1371 } 1372 1373 private static void writeErrorFileIndex() throws IOException { 1374 PrintWriter generated_html_index = FileUtilities.openUTF8Writer(ErrorFile.generated_html_directory, "index.html"); 1375 ConsoleCheckCLDR.ErrorFile.showIndexHead("Error Report Index", "", generated_html_index); 1376 ConsoleCheckCLDR.ErrorFile.showErrorFileIndex(generated_html_index); 1377 generated_html_index.close(); 1378 showSections(); 1379 } 1380 1381 private static void writeErrorCountsText() { 1382 // if (ErrorFile.htmlErrorsPerLocale.total() != 0) { 1383 1384 // do the plain text file 1385 ErrorFile.generated_html_count.print(ConsoleCheckCLDR.lastHtmlLocaleID + ";\tcounts"); 1386 for (ConsoleCheckCLDR.ErrorType type : ConsoleCheckCLDR.ErrorType.showInSummary) { 1387 ErrorFile.generated_html_count.print(";\t" + type + "=" + ErrorFile.htmlErrorsPerLocale.getCount(type)); 1388 } 1389 ErrorFile.generated_html_count.println(); 1390 ErrorFile.generated_html_count.flush(); 1391 1392 // now store the data for the index 1393 ErrorFile.errorFileIndexData.put(ConsoleCheckCLDR.lastHtmlLocaleID, 1394 new Pair<String, Counter<ErrorType>>(ConsoleCheckCLDR.lastHtmlLocaleID, ErrorFile.htmlErrorsPerLocale)); 1395 ErrorFile.htmlErrorsPerLocale = new Counter<ErrorType>(); 1396 // } 1397 } 1398 1399 /* 1400 * static Counter<Organization> missingOrganizationCounter = new Counter<Organization>(true); 1401 * static Counter<Organization> goodOrganizationCounter = new Counter<Organization>(true); 1402 * static Counter<Organization> conflictedOrganizations = new Counter<Organization>(true); 1403 * static Counter<VoteResolver.Status> winningStatusCounter = new Counter<VoteResolver.Status>(true); 1404 */ 1405 1406 static Counter<ErrorType> htmlErrorsPerLocale = new Counter<ErrorType>(); // ConsoleCheckCLDR.ErrorCount(); 1407 static PrintWriter generated_html_count = null; 1408 private static TreeMap<String, Pair<String, Counter<ErrorType>>> errorFileIndexData = new TreeMap<String, Pair<String, Counter<ErrorType>>>(); 1409 1410 // private static ConsoleCheckCLDR.ErrorCount htmlErrorsPerBaseLanguage = new ConsoleCheckCLDR.ErrorCount(); 1411 static PrintWriter errorFileWriter = null; 1412 private static String htmlOpenedFileLanguage = null; 1413 private static String htmlOpenedFileLocale = null; 1414 private static final String ERROR_CHART_HEADER = org.unicode.cldr.test.HelpMessages 1415 .getChartMessages("error_locale_header"); 1416 // "Please review and correct them. " + 1417 // "Note that errors in <i>sublocales</i> are often fixed by fixing the main locale.</p>" + 1418 // Utility.LINE_SEPARATOR + 1419 // "<p><i>This list is only generated daily, and so may not reflect fixes you have made until tomorrow. " + 1420 // "(There were production problems in integrating it fully into the Survey tool. " + 1421 // "However, it should let you see the problems and make sure that they get taken care of.)</i></p>" + 1422 // "<p>Coverage depends on your organizations goals: the highest tier languages should include up through all Modern values.</p>" 1423 // + Utility.LINE_SEPARATOR; 1424 static String generated_html_directory = null; 1425 public static Counter<Row.R4<String, ErrorType, Subtype, String>> sectionToProblemsToLocaleToCount = new Counter<Row.R4<String, ErrorType, Subtype, String>>(); 1426 } 1427 1428 private static void showSummary(CheckCLDR checkCldr, String localeID, Level level, String value) { 1429 String line = "# " + getLocaleAndName(localeID) + "\tSummary\t" + level + "\t" + value; 1430 System.out.println(line); 1431 // if (generated_html != null) { 1432 // line = TransliteratorUtilities.toHTML.transform(line); 1433 // line = line.replace("\t", "</td><td>"); 1434 // generated_html.println("<table><tr><td>" + line + "</td></tr></table>"); 1435 // } 1436 } 1437 1438 private static void showExamples(CheckCLDR checkCldr, String prettyPath, String localeID, 1439 ExampleGenerator exampleGenerator, String path, String value, String fullPath, String example, 1440 ExampleContext exampleContext) { 1441 if (example != null) { 1442 showValue(checkCldr.getCldrFileToCheck(), prettyPath, localeID, example, path, value, fullPath, "ok", 1443 Subtype.none, exampleContext); 1444 } 1445 } 1446 1447 private static void addPrettyPaths(CLDRFile file, Matcher pathFilter, PathHeader.Factory pathHeaderFactory, 1448 boolean noaliases, boolean filterDraft, Collection<String> target) { 1449 // Status pathStatus = new Status(); 1450 for (Iterator<String> pit = file.iterator(pathFilter); pit.hasNext();) { 1451 String path = pit.next(); 1452 if (file.isPathExcludedForSurvey(path)) { 1453 continue; 1454 } 1455 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1456 } 1457 } 1458 1459 private static void addPrettyPaths(CLDRFile file, Collection<String> paths, Matcher pathFilter, 1460 PathHeader.Factory pathHeaderFactory, boolean noaliases, boolean filterDraft, Collection<String> target) { 1461 // Status pathStatus = new Status(); 1462 for (String path : paths) { 1463 if (pathFilter != null && !pathFilter.reset(path).matches()) continue; 1464 addPrettyPath(file, pathHeaderFactory, noaliases, filterDraft, target, path); 1465 } 1466 } 1467 1468 private static void addPrettyPath(CLDRFile file, PathHeader.Factory pathHeaderFactory, boolean noaliases, 1469 boolean filterDraft, Collection<String> target, String path) { 1470 if (noaliases && XMLSource.Alias.isAliasPath(path)) { // this is just for console testing, the survey tool 1471 // shouldn't do it. 1472 return; 1473 // file.getSourceLocaleID(path, pathStatus); 1474 // if (!path.equals(pathStatus.pathWhereFound)) { 1475 // continue; 1476 // } 1477 } 1478 if (filterDraft) { 1479 String newPath = CLDRFile.getNondraftNonaltXPath(path); 1480 if (!newPath.equals(path)) { 1481 String value = file.getStringValue(newPath); 1482 if (value != null) { 1483 return; 1484 } 1485 } 1486 } 1487 String prettyPath = pathHeaderFactory.fromPath(path).toString(); // prettyPathMaker.getPrettyPath(path, true); 1488 // // get sortable version 1489 target.add(prettyPath); 1490 } 1491 1492 public static synchronized void setDisplayInformation(CLDRFile inputDisplayInformation, 1493 ExampleGenerator inputExampleGenerator) { 1494 CheckCLDR.setDisplayInformation(inputDisplayInformation); 1495 englishExampleGenerator = inputExampleGenerator; 1496 } 1497 1498 public static synchronized void setExampleGenerator(ExampleGenerator inputExampleGenerator) { 1499 englishExampleGenerator = inputExampleGenerator; 1500 } 1501 1502 public static synchronized ExampleGenerator getExampleGenerator() { 1503 return englishExampleGenerator; 1504 } 1505 1506 private static ExampleGenerator englishExampleGenerator; 1507 private static Object lastLocaleID = null; 1508 1509 static Matcher coverageMatcher = PatternCache.get("meet ([a-z]*) coverage").matcher(""); // HACK TODO fix 1510 1511 private static void showHeaderLine() { 1512 if (SHOW_LOCALE) { 1513 if (idView) { 1514 System.out 1515 .println("Locale\tID\tDesc.\tEng.Value\tEng.Ex.\tLoc.Value\tLoc.Ex\terror/warning type\tError/Warning Msg"); 1516 } else { 1517 System.out 1518 .println( 1519 "Locale\tStatus\tPPath\tEng.Value\tEng.Ex.\tLoc.Value\tfill-in\tLoc.Ex\terror/warning type\tError/Warning Msg\tFull Path\tAliasedSource/Path?"); 1520 } 1521 } 1522 } 1523 1524 private static PathDescription pathDescription = null; 1525 1526 private static String getIdString(CLDRFile cldrFile, String path, String value) { 1527 if (pathDescription == null) { 1528 pathDescription = new PathDescription(supplementalDataInfo, english, null, null, 1529 PathDescription.ErrorHandling.CONTINUE); 1530 } 1531 final String description = pathDescription.getDescription(path, value, null, null); 1532 return "\t" + StringId.getId(path) + "" + "\t" + description + ""; 1533 } 1534 1535 private static void showValue(CLDRFile cldrFile, String prettyPath, String localeID, String example, 1536 String path, String value, String fullPath, String statusString, 1537 Subtype subType, ExampleContext exampleContext) { 1538 ErrorType shortStatus = ErrorType.fromStatusString(statusString); 1539 subtotalCount.add(shortStatus, 1); 1540 totalCount.add(shortStatus, 1); 1541 if (subType == null) { 1542 subType = Subtype.none; 1543 } 1544 1545 if (ErrorFile.errorFileWriter == null) { 1546 example = example == null ? "" : example; 1547 String englishExample = null; 1548 final String englishPathValue = path == null ? null : getEnglishPathValue(path); 1549 if (SHOW_EXAMPLES && path != null) { 1550 englishExample = ExampleGenerator.simplify(getExampleGenerator().getExampleHtml(path, englishPathValue, 1551 exampleContext, ExampleType.ENGLISH)); 1552 } 1553 englishExample = englishExample == null ? "" : englishExample; 1554 String cleanPrettyPath = path == null ? null : prettyPath; // prettyPathMaker.getOutputForm(prettyPath); 1555 Status status = new Status(); 1556 String sourceLocaleID = path == null ? null : cldrFile.getSourceLocaleID(path, status); 1557 String fillinValue = path == null ? null : cldrFile.getFillInValue(path); 1558 fillinValue = fillinValue == null ? "" : fillinValue.equals(value) ? "=" : fillinValue; 1559 1560 final String otherSource = path == null ? null 1561 : (sourceLocaleID.equals(localeID) ? "" 1562 : "\t" + sourceLocaleID); 1563 final String otherPath = path == null ? null 1564 : (status.pathWhereFound.equals(path) ? "" 1565 : "\t" + status.pathWhereFound); 1566 1567 String idViewString = idView ? (path == null ? "\tNO_ID" : getIdString(cldrFile, path, value)) : ""; 1568 System.out.println( 1569 getLocaleAndName(localeID) 1570 + (idViewString.isEmpty() ? 1571 // + "\t" + subtotalCount.getCount(shortStatus) 1572 "\t" + shortStatus 1573 + "\t" + cleanPrettyPath + "" 1574 + "\t" + englishPathValue + "" 1575 + "\t" + englishExample + "" 1576 + "\t" + value + "" 1577 + "\t" + fillinValue + "" 1578 + "\t" + example + "" 1579 + "\t" + subType + "" 1580 + "\t" + statusString + "" 1581 + "\t" + fullPath 1582 + otherSource 1583 + otherPath 1584 : idViewString 1585 + "\t" + englishPathValue + "" 1586 + "\t" + englishExample + "" 1587 + "\t" + value + "" 1588 + "\t" + example + "" 1589 + "\t" + subType + "" 1590 + "\t" + statusString + "")); 1591 } else if (ErrorFile.errorFileWriter != null) { 1592 if (shortStatus == ErrorType.contributed) { 1593 return; 1594 } 1595 if (shortStatus == ErrorType.posix) { 1596 shortStatus = ErrorType.minimal; 1597 } 1598 if (!localeID.equals(lastHtmlLocaleID)) { 1599 ErrorFile.writeErrorCountsText(); 1600 // startGeneratedTable(generated_html, generated_html_table); 1601 lastHtmlLocaleID = localeID; 1602 } 1603 addError(localeID, path, shortStatus); 1604 // ErrorFile.htmlErrorsPerBaseLanguage.increment(shortStatus); 1605 1606 // String menuPath = path == null ? null : PathUtilities.xpathToMenu(path); 1607 // String link = path == null ? null : "http://unicode.org/cldr/apps/survey?_=" + localeID + "&x=" + 1608 // menuPath; 1609 ErrorFile.addDataToErrorFile(localeID, path, value, shortStatus, subType); 1610 } 1611 if (PATH_IN_COUNT && ErrorFile.generated_html_count != null) { 1612 ErrorFile.generated_html_count.println(lastHtmlLocaleID + ";\tpath:\t" + path); 1613 } 1614 } 1615 1616 private static void addError(String localeID, String path, ErrorType shortStatus) { 1617 if (ErrorType.showInSummary.contains(shortStatus)) { 1618 ErrorFile.htmlErrorsPerLocale.increment(shortStatus); 1619 } 1620 } 1621 1622 static String lastHtmlLocaleID = ""; 1623 private static VoteResolver<String> voteResolver; 1624 private static String resolveVotesDirectory; 1625 private static boolean idView; 1626 private static SupplementalDataInfo supplementalDataInfo; 1627 private static CLDRFile english; 1628 1629 public static class PathShower { 1630 String localeID; 1631 boolean newLocale = true; 1632 String lastPath; 1633 String[] lastSplitPath; 1634 boolean showEnglish; 1635 String splitChar = "/"; 1636 1637 static final String lead = "****************************************"; 1638 1639 public void set(String localeID) { 1640 this.localeID = localeID; 1641 newLocale = true; 1642 LocaleIDParser localeIDParser = new LocaleIDParser(); 1643 showEnglish = !localeIDParser.set(localeID).getLanguageScript().equals("en"); 1644 // localeID.equals(CheckCLDR.displayInformation.getLocaleID()); 1645 lastPath = null; 1646 lastSplitPath = null; 1647 } 1648 1649 private void showHeader(String path, String value) { 1650 if (newLocale) { 1651 System.out.println("Locale:\t" + getLocaleAndName(localeID)); 1652 newLocale = false; 1653 } 1654 if (path.equals(lastPath)) return; 1655 1656 // This logic keeps us from splitting on an attribute value that contains a / 1657 // such as time zone names. 1658 1659 StringBuffer newPath = new StringBuffer(); 1660 boolean inQuotes = false; 1661 for (int i = 0; i < path.length(); i++) { 1662 if ((path.charAt(i) == '/') && !inQuotes) 1663 newPath.append('%'); 1664 else 1665 newPath.append(path.charAt(i)); 1666 1667 if (path.charAt(i) == '\"') 1668 inQuotes = !inQuotes; 1669 } 1670 1671 String[] splitPath = newPath.toString().split("%"); 1672 1673 for (int i = 0; i < splitPath.length; ++i) { 1674 if (lastSplitPath != null && i < lastSplitPath.length && splitPath[i].equals(lastSplitPath[i])) { 1675 continue; 1676 } 1677 lastSplitPath = null; // mark so we continue printing now 1678 System.out.print(lead.substring(0, i)); 1679 System.out.print(splitPath[i]); 1680 if (i == splitPath.length - 1) { 1681 showValue(path, value, showEnglish, localeID); 1682 } else { 1683 System.out.print(":"); 1684 } 1685 System.out.println(); 1686 } 1687 // String prettierPath = path; 1688 // if (false) { 1689 // prettierPath = prettyPath.transliterate(path); 1690 // } 1691 1692 lastPath = path; 1693 lastSplitPath = splitPath; 1694 } 1695 1696 public String getSplitChar() { 1697 return splitChar; 1698 } 1699 1700 public PathShower setSplitChar(String splitChar) { 1701 this.splitChar = splitChar; 1702 return this; 1703 } 1704 } 1705 1706 private static void showValue(String path, String value, boolean showEnglish, String localeID) { 1707 System.out.println("\tValue:\t" + value + (showEnglish ? "\t" + getEnglishPathValue(path) : "") + "\tLocale:\t" 1708 + localeID); 1709 } 1710 1711 private static String getEnglishPathValue(String path) { 1712 String englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path); 1713 if (englishValue == null) { 1714 String path2 = CLDRFile.getNondraftNonaltXPath(path); 1715 englishValue = CheckCLDR.getDisplayInformation().getWinningValue(path2); 1716 } 1717 return englishValue; 1718 } 1719 1720 /** 1721 * Utility for getting information. 1722 * 1723 * @param locale 1724 * @return 1725 */ 1726 public static String getLocaleAndName(String locale) { 1727 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1728 if (localizedName == null || localizedName.equals(locale)) return locale; 1729 return locale + " [" + localizedName + "]"; 1730 } 1731 1732 /** 1733 * Utility for getting information. 1734 * 1735 * @param locale 1736 * @param linkToXml 1737 * TODO 1738 * @return 1739 */ 1740 public static String getNameAndLocale(String locale, boolean linkToXml) { 1741 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1742 if (localizedName == null || localizedName.equals(locale)) return locale; 1743 if (linkToXml) { 1744 locale = "<a href='http://unicode.org/cldr/data/common/main/" + locale + ".xml'>" + locale + "</a>"; 1745 } 1746 return localizedName + " [" + locale + "]"; 1747 } 1748 1749 public static String getLocaleName(String locale) { 1750 String localizedName = CheckCLDR.getDisplayInformation().getName(locale); 1751 if (localizedName == null || localizedName.equals(locale)) return locale; 1752 return localizedName; 1753 } 1754 1755 public static String getLinkedLocale(String locale) { 1756 return "<a href='http://unicode.org/cldr/apps/survey?_=" + locale + "'>" + locale + "</a>"; 1757 } 1758 } 1759