1 /* 2 ****************************************************************************** 3 * Copyright (C) 2005-2014, International Business Machines Corporation and * 4 * others. All Rights Reserved. * 5 ****************************************************************************** 6 */ 7 8 package org.unicode.cldr.test; 9 10 import java.io.BufferedReader; 11 import java.io.IOException; 12 import java.text.ParsePosition; 13 import java.util.ArrayList; 14 import java.util.Arrays; 15 import java.util.EnumSet; 16 import java.util.HashMap; 17 import java.util.Iterator; 18 import java.util.List; 19 import java.util.Locale; 20 import java.util.Map; 21 import java.util.Set; 22 import java.util.TreeSet; 23 import java.util.regex.Matcher; 24 import java.util.regex.Pattern; 25 26 import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype; 27 import org.unicode.cldr.util.CLDRFile; 28 import org.unicode.cldr.util.CLDRInfo.CandidateInfo; 29 import org.unicode.cldr.util.CLDRInfo.PathValueInfo; 30 import org.unicode.cldr.util.CLDRInfo.UserInfo; 31 import org.unicode.cldr.util.CLDRLocale; 32 import org.unicode.cldr.util.CldrUtility; 33 import org.unicode.cldr.util.Factory; 34 import org.unicode.cldr.util.InternalCldrException; 35 import org.unicode.cldr.util.Level; 36 import org.unicode.cldr.util.PathHeader; 37 import org.unicode.cldr.util.PathHeader.SurveyToolStatus; 38 import org.unicode.cldr.util.PatternCache; 39 import org.unicode.cldr.util.RegexFileParser; 40 import org.unicode.cldr.util.RegexFileParser.RegexLineParser; 41 import org.unicode.cldr.util.StandardCodes; 42 import org.unicode.cldr.util.TransliteratorUtilities; 43 import org.unicode.cldr.util.VoteResolver; 44 45 import com.ibm.icu.dev.util.ElapsedTimer; 46 import com.ibm.icu.impl.Row.R3; 47 import com.ibm.icu.text.ListFormatter; 48 import com.ibm.icu.text.MessageFormat; 49 import com.ibm.icu.text.Transliterator; 50 import com.ibm.icu.util.ICUUncheckedIOException; 51 52 /** 53 * This class provides a foundation for both console-driven CLDR tests, and 54 * Survey Tool Tests. 55 * <p> 56 * To add a test, subclass CLDRFile and override handleCheck and possibly setCldrFileToCheck. Then put the test into 57 * getCheckAll. 58 * <p> 59 * To use the test, take a look at the main in ConsoleCheckCLDR. Note that you need to call setDisplayInformation with 60 * the CLDRFile for the locale that you want the display information (eg names for codes) to be in.<br> 61 * Some options are passed in the Map options. Examples: boolean SHOW_TIMES = options.containsKey("SHOW_TIMES"); // for 62 * printing times for doing setCldrFileToCheck. 63 * <p> 64 * Some errors/warnings will be explicitly filtered out when calling CheckCLDR's check() method. 65 * The full list of filters can be found in org/unicode/cldr/util/data/CheckCLDR-exceptions.txt. 66 * 67 * @author davis 68 */ 69 abstract public class CheckCLDR { 70 private static CLDRFile displayInformation; 71 72 private CLDRFile cldrFileToCheck; 73 private CLDRFile englishFile = null; 74 75 private boolean skipTest = false; 76 private Phase phase; 77 private Map<Subtype, List<Pattern>> filtersForLocale = new HashMap<Subtype, List<Pattern>>(); 78 79 public enum InputMethod { 80 DIRECT, BULK 81 } 82 83 public enum StatusAction { 84 /** 85 * Allow voting and add new values (in Change column). 86 */ 87 ALLOW, 88 /** 89 * Allow voting and ticket (in Change column). 90 */ 91 ALLOW_VOTING_AND_TICKET, 92 /** 93 * Allow voting but no add new values (in Change column). 94 */ 95 ALLOW_VOTING_BUT_NO_ADD, 96 /** 97 * Only allow filing a ticket. 98 */ 99 ALLOW_TICKET_ONLY, 100 /** 101 * Disallow (for various reasons) 102 */ 103 FORBID_ERRORS(true), FORBID_READONLY(true), FORBID_UNLESS_DATA_SUBMISSION(true), FORBID_COVERAGE(true), FORBID_NEEDS_TICKET(true); 104 105 private final boolean isForbidden; 106 107 private StatusAction() { 108 isForbidden = false; 109 }; 110 111 private StatusAction(boolean isForbidden) { 112 this.isForbidden = isForbidden; 113 }; 114 115 public boolean isForbidden() { 116 return isForbidden; 117 } 118 119 public boolean canShow() { 120 return !isForbidden; 121 } 122 123 /** 124 * @deprecated 125 */ 126 public static final StatusAction FORBID = FORBID_READONLY; 127 /** 128 * @deprecated 129 */ 130 public static final StatusAction SHOW_VOTING_AND_ADD = ALLOW; 131 /** 132 * @deprecated 133 */ 134 public static final StatusAction SHOW_VOTING_AND_TICKET = ALLOW_VOTING_AND_TICKET; 135 /** 136 * @deprecated 137 */ 138 public static final StatusAction SHOW_VOTING_BUT_NO_ADD = ALLOW_VOTING_BUT_NO_ADD; 139 /** 140 * @deprecated 141 */ 142 public static final StatusAction FORBID_HAS_ERROR = FORBID_ERRORS; 143 } 144 145 private static final HashMap<String, Phase> PHASE_NAMES = new HashMap<String, Phase>(); 146 147 public enum Phase { 148 BUILD, SUBMISSION, VETTING, FINAL_TESTING("RESOLUTION"); 149 Phase(String... alternateName) { 150 for (String name : alternateName) { 151 PHASE_NAMES.put(name.toUpperCase(Locale.ENGLISH), this); 152 } 153 } 154 155 public static Phase forString(String value) { 156 if (value == null) { 157 return org.unicode.cldr.util.CLDRConfig.getInstance().getPhase(); 158 } 159 value = value.toUpperCase(Locale.ENGLISH); 160 Phase result = PHASE_NAMES.get(value); 161 return result != null ? result 162 : Phase.valueOf(value); 163 } 164 165 /** 166 * Return whether or not to show a row, and if so, how. 167 * 168 * @param pathValueInfo 169 * @param inputMethod 170 * @param status 171 * @param userInfo 172 * null if there is no userInfo (nobody logged in). 173 * @return 174 */ 175 public StatusAction getShowRowAction( 176 PathValueInfo pathValueInfo, 177 InputMethod inputMethod, 178 PathHeader.SurveyToolStatus status, 179 UserInfo userInfo // can get voterInfo from this. 180 ) { 181 182 // always forbid deprecated items - don't show. 183 if (status == SurveyToolStatus.DEPRECATED) { 184 return StatusAction.FORBID_READONLY; 185 } 186 187 if (status == SurveyToolStatus.READ_ONLY) { 188 return StatusAction.ALLOW_TICKET_ONLY; 189 } 190 191 // always forbid bulk import except in data submission. 192 if (inputMethod == InputMethod.BULK && this != Phase.SUBMISSION) { 193 return StatusAction.FORBID_UNLESS_DATA_SUBMISSION; 194 } 195 196 // if TC+, allow anything else, even suppressed items and errors 197 if (userInfo != null && userInfo.getVoterInfo().getLevel().compareTo(VoteResolver.Level.tc) >= 0) { 198 return StatusAction.ALLOW; 199 } 200 201 // if the coverage level is optional, disallow everything 202 if (pathValueInfo.getCoverageLevel().compareTo(Level.COMPREHENSIVE) > 0) { 203 return StatusAction.FORBID_COVERAGE; 204 } 205 206 if (status == SurveyToolStatus.HIDE) { 207 return StatusAction.FORBID_READONLY; 208 } 209 210 if (this == Phase.SUBMISSION) { 211 return (status == SurveyToolStatus.READ_WRITE || status == SurveyToolStatus.LTR_ALWAYS) 212 ? StatusAction.ALLOW 213 : StatusAction.ALLOW_VOTING_AND_TICKET; 214 } 215 216 // We are not in submission. 217 // Only allow ADD if we have an error or warning 218 ValueStatus valueStatus = ValueStatus.NONE; 219 CandidateInfo winner = pathValueInfo.getCurrentItem(); 220 // Only check winning value for errors/warnings per ticket #8677 221 // We used to check all candidates. 222 // for (CandidateInfo value : pathValueInfo.getValues()) { 223 valueStatus = getValueStatus(winner, valueStatus); 224 if (valueStatus != ValueStatus.NONE) { 225 return (status == SurveyToolStatus.READ_WRITE || status == SurveyToolStatus.LTR_ALWAYS) 226 ? StatusAction.ALLOW 227 : StatusAction.ALLOW_VOTING_AND_TICKET; 228 } 229 // } 230 231 // No warnings, so allow just voting. 232 return StatusAction.ALLOW_VOTING_BUT_NO_ADD; 233 } 234 235 /** 236 * getAcceptNewItemAction. MUST only be called if getShowRowAction(...).canShow() 237 * TODO Consider moving Phase, StatusAction, etc into CLDRInfo. 238 * 239 * @param enteredValue 240 * If null, means an abstention. 241 * If voting for an existing value, pathValueInfo.getValues().contains(enteredValue) MUST be true 242 * @param pathValueInfo 243 * @param inputMethod 244 * @param status 245 * @param userInfo 246 * @return 247 */ 248 public StatusAction getAcceptNewItemAction( 249 CandidateInfo enteredValue, 250 PathValueInfo pathValueInfo, 251 InputMethod inputMethod, 252 PathHeader.SurveyToolStatus status, 253 UserInfo userInfo // can get voterInfo from this. 254 ) { 255 if (status != SurveyToolStatus.READ_WRITE && status != SurveyToolStatus.LTR_ALWAYS) { 256 return StatusAction.FORBID_READONLY; // not writable. 257 } 258 259 // only logged in users can add items. 260 if (userInfo == null) { 261 return StatusAction.FORBID_ERRORS; 262 } 263 264 // we can always abstain 265 if (enteredValue == null) { 266 return StatusAction.ALLOW; 267 } 268 269 // if TC+, allow anything else, even suppressed items and errors 270 if (userInfo.getVoterInfo().getLevel().compareTo(VoteResolver.Level.tc) >= 0) { 271 return StatusAction.ALLOW; 272 } 273 274 // Disallow errors. 275 ValueStatus valueStatus = getValueStatus(enteredValue, ValueStatus.NONE); 276 if (valueStatus == ValueStatus.ERROR) { 277 return StatusAction.FORBID_ERRORS; 278 } 279 280 // Allow items if submission 281 if (this == Phase.SUBMISSION) { 282 return StatusAction.ALLOW; 283 } 284 285 // Voting for an existing value is ok 286 valueStatus = ValueStatus.NONE; 287 for (CandidateInfo value : pathValueInfo.getValues()) { 288 if (value == enteredValue) { 289 return StatusAction.ALLOW; 290 } 291 valueStatus = getValueStatus(value, valueStatus); 292 } 293 294 // If there were any errors/warnings on other values, allow 295 if (valueStatus != ValueStatus.NONE) { 296 return StatusAction.ALLOW; 297 } 298 299 // Otherwise (we are vetting, but with no errors or warnings) 300 // DISALLOW NEW STUFF 301 302 return StatusAction.FORBID_UNLESS_DATA_SUBMISSION; 303 } 304 305 enum ValueStatus { 306 ERROR, WARNING, NONE 307 } 308 309 private ValueStatus getValueStatus(CandidateInfo value, ValueStatus previous) { 310 if (previous == ValueStatus.ERROR || value == null) { 311 return previous; 312 } 313 314 for (CheckStatus item : value.getCheckStatusList()) { 315 CheckStatus.Type type = item.getType(); 316 if (type.equals(CheckStatus.Type.Error)) { 317 if (CheckStatus.crossCheckSubtypes.contains(item.getSubtype())) { 318 return ValueStatus.WARNING; 319 } else { 320 return ValueStatus.ERROR; 321 } 322 } else if (type.equals(CheckStatus.Type.Warning)) { 323 previous = ValueStatus.WARNING; 324 } 325 } 326 return previous; 327 } 328 } 329 330 public static final class Options implements Comparable<Options> { 331 332 public enum Option { 333 locale, CoverageLevel_requiredLevel("CoverageLevel.requiredLevel"), CoverageLevel_localeType( 334 "CoverageLevel.localeType"), SHOW_TIMES, phase, lgWarningCheck, CheckCoverage_skip("CheckCoverage.skip"), exemplarErrors; 335 336 private String key; 337 338 public String getKey() { 339 return key; 340 } 341 342 Option(String key) { 343 this.key = key; 344 } 345 346 Option() { 347 this.key = name(); 348 } 349 }; 350 351 private static StandardCodes sc = StandardCodes.make(); 352 353 private final boolean DEBUG_OPTS = false; 354 355 String options[] = new String[Option.values().length]; 356 CLDRLocale locale = null; 357 358 private final String key; // for fast compare 359 360 /** 361 * Adopt some other map 362 * @param fromOptions 363 */ 364 public Options(Map<String, String> fromOptions) { 365 clear(); 366 setAll(fromOptions); 367 key = null; // no key = slow compare 368 } 369 370 private void setAll(Map<String, String> fromOptions) { 371 for (Map.Entry<String, String> e : fromOptions.entrySet()) { 372 set(e.getKey(), e.getValue()); 373 } 374 } 375 376 /** 377 * @param key 378 * @param value 379 */ 380 public void set(String key, String value) { 381 // TODO- cache the map 382 for (Option o : Option.values()) { 383 if (o.getKey().equals(key)) { 384 set(o, value); 385 return; 386 } 387 } 388 throw new IllegalArgumentException("Unknown CLDR option: '" + key + "' - valid keys are: " + Options.getValidKeys()); 389 } 390 391 private static String getValidKeys() { 392 Set<String> allkeys = new TreeSet<String>(); 393 for (Option o : Option.values()) { 394 allkeys.add(o.getKey()); 395 } 396 return ListFormatter.getInstance().format(allkeys); 397 } 398 399 public Options() { 400 clear(); 401 key = "".intern(); // null Options. 402 } 403 404 /** 405 * Deep clone 406 * @param options2 407 */ 408 public Options(Options options2) { 409 this.options = Arrays.copyOf(options2.options, options2.options.length); 410 this.key = options2.key; 411 } 412 413 public Options(CLDRLocale locale, CheckCLDR.Phase testPhase, String requiredLevel, String localeType) { 414 this.locale = locale; 415 options = new String[Option.values().length]; 416 StringBuilder sb = new StringBuilder(); 417 set(Option.locale, locale.getBaseName()); 418 sb.append(locale.getBaseName()).append('/'); 419 set(Option.CoverageLevel_requiredLevel, requiredLevel); 420 sb.append(requiredLevel).append('/'); 421 set(Option.CoverageLevel_localeType, localeType); 422 sb.append(localeType).append('/'); 423 set(Option.phase, testPhase.name().toLowerCase()); 424 sb.append(localeType).append('/'); 425 key = sb.toString().intern(); 426 } 427 428 @Override 429 public Options clone() { 430 return new Options(this); 431 } 432 433 @Override 434 public boolean equals(Object other) { 435 if (this == other) return true; 436 if (!(other instanceof Options)) return false; 437 if (this.key != null && ((Options) other).key != null) { 438 return (this.key == ((Options) other).key); 439 } else { 440 return this.compareTo((Options) other) == 0; 441 } 442 } 443 444 private Options clear(Option o) { 445 set(o, null); 446 return this; 447 } 448 449 private Options clear() { 450 for (int i = 0; i < options.length; i++) { 451 options[i] = null; 452 } 453 return this; 454 } 455 456 private Options set(Option o, String v) { 457 options[o.ordinal()] = v; 458 if (DEBUG_OPTS) System.err.println("Setting " + o + " = " + v); 459 return this; 460 } 461 462 public String get(Option o) { 463 final String v = options[o.ordinal()]; 464 if (DEBUG_OPTS) System.err.println("Getting " + o + " = " + v); 465 return v; 466 } 467 468 public CLDRLocale getLocale() { 469 if (locale != null) return locale; 470 return CLDRLocale.getInstance(get(Option.locale)); 471 } 472 473 public Level getRequiredLevel(String localeID) { 474 Level result; 475 // see if there is an explicit level 476 String localeType = get(Option.CoverageLevel_requiredLevel); 477 if (localeType != null) { 478 result = Level.get(localeType); 479 if (result != Level.UNDETERMINED) { 480 return result; 481 } 482 } 483 // otherwise, see if there is an organization level 484 return sc.getLocaleCoverageLevel("Cldr", localeID); 485 } 486 487 public boolean contains(Option o) { 488 String s = get(o); 489 return (s != null && !s.isEmpty()); 490 } 491 492 @Override 493 public int compareTo(Options other) { 494 if (other == this) return 0; 495 if (key != null && other.key != null) { 496 if (key == other.key) return 0; 497 return key.compareTo(other.key); 498 } 499 for (int i = 0; i < options.length; i++) { 500 final String s1 = options[i]; 501 final String s2 = other.options[i]; 502 if (s1 == null && s2 == null) { 503 // no difference 504 } else if (s1 == null) { 505 return -1; 506 } else if (s2 == null) { 507 return 1; 508 } else { 509 int rv = s1.compareTo(s2); 510 if (rv != 0) { 511 return rv; 512 } 513 } 514 } 515 return 0; 516 } 517 518 @Override 519 public int hashCode() { 520 if (key != null) return key.hashCode(); 521 522 int h = 1; 523 for (int i = 0; i < options.length; i++) { 524 if (options[i] == null) { 525 h *= 11; 526 } else { 527 h = (h * 11) + options[i].hashCode(); 528 } 529 } 530 return h; 531 } 532 533 @Override 534 public String toString() { 535 if (key != null) return "Options:" + key; 536 StringBuilder sb = new StringBuilder(); 537 for (Option o : Option.values()) { 538 if (options[o.ordinal()] != null) { 539 sb.append(o) 540 .append('=') 541 .append(options[o.ordinal()]) 542 .append(' '); 543 } 544 } 545 return sb.toString(); 546 } 547 }; 548 549 public boolean isSkipTest() { 550 return skipTest; 551 } 552 553 // this should only be set for the test in setCldrFileToCheck 554 public void setSkipTest(boolean skipTest) { 555 this.skipTest = skipTest; 556 } 557 558 /** 559 * Here is where the list of all checks is found. 560 * 561 * @param nameMatcher 562 * Regex pattern that determines which checks are run, 563 * based on their class name (such as .* for all checks, .*Collisions.* for CheckDisplayCollisions, etc.) 564 * @return 565 */ 566 public static CompoundCheckCLDR getCheckAll(Factory factory, String nameMatcher) { 567 return new CompoundCheckCLDR() 568 .setFilter(Pattern.compile(nameMatcher, Pattern.CASE_INSENSITIVE).matcher("")) 569 //.add(new CheckAttributeValues(factory)) 570 .add(new CheckChildren(factory)) 571 .add(new CheckCoverage(factory)) 572 .add(new CheckDates(factory)) 573 .add(new CheckForCopy(factory)) 574 .add(new CheckDisplayCollisions(factory)) 575 .add(new CheckExemplars(factory)) 576 .add(new CheckForExemplars(factory)) 577 .add(new CheckForInheritanceMarkers()) 578 .add(new CheckNames()) 579 .add(new CheckNumbers(factory)) 580 // .add(new CheckZones()) // this doesn't work; many spurious errors that user can't correct 581 .add(new CheckMetazones()) 582 .add(new CheckLogicalGroupings(factory)) 583 .add(new CheckAlt()) 584 .add(new CheckCurrencies()) 585 .add(new CheckCasing()) 586 .add(new CheckConsistentCasing(factory)) // this doesn't work; many spurious errors that user can't correct 587 .add(new CheckQuotes()) 588 .add(new CheckUnits()) 589 .add(new CheckWidths()) 590 .add(new CheckPlaceHolders()) 591 .add(new CheckNew(factory)) // this is at the end; it will check for other certain other errors and warnings and 592 // not add a message if there are any. 593 ; 594 } 595 596 /** 597 * These determine what language is used to display information. Must be set before use. 598 * 599 * @param locale 600 * @return 601 */ 602 public static synchronized CLDRFile getDisplayInformation() { 603 return displayInformation; 604 } 605 606 public static synchronized void setDisplayInformation(CLDRFile inputDisplayInformation) { 607 displayInformation = inputDisplayInformation; 608 } 609 610 /** 611 * [Warnings - please zoom in] dates/timeZoneNames/singleCountries 612 * (empty) 613 * [refs][hide] Ref: [Zoom...] 614 * [Warnings - please zoom in] dates/timeZoneNames/hours {0}/{1} {0}/{1} 615 * [refs][hide] Ref: [Zoom...] 616 * [Warnings - please zoom in] dates/timeZoneNames/hour +HH:mm;-HH:mm 617 * +HH:mm;-HH:mm 618 * [refs][hide] Ref: [Zoom...] 619 * [ok] layout/orientation (empty) 620 * [refs][hide] Ref: [Zoom...] 621 * [ok] dates/localizedPatternChars GyMdkHmsSEDFwWahKzYeugAZvcL 622 * GaMjkHmsSEDFwWxhKzAeugXZvcL 623 * [refs][hide] Ref: [Zoom...] 624 */ 625 626 /** 627 * Get the CLDRFile. 628 * 629 * @param cldrFileToCheck 630 */ 631 public final CLDRFile getCldrFileToCheck() { 632 return cldrFileToCheck; 633 } 634 635 /** 636 * Don't override this, use the other setCldrFileToCheck which takes an Options instead of a Map<> 637 * @param cldrFileToCheck 638 * @param options 639 * @param possibleErrors 640 * @return 641 * @see #setCldrFileToCheck(CLDRFile, Options, List) 642 */ 643 final public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Map<String, String> options, 644 List<CheckStatus> possibleErrors) { 645 return setCldrFileToCheck(cldrFileToCheck, new Options(options), possibleErrors); 646 } 647 648 /** 649 * Set the CLDRFile. Must be done before calling check. If null is called, just skip 650 * Often subclassed for initializing. If so, make the first 2 lines: 651 * if (cldrFileToCheck == null) return this; 652 * super.setCldrFileToCheck(cldrFileToCheck); 653 * do stuff 654 * 655 * @param cldrFileToCheck 656 * @param options (not currently used) 657 * @param possibleErrors 658 * TODO 659 */ 660 public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, 661 List<CheckStatus> possibleErrors) { 662 this.cldrFileToCheck = cldrFileToCheck; 663 664 // Shortlist error filters for this locale. 665 loadFilters(); 666 String locale = cldrFileToCheck.getLocaleID(); 667 filtersForLocale.clear(); 668 for (R3<Pattern, Subtype, Pattern> filter : allFilters) { 669 if (filter.get0() == null || !filter.get0().matcher(locale).matches()) continue; 670 Subtype subtype = filter.get1(); 671 List<Pattern> xpaths = filtersForLocale.get(subtype); 672 if (xpaths == null) { 673 filtersForLocale.put(subtype, xpaths = new ArrayList<Pattern>()); 674 } 675 xpaths.add(filter.get2()); 676 } 677 return this; 678 } 679 680 /** 681 * Status value returned from check 682 */ 683 public static class CheckStatus { 684 public static final Type alertType = Type.Comment, 685 warningType = Type.Warning, 686 errorType = Type.Error, 687 exampleType = Type.Example, 688 demoType = Type.Demo; 689 690 public enum Type { 691 Comment, Warning, Error, Example, Demo 692 }; 693 694 public enum Subtype { 695 none, noUnproposedVariant, deprecatedAttribute, illegalPlural, invalidLocale, incorrectCasing, 696 valueMustBeOverridden, 697 valueAlwaysOverridden, nullChildFile, internalError, coverageLevel, missingPluralInfo, 698 currencySymbolTooWide, incorrectDatePattern, abbreviatedDateFieldTooWide, displayCollision, 699 illegalExemplarSet, missingAuxiliaryExemplars, extraPlaceholders, missingPlaceholders, 700 shouldntHavePlaceholders, couldNotAccessExemplars, noExemplarCharacters, modifiedEnglishValue, 701 invalidCurrencyMatchSet, multipleMetazoneMappings, noMetazoneMapping, noMetazoneMappingAfter1970, 702 noMetazoneMappingBeforeNow, cannotCreateZoneFormatter, insufficientCoverage, missingLanguageTerritoryInfo, 703 missingEuroCountryInfo, deprecatedAttributeWithReplacement, missingOrExtraDateField, internalUnicodeSetFormattingError, 704 auxiliaryExemplarsOverlap, missingPunctuationCharacters, 705 706 charactersNotInCurrencyExemplars, asciiCharactersNotInCurrencyExemplars, 707 charactersNotInMainOrAuxiliaryExemplars, asciiCharactersNotInMainOrAuxiliaryExemplars, 708 709 narrowDateFieldTooWide, illegalCharactersInExemplars, orientationDisagreesWithExemplars, 710 inconsistentDatePattern, inconsistentTimePattern, missingDatePattern, illegalDatePattern, 711 missingMainExemplars, mustNotStartOrEndWithSpace, illegalCharactersInNumberPattern, 712 numberPatternNotCanonical, currencyPatternMissingCurrencySymbol, missingMinusSign, 713 badNumericType, percentPatternMissingPercentSymbol, illegalNumberFormat, unexpectedAttributeValue, 714 metazoneContainsDigit, tooManyGroupingSeparators, inconsistentPluralFormat, sameAsEnglishOrCode, 715 dateSymbolCollision, incompleteLogicalGroup, extraMetazoneString, inconsistentDraftStatus, 716 errorOrWarningInLogicalGroup, valueTooWide, valueTooNarrow, nameContainsYear, patternCannotContainDigits, 717 patternContainsInvalidCharacters, parenthesesNotAllowed, illegalNumberingSystem, unexpectedOrderOfEraYear, 718 invalidPlaceHolder, asciiQuotesNotAllowed, badMinimumGroupingDigits, inconsistentPeriods, 719 inheritanceMarkerNotAllowed, invalidDurationUnitPattern, invalidDelimiter, illegalCharactersInPattern, 720 badParseLenient, tooManyValues; 721 722 public String toString() { 723 return TO_STRING.matcher(name()).replaceAll(" $1").toLowerCase(); 724 } 725 726 static Pattern TO_STRING = PatternCache.get("([A-Z])"); 727 }; 728 729 public static EnumSet<Subtype> crossCheckSubtypes = EnumSet.of( 730 Subtype.dateSymbolCollision, 731 Subtype.displayCollision, 732 Subtype.inconsistentDraftStatus, 733 Subtype.incompleteLogicalGroup, 734 Subtype.inconsistentPeriods, 735 Subtype.abbreviatedDateFieldTooWide, 736 Subtype.narrowDateFieldTooWide, 737 Subtype.coverageLevel); 738 739 private Type type; 740 private Subtype subtype = Subtype.none; 741 private String messageFormat; 742 private Object[] parameters; 743 private String htmlMessage; 744 private CheckCLDR cause; 745 private boolean checkOnSubmit = true; 746 747 public CheckStatus() { 748 749 } 750 751 public boolean isCheckOnSubmit() { 752 return checkOnSubmit; 753 } 754 755 public CheckStatus setCheckOnSubmit(boolean dependent) { 756 this.checkOnSubmit = dependent; 757 return this; 758 } 759 760 public Type getType() { 761 return type; 762 } 763 764 public CheckStatus setMainType(CheckStatus.Type type) { 765 this.type = type; 766 return this; 767 } 768 769 public String getMessage() { 770 String message = messageFormat; 771 if (messageFormat != null && parameters != null) { 772 try { 773 String fixedApos = MessageFormat.autoQuoteApostrophe(messageFormat); 774 MessageFormat format = new MessageFormat(fixedApos); 775 message = format.format(parameters); 776 } catch (Exception e) { 777 message = messageFormat; 778 System.err.println("MessageFormat Failure: " + subtype + "; " + messageFormat + "; " 779 + (parameters == null ? null : Arrays.asList(parameters))); 780 // throw new IllegalArgumentException(subtype + "; " + messageFormat + "; " 781 // + (parameters == null ? null : Arrays.asList(parameters)), e); 782 } 783 } 784 Exception[] exceptionParameters = getExceptionParameters(); 785 if (exceptionParameters != null) { 786 for (Exception exception : exceptionParameters) { 787 message += "; " + exception.getMessage(); // + " \t(" + exception.getClass().getName() + ")"; 788 // for (StackTraceElement item : exception.getStackTrace()) { 789 // message += "\n\t" + item; 790 // } 791 } 792 } 793 return message.replace('\t', ' '); 794 } 795 796 /** 797 * @deprecated 798 */ 799 public String getHTMLMessage() { 800 return htmlMessage; 801 } 802 803 /** 804 * @deprecated 805 */ 806 public CheckStatus setHTMLMessage(String message) { 807 htmlMessage = message; 808 return this; 809 } 810 811 public CheckStatus setMessage(String message) { 812 if (cause == null) { 813 throw new IllegalArgumentException("Must have cause set."); 814 } 815 this.messageFormat = message; 816 this.parameters = null; 817 return this; 818 } 819 820 public CheckStatus setMessage(String message, Object... messageArguments) { 821 if (cause == null) { 822 throw new IllegalArgumentException("Must have cause set."); 823 } 824 this.messageFormat = message; 825 this.parameters = messageArguments; 826 return this; 827 } 828 829 public String toString() { 830 return getType() + ": " + getMessage(); 831 } 832 833 /** 834 * Warning: don't change the contents of the parameters after retrieving. 835 */ 836 public Object[] getParameters() { 837 return parameters; 838 } 839 840 /** 841 * Returns any Exception parameters in the status, or null if there are none. 842 * 843 * @return 844 */ 845 public Exception[] getExceptionParameters() { 846 if (parameters == null) { 847 return null; 848 } 849 850 List<Exception> errors = new ArrayList<Exception>(); 851 for (Object o : parameters) { 852 if (o instanceof Exception) { 853 errors.add((Exception) o); 854 } 855 } 856 if (errors.size() == 0) { 857 return null; 858 } 859 return errors.toArray(new Exception[errors.size()]); 860 } 861 862 /** 863 * Warning: don't change the contents of the parameters after passing in. 864 */ 865 public CheckStatus setParameters(Object[] parameters) { 866 if (cause == null) { 867 throw new IllegalArgumentException("Must have cause set."); 868 } 869 this.parameters = parameters; 870 return this; 871 } 872 873 public SimpleDemo getDemo() { 874 return null; 875 } 876 877 public CheckCLDR getCause() { 878 return cause; 879 } 880 881 public CheckStatus setCause(CheckCLDR cause) { 882 this.cause = cause; 883 return this; 884 } 885 886 public Subtype getSubtype() { 887 return subtype; 888 } 889 890 public CheckStatus setSubtype(Subtype subtype) { 891 this.subtype = subtype; 892 return this; 893 } 894 895 /** 896 * Convenience function: return true if any items in this list are of errorType 897 * 898 * @param result 899 * the list to check (could be null for empty) 900 * @return true if any items in result are of errorType 901 */ 902 public static final boolean hasError(List<CheckStatus> result) { 903 return hasType(result, errorType); 904 } 905 906 /** 907 * Convenience function: return true if any items in this list are of errorType 908 * 909 * @param result 910 * the list to check (could be null for empty) 911 * @return true if any items in result are of errorType 912 */ 913 public static boolean hasType(List<CheckStatus> result, Type type) { 914 if (result == null) return false; 915 for (CheckStatus s : result) { 916 if (s.getType().equals(type)) { 917 return true; 918 } 919 } 920 return false; 921 } 922 } 923 924 public static abstract class SimpleDemo { 925 Map<String, String> internalPostArguments = new HashMap<String, String>(); 926 927 /** 928 * @param postArguments 929 * A read-write map containing post-style arguments. eg TEXTBOX=abcd, etc. <br> 930 * The first time this is called, the Map should be empty. 931 * @return true if the map has been changed 932 */ 933 public abstract String getHTML(Map<String, String> postArguments) throws Exception; 934 935 /** 936 * Only here for compatibiltiy. Use the other getHTML instead 937 */ 938 public final String getHTML(String path, String fullPath, String value) throws Exception { 939 return getHTML(internalPostArguments); 940 } 941 942 /** 943 * THIS IS ONLY FOR COMPATIBILITY: you can call this, then the non-postArguments form of getHTML; or better, 944 * call 945 * getHTML with the postArguments. 946 * 947 * @param postArguments 948 * A read-write map containing post-style arguments. eg TEXTBOX=abcd, etc. 949 * @return true if the map has been changed 950 */ 951 public final boolean processPost(Map<String, String> postArguments) { 952 internalPostArguments.clear(); 953 internalPostArguments.putAll(postArguments); 954 return true; 955 } 956 // /** 957 // * Utility for setting map. Use the paradigm in CheckNumbers. 958 // */ 959 // public boolean putIfDifferent(Map inout, String key, String value) { 960 // Object oldValue = inout.put(key, value); 961 // return !value.equals(oldValue); 962 // } 963 } 964 965 public static abstract class FormatDemo extends SimpleDemo { 966 protected String currentPattern, currentInput, currentFormatted, currentReparsed; 967 protected ParsePosition parsePosition = new ParsePosition(0); 968 969 protected abstract String getPattern(); 970 971 protected abstract String getSampleInput(); 972 973 protected abstract void getArguments(Map<String, String> postArguments); 974 975 public String getHTML(Map<String, String> postArguments) throws Exception { 976 getArguments(postArguments); 977 StringBuffer htmlMessage = new StringBuffer(); 978 FormatDemo.appendTitle(htmlMessage); 979 FormatDemo.appendLine(htmlMessage, currentPattern, currentInput, currentFormatted, currentReparsed); 980 htmlMessage.append("</table>"); 981 return htmlMessage.toString(); 982 } 983 984 public String getPlainText(Map<String, String> postArguments) { 985 getArguments(postArguments); 986 return MessageFormat.format("<\"\u200E{0}\u200E\", \"{1}\"> \u2192 \"\u200E{2}\u200E\" \u2192 \"{3}\"", 987 (Object[]) new String[] { currentPattern, currentInput, currentFormatted, currentReparsed }); 988 } 989 990 /** 991 * @param htmlMessage 992 * @param pattern 993 * @param input 994 * @param formatted 995 * @param reparsed 996 */ 997 public static void appendLine(StringBuffer htmlMessage, String pattern, String input, String formatted, 998 String reparsed) { 999 htmlMessage.append("<tr><td><input type='text' name='pattern' value='") 1000 .append(TransliteratorUtilities.toXML.transliterate(pattern)) 1001 .append("'></td><td><input type='text' name='input' value='") 1002 .append(TransliteratorUtilities.toXML.transliterate(input)) 1003 .append("'></td><td>") 1004 .append("<input type='submit' value='Test' name='Test'>") 1005 .append("</td><td>" + "<input type='text' name='formatted' value='") 1006 .append(TransliteratorUtilities.toXML.transliterate(formatted)) 1007 .append("'></td><td>" + "<input type='text' name='reparsed' value='") 1008 .append(TransliteratorUtilities.toXML.transliterate(reparsed)) 1009 .append("'></td></tr>"); 1010 } 1011 1012 /** 1013 * @param htmlMessage 1014 */ 1015 public static void appendTitle(StringBuffer htmlMessage) { 1016 htmlMessage.append("<table border='1' cellspacing='0' cellpadding='2'" + 1017 // " style='border-collapse: collapse' style='width: 100%'" + 1018 "><tr>" + 1019 "<th>Pattern</th>" + 1020 "<th>Unlocalized Input</th>" + 1021 "<th></th>" + 1022 "<th>Localized Format</th>" + 1023 "<th>Re-Parsed</th>" + 1024 "</tr>"); 1025 } 1026 } 1027 1028 /** 1029 * Wraps the options in an Options and delegates. 1030 * @param path 1031 * Must be a distinguished path, such as what comes out of CLDRFile.iterator() 1032 * @param fullPath 1033 * Must be the full path 1034 * @param value 1035 * the value associated with the path 1036 * @param options 1037 * A set of test-specific options. Set these with code of the form:<br> 1038 * options.put("CoverageLevel.localeType", "G0")<br> 1039 * That is, the key is of the form <testname>.<optiontype>, and the value is of the form <optionvalue>.<br> 1040 * There is one general option; the following will select only the tests that should be run during this 1041 * phase.<br> 1042 * options.put("phase", Phase.<something>); 1043 * It can be used for new data entry. 1044 * @param result 1045 * @return 1046 * @deprecated use CheckCLDR#check(String, String, String, Options, List) 1047 */ 1048 @Deprecated 1049 public final CheckCLDR check(String path, String fullPath, String value, Map<String, String> options, 1050 List<CheckStatus> result) { 1051 return check(path, fullPath, value, new Options(options), result); 1052 } 1053 1054 /** 1055 * Checks the path/value in the cldrFileToCheck for correctness, according to some criterion. 1056 * If the path is relevant to the check, there is an alert or warning, then a CheckStatus is added to List. 1057 * 1058 * @param path 1059 * Must be a distinguished path, such as what comes out of CLDRFile.iterator() 1060 * @param fullPath 1061 * Must be the full path 1062 * @param value 1063 * the value associated with the path 1064 * @param result 1065 */ 1066 public final CheckCLDR check(String path, String fullPath, String value, Options options, 1067 List<CheckStatus> result) { 1068 if (cldrFileToCheck == null) { 1069 throw new InternalCldrException("CheckCLDR problem: cldrFileToCheck must not be null"); 1070 } 1071 if (path == null) { 1072 throw new InternalCldrException("CheckCLDR problem: path must not be null"); 1073 } 1074 // if (fullPath == null) { 1075 // throw new InternalError("CheckCLDR problem: fullPath must not be null"); 1076 // } 1077 // if (value == null) { 1078 // throw new InternalError("CheckCLDR problem: value must not be null"); 1079 // } 1080 result.clear(); 1081 1082 /* 1083 * If the item is non-winning, and either inherited or it is code-fallback, then don't run 1084 * any tests on this item. See http://unicode.org/cldr/trac/ticket/7574 1085 * 1086 * The following conditional formerly used "value == ..." and "value != ...", which in Java doesn't 1087 * mean what it does in some other languages. The condition has been changed to use the equals() method. 1088 * Since value can be null, check for that first. 1089 */ 1090 // if (value == cldrFileToCheck.getBaileyValue(path, null, null) && value != cldrFileToCheck.getWinningValue(path)) { 1091 if (value != null 1092 && value.equals(cldrFileToCheck.getBaileyValue(path, null, null)) 1093 && !value.equals(cldrFileToCheck.getWinningValue(path))) { 1094 return this; 1095 } 1096 // If we're being asked to run tests for an inheritance marker, then we need to change it 1097 // to the "real" value first before running tests. Testing the value CldrUtility.INHERITANCE_MARKER ("") doesn't make sense. 1098 if (CldrUtility.INHERITANCE_MARKER.equals(value)) { 1099 value = cldrFileToCheck.getConstructedBaileyValue(path, null, null); 1100 // If it hasn't changed, then don't run any tests. 1101 if (CldrUtility.INHERITANCE_MARKER.equals(value)) { 1102 return this; 1103 } 1104 } 1105 CheckCLDR instance = handleCheck(path, fullPath, value, options, result); 1106 Iterator<CheckStatus> iterator = result.iterator(); 1107 // Filter out any errors/warnings that match the filter list in CheckCLDR-exceptions.txt. 1108 while (iterator.hasNext()) { 1109 CheckStatus status = iterator.next(); 1110 if (shouldExcludeStatus(fullPath, status)) { 1111 iterator.remove(); 1112 } 1113 } 1114 return instance; 1115 } 1116 1117 /** 1118 * @deprecated use {@link #getExamples(String, String, String, Options, List)} 1119 * @param path 1120 * @param fullPath 1121 * @param value 1122 * @param options 1123 * @param result 1124 * @return 1125 */ 1126 @Deprecated 1127 public final CheckCLDR getExamples(String path, String fullPath, String value, Map<String, String> options, 1128 List<CheckStatus> result) { 1129 return getExamples(path, fullPath, value, new Options(options), result); 1130 } 1131 1132 /** 1133 * Returns any examples in the result parameter. Both examples and demos can 1134 * be returned. A demo will have getType() == CheckStatus.demoType. In that 1135 * case, there will be no getMessage or getHTMLMessage available; instead, 1136 * call getDemo() to get the demo, then call getHTML() to get the initial 1137 * HTML. 1138 */ 1139 public final CheckCLDR getExamples(String path, String fullPath, String value, Options options, 1140 List<CheckStatus> result) { 1141 result.clear(); 1142 return handleGetExamples(path, fullPath, value, options, result); 1143 } 1144 1145 protected CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options2, 1146 List<CheckStatus> result) { 1147 return this; // NOOP unless overridden 1148 } 1149 1150 /** 1151 * This is what the subclasses override. 1152 * If they ever use pathParts or fullPathParts, they need to call initialize() with the respective 1153 * path. Otherwise they must NOT change pathParts or fullPathParts. 1154 * <p> 1155 * If something is found, a CheckStatus is added to result. This can be done multiple times in one call, if multiple 1156 * errors or warnings are found. The CheckStatus may return warnings, errors, examples, or demos. We may expand that 1157 * in the future. 1158 * <p> 1159 * The code to add the CheckStatus will look something like:: 1160 * 1161 * <pre> 1162 * result.add(new CheckStatus() 1163 * .setType(CheckStatus.errorType) 1164 * .setMessage("Value should be {0}", new Object[] { pattern })); 1165 * </pre> 1166 * 1167 * @param options 1168 * TODO 1169 */ 1170 abstract public CheckCLDR handleCheck(String path, String fullPath, String value, 1171 Options options, List<CheckStatus> result); 1172 1173 /** 1174 * Only for use in ConsoleCheck, for debugging 1175 */ 1176 public void handleFinish() { 1177 } 1178 1179 /** 1180 * Internal class used to bundle up a number of Checks. 1181 * 1182 * @author davis 1183 * 1184 */ 1185 static class CompoundCheckCLDR extends CheckCLDR { 1186 private Matcher filter; 1187 private List<CheckCLDR> checkList = new ArrayList<CheckCLDR>(); 1188 private List<CheckCLDR> filteredCheckList = new ArrayList<CheckCLDR>(); 1189 1190 public CompoundCheckCLDR add(CheckCLDR item) { 1191 checkList.add(item); 1192 if (filter == null) { 1193 filteredCheckList.add(item); 1194 } else { 1195 final String className = item.getClass().getName(); 1196 if (filter.reset(className).find()) { 1197 filteredCheckList.add(item); 1198 } 1199 } 1200 return this; 1201 } 1202 1203 public CheckCLDR handleCheck(String path, String fullPath, String value, 1204 Options options, List<CheckStatus> result) { 1205 result.clear(); 1206 // If we're being asked to run tests for an inheritance marker, then we need to change it 1207 // to the "real" value first before running tests. Testing the value CldrUtility.INHERITANCE_MARKER ("") doesn't make sense. 1208 if (CldrUtility.INHERITANCE_MARKER.equals(value)) { 1209 value = getCldrFileToCheck().getConstructedBaileyValue(path, null, null); 1210 } 1211 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) { 1212 CheckCLDR item = it.next(); 1213 // skip proposed items in final testing. 1214 if (Phase.FINAL_TESTING == item.getPhase()) { 1215 if (path.contains("proposed") && path.contains("[@alt=")) { 1216 continue; 1217 } 1218 } 1219 try { 1220 if (!item.isSkipTest()) { 1221 item.handleCheck(path, fullPath, value, options, result); 1222 } 1223 } catch (Exception e) { 1224 addError(result, item, e); 1225 return this; 1226 } 1227 } 1228 return this; 1229 } 1230 1231 @Override 1232 public void handleFinish() { 1233 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) { 1234 CheckCLDR item = it.next(); 1235 item.handleFinish(); 1236 } 1237 } 1238 1239 protected CheckCLDR handleGetExamples(String path, String fullPath, String value, Options options, 1240 List<CheckStatus> result) { 1241 result.clear(); 1242 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) { 1243 CheckCLDR item = it.next(); 1244 try { 1245 item.handleGetExamples(path, fullPath, value, options, result); 1246 } catch (Exception e) { 1247 addError(result, item, e); 1248 return this; 1249 } 1250 } 1251 return this; 1252 } 1253 1254 private void addError(List<CheckStatus> result, CheckCLDR item, Exception e) { 1255 result.add(new CheckStatus() 1256 .setCause(this) 1257 .setMainType(CheckStatus.errorType) 1258 .setSubtype(Subtype.internalError) 1259 .setMessage("Internal error in {0}. Exception: {1}, Message: {2}, Trace: {3}", 1260 new Object[] { item.getClass().getName(), e.getClass().getName(), e, 1261 Arrays.asList(e.getStackTrace()) 1262 })); 1263 } 1264 1265 public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, 1266 List<CheckStatus> possibleErrors) { 1267 ElapsedTimer testTime = null, testOverallTime = null; 1268 if (cldrFileToCheck == null) return this; 1269 boolean SHOW_TIMES = options.contains(Options.Option.SHOW_TIMES); 1270 setPhase(Phase.forString(options.get(Options.Option.phase))); 1271 if (SHOW_TIMES) testOverallTime = new ElapsedTimer("Test setup time for setCldrFileToCheck: {0}"); 1272 super.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 1273 possibleErrors.clear(); 1274 1275 for (Iterator<CheckCLDR> it = filteredCheckList.iterator(); it.hasNext();) { 1276 CheckCLDR item = (CheckCLDR) it.next(); 1277 if (SHOW_TIMES) 1278 testTime = new ElapsedTimer("Test setup time for " + item.getClass().toString() + ": {0}"); 1279 try { 1280 item.setPhase(getPhase()); 1281 item.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 1282 if (SHOW_TIMES) { 1283 if (item.isSkipTest()) { 1284 System.out.println("Disabled : " + testTime); 1285 } else { 1286 System.out.println("OK : " + testTime); 1287 } 1288 } 1289 } catch (RuntimeException e) { 1290 addError(possibleErrors, item, e); 1291 if (SHOW_TIMES) System.out.println("ERR: " + testTime + " - " + e.toString()); 1292 } 1293 } 1294 if (SHOW_TIMES) System.out.println("Overall: " + testOverallTime + ": {0}"); 1295 return this; 1296 } 1297 1298 public Matcher getFilter() { 1299 return filter; 1300 } 1301 1302 public CompoundCheckCLDR setFilter(Matcher filter) { 1303 this.filter = filter; 1304 filteredCheckList.clear(); 1305 for (Iterator<CheckCLDR> it = checkList.iterator(); it.hasNext();) { 1306 CheckCLDR item = it.next(); 1307 if (filter == null || filter.reset(item.getClass().getName()).matches()) { 1308 filteredCheckList.add(item); 1309 item.setCldrFileToCheck(getCldrFileToCheck(), (Options) null, null); 1310 } 1311 } 1312 return this; 1313 } 1314 1315 public String getFilteredTests() { 1316 return filteredCheckList.toString(); 1317 } 1318 1319 public List<CheckCLDR> getFilteredTestList() { 1320 return filteredCheckList; 1321 } 1322 } 1323 1324 // static Transliterator prettyPath = getTransliteratorFromFile("ID", "prettyPath.txt"); 1325 1326 public static Transliterator getTransliteratorFromFile(String ID, String file) { 1327 try { 1328 BufferedReader br = CldrUtility.getUTF8Data(file); 1329 StringBuffer input = new StringBuffer(); 1330 while (true) { 1331 String line = br.readLine(); 1332 if (line == null) break; 1333 if (line.startsWith("\uFEFF")) line = line.substring(1); // remove BOM 1334 input.append(line); 1335 input.append('\n'); 1336 } 1337 return Transliterator.createFromRules(ID, input.toString(), Transliterator.FORWARD); 1338 } catch (IOException e) { 1339 throw new ICUUncheckedIOException("Can't open transliterator file " + file, e); 1340 } 1341 } 1342 1343 public Phase getPhase() { 1344 return phase; 1345 } 1346 1347 public void setPhase(Phase phase) { 1348 this.phase = phase; 1349 } 1350 1351 /** 1352 * A map of error/warning types to their filters. 1353 */ 1354 private static List<R3<Pattern, Subtype, Pattern>> allFilters; 1355 1356 /** 1357 * Loads the set of filters used for CheckCLDR results. 1358 */ 1359 private void loadFilters() { 1360 if (allFilters != null) return; 1361 allFilters = new ArrayList<R3<Pattern, Subtype, Pattern>>(); 1362 RegexFileParser fileParser = new RegexFileParser(); 1363 fileParser.setLineParser(new RegexLineParser() { 1364 @Override 1365 public void parse(String line) { 1366 String[] fields = line.split("\\s*;\\s*"); 1367 Subtype subtype = Subtype.valueOf(fields[0]); 1368 Pattern locale = PatternCache.get(fields[1]); 1369 Pattern xpathRegex = PatternCache.get(fields[2].replaceAll("\\[@", "\\\\[@")); 1370 allFilters.add(new R3<Pattern, Subtype, Pattern>(locale, subtype, xpathRegex)); 1371 } 1372 }); 1373 fileParser.parse(CheckCLDR.class, "/org/unicode/cldr/util/data/CheckCLDR-exceptions.txt"); 1374 } 1375 1376 /** 1377 * Checks if a status should be excluded from the list of results returned 1378 * from CheckCLDR. 1379 * @param xpath the xpath that the status belongs to 1380 * @param status the status 1381 * @return true if the status should be included 1382 */ 1383 private boolean shouldExcludeStatus(String xpath, CheckStatus status) { 1384 List<Pattern> xpathPatterns = filtersForLocale.get(status.getSubtype()); 1385 if (xpathPatterns == null) return false; 1386 for (Pattern xpathPattern : xpathPatterns) { 1387 if (xpathPattern.matcher(xpath).matches()) { 1388 return true; 1389 } 1390 } 1391 return false; 1392 } 1393 1394 public CLDRFile getEnglishFile() { 1395 return englishFile; 1396 } 1397 1398 public void setEnglishFile(CLDRFile englishFile) { 1399 this.englishFile = englishFile; 1400 } 1401 } 1402
