1 package org.unicode.cldr.util; 2 3 import java.util.Collection; 4 import java.util.Collections; 5 import java.util.EnumMap; 6 import java.util.EnumSet; 7 import java.util.LinkedHashMap; 8 import java.util.LinkedHashSet; 9 import java.util.List; 10 import java.util.Locale; 11 import java.util.Map; 12 import java.util.Map.Entry; 13 import java.util.Objects; 14 import java.util.Set; 15 import java.util.TreeMap; 16 import java.util.TreeSet; 17 import java.util.regex.Pattern; 18 19 import org.unicode.cldr.util.LanguageInfo.CldrDir; 20 import org.unicode.cldr.util.StandardCodes.LstrType; 21 import org.unicode.cldr.util.SupplementalDataInfo.AttributeValidityInfo; 22 23 import com.google.common.base.Splitter; 24 import com.google.common.collect.ComparisonChain; 25 import com.ibm.icu.impl.Relation; 26 import com.ibm.icu.impl.Row; 27 import com.ibm.icu.impl.Row.R2; 28 import com.ibm.icu.impl.Row.R3; 29 import com.ibm.icu.text.UnicodeSet; 30 import com.ibm.icu.util.ICUException; 31 import com.ibm.icu.util.Output; 32 33 public class AttributeValueValidity { 34 35 public enum Status { 36 ok, deprecated, illegal, noTest 37 } 38 39 public enum LocaleSpecific { 40 pluralCardinal, pluralOrdinal, dayPeriodFormat, dayPeriodSelection 41 } 42 43 static final Splitter BAR = Splitter.on('|').trimResults().omitEmptyStrings(); 44 static final Splitter SPACE = Splitter.on(PatternCache.get("\\s+")).trimResults().omitEmptyStrings(); 45 46 private static final Set<DtdType> ALL_DTDs = Collections.unmodifiableSet(EnumSet.allOf(DtdType.class)); 47 48 private static final SupplementalDataInfo supplementalData = CLDRConfig.getInstance().getSupplementalDataInfo(); 49 50 private static Map<DtdType, Map<String, Map<String, MatcherPattern>>> dtd_element_attribute_validity = new EnumMap<>(DtdType.class); 51 private static Map<String, MatcherPattern> common_attribute_validity = new LinkedHashMap<String, MatcherPattern>(); 52 private static Map<String, MatcherPattern> variables = new LinkedHashMap<String, MatcherPattern>(); 53 private static final RegexMatcher NOT_DONE_YET = new RegexMatcher(".*", Pattern.COMMENTS); 54 private static final Map<AttributeValidityInfo, String> failures = new LinkedHashMap<>(); 55 private static final boolean DEBUG = false; 56 57 static { 58 59 Relation<R2<String, String>, String> bcp47Aliases = supplementalData.getBcp47Aliases(); 60 Set<String> bcp47Keys = new LinkedHashSet<>(); 61 Set<String> bcp47Values = new LinkedHashSet<>(); 62 for (Entry<String, Set<String>> keyValues : supplementalData.getBcp47Keys().keyValuesSet()) { 63 Set<String> fullValues = new TreeSet<>(); 64 String key = keyValues.getKey(); 65 bcp47Keys.add(key); 66 67 Set<String> rawValues = keyValues.getValue(); 68 69 for (String value : rawValues) { 70 if (key.equals("cu")) { // Currency codes are in upper case. 71 fullValues.add(value.toUpperCase()); 72 } else { 73 fullValues.add(value); 74 } 75 R2<String, String> keyValue = R2.of(key, value); 76 Set<String> aliases = bcp47Aliases.getAll(keyValue); 77 if (aliases != null) { 78 fullValues.addAll(aliases); 79 } 80 } 81 // Special case exception for generic calendar, since we don't want to expose it in bcp47 82 if (key.equals("ca")) { 83 fullValues.add("generic"); 84 } 85 fullValues = Collections.unmodifiableSet(fullValues); 86 addCollectionVariable("$_bcp47_" + key, fullValues); 87 88 // add aliased keys 89 Set<String> aliases = supplementalData.getBcp47Aliases().getAll(Row.of(key, "")); 90 if (aliases != null) { 91 for (String aliasKey : aliases) { 92 bcp47Keys.add(aliasKey); 93 addCollectionVariable("$_bcp47_" + aliasKey, fullValues); 94 } 95 } 96 bcp47Values.addAll(fullValues); 97 } 98 bcp47Keys.add("x"); // special-case private use 99 bcp47Keys.add("x0"); // special-case, has no subtypes 100 addCollectionVariable("$_bcp47_keys", bcp47Keys); 101 addCollectionVariable("$_bcp47_value", bcp47Values); 102 103 Validity validity = Validity.getInstance(); 104 for (LstrType key : LstrType.values()) { 105 final Map<Validity.Status, Set<String>> statusToCodes = validity.getStatusToCodes(key); 106 if (statusToCodes == null) { 107 continue; 108 } 109 String keyName = "$_" + key; 110 Set<String> all = new LinkedHashSet<>(); 111 Set<String> prefix = new LinkedHashSet<>(); 112 Set<String> suffix = new LinkedHashSet<>(); 113 Set<String> regularAndUnknown = new LinkedHashSet<>(); 114 for (Entry<Validity.Status, Set<String>> item2 : statusToCodes.entrySet()) { 115 Validity.Status status = item2.getKey(); 116 Set<String> validItems = item2.getValue(); 117 if (key == LstrType.variant) { // uppercased in CLDR 118 Set<String> temp2 = new LinkedHashSet<>(validItems); 119 for (String item : validItems) { 120 temp2.add(item.toUpperCase(Locale.ROOT)); 121 } 122 validItems = temp2; 123 } else if (key == LstrType.subdivision) { 124 for (String item : validItems) { 125 if (item.contains("-")) { 126 List<String> parts = Splitter.on('-').splitToList(item); 127 prefix.add(parts.get(0)); 128 suffix.add(parts.get(1)); 129 } else { 130 int prefixWidth = item.charAt(0) < 'A' ? 3 : 2; 131 prefix.add(item.substring(0, prefixWidth)); 132 suffix.add(item.substring(prefixWidth)); 133 } 134 } 135 } 136 all.addAll(validItems); 137 if (status == Validity.Status.regular || status == Validity.Status.special || status == Validity.Status.unknown) { 138 regularAndUnknown.addAll(validItems); 139 } 140 addCollectionVariable(keyName + "_" + status, validItems); 141 // MatcherPattern m = new MatcherPattern(key.toString(), validItems.toString(), new CollectionMatcher(validItems)); 142 // variables.put(keyName+"_"+status, m); 143 } 144 if (key == LstrType.subdivision) { 145 addCollectionVariable(keyName + "_prefix", prefix); 146 addCollectionVariable(keyName + "_suffix", suffix); 147 } 148 addCollectionVariable(keyName, all); 149 addCollectionVariable(keyName + "_plus", regularAndUnknown); 150 151 // MatcherPattern m = new MatcherPattern(key.toString(), all.toString(), new CollectionMatcher(all)); 152 // variables.put(keyName, m); 153 // MatcherPattern m2 = new MatcherPattern(key.toString(), regularAndUnknown.toString(), new CollectionMatcher(regularAndUnknown)); 154 // variables.put(keyName + "_plus", m2); 155 } 156 157 Set<String> main = new LinkedHashSet<>(); 158 main.addAll(StandardCodes.LstrType.language.specials); 159 Set<String> coverage = new LinkedHashSet<>(); 160 Set<String> large_official = new LinkedHashSet<>(); 161 final LocaleIDParser lip = new LocaleIDParser(); 162 163 for (String language : LanguageInfo.getAvailable()) { 164 LanguageInfo info = LanguageInfo.get(language); 165 CldrDir cldrDir = info.getCldrDir(); 166 String base = lip.set(language).getLanguage(); 167 if (cldrDir == CldrDir.main || cldrDir == CldrDir.base) { 168 main.add(base); 169 } 170 if (info.getCldrLevel() == Level.MODERN) { 171 coverage.add(base); 172 } 173 if (info.getLiteratePopulation() > 1000000 && !info.getStatusToRegions().isEmpty()) { 174 large_official.add(base); 175 } 176 } 177 addCollectionVariable("$_language_main", main); 178 addCollectionVariable("$_language_coverage", coverage); 179 addCollectionVariable("$_language_large_official", large_official); 180 Set<String> cldrLang = new TreeSet<>(main); 181 cldrLang.addAll(coverage); 182 cldrLang.addAll(large_official); 183 addCollectionVariable("$_language_cldr", large_official); 184 // System.out.println("\ncldrLang:\n" + Joiner.on(' ').join(cldrLang)); 185 186 Map<String, R2<String, String>> rawVariables = supplementalData.getValidityInfo(); 187 for (Entry<String, R2<String, String>> item : rawVariables.entrySet()) { 188 String id = item.getKey(); 189 String type = item.getValue().get0(); 190 String value = item.getValue().get1(); 191 MatcherPattern mp = getMatcherPattern2(type, value); 192 if (mp != null) { 193 variables.put(id, mp); 194 // variableReplacer.add(id, value); 195 } else { 196 throw new IllegalArgumentException("Duplicate element " + mp); 197 } 198 } 199 //System.out.println("Variables: " + variables.keySet()); 200 201 Map<AttributeValidityInfo, String> rawAttributeValueInfo = supplementalData.getAttributeValidity(); 202 int x = 0; 203 for (Entry<AttributeValidityInfo, String> entry : rawAttributeValueInfo.entrySet()) { 204 AttributeValidityInfo item = entry.getKey(); 205 String value = entry.getValue(); 206 //System.out.println(item); 207 MatcherPattern mp = getMatcherPattern2(item.getType(), value); 208 if (mp == null) { 209 getMatcherPattern2(item.getType(), value); // for debugging 210 failures.put(item, value); 211 continue; 212 } 213 Set<DtdType> dtds = item.getDtds(); 214 if (dtds == null) { 215 dtds = ALL_DTDs; 216 } 217 for (DtdType dtdType : dtds) { 218 DtdData data = DtdData.getInstance(dtdType); 219 Map<String, Map<String, MatcherPattern>> element_attribute_validity = dtd_element_attribute_validity.get(dtdType); 220 if (element_attribute_validity == null) { 221 dtd_element_attribute_validity.put(dtdType, element_attribute_validity = new TreeMap<String, Map<String, MatcherPattern>>()); 222 } 223 224 // <attributeValues dtds="supplementalData" elements="currency" attributes="before from to">$currencyDate</attributeValues> 225 226 Set<String> attributeList = item.getAttributes(); 227 Set<String> elementList = item.getElements(); 228 if (elementList.size() == 0) { 229 addAttributes(attributeList, common_attribute_validity, mp); 230 } else { 231 for (String element : elementList) { 232 // check if unnecessary 233 DtdData.Element elementInfo = data.getElementFromName().get(element); 234 if (elementInfo == null) { 235 throw new ICUException( 236 "Illegal <attributeValues>, element not valid: " 237 + dtdType 238 + ", element: " + element); 239 } else { 240 for (String attribute : attributeList) { 241 DtdData.Attribute attributeInfo = elementInfo.getAttributeNamed(attribute); 242 if (attributeInfo == null) { 243 throw new ICUException( 244 "Illegal <attributeValues>, attribute not valid: " 245 + dtdType 246 + ", element: " + element 247 + ", attribute: " + attribute); 248 } else if (!attributeInfo.values.isEmpty()) { 249 // if (false) { 250 // System.out.println("Unnecessary <attributeValues >, the DTD has specific list: element: " + element + ", attribute: " + attribute + ", " + attributeInfo.values); 251 // } 252 } 253 } 254 } 255 // System.out.println("\t" + element); 256 Map<String, MatcherPattern> attribute_validity = element_attribute_validity.get(element); 257 if (attribute_validity == null) { 258 element_attribute_validity.put(element, attribute_validity = new TreeMap<String, MatcherPattern>()); 259 } 260 addAttributes(attributeList, attribute_validity, mp); 261 } 262 } 263 } 264 } 265 // show values 266 // for (Entry<DtdType, Map<String, Map<String, MatcherPattern>>> entry1 : dtd_element_attribute_validity.entrySet()) { 267 // final DtdType dtdType = entry1.getKey(); 268 // Map<String, Map<String, MatcherPattern>> element_attribute_validity = entry1.getValue(); 269 // DtdData dtdData2 = DtdData.getInstance(dtdType); 270 // for (Element element : dtdData2.getElements()) { 271 // Set<Attribute> attributes = element.getAttributes().keySet(); 272 // 273 // } 274 // for (Entry<String, Map<String, MatcherPattern>> entry2 : entry1.getValue().entrySet()) { 275 // for (Entry<String, MatcherPattern> entry3 : entry2.getValue().entrySet()) { 276 // System.out.println(dtdType + "\t" + entry2.getKey() + "\t" + entry3.getKey() + "\t" + entry3.getValue()); 277 // } 278 // } 279 // } 280 281 // private LocaleIDParser localeIDParser = new LocaleIDParser(); 282 // 283 // @Override 284 // public CheckCLDR setCldrFileToCheck(CLDRFile cldrFileToCheck, Options options, 285 // List<CheckStatus> possibleErrors) { 286 // if (cldrFileToCheck == null) return this; 287 // if (Phase.FINAL_TESTING == getPhase() || Phase.BUILD == getPhase()) { 288 // setSkipTest(false); // ok 289 // } else { 290 // setSkipTest(true); 291 // return this; 292 // } 293 // 294 // pluralInfo = supplementalData.getPlurals(PluralType.cardinal, cldrFileToCheck.getLocaleID()); 295 // super.setCldrFileToCheck(cldrFileToCheck, options, possibleErrors); 296 // isEnglish = "en".equals(localeIDParser.set(cldrFileToCheck.getLocaleID()).getLanguage()); 297 // synchronized (elementOrder) { 298 // if (!initialized) { 299 // getMetadata(); 300 // initialized = true; 301 // localeMatcher = LocaleMatcher.make(); 302 // } 303 // } 304 // if (!localeMatcher.matches(cldrFileToCheck.getLocaleID())) { 305 // possibleErrors.add(new CheckStatus() 306 // .setCause(null).setMainType(CheckStatus.errorType).setSubtype(Subtype.invalidLocale) 307 // .setMessage("Invalid Locale {0}", 308 // new Object[] { cldrFileToCheck.getLocaleID() })); 309 // 310 // } 311 // return this; 312 // } 313 } 314 315 private static void addCollectionVariable(String name, Set<String> validItems) { 316 variables.put(name, new CollectionMatcher(validItems)); 317 } 318 319 public static Relation<String, String> getAllPossibleMissing(DtdType dtdType) { 320 Relation<String, String> missing = Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class); 321 322 if (dtdType == DtdType.ldmlICU) { 323 return missing; 324 } 325 326 DtdData dtdData2 = DtdData.getInstance(dtdType); 327 Map<String, Map<String, MatcherPattern>> element_attribute_validity = CldrUtility.ifNull( 328 dtd_element_attribute_validity.get(dtdType), 329 Collections.<String, Map<String, MatcherPattern>> emptyMap()); 330 331 for (DtdData.Element element : dtdData2.getElements()) { 332 if (element.isDeprecated()) { 333 continue; 334 } 335 Map<String, MatcherPattern> attribute_validity = CldrUtility.ifNull( 336 element_attribute_validity.get(element.name), 337 Collections.<String, MatcherPattern> emptyMap()); 338 for (DtdData.Attribute attribute : element.getAttributes().keySet()) { 339 if (attribute.isDeprecated()) { 340 continue; 341 } 342 if (!attribute.values.isEmpty()) { 343 continue; 344 } 345 MatcherPattern validity = attribute_validity.get(attribute.name); 346 if (validity != null) { 347 continue; 348 } 349 // <attributeValues attributes="alt" type="choice">$alt</attributeValues> 350 // <attributeValues dtds="supplementalData" elements="character" attributes="value" type="regex">.</attributeValues> 351 missing.put(attribute.name, 352 new AttributeValueSpec(dtdType, element.name, attribute.name, "$xxx").toString()); 353 } 354 } 355 return missing; 356 } 357 358 public static abstract class MatcherPattern { 359 360 public abstract boolean matches(String value, Output<String> reason); 361 362 public String getPattern() { 363 String temp = _getPattern(); 364 return temp.length() <= MAX_STRING ? temp : temp.substring(0, MAX_STRING) + ""; 365 } 366 367 public abstract String _getPattern(); 368 369 public String toString() { 370 return getClass().getName() + "\t" + getPattern(); 371 } 372 } 373 374 // private static MatcherPattern getBcp47MatcherPattern(String key) { 375 // // <key type="calendar">Calendar</key> 376 // // <type key="calendar" type="chinese">Chinese Calendar</type> 377 // 378 // //<attributeValues elements="key" attributes="type" type="bcp47">key</attributeValues> 379 // //<attributeValues elements="type" attributes="key" type="bcp47">key</attributeValues> 380 // //<attributeValues elements="type" attributes="type" type="bcp47">use-key</attributeValues> 381 // 382 // Set<String> values; 383 // if (key.equals("key")) { 384 // values = BCP47_KEY_VALUES.keySet(); 385 // } else { 386 // values = BCP47_KEY_VALUES.get(key); 387 // } 388 // return new CollectionMatcher(values); 389 // } 390 391 enum MatcherTypes { 392 single, choice, list, unicodeSet, unicodeSetOrString, regex, locale, bcp47, subdivision, localeSpecific, TODO; 393 } 394 395 private static MatcherPattern getMatcherPattern2(String type, String value) { 396 final MatcherTypes matcherType = type == null ? MatcherTypes.single : MatcherTypes.valueOf(type); 397 398 if (matcherType != MatcherTypes.TODO && value.startsWith("$")) { 399 MatcherPattern result = getVariable(matcherType, value); 400 if (result != null) { 401 return result; 402 } 403 throw new IllegalArgumentException("Unknown variable: " + value); 404 } 405 406 MatcherPattern result; 407 408 switch (matcherType) { 409 case single: 410 result = new CollectionMatcher(Collections.singleton(value.trim())); 411 break; 412 case choice: 413 result = new CollectionMatcher(SPACE.splitToList(value)); 414 break; 415 case unicodeSet: 416 result = new UnicodeSetMatcher(new UnicodeSet(value)); 417 break; 418 case unicodeSetOrString: 419 result = new UnicodeSetOrStringMatcher(new UnicodeSet(value)); 420 break; 421 // case bcp47: 422 // return getBcp47MatcherPattern(value); 423 case regex: 424 result = new RegexMatcher(value, Pattern.COMMENTS); // Pattern.COMMENTS to get whitespace 425 break; 426 case locale: 427 result = value.equals("all") ? LocaleMatcher.ALL_LANGUAGES : LocaleMatcher.REGULAR; 428 break; 429 case localeSpecific: 430 result = LocaleSpecificMatcher.getInstance(value); 431 break; 432 case TODO: 433 result = NOT_DONE_YET; 434 break; 435 case list: 436 result = new ListMatcher(new CollectionMatcher(SPACE.splitToList(value))); 437 break; 438 default: 439 return null; 440 } 441 442 return result; 443 } 444 445 private static MatcherPattern getVariable(final MatcherTypes matcherType, String value) { 446 List<String> values = BAR.splitToList(value); //value.trim().split("|"); 447 MatcherPattern[] reasons = new MatcherPattern[values.size()]; 448 for (int i = 0; i < values.size(); ++i) { 449 reasons[i] = getNonNullVariable(values.get(i)); 450 } 451 MatcherPattern result; 452 453 if (reasons.length == 1) { 454 result = reasons[0]; 455 } else { 456 result = new OrMatcher(reasons); 457 } 458 if (matcherType == MatcherTypes.list) { 459 result = new ListMatcher(result); 460 } 461 return result; 462 } 463 464 private static void addAttributes(Set<String> attributes, Map<String, MatcherPattern> attribute_validity, MatcherPattern mp) { 465 for (String attribute : attributes) { 466 MatcherPattern old = attribute_validity.get(attribute); 467 if (old != null) { 468 mp = new OrMatcher(old, mp); 469 } 470 attribute_validity.put(attribute, mp); 471 } 472 } 473 474 public static class RegexMatcher extends MatcherPattern { 475 476 private java.util.regex.Matcher matcher; 477 478 public RegexMatcher(String pattern, int flags) { 479 matcher = Pattern.compile(pattern, flags).matcher(""); 480 } 481 482 public boolean matches(String value, Output<String> reason) { 483 matcher.reset(value.toString()); 484 boolean result = matcher.matches(); 485 if (!result && reason != null) { 486 reason.value = RegexUtilities.showMismatch(matcher, value.toString()); 487 } 488 return result; 489 } 490 491 @Override 492 public String _getPattern() { 493 return matcher.toString(); 494 } 495 } 496 497 private static EnumMap<LocaleSpecific, Set<String>> LOCALE_SPECIFIC = null; 498 499 /** WARNING, not thread-safe. Needs cleanup **/ 500 public static void setLocaleSpecifics(EnumMap<LocaleSpecific, Set<String>> newValues) { 501 LOCALE_SPECIFIC = newValues; 502 } 503 504 public static class LocaleSpecificMatcher extends MatcherPattern { 505 final LocaleSpecific ls; 506 507 public LocaleSpecificMatcher(LocaleSpecific ls) { 508 this.ls = ls; 509 } 510 511 public static LocaleSpecificMatcher getInstance(String value) { 512 return new LocaleSpecificMatcher(LocaleSpecific.valueOf(value)); 513 } 514 515 public boolean matches(String value) { 516 return LOCALE_SPECIFIC.get(ls).contains(value); 517 } 518 519 static final int MAX_STRING = 64; 520 521 public boolean matches(String value, Output<String> reason) { 522 boolean result = LOCALE_SPECIFIC.get(ls).contains(value); 523 if (!result && reason != null) { 524 reason.value = " " + getPattern(); 525 } 526 return result; 527 } 528 529 @Override 530 public String _getPattern() { 531 return LOCALE_SPECIFIC.get(ls).toString(); 532 } 533 } 534 535 static final int MAX_STRING = 64; 536 537 public static class CollectionMatcher extends MatcherPattern { 538 private final Collection<String> collection; 539 540 public CollectionMatcher(Collection<String> collection) { 541 this.collection = Collections.unmodifiableCollection(new LinkedHashSet<>(collection)); 542 } 543 544 @Override 545 public boolean matches(String value, Output<String> reason) { 546 boolean result = collection.contains(value); 547 if (!result && reason != null) { 548 reason.value = " " + getPattern(); 549 } 550 return result; 551 } 552 553 @Override 554 public String _getPattern() { 555 return collection.toString(); 556 } 557 } 558 559 public static class UnicodeSetMatcher extends MatcherPattern { 560 private final UnicodeSet collection; 561 562 public UnicodeSetMatcher(UnicodeSet collection) { 563 this.collection = collection.freeze(); 564 } 565 566 @Override 567 public boolean matches(String value, Output<String> reason) { 568 boolean result = false; 569 try { 570 UnicodeSet valueSet = new UnicodeSet(value); 571 result = collection.containsAll(valueSet); 572 if (!result && reason != null) { 573 reason.value = " " + getPattern(); 574 } 575 } catch (Exception e) { 576 reason.value = " illegal pattern " + getPattern() + ": " + value; 577 } 578 return result; 579 } 580 581 @Override 582 public String _getPattern() { 583 return collection.toPattern(false); 584 } 585 } 586 587 public static class UnicodeSetOrStringMatcher extends MatcherPattern { 588 private final UnicodeSet collection; 589 590 public UnicodeSetOrStringMatcher(UnicodeSet collection) { 591 this.collection = collection.freeze(); 592 } 593 594 @Override 595 public boolean matches(String value, Output<String> reason) { 596 boolean result = false; 597 if (UnicodeSet.resemblesPattern(value, 0)) { 598 try { 599 UnicodeSet valueSet = new UnicodeSet(value); 600 result = collection.containsAll(valueSet); 601 if (!result && reason != null) { 602 reason.value = " " + getPattern(); 603 } 604 } catch (Exception e) { 605 reason.value = " illegal pattern " + getPattern() + ": " + value; 606 } 607 } else { 608 result = collection.contains(value); 609 if (!result && reason != null) { 610 reason.value = " " + getPattern(); 611 } 612 } 613 return result; 614 } 615 616 @Override 617 public String _getPattern() { 618 return collection.toPattern(false); 619 } 620 } 621 622 public static class OrMatcher extends MatcherPattern { 623 private final MatcherPattern[] operands; 624 625 public OrMatcher(MatcherPattern... operands) { 626 for (MatcherPattern operand : operands) { 627 if (operand == null) { 628 throw new NullPointerException(); 629 } 630 } 631 this.operands = operands; 632 } 633 634 public boolean matches(String value, Output<String> reason) { 635 StringBuilder fullReason = reason == null ? null : new StringBuilder(); 636 for (MatcherPattern operand : operands) { 637 if (operand.matches(value, reason)) { 638 return true; 639 } 640 if (fullReason != null) { 641 if (fullReason.length() != 0) { 642 fullReason.append("&"); 643 } 644 fullReason.append(reason.value); 645 } 646 } 647 if (fullReason != null) { 648 reason.value = fullReason.toString(); 649 } 650 return false; 651 } 652 653 @Override 654 public String _getPattern() { 655 StringBuffer result = new StringBuffer(); 656 for (MatcherPattern operand : operands) { 657 if (result.length() != 0) { 658 result.append('|'); 659 } 660 result.append(operand._getPattern()); 661 } 662 return result.toString(); 663 } 664 } 665 666 public static class ListMatcher extends MatcherPattern { 667 private MatcherPattern other; 668 669 public ListMatcher(MatcherPattern other) { 670 this.other = other; 671 } 672 673 public boolean matches(String value, Output<String> reason) { 674 List<String> values = SPACE.splitToList(value); 675 if (values.isEmpty()) return true; 676 for (String valueItem : values) { 677 if (!other.matches(valueItem, reason)) { 678 if (reason != null) { 679 reason.value = "" + valueItem + " " + other.getPattern(); 680 } 681 return false; 682 } 683 } 684 return true; 685 } 686 687 @Override 688 public String _getPattern() { 689 return "List of " + other._getPattern(); 690 } 691 } 692 693 public static class LocaleMatcher extends MatcherPattern { 694 //final ObjectMatcherReason grandfathered = getNonNullVariable("$grandfathered").matcher; 695 final MatcherPattern language; 696 final MatcherPattern script = getNonNullVariable("$_script"); 697 final MatcherPattern territory = getNonNullVariable("$_region"); 698 final MatcherPattern variant = getNonNullVariable("$_variant"); 699 final LocaleIDParser lip = new LocaleIDParser(); 700 701 public static LocaleMatcher REGULAR = new LocaleMatcher("$_language_plus"); 702 public static LocaleMatcher ALL_LANGUAGES = new LocaleMatcher("$_language"); 703 704 private LocaleMatcher(String variable) { 705 language = getNonNullVariable(variable); 706 } 707 708 public boolean matches(String value, Output<String> reason) { 709 // if (grandfathered.matches(value, reason)) { 710 // return true; 711 // } 712 lip.set((String) value); 713 String field = lip.getLanguage(); 714 if (!language.matches(field, reason)) { 715 if (reason != null) { 716 reason.value = "invalid base language"; 717 } 718 return false; 719 } 720 field = lip.getScript(); 721 if (field.length() != 0 && !script.matches(field, reason)) { 722 if (reason != null) { 723 reason.value = "invalid script"; 724 } 725 return false; 726 } 727 field = lip.getRegion(); 728 if (field.length() != 0 && !territory.matches(field, reason)) { 729 if (reason != null) { 730 reason.value = "invalid region"; 731 } 732 return false; 733 } 734 String[] fields = lip.getVariants(); 735 for (int i = 0; i < fields.length; ++i) { 736 if (!variant.matches(fields[i], reason)) { 737 if (reason != null) { 738 reason.value = "invalid variant"; 739 } 740 return false; 741 } 742 } 743 return true; 744 } 745 746 @Override 747 public String _getPattern() { 748 return "Unicode_Language_Subtag"; 749 } 750 } 751 752 public static final class AttributeValueSpec implements Comparable<AttributeValueSpec> { 753 public AttributeValueSpec(DtdType type, String element, String attribute, String attributeValue) { 754 this.type = type; 755 this.element = element; 756 this.attribute = attribute; 757 this.attributeValue = attributeValue; 758 } 759 760 public final DtdType type; 761 public final String element; 762 public final String attribute; 763 public final String attributeValue; 764 765 @Override 766 public int hashCode() { 767 return Objects.hash(type, element, attribute, attributeValue); 768 } 769 770 @Override 771 public boolean equals(Object obj) { 772 AttributeValueSpec other = (AttributeValueSpec) obj; 773 return CldrUtility.deepEquals( 774 type, other.type, 775 element, other.element, 776 attribute, other.attribute, 777 attributeValue, other.attributeValue); 778 } 779 780 @Override 781 public int compareTo(AttributeValueSpec other) { 782 return ComparisonChain.start() 783 .compare(type, other.type) 784 .compare(element, other.element) 785 .compare(attribute, other.attribute) 786 .compare(attributeValue, other.attributeValue) 787 .result(); 788 } 789 790 @Override 791 public String toString() { 792 return "<attributeValues" 793 + " dtds='" + type + "\'" 794 + " elements='" + element + "\'" 795 + " attributes='" + attribute + "\'" 796 + " type='TODO\'>" 797 + attributeValue 798 + "</attributeValues>"; 799 } 800 } 801 802 /** 803 * return Status 804 * @param attribute_validity 805 * @param attribute 806 * @param attributeValue 807 * @param result 808 * @return 809 */ 810 private static Status check(Map<String, MatcherPattern> attribute_validity, 811 String element, String attribute, String attributeValue, 812 Output<String> reason) { 813 814 if (attribute_validity == null) { 815 return Status.noTest; // no test 816 } 817 MatcherPattern matcherPattern = attribute_validity.get(attribute); 818 if (matcherPattern == null) { 819 return Status.noTest; // no test 820 } 821 if (matcherPattern.matches(attributeValue, reason)) { 822 return Status.ok; 823 } 824 return Status.illegal; 825 } 826 827 public static Status check(DtdData dtdData, String element, String attribute, String attributeValue, Output<String> reason) { 828 if (dtdData.isDeprecated(element, attribute, attributeValue)) { 829 return Status.deprecated; 830 } 831 Status haveTest = check(common_attribute_validity, element, attribute, attributeValue, reason); 832 833 if (haveTest == Status.noTest) { 834 final Map<String, Map<String, MatcherPattern>> element_attribute_validity = dtd_element_attribute_validity.get(dtdData.dtdType); 835 if (element_attribute_validity == null) { 836 return Status.noTest; 837 } 838 839 Map<String, MatcherPattern> attribute_validity = element_attribute_validity.get(element); 840 if (attribute_validity == null) { 841 return Status.noTest; 842 } 843 844 haveTest = check(attribute_validity, element, attribute, attributeValue, reason); 845 } 846 return haveTest; 847 } 848 849 public static Set<R3<DtdType, String, String>> getTodoTests() { 850 Set<Row.R3<DtdType, String, String>> result = new LinkedHashSet<>(); 851 for (Entry<DtdType, Map<String, Map<String, MatcherPattern>>> entry1 : dtd_element_attribute_validity.entrySet()) { 852 for (Entry<String, Map<String, MatcherPattern>> entry2 : entry1.getValue().entrySet()) { 853 for (Entry<String, MatcherPattern> entry3 : entry2.getValue().entrySet()) { 854 if (entry3.getValue() == NOT_DONE_YET) { 855 result.add(Row.of(entry1.getKey(), entry2.getKey(), entry3.getKey())); 856 } 857 } 858 } 859 } 860 return result; 861 } 862 863 public static Map<AttributeValidityInfo, String> getReadFailures() { 864 return Collections.unmodifiableMap(failures); 865 } 866 867 public static MatcherPattern getMatcherPattern(String variable) { 868 return variables.get(variable); 869 } 870 871 private static MatcherPattern getNonNullVariable(String variable) { 872 MatcherPattern result = variables.get(variable); 873 if (result == null) { 874 throw new NullPointerException(); 875 } 876 return result; 877 } 878 879 public static Set<String> getMatcherPatternIds() { 880 return Collections.unmodifiableSet(variables.keySet()); 881 } 882 883 public static void main(String[] args) { 884 for (DtdType type : DtdType.values()) { 885 Relation<String, String> missing = getAllPossibleMissing(type); 886 for (Entry<String, String> x : missing.keyValueSet()) { 887 System.out.println(type + "\t" + CldrUtility.toString(x)); 888 } 889 } 890 } 891 } 892
