1 package org.unicode.cldr.test; 2 3 import java.io.File; 4 import java.io.IOException; 5 import java.io.InputStream; 6 import java.util.Arrays; 7 import java.util.Date; 8 import java.util.EnumSet; 9 import java.util.HashSet; 10 import java.util.Iterator; 11 import java.util.Objects; 12 import java.util.Set; 13 import java.util.TreeMap; 14 import java.util.TreeSet; 15 import java.util.regex.Matcher; 16 17 import org.unicode.cldr.tool.ToolConfig; 18 import org.unicode.cldr.util.CLDRConfig; 19 import org.unicode.cldr.util.CLDRFile; 20 import org.unicode.cldr.util.CLDRPaths; 21 import org.unicode.cldr.util.CldrUtility; 22 import org.unicode.cldr.util.DateTimeFormats; 23 import org.unicode.cldr.util.DtdType; 24 import org.unicode.cldr.util.Factory; 25 import org.unicode.cldr.util.InputStreamFactory; 26 import org.unicode.cldr.util.LanguageTagParser; 27 import org.unicode.cldr.util.Level; 28 import org.unicode.cldr.util.Organization; 29 import org.unicode.cldr.util.PatternCache; 30 import org.unicode.cldr.util.PrettyPath; 31 import org.unicode.cldr.util.StandardCodes; 32 import org.unicode.cldr.util.XMLFileReader; 33 import org.unicode.cldr.util.XPathParts; 34 import org.xml.sax.ErrorHandler; 35 import org.xml.sax.InputSource; 36 import org.xml.sax.SAXException; 37 import org.xml.sax.SAXParseException; 38 import org.xml.sax.XMLReader; 39 40 import com.ibm.icu.impl.Relation; 41 import com.ibm.icu.text.DateFormatSymbols; 42 import com.ibm.icu.text.SimpleDateFormat; 43 import com.ibm.icu.util.ULocale; 44 45 /** 46 * Simple test that loads each file in the cldr directory, thus verifying that 47 * the DTD works, and also checks that the PrettyPaths work. 48 * 49 * @author markdavis 50 */ 51 public class QuickCheck { 52 private static final Set<String> skipAttributes = new HashSet<String>(Arrays.asList(new String[] { 53 "alt", "draft", "references" })); 54 55 private static String localeRegex; 56 57 private static boolean showInfo = false; 58 59 private static String commonDirectory; 60 private static String mainDirectory; 61 62 private static boolean resolved; 63 64 private static Exception[] internalException = new Exception[1]; 65 66 private static boolean verbose; 67 68 public static void main(String[] args) throws IOException { 69 CLDRConfig testInfo = ToolConfig.getToolInstance(); 70 Factory factory = testInfo.getCldrFactory(); 71 checkStock(factory); 72 if (true) return; 73 verbose = CldrUtility.getProperty("verbose", "false", "true").matches("(?i)T|TRUE"); 74 localeRegex = CldrUtility.getProperty("locale", ".*"); 75 76 showInfo = CldrUtility.getProperty("showinfo", "false", "true").matches("(?i)T|TRUE"); 77 78 commonDirectory = CLDRPaths.COMMON_DIRECTORY; // Utility.getProperty("common", Utility.COMMON_DIRECTORY); 79 // if (commonDirectory == null) commonDirectory = Utility.COMMON_DIRECTORY 80 // System.out.println("Main Source Directory: " + commonDirectory + 81 // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)"); 82 83 mainDirectory = CldrUtility.getProperty("main", CLDRPaths.COMMON_DIRECTORY + "/main"); 84 // System.out.println("Main Source Directory: " + commonDirectory + 85 // "\t\t(to change, use -DSOURCE=xxx, eg -DSOURCE=C:/cvsdata/unicode/cldr/incoming/proposed/main)"); 86 87 resolved = CldrUtility.getProperty("resolved", "false", "true").matches("(?i)T|TRUE"); 88 89 boolean paths = CldrUtility.getProperty("paths", "true").matches("(?i)T|TRUE"); 90 91 pretty = CldrUtility.getProperty("pretty", "true").matches("(?i)T|TRUE"); 92 93 double startTime = System.currentTimeMillis(); 94 checkDtds(); 95 double deltaTime = System.currentTimeMillis() - startTime; 96 System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds"); 97 98 if (paths) { 99 System.out.println("Checking paths"); 100 checkPaths(); 101 deltaTime = System.currentTimeMillis() - startTime; 102 System.out.println("Elapsed: " + deltaTime / 1000.0 + " seconds"); 103 System.out.println("Basic Test Passes"); 104 } 105 } 106 107 private static void checkDtds() throws IOException { 108 checkDtds(commonDirectory + "supplemental"); 109 checkDtds(commonDirectory + "collation"); 110 checkDtds(commonDirectory + "main"); 111 checkDtds(commonDirectory + "rbnf"); 112 checkDtds(commonDirectory + "segments"); 113 checkDtds(commonDirectory + "../test"); 114 checkDtds(commonDirectory + "transforms"); 115 } 116 117 private static void checkDtds(String directory) throws IOException { 118 File directoryFile = new File(directory); 119 File[] listFiles = directoryFile.listFiles(); 120 String canonicalPath = directoryFile.getCanonicalPath(); 121 if (listFiles == null) { 122 throw new IllegalArgumentException("Empty directory: " + canonicalPath); 123 } 124 System.out.println("Checking files for DTD errors in: " + canonicalPath); 125 for (File fileName : listFiles) { 126 if (!fileName.toString().endsWith(".xml")) { 127 continue; 128 } 129 check(fileName); 130 } 131 } 132 133 static class MyErrorHandler implements ErrorHandler { 134 public void error(SAXParseException exception) throws SAXException { 135 System.out.println("\nerror: " + XMLFileReader.showSAX(exception)); 136 throw exception; 137 } 138 139 public void fatalError(SAXParseException exception) throws SAXException { 140 System.out.println("\nfatalError: " + XMLFileReader.showSAX(exception)); 141 throw exception; 142 } 143 144 public void warning(SAXParseException exception) throws SAXException { 145 System.out.println("\nwarning: " + XMLFileReader.showSAX(exception)); 146 throw exception; 147 } 148 } 149 150 public static void check(File systemID) { 151 try (InputStream fis = InputStreamFactory.createInputStream(systemID)) { 152 // FileInputStream fis = new FileInputStream(systemID); 153 XMLReader xmlReader = XMLFileReader.createXMLReader(true); 154 xmlReader.setErrorHandler(new MyErrorHandler()); 155 InputSource is = new InputSource(fis); 156 is.setSystemId(systemID.toString()); 157 xmlReader.parse(is); 158 // fis.close(); 159 } catch (SAXException | IOException e) { // SAXParseException is a Subtype of SaxException 160 System.out.println("\t" + "Can't read " + systemID); 161 System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 162 } 163 // catch (SAXException e) { 164 // System.out.println("\t" + "Can't read " + systemID); 165 // System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 166 // } catch (IOException e) { 167 // System.out.println("\t" + "Can't read " + systemID); 168 // System.out.println("\t" + e.getClass() + "\t" + e.getMessage()); 169 // } 170 } 171 172 static Matcher skipPaths = PatternCache.get("/identity" + "|/alias" + "|\\[@alt=\"proposed").matcher(""); 173 174 private static boolean pretty; 175 176 private static void checkPaths() { 177 Relation<String, String> distinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null); 178 Relation<String, String> nonDistinguishing = Relation.<String, String> of(new TreeMap<String, Set<String>>(), TreeSet.class, null); 179 XPathParts parts = new XPathParts(); 180 Factory cldrFactory = Factory.make(mainDirectory, localeRegex); 181 CLDRFile english = cldrFactory.make("en", true); 182 183 Relation<String, String> pathToLocale = Relation.of( 184 new TreeMap<String, Set<String>>(CLDRFile.getComparator(DtdType.ldml)), 185 TreeSet.class, null); 186 for (String locale : cldrFactory.getAvailable()) { 187 // if (locale.equals("root") && !localeRegex.equals("root")) 188 // continue; 189 CLDRFile file; 190 try { 191 file = cldrFactory.make(locale, resolved); 192 } catch (Exception e) { 193 System.out.println("\nfatalError: " + e.getMessage()); 194 continue; 195 } 196 if (file.isNonInheriting()) 197 continue; 198 DisplayAndInputProcessor displayAndInputProcessor = new DisplayAndInputProcessor(file, false); 199 200 System.out.println(locale + "\t-\t" + english.getName(locale)); 201 DtdType dtdType = null; 202 203 for (Iterator<String> it = file.iterator(); it.hasNext();) { 204 String path = it.next(); 205 if (path.endsWith("/alias")) { 206 continue; 207 } 208 String value = file.getStringValue(path); 209 if (value == null) { 210 throw new IllegalArgumentException(locale + "\tError: in null value at " + path); 211 } 212 String displayValue = displayAndInputProcessor.processForDisplay(path, value); 213 if (!displayValue.equals(value)) { 214 System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes display value <" + value 215 + ">\t=>\t<" + displayValue + ">\t\t" + path); 216 } 217 String inputValue = displayAndInputProcessor.processInput(path, value, internalException); 218 if (internalException[0] != null) { 219 System.out.println("\t" + locale + "\tdisplayAndInputProcessor internal error <" + value 220 + ">\t=>\t<" + inputValue + ">\t\t" + path); 221 internalException[0].printStackTrace(System.out); 222 } 223 if (verbose && !inputValue.equals(value)) { 224 displayAndInputProcessor.processInput(path, value, internalException); // for debugging 225 System.out.println("\t" + locale + "\tdisplayAndInputProcessor changes input value <" + value 226 + ">\t=>\t<" + inputValue + ">\t\t" + path); 227 } 228 229 pathToLocale.put(path, locale); 230 231 // also check for non-distinguishing attributes 232 if (path.contains("/identity")) continue; 233 234 // make sure we don't have problem alts 235 if (path.contains("proposed")) { 236 String sourceLocale = file.getSourceLocaleID(path, null); 237 if (locale.equals(sourceLocale)) { 238 String nonAltPath = CLDRFile.getNondraftNonaltXPath(path); 239 if (!path.equals(nonAltPath)) { 240 String nonAltLocale = file.getSourceLocaleID(nonAltPath, null); 241 String nonAltValue = file.getStringValue(nonAltPath); 242 if (nonAltValue == null || !locale.equals(nonAltLocale)) { 243 System.out.println("\t" + locale + "\tProblem alt=proposed <" + value + ">\t\t" + path); 244 } 245 } 246 } 247 } 248 249 String fullPath = file.getFullXPath(path); 250 parts.set(fullPath); 251 if (dtdType == null) { 252 dtdType = DtdType.valueOf(parts.getElement(0)); 253 } 254 for (int i = 0; i < parts.size(); ++i) { 255 if (parts.getAttributeCount(i) == 0) continue; 256 String element = parts.getElement(i); 257 for (String attribute : parts.getAttributeKeys(i)) { 258 if (skipAttributes.contains(attribute)) continue; 259 if (CLDRFile.isDistinguishing(dtdType, element, attribute)) { 260 distinguishing.put(element, attribute); 261 } else { 262 nonDistinguishing.put(element, attribute); 263 } 264 } 265 } 266 } 267 } 268 System.out.println(); 269 270 System.out.format("Distinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, distinguishing); 271 System.out.format("Nondistinguishing Elements: %s" + CldrUtility.LINE_SEPARATOR, nonDistinguishing); 272 System.out.format("Skipped %s" + CldrUtility.LINE_SEPARATOR, skipAttributes); 273 274 if (pretty) { 275 if (showInfo) { 276 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Path to PrettyPath mapping" 277 + CldrUtility.LINE_SEPARATOR); 278 } 279 PrettyPath prettyPath = new PrettyPath().setShowErrors(true); 280 Set<String> badPaths = new TreeSet<String>(); 281 for (String path : pathToLocale.keySet()) { 282 String prettied = prettyPath.getPrettyPath(path, false); 283 if (showInfo) System.out.println(prettied + "\t\t" + path); 284 if (prettied.contains("%%") && !path.contains("/alias")) { 285 badPaths.add(path); 286 } 287 } 288 // now remove root 289 290 if (showInfo) { 291 System.out.println(CldrUtility.LINE_SEPARATOR + "Showing Paths not in root" 292 + CldrUtility.LINE_SEPARATOR); 293 } 294 295 CLDRFile root = cldrFactory.make("root", true); 296 for (Iterator<String> it = root.iterator(); it.hasNext();) { 297 pathToLocale.removeAll(it.next()); 298 } 299 if (showInfo) for (String path : pathToLocale.keySet()) { 300 if (skipPaths.reset(path).find()) { 301 continue; 302 } 303 System.out.println(path + "\t" + pathToLocale.getAll(path)); 304 } 305 306 if (badPaths.size() != 0) { 307 System.out.println("Error: " + badPaths.size() 308 + " Paths were not prettied: use -DSHOW and look for ones with %% in them."); 309 } 310 } 311 } 312 313 static void checkStock(Factory factory) { 314 String[][] items = { 315 { "full", "yMMMMEEEEd", "jmmsszzzz" }, 316 { "long", "yMMMMd", "jmmssz" }, 317 { "medium", "yMMMd", "jmmss" }, 318 { "short", "yMd", "jmm" }, 319 }; 320 String calendarID = "gregorian"; 321 String datetimePathPrefix = "//ldml/dates/calendars/calendar[@type=\"" + calendarID + "\"]/"; 322 323 int total = 0; 324 int mismatch = 0; 325 LanguageTagParser ltp = new LanguageTagParser(); 326 Iterable<String> locales = StandardCodes.make().getLocaleCoverageLocales(Organization.cldr, EnumSet.of(Level.MODERN)); 327 for (String locale : locales) { 328 if (!ltp.set(locale).getRegion().isEmpty()) { 329 continue; 330 } 331 CLDRFile file = factory.make(locale, false); 332 DateTimeFormats dtf = new DateTimeFormats(); 333 dtf.set(file, "gregorian", false); 334 for (String[] stockInfo : items) { 335 String length = stockInfo[0]; 336 //ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="full"]/dateFormat[@type="standard"]/pattern[@type="standard"] 337 String path = datetimePathPrefix + "dateFormats/dateFormatLength[@type=\"" + 338 length + "\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"; 339 String stockDatePattern = file.getStringValue(path); 340 String flexibleDatePattern = dtf.getBestPattern(stockInfo[1]); 341 mismatch += showStatus(++total, locale, "date", length, stockInfo[1], stockDatePattern, flexibleDatePattern); 342 path = datetimePathPrefix + "timeFormats/timeFormatLength[@type=\"" + length + 343 "\"]/timeFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"; 344 String stockTimePattern = file.getStringValue(path); 345 String flexibleTimePattern = dtf.getBestPattern(stockInfo[2]); 346 mismatch += showStatus(++total, locale, "time", length, stockInfo[2], stockTimePattern, flexibleTimePattern); 347 } 348 } 349 System.out.println("Mismatches:\t" + mismatch + "\tTotal:\t" + total); 350 } 351 352 static final Date SAMPLE_DATE = new Date(2013 - 1900, 1 - 1, 29, 13, 59, 59); 353 354 private static int showStatus(int total, String locale, String type, String length, 355 String skeleton, String stockPattern, String flexiblePattern) { 356 ULocale ulocale = new ULocale(locale); 357 DateFormatSymbols dfs = new DateFormatSymbols(ulocale); // just use ICU for now 358 boolean areSame = Objects.equals(stockPattern, flexiblePattern); 359 System.out.println(total 360 + "\t" + (areSame ? "ok" : "diff") 361 + "\t" + locale 362 + "\t" + type 363 + "\t" + length 364 + "\t" + skeleton 365 + "\t" + stockPattern 366 + "\t" + (areSame ? "" : flexiblePattern) 367 + "\t'" + new SimpleDateFormat(stockPattern, dfs, ulocale).format(SAMPLE_DATE) 368 + "\t'" + (areSame ? "" : new SimpleDateFormat(flexiblePattern, dfs, ulocale).format(SAMPLE_DATE))); 369 return areSame ? 0 : 1; 370 } 371 372 }
