Home | History | Annotate | Download | only in tool
      1 package org.unicode.cldr.tool;
      2 
      3 import java.io.File;
      4 import java.io.IOException;
      5 import java.io.PrintWriter;
      6 import java.util.ArrayList;
      7 import java.util.Arrays;
      8 import java.util.Collection;
      9 import java.util.Collections;
     10 import java.util.EnumMap;
     11 import java.util.EnumSet;
     12 import java.util.HashMap;
     13 import java.util.HashSet;
     14 import java.util.Iterator;
     15 import java.util.LinkedHashMap;
     16 import java.util.LinkedHashSet;
     17 import java.util.List;
     18 import java.util.Locale;
     19 import java.util.Map;
     20 import java.util.Map.Entry;
     21 import java.util.Set;
     22 import java.util.TreeMap;
     23 import java.util.TreeSet;
     24 import java.util.regex.Matcher;
     25 
     26 import org.unicode.cldr.draft.FileUtilities;
     27 import org.unicode.cldr.test.CheckCLDR.InputMethod;
     28 import org.unicode.cldr.test.CheckCLDR.Phase;
     29 import org.unicode.cldr.test.CheckCLDR.StatusAction;
     30 import org.unicode.cldr.test.CoverageLevel2;
     31 import org.unicode.cldr.tool.FormattedFileWriter.Anchors;
     32 import org.unicode.cldr.tool.Option.Options;
     33 import org.unicode.cldr.util.Annotations;
     34 import org.unicode.cldr.util.CLDRConfig;
     35 import org.unicode.cldr.util.CLDRFile;
     36 import org.unicode.cldr.util.CLDRFile.DraftStatus;
     37 import org.unicode.cldr.util.CLDRFile.Status;
     38 import org.unicode.cldr.util.CLDRInfo.CandidateInfo;
     39 import org.unicode.cldr.util.CLDRInfo.PathValueInfo;
     40 import org.unicode.cldr.util.CLDRInfo.UserInfo;
     41 import org.unicode.cldr.util.CLDRLocale;
     42 import org.unicode.cldr.util.CLDRPaths;
     43 import org.unicode.cldr.util.CLDRURLS;
     44 import org.unicode.cldr.util.CldrUtility;
     45 import org.unicode.cldr.util.CoreCoverageInfo;
     46 import org.unicode.cldr.util.CoreCoverageInfo.CoreItems;
     47 import org.unicode.cldr.util.Counter;
     48 import org.unicode.cldr.util.Counter2;
     49 import org.unicode.cldr.util.CoverageInfo;
     50 import org.unicode.cldr.util.DtdType;
     51 import org.unicode.cldr.util.LanguageTagCanonicalizer;
     52 import org.unicode.cldr.util.LanguageTagParser;
     53 import org.unicode.cldr.util.Level;
     54 import org.unicode.cldr.util.Organization;
     55 import org.unicode.cldr.util.PathHeader;
     56 import org.unicode.cldr.util.PathHeader.Factory;
     57 import org.unicode.cldr.util.PathHeader.SurveyToolStatus;
     58 import org.unicode.cldr.util.PatternCache;
     59 import org.unicode.cldr.util.RegexLookup;
     60 import org.unicode.cldr.util.RegexLookup.LookupType;
     61 import org.unicode.cldr.util.SimpleFactory;
     62 import org.unicode.cldr.util.StandardCodes;
     63 import org.unicode.cldr.util.SupplementalDataInfo;
     64 import org.unicode.cldr.util.VettingViewer;
     65 import org.unicode.cldr.util.VettingViewer.MissingStatus;
     66 import org.unicode.cldr.util.VoteResolver.VoterInfo;
     67 
     68 import com.google.common.collect.ImmutableSet;
     69 import com.google.common.collect.LinkedHashMultimap;
     70 import com.google.common.collect.Multimap;
     71 import com.google.common.collect.Ordering;
     72 import com.ibm.icu.dev.util.CollectionUtilities;
     73 import com.ibm.icu.dev.util.UnicodeMap;
     74 import com.ibm.icu.impl.Relation;
     75 import com.ibm.icu.lang.UCharacter;
     76 import com.ibm.icu.text.NumberFormat;
     77 import com.ibm.icu.text.UnicodeSet;
     78 import com.ibm.icu.util.ICUUncheckedIOException;
     79 
     80 public class ShowLocaleCoverage {
     81     private static final String SPREADSHEET_MISSING = "#LCode\tEnglish Name\tScript\tEnglish Value\tNative Value\tCldr Target\tPath Level\tStatus\tAction\tSTStatus\tST Link\tSection\tPage\tHeader\tCode\tPath";
     82     private static final boolean DEBUG = false;
     83     private static final char DEBUG_FILTER = 0; // use letter to only load locales starting with that letter
     84 
     85     private static final String LATEST = ToolConstants.CHART_VERSION;
     86     private static final double CORE_SIZE = CoreItems.values().length - CoreItems.ONLY_RECOMMENDED.size();
     87     public static CLDRConfig testInfo = ToolConfig.getToolInstance();
     88     private static final StandardCodes SC = testInfo.getStandardCodes();
     89     private static final SupplementalDataInfo SUPPLEMENTAL_DATA_INFO = testInfo.getSupplementalDataInfo();
     90     private static final StandardCodes STANDARD_CODES = SC;
     91 
     92     static org.unicode.cldr.util.Factory factory = testInfo.getCommonAndSeedAndMainAndAnnotationsFactory();
     93     private static final CLDRFile ENGLISH = factory.make("en", true);
     94 
     95     private static UnicodeSet ENG_ANN = Annotations.getData("en").keySet();
     96 
     97     // added info using pattern in VettingViewer.
     98 
     99     static final RegexLookup<Boolean> HACK = RegexLookup.<Boolean> of(LookupType.STANDARD, RegexLookup.RegexFinderTransformPath)
    100         .add("//ldml/localeDisplayNames/keys/key[@type=\"(d0|em|fw|i0|k0|lw|m0|rg|s0|ss|t0|x0)\"]", true)
    101         .add("//ldml/localeDisplayNames/types/type[@key=\"(em|fw|kr|lw|ss)\"].*", true)
    102         .add("//ldml/localeDisplayNames/languages/language[@type=\".*_.*\"]", true)
    103         .add("//ldml/localeDisplayNames/languages/language[@type=\".*\"][@alt=\".*\"]", true)
    104         .add("//ldml/localeDisplayNames/territories/territory[@type=\".*\"][@alt=\".*\"]", true)
    105         .add("//ldml/localeDisplayNames/territories/territory[@type=\"EZ\"]", true);
    106 
    107     //private static final String OUT_DIRECTORY = CLDRPaths.GEN_DIRECTORY + "/coverage/"; // CldrUtility.MAIN_DIRECTORY;
    108 
    109     final static Options myOptions = new Options();
    110 
    111     enum MyOptions {
    112         filter(".+", ".*", "Filter the information based on id, using a regex argument."),
    113         //        draftStatus(".+", "unconfirmed", "Filter the information to a minimum draft status."),
    114         chart(null, null, "chart only"),
    115         growth("true", "true", "Compute growth data"),
    116         organization(".+", null, "Only locales for organization"),
    117         version(".+",
    118             LATEST, "To get different versions"),
    119         rawData(null, null, "Output the raw data from all coverage levels"),
    120         targetDir(".*",
    121             CLDRPaths.GEN_DIRECTORY + "/statistics/", "target output file."),
    122         directories("(.*:)?[a-z]+(,[a-z]+)*", "common",
    123             "Space-delimited list of main source directories: common,seed,exemplar.\n" +
    124             "Optional, <baseDir>:common,seed"),;
    125 
    126         // targetDirectory(".+", CldrUtility.CHART_DIRECTORY + "keyboards/", "The target directory."),
    127         // layouts(null, null, "Only create html files for keyboard layouts"),
    128         // repertoire(null, null, "Only create html files for repertoire"), ;
    129         // boilerplate
    130         final Option option;
    131 
    132         MyOptions(String argumentPattern, String defaultArgument, String helpText) {
    133             option = myOptions.add(this, argumentPattern, defaultArgument, helpText);
    134         }
    135     }
    136 
    137     static final RegexLookup<Boolean> SUPPRESS_PATHS_CAN_BE_EMPTY = new RegexLookup<Boolean>()
    138         .add("\\[@alt=\"accounting\"]", true)
    139         .add("\\[@alt=\"variant\"]", true)
    140         .add("^//ldml/localeDisplayNames/territories/territory.*@alt=\"short", true)
    141         .add("^//ldml/localeDisplayNames/languages/language.*_", true)
    142         .add("^//ldml/numbers/currencies/currency.*/symbol", true)
    143         .add("^//ldml/characters/exemplarCharacters", true);
    144 
    145     static DraftStatus minimumDraftStatus = DraftStatus.unconfirmed;
    146     static final Factory pathHeaderFactory = PathHeader.getFactory(ENGLISH);
    147 
    148     static boolean RAW_DATA = true;
    149     private static Set<String> COMMON_LOCALES;
    150 
    151     public static void main(String[] args) throws IOException {
    152         myOptions.parse(MyOptions.filter, args, true);
    153 
    154         if (MyOptions.chart.option.doesOccur()) {
    155             showCoverage(null);
    156             return;
    157         }
    158 
    159         Matcher matcher = PatternCache.get(MyOptions.filter.option.getValue()).matcher("");
    160 
    161         if (MyOptions.growth.option.doesOccur()) {
    162             try (PrintWriter out = FileUtilities.openUTF8Writer(CLDRPaths.CHART_DIRECTORY + "tsv/", "locale-growth.tsv")) {
    163                 doGrowth(matcher, out);
    164                 return;
    165             }
    166         }
    167 
    168         Set<String> locales = null;
    169         String organization = MyOptions.organization.option.getValue();
    170         boolean useOrgLevel = MyOptions.organization.option.doesOccur();
    171         if (useOrgLevel) {
    172             locales = STANDARD_CODES.getLocaleCoverageLocales(organization);
    173         }
    174 
    175         if (MyOptions.version.option.doesOccur()) {
    176             String number = MyOptions.version.option.getValue().trim();
    177             if (!number.contains(".")) {
    178                 number += ".0";
    179             }
    180             factory = org.unicode.cldr.util.Factory.make(
    181                 CLDRPaths.ARCHIVE_DIRECTORY + "cldr-" + number + "/common/main/", ".*");
    182         } else {
    183             if (MyOptions.directories.option.doesOccur()) {
    184                 String directories = MyOptions.directories.option.getValue().trim();
    185                 CLDRConfig cldrConfig = CLDRConfig.getInstance();
    186                 String base = null;
    187                 int colonPos = directories.indexOf(':');
    188                 if (colonPos >= 0) {
    189                     base = directories.substring(0, colonPos).trim();
    190                     directories = directories.substring(colonPos + 1).trim();
    191                 } else {
    192                     base = cldrConfig.getCldrBaseDirectory().toString();
    193                 }
    194                 String[] items = directories.split(",\\s*");
    195                 File[] fullDirectories = new File[items.length];
    196                 int i = 0;
    197                 for (String item : items) {
    198                     fullDirectories[i++] = new File(base + "/" + item + "/main");
    199                 }
    200                 factory = SimpleFactory.make(fullDirectories, ".*");
    201                 COMMON_LOCALES = SimpleFactory.make(base + "/" + "common" + "/main", ".*").getAvailableLanguages();
    202             }
    203         }
    204         fixCommonLocales();
    205 
    206         RAW_DATA = MyOptions.rawData.option.doesOccur();
    207 
    208         //showEnglish();
    209 
    210         showCoverage(null, matcher, locales, useOrgLevel);
    211     }
    212 
    213     public static void fixCommonLocales() {
    214         if (COMMON_LOCALES == null) {
    215             COMMON_LOCALES = factory.getAvailableLanguages();
    216         }
    217     }
    218 
    219     private static void doGrowth(Matcher matcher, PrintWriter out) {
    220         TreeMap<String, List<Double>> growthData = new TreeMap<>(Ordering.natural().reverse()); // sort by version, descending
    221 //        if (DEBUG) {
    222 //            for (String dir : new File(CLDRPaths.ARCHIVE_DIRECTORY).list()) {
    223 //                if (!dir.startsWith("cldr")) {
    224 //                    continue;
    225 //                }
    226 //                String version = getNormalizedVersion(dir);
    227 //                if (version == null) {
    228 //                    continue;
    229 //                }
    230 //                org.unicode.cldr.util.Factory newFactory = org.unicode.cldr.util.Factory.make(
    231 //                    CLDRPaths.ARCHIVE_DIRECTORY + "/" + dir + "/common/main/", ".*");
    232 //                System.out.println("Reading: " + version);
    233 //                Map<String, FoundAndTotal> currentData = addGrowth(newFactory, matcher);
    234 //                System.out.println("Read: " + version + "\t" + currentData);
    235 //                break;
    236 //            }
    237 //        }
    238         Map<String, FoundAndTotal> latestData = addGrowth(factory, null, matcher, DEBUG);
    239         addCompletionList(getYearFromVersion(LATEST, false), getCompletion(latestData, latestData), growthData);
    240         if (DEBUG) System.out.println(latestData);
    241         //System.out.println(growthData);
    242         List<String> dirs = new ArrayList<>(Arrays.asList(new File(CLDRPaths.ARCHIVE_DIRECTORY).list()));
    243         Collections.reverse(dirs);
    244         for (String dir : dirs) {
    245             if (!dir.startsWith("cldr")) {
    246                 continue;
    247             }
    248             String version = getNormalizedVersion(dir);
    249             if (version == null) {
    250                 continue;
    251             }
    252 //            if (version.compareTo("12") < 0) {
    253 //                continue;
    254 //            }
    255             System.out.println("Reading: " + version);
    256             if (version.equals("2008")) {
    257                 int debug = 0;
    258             }
    259             Map<String, FoundAndTotal> currentData = addGrowth(factory, dir, matcher, false);
    260             System.out.println("Read: " + version + "\t" + currentData);
    261             Counter2<String> completionData = getCompletion(latestData, currentData);
    262             //System.out.println(version + "\t" + completionData);
    263             addCompletionList(version, completionData, growthData);
    264             if (DEBUG) System.out.println(currentData);
    265         }
    266         boolean first = true;
    267         for (Entry<String, List<Double>> entry : growthData.entrySet()) {
    268             if (first) {
    269                 for (int i = 0; i < entry.getValue().size(); ++i) {
    270                     out.print("\t" + i);
    271                 }
    272                 out.println();
    273                 first = false;
    274             }
    275             out.println(entry.getKey() + "\t" + CollectionUtilities.join(entry.getValue(), "\t"));
    276         }
    277     }
    278 
    279     static final Map<String, String> versionToYear = new HashMap<>();
    280     static {
    281         int[][] mapping = {
    282             { 34, 2018 },
    283             { 32, 2017 },
    284             { 30, 2016 },
    285             { 28, 2015 },
    286             { 26, 2014 },
    287             { 24, 2013 },
    288             { 22, 2012 },
    289             { 20, 2011 },
    290             { 19, 2010 },
    291             { 17, 2009 },
    292             { 16, 2008 },
    293             { 15, 2007 },
    294             { 14, 2006 },
    295             { 13, 2005 },
    296             { 12, 2004 },
    297             { 10, 2003 },
    298         };
    299         for (int[] row : mapping) {
    300             versionToYear.put(String.valueOf(row[0]), String.valueOf(row[1]));
    301         }
    302     }
    303 
    304     public static String getNormalizedVersion(String dir) {
    305         String rawVersion = dir.substring(dir.indexOf('-') + 1);
    306         int firstDot = rawVersion.indexOf('.');
    307         int secondDot = rawVersion.indexOf('.', firstDot + 1);
    308         if (secondDot > 0) {
    309             rawVersion = rawVersion.substring(0, firstDot) + rawVersion.substring(firstDot + 1, secondDot);
    310         } else {
    311             rawVersion = rawVersion.substring(0, firstDot);
    312         }
    313         String result = getYearFromVersion(rawVersion, true);
    314         return result == null ? null : result.toString();
    315     }
    316 
    317     private static String getYearFromVersion(String version, boolean allowNull) {
    318         String result = versionToYear.get(version);
    319         if (!allowNull && result == null) {
    320             throw new IllegalArgumentException("No year for version: " + version);
    321         }
    322         return result;
    323     }
    324 
    325     public static void addCompletionList(String version, Counter2<String> completionData, TreeMap<String, List<Double>> growthData) {
    326         List<Double> x = new ArrayList<>();
    327         for (String key : completionData.getKeysetSortedByCount(false)) {
    328             x.add(completionData.getCount(key));
    329         }
    330         growthData.put(version, x);
    331         System.out.println(version + "\t" + x.size());
    332     }
    333 
    334     public static Counter2<String> getCompletion(Map<String, FoundAndTotal> latestData, Map<String, FoundAndTotal> currentData) {
    335         Counter2<String> completionData = new Counter2<>();
    336         for (Entry<String, FoundAndTotal> entry : latestData.entrySet()) {
    337             final String locale = entry.getKey();
    338             final FoundAndTotal currentRecord = currentData.get(locale);
    339             if (currentRecord == null) {
    340                 continue;
    341             }
    342             double total = entry.getValue().total;
    343             if (total == 0) {
    344                 continue;
    345             }
    346             double completion = currentRecord.found / total;
    347             completionData.add(locale, completion);
    348         }
    349         return completionData;
    350     }
    351 
    352     static class FoundAndTotal {
    353         final int found;
    354         final int total;
    355 
    356         public FoundAndTotal(Counter<Level>... counters) {
    357             final int[] count = { 0, 0, 0 };
    358             for (Level level : Level.values()) {
    359                 if (level == Level.COMPREHENSIVE || level == Level.OPTIONAL) {
    360                     continue;
    361                 }
    362                 int i = 0;
    363                 for (Counter<Level> counter : counters) {
    364                     count[i++] += counter.get(level);
    365                 }
    366             }
    367             found = count[0];
    368             total = found + count[1] + count[2];
    369         }
    370 
    371         @Override
    372         public String toString() {
    373             return found + "/" + total;
    374         }
    375     }
    376 
    377     private static Map<String, FoundAndTotal> addGrowth(org.unicode.cldr.util.Factory latestFactory, String dir, Matcher matcher, boolean showMissing) {
    378         org.unicode.cldr.util.Factory newFactory = dir == null ? factory
    379             : org.unicode.cldr.util.Factory.make(
    380                 CLDRPaths.ARCHIVE_DIRECTORY + "/" + dir + "/common/main/", ".*");
    381         Map<String, FoundAndTotal> data = new HashMap<>();
    382         char c = 0;
    383         Set<String> latestAvailable = newFactory.getAvailableLanguages();
    384         for (String locale : newFactory.getAvailableLanguages()) {
    385             if (!matcher.reset(locale).matches()) {
    386                 continue;
    387             }
    388             if (!latestAvailable.contains(locale)) {
    389                 continue;
    390             }
    391             if (SUPPLEMENTAL_DATA_INFO.getDefaultContentLocales().contains(locale)
    392                 || locale.equals("root")
    393                 || locale.equals("supplementalData")) {
    394                 continue;
    395             }
    396             char nc = locale.charAt(0);
    397             if (nc != c) {
    398                 System.out.println("\t" + locale);
    399                 c = nc;
    400             }
    401             if (DEBUG_FILTER != 0 && DEBUG_FILTER != nc) {
    402                 continue;
    403             }
    404             CLDRFile latestFile = null;
    405             try {
    406                 latestFile = latestFactory.make(locale, true);
    407             } catch (Exception e2) {
    408                 continue;
    409             }
    410             final CLDRFile file = newFactory.make(locale, true);
    411             // HACK check bogus
    412 //            Collection<String> extra = file.getExtraPaths();
    413 //
    414 //            final Iterable<String> fullIterable = file.fullIterable();
    415 //            for (String path : fullIterable) {
    416 //                if (path.contains("\"one[@")) {
    417 //                    boolean inside = extra.contains(path);
    418 //                    Status status = new Status();
    419 //                    String loc = file.getSourceLocaleID(path, status );
    420 //                    int debug = 0;
    421 //                }
    422 //            }
    423             // END HACK
    424             Counter<Level> foundCounter = new Counter<Level>();
    425             Counter<Level> unconfirmedCounter = new Counter<Level>();
    426             Counter<Level> missingCounter = new Counter<Level>();
    427             Set<String> unconfirmedPaths = null;
    428             Relation<MissingStatus, String> missingPaths = null;
    429             unconfirmedPaths = new LinkedHashSet<>();
    430             missingPaths = Relation.of(new LinkedHashMap(), LinkedHashSet.class);
    431             VettingViewer.getStatus(latestFile.fullIterable(), file,
    432                 pathHeaderFactory, foundCounter, unconfirmedCounter,
    433                 missingCounter, missingPaths, unconfirmedPaths);
    434 
    435             // HACK
    436             Set<Entry<MissingStatus, String>> missingRemovals = new HashSet<>();
    437             for (Entry<MissingStatus, String> e : missingPaths.keyValueSet()) {
    438                 if (e.getKey() == MissingStatus.ABSENT) {
    439                     final String path = e.getValue();
    440                     if (HACK.get(path) != null) {
    441                         missingRemovals.add(e);
    442                         missingCounter.add(Level.MODERN, -1);
    443                         foundCounter.add(Level.MODERN, 1);
    444                     } else {
    445                         Status status = new Status();
    446                         String loc = file.getSourceLocaleID(path, status);
    447                         int debug = 0;
    448                     }
    449                 }
    450             }
    451             for (Entry<MissingStatus, String> e : missingRemovals) {
    452                 missingPaths.remove(e.getKey(), e.getValue());
    453             }
    454             // END HACK
    455 
    456             if (showMissing) {
    457                 int count = 0;
    458                 for (String s : unconfirmedPaths) {
    459                     System.out.println(++count + "\t" + locale + "\tunconfirmed\t" + s);
    460                 }
    461                 for (Entry<MissingStatus, String> e : missingPaths.keyValueSet()) {
    462                     String path = e.getValue();
    463                     Status status = new Status();
    464                     String loc = file.getSourceLocaleID(path, status);
    465                     int debug = 0;
    466 
    467                     System.out.println(++count + "\t" + locale + "\t" + CldrUtility.toString(e));
    468                 }
    469                 int debug = 0;
    470             }
    471 
    472             // add annotations
    473             System.out.println(locale + " annotations");
    474             try {
    475                 UnicodeMap<Annotations> annotations = dir == null ? Annotations.getData(locale)
    476                     : Annotations.getData(CLDRPaths.ARCHIVE_DIRECTORY + "/" + dir + "/common/annotations/", locale);
    477                 for (String cp : ENG_ANN) {
    478                     Annotations annotation = annotations.get(cp);
    479                     if (annotation == null) {
    480                         missingCounter.add(Level.MODERN, 1);
    481                     } else if (annotation.getShortName() == null) {
    482                         missingCounter.add(Level.MODERN, 1);
    483                     } else {
    484                         foundCounter.add(Level.MODERN, 1);
    485                     }
    486                 }
    487             } catch (Exception e1) {
    488                 missingCounter.add(Level.MODERN, ENG_ANN.size());
    489             }
    490 
    491             data.put(locale, new FoundAndTotal(foundCounter, unconfirmedCounter, missingCounter));
    492         }
    493         return Collections.unmodifiableMap(data);
    494     }
    495 
    496     public static void showCoverage(Anchors anchors) throws IOException {
    497         showCoverage(anchors, PatternCache.get(".*").matcher(""), null, false);
    498     }
    499 
    500     public static void showCoverage(Anchors anchors, Matcher matcher, Set<String> locales, boolean useOrgLevel) throws IOException {
    501         final String title = "Locale Coverage";
    502         try (PrintWriter pw = new PrintWriter(new FormattedFileWriter(null, title, null, anchors));
    503             PrintWriter tsv_summary = FileUtilities.openUTF8Writer(CLDRPaths.CHART_DIRECTORY + "tsv/", "locale-coverage.tsv");
    504             PrintWriter tsv_missing = FileUtilities.openUTF8Writer(CLDRPaths.CHART_DIRECTORY + "tsv/", "locale-missing.tsv");
    505             ){
    506             printData(pw, tsv_summary, tsv_missing, locales, matcher, useOrgLevel);
    507             new ShowPlurals().appendBlanksForScrolling(pw);
    508         }
    509     }
    510 
    511 //    public static void showEnglish() {
    512 //        Map<PathHeader,String> sorted = new TreeMap<>();
    513 //        CoverageInfo coverageInfo=CLDRConfig.getInstance().getCoverageInfo();
    514 //        for (String path : ENGLISH) {
    515 ////            Level currentLevel = SUPPLEMENTAL_DATA_INFO.getCoverageLevel(path, "en");
    516 //            Level currentLevel=coverageInfo.getCoverageLevel(path, "en");
    517 //            if (currentLevel.compareTo(Level.MINIMAL) <= 0) {
    518 //                PathHeader ph = pathHeaderFactory.fromPath(path);
    519 //                sorted.put(ph, currentLevel + "\t" + ENGLISH.getStringValue(path));
    520 //            }
    521 //        }
    522 //        for (Entry<PathHeader, String> entry : sorted.entrySet()) {
    523 //            System.out.println(entry.getKey() + "\t" + entry.getValue());
    524 //        }
    525 //    }
    526 
    527     static class IterableFilter implements Iterable<String> {
    528         private Iterable<String> source;
    529 
    530         IterableFilter(Iterable<String> source) {
    531             this.source = source;
    532         }
    533 
    534         /**
    535          * When some paths are defined after submission, we need to change them to COMPREHENSIVE in computing the vetting status.
    536          */
    537 
    538         static final Set<String> SUPPRESS_PATHS_AFTER_SUBMISSION = ImmutableSet.of(
    539             "//ldml/localeDisplayNames/languages/language[@type=\"ccp\"]",
    540             "//ldml/localeDisplayNames/territories/territory[@type=\"XA\"]",
    541             "//ldml/localeDisplayNames/territories/territory[@type=\"XB\"]",
    542             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"Gy\"]/greatestDifference[@id=\"G\"]",
    543             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"Gy\"]/greatestDifference[@id=\"y\"]",
    544             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyM\"]/greatestDifference[@id=\"G\"]",
    545             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyM\"]/greatestDifference[@id=\"M\"]",
    546             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyM\"]/greatestDifference[@id=\"y\"]",
    547             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"d\"]",
    548             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"G\"]",
    549             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"M\"]",
    550             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"y\"]",
    551             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"d\"]",
    552             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"G\"]",
    553             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"M\"]",
    554             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"y\"]",
    555             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMM\"]/greatestDifference[@id=\"G\"]",
    556             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMM\"]/greatestDifference[@id=\"M\"]",
    557             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMM\"]/greatestDifference[@id=\"y\"]",
    558             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"d\"]",
    559             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"G\"]",
    560             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"M\"]",
    561             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"y\"]",
    562             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"d\"]",
    563             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"G\"]",
    564             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"M\"]",
    565             "//ldml/dates/calendars/calendar[@type=\"generic\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"y\"]",
    566             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"Gy\"]/greatestDifference[@id=\"G\"]",
    567             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"Gy\"]/greatestDifference[@id=\"y\"]",
    568             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyM\"]/greatestDifference[@id=\"G\"]",
    569             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyM\"]/greatestDifference[@id=\"M\"]",
    570             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyM\"]/greatestDifference[@id=\"y\"]",
    571             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"d\"]",
    572             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"G\"]",
    573             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"M\"]",
    574             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMd\"]/greatestDifference[@id=\"y\"]",
    575             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"d\"]",
    576             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"G\"]",
    577             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"M\"]",
    578             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMEd\"]/greatestDifference[@id=\"y\"]",
    579             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMM\"]/greatestDifference[@id=\"G\"]",
    580             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMM\"]/greatestDifference[@id=\"M\"]",
    581             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMM\"]/greatestDifference[@id=\"y\"]",
    582             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"d\"]",
    583             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"G\"]",
    584             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"M\"]",
    585             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMd\"]/greatestDifference[@id=\"y\"]",
    586             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"d\"]",
    587             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"G\"]",
    588             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"M\"]",
    589             "//ldml/dates/calendars/calendar[@type=\"gregorian\"]/dateTimeFormats/intervalFormats/intervalFormatItem[@id=\"GyMMMEd\"]/greatestDifference[@id=\"y\"]"
    590             );
    591         @Override
    592         public Iterator<String> iterator() {
    593             return new IteratorFilter(source.iterator());
    594         }
    595 
    596         static class IteratorFilter implements Iterator<String> {
    597             Iterator<String> source;
    598             String peek;
    599 
    600             public IteratorFilter(Iterator<String> source) {
    601                 this.source = source;
    602                 fillPeek();
    603             }
    604             @Override
    605             public boolean hasNext() {
    606                 return peek != null;
    607             }
    608             @Override
    609             public String next() {
    610                 String result = peek;
    611                 fillPeek();
    612                 return result;
    613             }
    614 
    615             private void fillPeek() {
    616                 peek = null;
    617                 while (source.hasNext()) {
    618                     peek = source.next();
    619                     // if it is ok to assess, then break
    620                     if (!SUPPRESS_PATHS_AFTER_SUBMISSION.contains(peek)
    621                         && SUPPRESS_PATHS_CAN_BE_EMPTY.get(peek) != Boolean.TRUE) {
    622                         break;
    623                     }
    624                     peek = null;
    625                 }
    626             }
    627         }
    628 
    629     }
    630     static void printData(PrintWriter pw, PrintWriter tsv_summary, PrintWriter tsv_missing, Set<String> locales, Matcher matcher, boolean useOrgLevel) {
    631 //        Set<String> checkModernLocales = STANDARD_CODES.getLocaleCoverageLocales("google", EnumSet.of(Level.MODERN));
    632         Set<String> checkModernLocales = STANDARD_CODES.getLocaleCoverageLocales(Organization.cldr, EnumSet.of(Level.MODERN));
    633         Set<String> availableLanguages = new TreeSet<>(factory.getAvailableLanguages());
    634         availableLanguages.addAll(checkModernLocales);
    635         Relation<String, String> languageToRegion = Relation.of(new TreeMap(), TreeSet.class);
    636         LanguageTagParser ltp = new LanguageTagParser();
    637         LanguageTagCanonicalizer ltc = new LanguageTagCanonicalizer(true);
    638         for (String locale : factory.getAvailable()) {
    639             String country = ltp.set(locale).getRegion();
    640             if (!country.isEmpty()) {
    641                 languageToRegion.put(ltc.transform(ltp.getLanguageScript()), country);
    642             }
    643         }
    644 
    645         fixCommonLocales();
    646 
    647         System.out.println(CollectionUtilities.join(languageToRegion.keyValuesSet(), "\n"));
    648 
    649         System.out.println("# Checking: " + availableLanguages);
    650 
    651         pw.println("<p style='text-align: left'>This chart shows the coverage levels for this release. </p>" +
    652             "<ol>"
    653             + "<li>Fields = fields found at a modern level</li>"
    654             + "<li>UC = unconfirmed values: typically treated as missing by implementations</li>"
    655             + "<li>Miss = missing values</li>"
    656             + "<li>Modern%, etc = fields/(fields + missing + unconfirmed)  at that level</li></ol>"
    657             + "<li>Core Missing = missing core fields  optionals marked with *</li></ol>"
    658             + "<p>A high-level summary of the meaning of the coverage values are at " +
    659             "<a target='_blank' href='http://www.unicode.org/reports/tr35/tr35-info.html#Coverage_Levels'>Coverage Levels</a>. " +
    660             "The Core values are described on " +
    661             "<a target='_blank' href='http://cldr.unicode.org/index/cldr-spec/minimaldata'>Core Data</a>." +
    662             "</p>");
    663 
    664         Relation<MissingStatus, String> missingPaths = Relation.of(new EnumMap<MissingStatus, Set<String>>(
    665             MissingStatus.class), TreeSet.class, CLDRFile.getComparator(DtdType.ldml));
    666         Set<String> unconfirmed = new TreeSet<String>(CLDRFile.getComparator(DtdType.ldml));
    667 
    668         //Map<String, String> likely = testInfo.getSupplementalDataInfo().getLikelySubtags();
    669         Set<String> defaultContents = SUPPLEMENTAL_DATA_INFO.getDefaultContentLocales();
    670 
    671         // Map<String,Counter<Level>> counts = new HashMap();
    672         //        System.out.print("Script\tEnglish\tNative\tCode\tCode*");
    673         //        for (Level level : Level.values()) {
    674         //            if (skipPrintingLevels.contains(level)) {
    675         //                continue;
    676         //            }
    677         //            System.out.print("\t" + level + " (f)\t(u)\t(m)");
    678         //        }
    679         //        System.out.println();
    680         // Factory pathHeaderFactory = PathHeader.getFactory(testInfo.getCldrFactory().make("en", true));
    681 
    682         tsv_missing.println(SPREADSHEET_MISSING);
    683 
    684         Counter<Level> foundCounter = new Counter<Level>();
    685         Counter<Level> unconfirmedCounter = new Counter<Level>();
    686         Counter<Level> missingCounter = new Counter<Level>();
    687 
    688         List<Level> reversedLevels = new ArrayList<>(EnumSet.allOf(Level.class));
    689         reversedLevels.remove(Level.COMPREHENSIVE);
    690         reversedLevels.remove(Level.UNDETERMINED);
    691         Collections.reverse(reversedLevels);
    692 
    693         PrintWriter out2;
    694         try {
    695             out2 = FileUtilities.openUTF8Writer(CLDRPaths.CHART_DIRECTORY + "tsv/", "showLocaleCoverage.tsv");
    696         } catch (IOException e1) {
    697             throw new ICUUncheckedIOException(e1);
    698         }
    699 
    700         out2.print("Code\tCom?\tEnglish Name\tNative Name\tScript\tSublocales\tStrings");
    701         for (Level level : reversedLevels) {
    702             out2.print("\t" + level + " %\t" + level + " UC%");
    703         }
    704         out2.println();
    705         //System.out.println("\tCore*\nCore* Missing");
    706         int localeCount = 0;
    707 
    708         final TablePrinter tablePrinter = new TablePrinter()
    709             .addColumn("Direct.", "class='source'", null, "class='source'", true)
    710             .setBreakSpans(true).setSpanRows(false)
    711             .addColumn("Code", "class='source'", CldrUtility.getDoubleLinkMsg(), "class='source'", true).setBreakSpans(true)
    712             .addColumn("English Name", "class='source'", null, "class='source'", true).setBreakSpans(true)
    713             .addColumn("Native Name", "class='source'", null, "class='source'", true).setBreakSpans(true)
    714             .addColumn("Script", "class='source'", null, "class='source'", true).setBreakSpans(true)
    715             .addColumn("CLDR target", "class='source'", null, "class='source'", true).setBreakSpans(true).setSortPriority(0).setSortAscending(false)
    716             .addColumn("Sublocales", "class='target'", null, "class='targetRight'", true).setBreakSpans(true)
    717             .setCellPattern("{0,number}")
    718             .addColumn("Fields", "class='target'", null, "class='targetRight'", true).setBreakSpans(true)
    719             .setCellPattern("{0,number}")
    720             .addColumn("UC", "class='target'", null, "class='targetRight'", true).setBreakSpans(true)
    721             .setCellPattern("{0,number}")
    722             .addColumn("Miss", "class='target'", null, "class='targetRight'", true).setBreakSpans(true)
    723             .setCellPattern("{0,number}")
    724             //.addColumn("Target Level", "class='target'", null, "class='target'", true).setBreakSpans(true)
    725             ;
    726         tsv_summary.println("Dir"
    727             + "\tCode"
    728             + "\tEnglish Name"
    729             + "\tNative Name"
    730             + "\tScript"
    731             + "\tCLDR target"
    732             + "\tSublocales"
    733             + "\tFields\tUC\tMissing"
    734             + "\tModern\tMiss +UC"
    735             + "\tModerate\tMiss +UC"
    736             + "\tBasic\tMiss +UC"
    737             + "\tCore\tMiss +UC"
    738             + "\tCore-Missing");
    739         NumberFormat tsvPercent = NumberFormat.getPercentInstance(Locale.ENGLISH);
    740         tsvPercent.setMaximumFractionDigits(2);
    741 
    742         for (Level level : reversedLevels) {
    743             String titleLevel = level.toString();
    744             tablePrinter.addColumn(UCharacter.toTitleCase(titleLevel, null) + "%", "class='target'", null, "class='targetRight'", true)
    745             .setCellPattern("{0,number,0.0%}")
    746             .setBreakSpans(true);
    747             switch(level) {
    748             case CORE:
    749                 tablePrinter.setSortPriority(4).setSortAscending(false);
    750                 break;
    751             case BASIC:
    752                 tablePrinter.setSortPriority(3).setSortAscending(false);
    753                 break;
    754             case MODERATE:
    755                 tablePrinter.setSortPriority(2).setSortAscending(false);
    756                 break;
    757             case MODERN:
    758                 tablePrinter.setSortPriority(1).setSortAscending(false);
    759                 break;
    760             }
    761 //            tablePrinter
    762 //            .addColumn(" UC%", "class='target'", null, "class='targetRight'", true)
    763 //            .setCellPattern("{0,number,0.0%}")
    764 //            .setBreakSpans(true)
    765             ;
    766         }
    767         tablePrinter.addColumn("Core Missing", "class='target'", null, "class='targetRight'", true)
    768         .setBreakSpans(true);
    769 
    770         long start = System.currentTimeMillis();
    771         LikelySubtags likelySubtags = new LikelySubtags();
    772 
    773         EnumMap<Level, Double> targetLevel = new EnumMap<>(Level.class);
    774         targetLevel.put(Level.CORE, 2 / 100d);
    775         targetLevel.put(Level.BASIC, 16 / 100d);
    776         targetLevel.put(Level.MODERATE, 33 / 100d);
    777         targetLevel.put(Level.MODERN, 100 / 100d);
    778 
    779 //        NumberFormat percentFormat = NumberFormat.getPercentInstance(ULocale.ENGLISH);
    780 //        percentFormat.setMaximumFractionDigits(2);
    781 //        percentFormat.setMinimumFractionDigits(2);
    782 //        NumberFormat intFormat = NumberFormat.getIntegerInstance(ULocale.ENGLISH);
    783 
    784         int counter = 0;
    785         for (String locale : availableLanguages) {
    786             try {
    787                 if (locale.contains("supplemental")) { // for old versions
    788                     continue;
    789                 }
    790                 if (locales != null && !locales.contains(locale)) {
    791                     String base = CLDRLocale.getInstance(locale).getLanguage();
    792                     if (!locales.contains(base)) {
    793                         continue;
    794                     }
    795                 }
    796                 if (!matcher.reset(locale).matches()) {
    797                     continue;
    798                 }
    799                 if (defaultContents.contains(locale) || "root".equals(locale) || "und".equals(locale)) {
    800                     continue;
    801                 }
    802 
    803                 boolean isSeed = new File(CLDRPaths.SEED_DIRECTORY, locale + ".xml").exists();
    804 
    805                 //boolean capture = locale.equals("en");
    806                 String region = ltp.set(locale).getRegion();
    807                 if (!region.isEmpty()) continue; // skip regions
    808 
    809                 final Level cldrLocaleLevelGoal = SC.getLocaleCoverageLevel(Organization.cldr.toString(), locale);
    810                 final boolean cldrLevelGoalModerateOrAbove = cldrLocaleLevelGoal.compareTo(Level.MODERATE) >= 0;
    811 
    812                 String isCommonLocale = Level.MODERN == cldrLocaleLevelGoal ? "C*"
    813                     : COMMON_LOCALES.contains(locale) ? "C"
    814                         : "";
    815 
    816                 String max = likelySubtags.maximize(locale);
    817                 String script = ltp.set(max).getScript();
    818 
    819                 String language = likelySubtags.minimize(locale);
    820 //                Level otherLevel = STANDARD_CODES.getLocaleCoverageLevel("apple", locale);
    821 //                if (otherLevel.compareTo(currentLevel) > 0
    822 //                    && otherLevel.compareTo(Level.MODERN) <= 0) {
    823 //                    currentLevel = otherLevel;
    824 //                }
    825 
    826                 missingPaths.clear();
    827                 unconfirmed.clear();
    828 
    829                 final CLDRFile file = factory.make(locale, true, minimumDraftStatus);
    830 
    831                 if (locale.equals("af")) {
    832                     int debug = 0;
    833                 }
    834 
    835                 Iterable<String> pathSource = new IterableFilter(file.fullIterable());
    836 
    837                 VettingViewer.getStatus(pathSource, file,
    838                     pathHeaderFactory, foundCounter, unconfirmedCounter,
    839                     missingCounter, missingPaths, unconfirmed);
    840 
    841                 Set<String> sublocales = languageToRegion.get(language);
    842                 if (sublocales == null) {
    843                     //System.err.println("No Sublocales: " + language);
    844                     sublocales = Collections.EMPTY_SET;
    845                 }
    846 
    847 //                List s = Lists.newArrayList(file.fullIterable());
    848 
    849                 String seedString = isSeed ? "seed" : "common";
    850                 tablePrinter.addRow()
    851                 .addCell(seedString)
    852                 .addCell(language)
    853                 .addCell(ENGLISH.getName(language))
    854                 .addCell(file.getName(language))
    855                 .addCell(script)
    856                 .addCell(cldrLocaleLevelGoal)
    857                 .addCell(sublocales.size());
    858 
    859                 tsv_summary
    860                 .append(seedString)
    861                 .append('\t').append(language)
    862                 .append('\t').append(ENGLISH.getName(language))
    863                 .append('\t').append(file.getName(language))
    864                 .append('\t').append(script)
    865                 .append('\t').append(cldrLocaleLevelGoal.toString())
    866                 .append('\t').append(sublocales.size()+"");
    867                 ;
    868 
    869 //                String header = language
    870 //                    + "\t" + isCommonLocale
    871 //                    + "\t" + ENGLISH.getName(language)
    872 //                    + "\t" + file.getName(language)
    873 //                    + "\t" + script
    874 //                    + "\t" + sublocales.size()
    875 //                    //+ "\t" + currentLevel
    876 //                    ;
    877 
    878                 int sumFound = 0;
    879                 int sumMissing = 0;
    880                 int sumUnconfirmed = 0;
    881 
    882                 // get the totals
    883 
    884                 EnumMap<Level, Integer> totals = new EnumMap<>(Level.class);
    885                 EnumMap<Level, Integer> confirmed = new EnumMap<>(Level.class);
    886 //                EnumMap<Level, Integer> unconfirmedByLevel = new EnumMap<>(Level.class);
    887                 Set<String> coreMissing = new LinkedHashSet<>();
    888 
    889                 if (locale.equals("af")) {
    890                     int debug = 0;
    891                 }
    892 
    893                 { // CORE
    894                     long missingExemplarCount = missingCounter.get(Level.CORE);
    895                     if (missingExemplarCount > 0) {
    896                         for (Entry<MissingStatus, String> statusAndPath : missingPaths.entrySet()) {
    897                             String path = statusAndPath.getValue();
    898                             if (path.startsWith("//ldml/characters/exemplarCharacters")) {
    899                                 PathHeader ph = pathHeaderFactory.fromPath(path);
    900                                 String problem = ph.getCode().replaceAll("Others: ","").replaceAll("Main Letters", "main-letters");
    901                                 coreMissing.add(problem);
    902                                 // String line = spreadsheetLine(locale, script, language, cldrLevelGoal, foundLevel, missingStatus.toString(), path, file.getStringValue(path));
    903                                 String line = spreadsheetLine(locale, script, language, cldrLocaleLevelGoal, Level.CORE, "ABSENT", path, "No " + problem + "");
    904                                 tsv_missing.println(line);
    905                             }
    906                         }
    907                     }
    908                     Multimap<CoreItems, String> detailedErrors = LinkedHashMultimap.create();
    909                     Set<CoreItems> coverage = new TreeSet<>(
    910                         CoreCoverageInfo.getCoreCoverageInfo(file, detailedErrors));
    911                     Set<CoreItems> missing = EnumSet.allOf(CoreItems.class);
    912                     missing.removeAll(coverage);
    913                     for (Entry<CoreItems, String> entry : detailedErrors.entries()) {
    914                         CoreItems coreItem = entry.getKey();
    915                         String value = entry.getValue();
    916                         coreMissing.add(coreItem.toString());
    917                         //String line = spreadsheetLine(language, script, "n/a", detailedErrors.get(entry).toString(), level, "ABSENT", "n/a", "n/a", "n/a");
    918                         if (cldrLevelGoalModerateOrAbove) {
    919                             String line = spreadsheetLine(locale, script, language, cldrLocaleLevelGoal, coreItem.desiredLevel, "ABSENT", value, "No " + coreItem + "");
    920                             tsv_missing.println(line);
    921                         }
    922                     }
    923                     missing.removeAll(CoreItems.ONLY_RECOMMENDED);
    924                     foundCounter.add(Level.CORE, coverage.size());
    925                     missingCounter.add(Level.CORE, missing.size());
    926 
    927 //                    sumFound += coverage.size();
    928 //                    sumMissing += missing.size();
    929 
    930 //                    confirmed.put(Level.CORE, (int) coverage.size());
    931 ////                    unconfirmedByLevel.put(level, (int)(foundCount + unconfirmedCount));
    932 //                    totals.put(Level.CORE, (int)(coverage.size() + missing.size()));
    933 
    934                 }
    935 
    936                 if (cldrLevelGoalModerateOrAbove) {
    937                     for (Entry<MissingStatus, String> entry : missingPaths.entrySet()) {
    938                         String path = entry.getValue();
    939 //                        if (SKIP_PATHS.get(path) == null) {
    940                         MissingStatus missingStatus = entry.getKey();
    941                         CoverageInfo coverageInfo = new CoverageInfo(SUPPLEMENTAL_DATA_INFO);
    942                         Level foundLevel = coverageInfo.getCoverageLevel(path, locale);
    943                         if (cldrLocaleLevelGoal.compareTo(foundLevel) >= 0) {
    944                             String line = spreadsheetLine(locale, script, language, cldrLocaleLevelGoal, foundLevel, missingStatus.toString(), path, file.getStringValue(path));
    945                             tsv_missing.println(line);
    946                         }
    947                     }
    948                 }
    949 
    950                 for (Level level : reversedLevels) {
    951                     long foundCount = foundCounter.get(level);
    952                     long unconfirmedCount = unconfirmedCounter.get(level);
    953                     long missingCount = missingCounter.get(level);
    954 
    955                     sumFound += foundCount;
    956                     sumUnconfirmed += unconfirmedCount;
    957                     sumMissing += missingCount;
    958 
    959                     confirmed.put(level, (int) foundCount);
    960 //                    unconfirmedByLevel.put(level, (int)(foundCount + unconfirmedCount));
    961                     totals.put(level, (int)(foundCount + unconfirmedCount + missingCount));
    962                 }
    963 
    964                 tsv_missing.flush();
    965 
    966                 double modernTotal = totals.get(Level.MODERN);
    967 
    968                 tablePrinter
    969                 .addCell(sumFound)
    970                 .addCell(sumUnconfirmed)
    971                 .addCell(sumMissing)
    972                 ;
    973 
    974                 tsv_summary
    975                 .append('\t').append(sumFound+"")
    976                 .append('\t').append(sumUnconfirmed+"")
    977                 .append('\t').append(sumMissing+"")
    978                 ;
    979 
    980 
    981 //                header += "\t" + sumFound;
    982 //                header += "\t" + (sumFound + sumUnconfirmed);
    983 
    984                 // print the totals
    985 
    986                 for (Level level : reversedLevels) {
    987                     if (useOrgLevel && cldrLocaleLevelGoal != level) {
    988                         continue;
    989                     }
    990                     int confirmedCoverage = confirmed.get(level);
    991 //                    int unconfirmedCoverage = unconfirmedByLevel.get(level);
    992                     double total = totals.get(level);
    993 
    994                     tablePrinter
    995                     .addCell(confirmedCoverage / total)
    996 //                    .addCell(unconfirmedCoverage / total)
    997                     ;
    998 
    999                     tsv_summary
   1000                     .append('\t').append(String.valueOf(confirmedCoverage))
   1001                     .append('\t').append(String.valueOf((int)total - confirmedCoverage))
   1002                     ;
   1003 
   1004 //                    if (RAW_DATA) {
   1005 //                        header += "\t" + confirmedCoverage / total
   1006 //                            + "\t" + unconfirmedCoverage / total;
   1007 //                    } else {
   1008 //                        Double factor = targetLevel.get(level) / (total / modernTotal);
   1009 //                        header += "\t" + factor * confirmedCoverage / modernTotal
   1010 ////                            + "\t" + factor * unconfirmedCoverage / modernTotal
   1011 //                            ;
   1012 //                    }
   1013                 }
   1014                 String coreMissingString =
   1015                     CollectionUtilities.join(coreMissing, ", ");
   1016 
   1017                 tablePrinter
   1018                 .addCell(coreMissingString)
   1019                 .finishRow();
   1020 
   1021                 tsv_summary
   1022                 .append('\t')
   1023                 .append(coreMissingString)
   1024                 .append('\n');
   1025 
   1026                 //out2.println(header + "\t" + coreValue + "\t" + CollectionUtilities.join(missing, ", "));
   1027 
   1028                 // Write missing paths (for >99% and specials
   1029 
   1030                 if (false) { // checkModernLocales.contains(locale)
   1031                     CoverageLevel2 coverageLevel2 = CoverageLevel2.getInstance(locale);
   1032 
   1033                     for (String path : unconfirmed) {
   1034                         Level level = coverageLevel2.getLevel(path);
   1035                         if (level.compareTo(cldrLocaleLevelGoal) > 0) {
   1036                             continue;
   1037                         }
   1038                         String line = spreadsheetLine(locale, script, language, cldrLocaleLevelGoal, level, "UNCONFIRMED", path, file.getStringValue(path));
   1039                         if (SUPPRESS_PATHS_CAN_BE_EMPTY.get(path) != null) {
   1040                             //System.out.println("\nSKIP: " + line);
   1041                         } else {
   1042                             tsv_missing.println(line);
   1043                         }
   1044                     }
   1045                     for (Entry<MissingStatus, String> entry : missingPaths.entrySet()) {
   1046                         String path = entry.getValue();
   1047                         Level level = coverageLevel2.getLevel(path);
   1048                         if (level.compareTo(cldrLocaleLevelGoal) > 0) {
   1049                             continue;
   1050                         }
   1051                         MissingStatus missingStatus = entry.getKey();
   1052                         String line = spreadsheetLine(locale, script, language, cldrLocaleLevelGoal, level, missingStatus.toString(), path, "???");
   1053                         if (SUPPRESS_PATHS_CAN_BE_EMPTY.get(path) != null) {
   1054                             //System.out.println("\nSKIP: " + line);
   1055                         } else {
   1056                             tsv_missing.println(line);
   1057                         }
   1058                     }
   1059                 }
   1060 
   1061                 localeCount++;
   1062             } catch (Exception e) {
   1063                 throw new IllegalArgumentException(e);
   1064             }
   1065         }
   1066         pw.println(tablePrinter.toTable());
   1067         out2.close();
   1068 
   1069         long end = System.currentTimeMillis();
   1070         System.out.println((end - start) + " millis = "
   1071             + ((end - start) / localeCount) + " millis/locale");
   1072 
   1073         //        CoverageLevel2 coverageLevel2 = CoverageLevel2.getInstance("en");
   1074         //
   1075         //        for (Entry<MissingStatus, Set<String>> entity : missingPaths.keyValuesSet()) {
   1076         //            for (PathHeader s : CldrUtility.transform(entity.getValue(), pathHeaderFactory, new TreeSet<PathHeader>())) {
   1077         //                System.out.println(entity.getKey() + "\t" + coverageLevel2.getLevel(s.getOriginalPath()) + "\t" + s
   1078         //                        + "\t\t" + s.getOriginalPath());
   1079         //            }
   1080         //        }
   1081     }
   1082 // userInfo.getVoterInfo().getLevel().compareTo(VoteResolver.Level.tc)
   1083     static final VoterInfo dummyVoterInfo = new VoterInfo(Organization.cldr, org.unicode.cldr.util.VoteResolver.Level.vetter, "somename");
   1084 
   1085     static final UserInfo dummyUserInfo = new UserInfo() {
   1086         public VoterInfo getVoterInfo() {
   1087             return dummyVoterInfo;
   1088         }
   1089     };
   1090     static final PathValueInfo dummyPathValueInfo = new PathValueInfo() {
   1091         // pathValueInfo.getCoverageLevel().compareTo(Level.COMPREHENSIVE)
   1092         public Collection<? extends CandidateInfo> getValues() {
   1093             throw new UnsupportedOperationException();
   1094         }
   1095         public CandidateInfo getCurrentItem() {
   1096             throw new UnsupportedOperationException();
   1097         }
   1098         public String getLastReleaseValue() {
   1099             throw new UnsupportedOperationException();
   1100         }
   1101         public Level getCoverageLevel() {
   1102             return Level.MODERN;
   1103         }
   1104         public boolean hadVotesSometimeThisRelease() {
   1105             throw new UnsupportedOperationException();
   1106         }
   1107     };
   1108 
   1109     public static String spreadsheetLine(String locale, String script, String language, Level cldrLocaleLevelGoal, Level itemLevel, String status, String path, String nativeValue) {
   1110         String phString = "n/a\tn/a\tn/a\tn/a";
   1111         String stLink = "n/a";
   1112         String englishValue = "n/a";
   1113         StatusAction action = null;
   1114         SurveyToolStatus surveyToolStatus = null;
   1115         try {
   1116             PathHeader ph = pathHeaderFactory.fromPath(path);
   1117             phString = ph.toString();
   1118             surveyToolStatus = ph.getSurveyToolStatus();
   1119             action = Phase.SUBMISSION.getShowRowAction(dummyPathValueInfo, InputMethod.DIRECT, surveyToolStatus, dummyUserInfo);
   1120             stLink = URLS.forXpath(locale, ph.getOriginalPath());
   1121             englishValue = ENGLISH.getStringValue(path);
   1122         } catch (Exception e) {
   1123         }
   1124         String line = language
   1125             + "\t" + ENGLISH.getName(language)
   1126             + "\t" + ENGLISH.getName("script", script)
   1127             + "\t" + englishValue
   1128             + "\t" + nativeValue
   1129             + "\t" + cldrLocaleLevelGoal
   1130             + "\t" + itemLevel
   1131             + "\t" + status
   1132             + "\t" + (action == null ? "?" : action.toString())
   1133             + "\t" + (surveyToolStatus == null ? "?" : surveyToolStatus.toString())
   1134             + "\t" + stLink
   1135             + "\t" + phString
   1136             + "\t" + path
   1137             ;
   1138         return line;
   1139     }
   1140 
   1141     private static CLDRURLS URLS = CLDRConfig.getInstance().urls();
   1142 
   1143 }
   1144