Home | History | Annotate | Download | only in tool 
      1 package org.unicode.cldr.tool;
      2 
      3 import java.util.HashMap;
      4 import java.util.LinkedHashSet;
      5 import java.util.List;
      6 import java.util.Locale;
      7 import java.util.Map;
      8 import java.util.Map.Entry;
      9 import java.util.Set;
     10 import java.util.TreeMap;
     11 import java.util.TreeSet;
     12 import java.util.regex.Pattern;
     13 
     14 import org.unicode.cldr.util.Builder;
     15 import org.unicode.cldr.util.CldrUtility;
     16 import org.unicode.cldr.util.FileProcessor;
     17 import org.unicode.cldr.util.Iso639Data;
     18 import org.unicode.cldr.util.IsoRegionData;
     19 import org.unicode.cldr.util.PatternCache;
     20 import org.unicode.cldr.util.StandardCodes;
     21 import org.unicode.cldr.util.SupplementalDataInfo;
     22 
     23 import com.ibm.icu.dev.util.CollectionUtilities;
     24 import com.ibm.icu.impl.Relation;
     25 import com.ibm.icu.impl.Row;
     26 import com.ibm.icu.impl.Row.R2;
     27 import com.ibm.icu.util.Output;
     28 
     29 public class LocaleReplacements {
     30     public static final Pattern WHITESPACE = PatternCache.get("\\s+");
     31 
     32     /**
     33      * eg language, eng, <overlong,en>
     34      */
     35     static Map<String, Map<String, Row.R2<Set<String>, String>>> type2item2replacementAndReason = new HashMap<String, Map<String, Row.R2<Set<String>, String>>>();
     36     static Map<String, Relation<String, Row.R2<String, Set<String>>>> type2reason2itemAndreplacement = new TreeMap<String, Relation<String, Row.R2<String, Set<String>>>>();
     37     static Relation<String, String> fixed = Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class);
     38 
     39     public String get(String old, Output<String> reason) {
     40         reason.value = null;
     41         return old;
     42     }
     43 
     44     static {
     45         Map<String, Map<String, Map<String, String>>> lstreg = StandardCodes.getLStreg();
     46         for (Entry<String, Map<String, Map<String, String>>> entry : lstreg.entrySet()) {
     47             String type = entry.getKey();
     48             Map<String, Map<String, String>> subtype2data = entry.getValue();
     49 
     50             for (Entry<String, Map<String, String>> itemAndData : subtype2data.entrySet()) {
     51                 final Map<String, String> value = itemAndData.getValue();
     52                 String deprecated = value.get("Deprecated");
     53                 if (deprecated != null) {
     54                     String preferredValue = value.get("Preferred-Value");
     55                     if (preferredValue == null) {
     56                         preferredValue = "";
     57                     }
     58                     final String key = itemAndData.getKey();
     59                     String type2 = type.equals("region") ? "territory" : type;
     60                     addType2item2reasonNreplacement(type2, key, preferredValue, "deprecated", false);
     61                 }
     62             }
     63         }
     64 
     65         for (String lang : Iso639Data.getAvailable()) {
     66             if (lang.length() != 2) continue;
     67             String alpha3 = Iso639Data.toAlpha3(lang);
     68             addType2item2reasonNreplacement("language", alpha3, lang, "overlong", false);
     69         }
     70         /*
     71          * return IsoRegionData.get_alpha3(region);
     72          * }
     73          * });
     74          * addRegions(english, territories, "AC,CP,DG,EA,EU,IC,TA".split(","), new Transform<String,String>() {
     75          * public String transform(String region) {
     76          * return IsoRegionData.getNumeric(region);
     77          */
     78         //Set<String> available2 = IsoRegionData.getAvailable();
     79 
     80         for (String region : IsoRegionData.getAvailable()) {
     81             String alpha3 = IsoRegionData.get_alpha3(region);
     82             addType2item2reasonNreplacement("territory", alpha3, region, "overlong", false);
     83             String numeric = IsoRegionData.getNumeric(region);
     84             addType2item2reasonNreplacement("territory", numeric, region, "overlong", false);
     85         }
     86 
     87         // Add overrides
     88         FileProcessor myReader = new FileProcessor() {
     89             @Override
     90             protected boolean handleLine(int lineCount, String line) {
     91                 addType2item2reasonNreplacement(line);
     92                 return true;
     93             }
     94         };
     95 
     96         myReader.process(CldrUtility.class, "data/localeReplacements.txt");
     97 
     98         // fix up the data by recursing
     99 
    100         for (Entry<String, Map<String, R2<Set<String>, String>>> entry : type2item2replacementAndReason.entrySet()) {
    101             //String type = entry.getKey();
    102             final Map<String, R2<Set<String>, String>> item2replacementAndReason = entry.getValue();
    103             while (true) {
    104                 boolean keepGoing = false;
    105                 for (Entry<String, R2<Set<String>, String>> entry2 : item2replacementAndReason.entrySet()) {
    106                     String item = entry2.getKey();
    107                     R2<Set<String>, String> replacementAndReason = entry2.getValue();
    108                     Set<String> replacements = replacementAndReason.get0();
    109                     //String reason = replacementAndReason.get1();
    110                     Set<String> newReplacements = new LinkedHashSet<String>(replacements.size());
    111                     boolean gotChange = false;
    112                     for (String oldRep : replacements) {
    113                         R2<Set<String>, String> newRepAndReason = item2replacementAndReason.get(oldRep);
    114                         if (newRepAndReason != null) {
    115                             fixed.put(item, oldRep + "\t-->\t" + newRepAndReason);
    116                             newReplacements.addAll(newRepAndReason.get0());
    117                             gotChange = true;
    118                         } else {
    119                             newReplacements.add(oldRep);
    120                         }
    121                     }
    122                     if (gotChange) {
    123                         replacementAndReason.set0(newReplacements);
    124                         keepGoing = true;
    125                     }
    126                 }
    127                 if (!keepGoing) {
    128                     break;
    129                 }
    130             }
    131         }
    132 
    133         for (Entry<String, Map<String, R2<Set<String>, String>>> entry : type2item2replacementAndReason.entrySet()) {
    134             String type = entry.getKey();
    135             final Map<String, R2<Set<String>, String>> item2replacementAndReason = entry.getValue();
    136             for (Entry<String, R2<Set<String>, String>> entry2 : item2replacementAndReason.entrySet()) {
    137                 String item = entry2.getKey();
    138                 R2<Set<String>, String> replacementAndReason = entry2.getValue();
    139                 Set<String> replacements = replacementAndReason.get0();
    140                 String reason = replacementAndReason.get1();
    141 
    142                 Relation<String, R2<String, Set<String>>> reason2item2replacement = type2reason2itemAndreplacement
    143                     .get(type);
    144                 if (reason2item2replacement == null) {
    145                     type2reason2itemAndreplacement.put(
    146                         type,
    147                         reason2item2replacement = Relation.of(new TreeMap<String, Set<R2<String, Set<String>>>>(),
    148                             TreeSet.class));
    149                 }
    150                 reason2item2replacement.put(reason, Row.of(item, replacements));
    151             }
    152         }
    153     }
    154 
    155     private static void addType2item2reasonNreplacement(String line) {
    156         String[] parts = WHITESPACE.split(line);
    157         if (parts.length < 4) {
    158             addType2item2reasonNreplacement(parts[0], parts[2], parts[1], "", true);
    159             return;
    160         }
    161         // language macrolanguage bxk luy
    162         for (int i = 3; i < parts.length; ++i) {
    163             addType2item2reasonNreplacement(parts[0], parts[2], parts[i], parts[1], true);
    164         }
    165     }
    166 
    167     private static void addType2item2reasonNreplacement(String type, String key, String preferredValue, String reason,
    168         boolean ignoreDuplicates) {
    169         if (key == null) {
    170             return;
    171         }
    172         if (type.equals("grandfathered") || type.equals("redundant")) {
    173             type = "language";
    174         }
    175 
    176         key = key.replace('-', '_');
    177         if (type.equals("variant")) {
    178             key = key.toUpperCase(Locale.US);
    179             preferredValue = preferredValue.toUpperCase(Locale.US);
    180         }
    181 
    182         Map<String, R2<Set<String>, String>> item2replacementAndReason = type2item2replacementAndReason.get(type);
    183         if (item2replacementAndReason == null) {
    184             type2item2replacementAndReason.put(type, item2replacementAndReason = new HashMap<String, R2<Set<String>, String>>());
    185         }
    186 
    187         R2<Set<String>, String> oldReplacementAndReason = item2replacementAndReason.get(key);
    188         if (oldReplacementAndReason != null) {
    189             final String message = "duplicateReplacement\t" + type + "\t" + key + "\told: "
    190                 + oldReplacementAndReason + "\tnew:" + preferredValue + ", " + reason;
    191             if (!ignoreDuplicates) {
    192                 throw new IllegalArgumentException(message);
    193             } else {
    194                 fixed.put(key, message);
    195                 Set<String> list = oldReplacementAndReason.get0();
    196                 list.add(preferredValue);
    197                 return;
    198             }
    199         }
    200         Set<String> list = new LinkedHashSet<String>(1);
    201         if (!preferredValue.isEmpty()) {
    202             list.add(preferredValue);
    203         }
    204         item2replacementAndReason.put(key, Row.of(list, reason));
    205     }
    206 
    207     public static void main(String[] args) {
    208         Map<String, Map<String, R2<List<String>, String>>> localeAliasInfo = SupplementalDataInfo.getInstance()
    209             .getLocaleAliasInfo();
    210 
    211         Set<String> newStuff = new TreeSet<String>();
    212         Set<String> oldStuff = new TreeSet<String>();
    213         for (Entry<String, Relation<String, R2<String, Set<String>>>> entry : type2reason2itemAndreplacement.entrySet()) {
    214             String type = entry.getKey();
    215             for (Entry<String, R2<String, Set<String>>> entry2 : entry.getValue().entrySet()) {
    216                 String reason = entry2.getKey();
    217                 R2<String, Set<String>> replacementAndReason = entry2.getValue();
    218                 String key = replacementAndReason.get0();
    219                 Set<String> replacements = replacementAndReason.get1();
    220                 final String message = type + "\t" + reason + "\t" + key + "\t"
    221                     + CollectionUtilities.join(replacements, " ");
    222                 // System.out.println(message);
    223                 newStuff.add(message);
    224             }
    225         }
    226         for (Entry<String, String> entry : fixed.entrySet()) {
    227             System.out.println(entry.getKey() + "\t" + entry.getValue());
    228         }
    229         // Returns type -> tag -> , like "language" -> "sh" -> <{"sr_Latn"}, reason>
    230         for (Entry<String, Map<String, R2<List<String>, String>>> entry : localeAliasInfo.entrySet()) {
    231             String type = entry.getKey();
    232             for (Entry<String, R2<List<String>, String>> entry2 : entry.getValue().entrySet()) {
    233                 String item = entry2.getKey();
    234                 R2<List<String>, String> replacementAndReason = entry2.getValue();
    235                 List<String> replacements = replacementAndReason.get0();
    236                 String reason = replacementAndReason.get1();
    237                 oldStuff.add(type + "\t" + reason + "\t" + item
    238                     + "\t" + (replacements == null ? "" : CollectionUtilities.join(replacements, " ")));
    239             }
    240         }
    241         Set<Row.R2<String, String>> merged = new TreeSet<Row.R2<String, String>>();
    242 
    243         Set<String> oldNotNew = Builder.with(new TreeSet<String>(oldStuff)).removeAll(newStuff).get();
    244         Set<String> newNotOld = Builder.with(new TreeSet<String>(newStuff)).removeAll(oldStuff).get();
    245         //Set<String> shared = Builder.with(new TreeSet<String>(oldStuff)).retainAll(newStuff).get();
    246         // for (String s : shared) {
    247         // merged.add(Row.of(s,"\tSAME"));
    248         // }
    249         for (String s : oldNotNew) {
    250             merged.add(Row.of(s, "\tOLD"));
    251         }
    252         for (String s : newNotOld) {
    253             merged.add(Row.of(s, "\tNEW"));
    254         }
    255         int i = 0;
    256         for (R2<String, String> s : merged) {
    257             System.out.println(++i + "\t" + s.get1() + "\t" + s.get0());
    258         }
    259         System.out.println("DONE");
    260     }
    261 }