Home | History | Annotate | Download | only in draft 
      1 package org.unicode.cldr.draft;
      2 
      3 import java.io.File;
      4 import java.io.IOException;
      5 import java.io.PrintWriter;
      6 import java.util.ArrayList;
      7 import java.util.Arrays;
      8 import java.util.Collection;
      9 import java.util.Collections;
     10 import java.util.HashSet;
     11 import java.util.Iterator;
     12 import java.util.LinkedHashMap;
     13 import java.util.LinkedHashSet;
     14 import java.util.List;
     15 import java.util.Map;
     16 import java.util.Set;
     17 import java.util.TreeMap;
     18 import java.util.TreeSet;
     19 
     20 import org.unicode.cldr.util.CLDRFile;
     21 import org.unicode.cldr.util.CLDRPaths;
     22 import org.unicode.cldr.util.DtdType;
     23 import org.unicode.cldr.util.ElementAttributeInfo;
     24 import org.unicode.cldr.util.Factory;
     25 import org.unicode.cldr.util.XPathParts;
     26 
     27 import com.ibm.icu.impl.Relation;
     28 import com.ibm.icu.impl.Row;
     29 import com.ibm.icu.impl.Row.R2;
     30 import com.ibm.icu.impl.Utility;
     31 import com.ibm.icu.util.ICUUncheckedIOException;
     32 
     33 public class JsonConverter {
     34 
     35     private static final String FILES = "el.*";
     36     private static final String MAIN_DIRECTORY = CLDRPaths.MAIN_DIRECTORY;// CldrUtility.SUPPLEMENTAL_DIRECTORY;
     37                                                                           // //CldrUtility.MAIN_DIRECTORY;
     38     private static final String OUT_DIRECTORY = CLDRPaths.GEN_DIRECTORY + "/jason/"; // CldrUtility.MAIN_DIRECTORY;
     39     private static boolean COMPACT = false;
     40     static final Set<String> REPLACING_BASE = !COMPACT ? Collections.EMPTY_SET : new HashSet<String>(
     41         Arrays.asList("type id key count".split("\\s")));
     42     static final Set<String> EXTRA_DISTINGUISHING = new HashSet<String>(
     43         Arrays.asList("locales territory desired supported".split("\\s")));
     44     static final Relation<String, String> mainInfo = ElementAttributeInfo.getInstance(DtdType.ldml)
     45         .getElement2Attributes();
     46     static final Relation<String, String> suppInfo = ElementAttributeInfo.getInstance(DtdType.supplementalData)
     47         .getElement2Attributes();
     48 
     49     public static void main(String[] args) throws IOException {
     50         final String subdirectory = new File(MAIN_DIRECTORY).getName();
     51         final Factory cldrFactory = Factory.make(MAIN_DIRECTORY, FILES);
     52         final Set<String> locales = new TreeSet<String>(cldrFactory.getAvailable());
     53         final XPathParts oldParts = new XPathParts();
     54         final XPathParts parts = new XPathParts();
     55         // ElementName elementName = new ElementName();
     56         // LinkedHashMap<String, String> nonDistinguishing = new LinkedHashMap<String, String>();
     57         for (String locale : locales) {
     58             System.out.println("Converting:\t" + locale);
     59             final CLDRFile file = cldrFactory.make(locale, false);
     60             Relation<String, String> element2Attributes = file.isNonInheriting() ? suppInfo : mainInfo;
     61             final Item main = new TableItem(null);
     62             DtdType dtdType = null;
     63             for (Iterator<String> it = file.iterator("", file.getComparator()); it.hasNext();) {
     64                 final String xpath = it.next();
     65                 final String fullXpath = file.getFullXPath(xpath);
     66                 String value = file.getStringValue(xpath);
     67                 oldParts.set(fullXpath);
     68                 if (dtdType == null) {
     69                     dtdType = DtdType.valueOf(parts.getElement(0));
     70                 }
     71                 rewrite(dtdType, oldParts, value, element2Attributes, parts);
     72                 System.out.println(parts);
     73                 Item current = main;
     74                 int size = parts.size();
     75 
     76                 for (int i = 0; i < size - 1; ++i) {
     77                     final String element = parts.getElement(i);
     78                     Map<String, String> actualAttributeKeys = parts.getAttributes(i);
     79                     Set<String> keySet = actualAttributeKeys.keySet();
     80                     if (keySet.size() != 0) {
     81                         Item temp = current.makeSubItem(element, Item.Type.unorderedItem);
     82                         for (String attribute : keySet) {
     83                             temp.put(attribute, actualAttributeKeys.get(attribute));
     84                         }
     85                     }
     86                     if (i < size - 2) {
     87                         current = current.makeSubItem(element,
     88                             actualAttributeKeys.containsKey("_q") ? Item.Type.orderedItem : Item.Type.unorderedItem);
     89                     } else {
     90                         current.put(element, parts.getElement(i + 1));
     91                     }
     92                 }
     93             }
     94             PrintWriter out = FileUtilities.openUTF8Writer(OUT_DIRECTORY + subdirectory, locale + ".json");
     95             main.print(out, 0);
     96             out.close();
     97         }
     98     }
     99 
    100     static Relation<String, String> extraDistinguishing = Relation.of(new TreeMap<String, Set<String>>(), LinkedHashSet.class);
    101     static {
    102         putAll(extraDistinguishing, "dayPeriodRule", "earlyMorning", "before", "from");
    103     }
    104 
    105     static <K, V> void putAll(Relation r, K key, V... values) {
    106         r.putAll(key, Arrays.asList(values));
    107     }
    108 
    109     private static boolean isDistinguishing(DtdType dtdType, final String element, final String attribute) {
    110         // <mapZone other="Afghanistan" territory="001" type="Asia/Kabul"/> result is the type!
    111         // <deprecatedItems elements="variant" attributes="type" values="BOKMAL NYNORSK AALAND POLYTONI"/>
    112         // ugly: if there are values, then everything else is distinguishing, ow if there are attibutes, elements are
    113         if (element.equals("deprecatedItems")) {
    114 
    115         }
    116         Set<String> extras = extraDistinguishing.getAll(element);
    117         if (extras != null && extras.contains(attribute)) return true;
    118         if (EXTRA_DISTINGUISHING.contains(attribute)) return true;
    119         return CLDRFile.isDistinguishing(dtdType, element, attribute);
    120     }
    121 
    122     private static void rewrite(DtdType dtdType, XPathParts parts, String value,
    123         Relation<String, String> element2Attributes, XPathParts out) {
    124         out.clear();
    125         int size = parts.size();
    126         for (int i = 1; i < size; ++i) {
    127             final String element = parts.getElement(i);
    128             out.addElement(element);
    129 
    130             // turn a path into a revised path. All distinguished attributes (including those not currently on the
    131             // string)
    132             // get turned into extra element/element pairs, starting with _
    133             // all non-distinguishing attributes get turned into separate children
    134             // a/b[@non="y"][@dist="x"]/w : z =>
    135             // a/b/_dist/x/_non=y
    136             // a/b/_dist/x/w=z
    137             Collection<String> actualAttributeKeys = parts.getAttributeKeys(i);
    138             boolean isOrdered = actualAttributeKeys.contains("_q");
    139             Set<String> possibleAttributeKeys = element2Attributes.getAll(element);
    140 
    141             for (final String attribute : actualAttributeKeys) {
    142                 String attributeValue = parts.getAttributeValue(i, attribute);
    143                 if (!isDistinguishing(dtdType, element, attribute)) {
    144                     out.addAttribute(attribute, attributeValue);
    145                 }
    146             }
    147             if (possibleAttributeKeys != null) {
    148                 for (final String attribute : possibleAttributeKeys) {
    149                     if (isDistinguishing(dtdType, element, attribute)) {
    150                         if (attribute.equals("alt")) continue; // TODO fix
    151                         String attributeValue = parts.getAttributeValue(i, attribute);
    152                         out.addElement("_" + attribute);
    153                         if (attributeValue == null) {
    154                             attributeValue = "?";
    155                         }
    156                         out.addElement(attributeValue);
    157                     }
    158                 }
    159             }
    160             if (isOrdered) {
    161                 Map<String, String> lastAttributes = out.getAttributes(-2);
    162                 lastAttributes.put("_q", "_q");
    163             }
    164         }
    165         if (value.length() > 0) {
    166             out.addElement(value);
    167         }
    168 
    169         if (!COMPACT) {
    170             return;
    171         }
    172         if (parts.getElement(-1).equals("type")) {
    173             String key = parts.getAttributeValue(-1, "key");
    174             if (key != null) {
    175                 parts.setElement(-2, key + "Key");
    176                 parts.putAttributeValue(-1, "key", null);
    177             }
    178             // fall thru
    179         }
    180         if (parts.getElement(1).equals("localeDisplayNames")) {
    181             String element2 = parts.getElement(2);
    182             if (!element2.endsWith("Pattern")) {
    183                 if (element2.endsWith("s")) {
    184                     element2 = element2.substring(0, element2.length() - 1);
    185                 }
    186                 parts.setElement(2, element2 + "Names");
    187             }
    188             parts.removeElement(1);
    189         }
    190         if (parts.getElement(1).equals("dates")) {
    191             parts.removeElement(1);
    192             String element1 = parts.getElement(1);
    193             if (element1.equals("timeZoneNames")) {
    194                 String main = parts.getElement(2);
    195                 if (main.equals("zone") || main.equals("metazone")) {
    196                     parts.setElement(1, main + "Names");
    197                 }
    198                 return;
    199             }
    200         }
    201         if (parts.getElement(1).equals("numbers") && parts.getElement(2).equals("currencies")) {
    202             parts.removeElement(1);
    203             return;
    204         }
    205     }
    206 
    207     static class ElementName {
    208         String oldBase;
    209         String base;
    210         boolean replacedBase;
    211         StringBuilder suffix = new StringBuilder();
    212 
    213         public void reset(String element) {
    214             suffix.setLength(0);
    215             base = oldBase = element;
    216             replacedBase = false;
    217         }
    218 
    219         public void add(String attribute, String attributeValue) {
    220             if (REPLACING_BASE.contains(attribute)) {
    221                 if (replacedBase) {
    222                     System.out.println("ERROR: Two replacement types on same element!!\t" + oldBase + "," + base + ","
    223                         + attribute + "," + attributeValue);
    224                 } else {
    225                     replacedBase = true;
    226                     base = attributeValue;
    227                     return;
    228                 }
    229             }
    230             suffix.append('$').append(attribute).append('=').append(attributeValue);
    231         }
    232 
    233         public String toString() {
    234             if (suffix == null) {
    235                 return base;
    236             }
    237             return base + suffix;
    238         }
    239     }
    240 
    241     static abstract class Item {
    242         protected Item parent;
    243 
    244         public Item(Item parent) {
    245             this.parent = parent;
    246         }
    247 
    248         public abstract int size();
    249 
    250         enum Type {
    251             unorderedItem, orderedItem
    252         }
    253 
    254         public abstract Appendable print(Appendable result, int i);
    255 
    256         protected Appendable indent(Appendable result, int i) throws IOException {
    257             return result.append(getIndent(i));
    258         }
    259 
    260         protected String getIndent(int i) {
    261             return Utility.repeat("    ", i);
    262         }
    263 
    264         public Appendable appendString(Appendable result, String string, int indent) throws IOException {
    265             result.append('"');
    266             for (int i = 0; i < string.length(); ++i) {
    267                 // http://www.json.org/
    268                 // any-Unicode-character-except-"-or-\-or-control-character
    269                 // uses UTF16
    270                 char ch = string.charAt(i);
    271                 switch (ch) {
    272                 case '\"':
    273                     result.append("\\\"");
    274                     break;
    275                 case '\\':
    276                     result.append("\\\\");
    277                     break;
    278                 case '/':
    279                     result.append("\\/");
    280                     break;
    281                 case '\b':
    282                     result.append("\\b");
    283                     break;
    284                 case '\f':
    285                     result.append("\\f");
    286                     break;
    287                 case '\n':
    288                     if (indent < 0) {
    289                         result.append("\\n");
    290                     } else {
    291                         result.append('\n').append(getIndent(indent));
    292                     }
    293                     break;
    294                 case '\r':
    295                     result.append("\\r");
    296                     break;
    297                 case '\t':
    298                     result.append("\\t");
    299                     break;
    300                 default:
    301                     if (ch <= 0x1F || 0x7F <= ch && ch <= 0x9F) {
    302                         result.append("\\u").append(Utility.hex(ch, 4));
    303                     } else {
    304                         result.append(ch);
    305                     }
    306                     break;
    307                 }
    308             }
    309             return result.append('"');
    310         }
    311 
    312         public String toString() {
    313             return print(new StringBuilder(), 0).toString();
    314         }
    315 
    316         protected Item create(Type ordered) {
    317             switch (ordered) {
    318             case unorderedItem:
    319                 return new TableItem(this);
    320             case orderedItem:
    321                 return new ArrayItem(this);
    322             default:
    323                 throw new UnsupportedOperationException();
    324             }
    325         }
    326 
    327         public abstract Item makeSubItem(String element, Type ordered);
    328 
    329         public abstract void put(String element, String value);
    330 
    331         public Item getRoot() {
    332             if (parent == null) {
    333                 return this;
    334             } else {
    335                 return parent.getRoot();
    336             }
    337         }
    338     }
    339 
    340     static class TableItem extends Item {
    341         public TableItem(Item parent) {
    342             super(parent);
    343         }
    344 
    345         private Map<String, Item> map = new LinkedHashMap<String, Item>();
    346 
    347         public Item get(String element) {
    348             return map.get(element);
    349         }
    350 
    351         public void put(String element, String value) {
    352             Item old = map.get(element);
    353             if (old != null) {
    354                 if (old instanceof StringItem) {
    355                     if (value.equals(((StringItem) old).value)) {
    356                         return;
    357                     }
    358                 }
    359                 throw new IllegalArgumentException("ERROR: Table already has object: " + element + ", " + old + ", "
    360                     + value + ", " + getRoot().toString());
    361             }
    362             map.put(element, new StringItem(value));
    363         }
    364 
    365         public Item makeSubItem(String element, Type ordered) {
    366             Item result = map.get(element);
    367             if (result != null) {
    368                 return result;
    369             }
    370             result = create(ordered);
    371             result.parent = this;
    372 
    373             map.put(element, result);
    374             return result;
    375         }
    376 
    377         public Appendable print(Appendable result, int i) {
    378             try {
    379                 if (map.size() == 0) {
    380                     result.append("{}");
    381                     return result;
    382                 }
    383                 result.append("{\n");
    384                 boolean first = true;
    385                 for (String key : map.keySet()) {
    386                     Item value = map.get(key);
    387                     if (first) {
    388                         first = false;
    389                     } else {
    390                         result.append(",\n");
    391                     }
    392                     indent(result, i + 1);
    393                     appendString(result, key, -1).append(" : ");
    394                     value.print(result, i + 1);
    395                 }
    396                 result.append("\n");
    397                 indent(result, i).append("}");
    398                 return result;
    399             } catch (IOException e) {
    400                 throw new ICUUncheckedIOException(e);
    401             }
    402         }
    403 
    404         @Override
    405         public int size() {
    406             return map.size();
    407         }
    408     }
    409 
    410     static class ArrayItem extends Item {
    411         public ArrayItem(Item parent) {
    412             super(parent);
    413         }
    414 
    415         private List<Row.R2<String, Item>> list = new ArrayList<Row.R2<String, Item>>();
    416 
    417         @Override
    418         public Appendable print(Appendable result, int i) {
    419             try {
    420                 if (list.size() == 0) {
    421                     result.append("[]");
    422                     return result;
    423                 }
    424 
    425                 result.append("[\n");
    426                 for (int j = 0; j < list.size(); ++j) {
    427                     if (j != 0) {
    428                         result.append(",\n");
    429                     }
    430                     indent(result, i + 1);
    431                     R2<String, Item> row = list.get(j);
    432                     result.append("{");
    433                     appendString(result, row.get0(), i + 1);
    434                     result.append(" : ");
    435                     row.get1().print(result, i + 1);
    436                     result.append("}");
    437                 }
    438                 result.append("\n");
    439                 indent(result, i).append("]");
    440                 return result;
    441             } catch (IOException e) {
    442                 throw new IllegalArgumentException(e);
    443             }
    444         }
    445 
    446         public Item makeSubItem(String element, Type ordered) {
    447             Item result = create(ordered);
    448             list.add(Row.of(element, result));
    449             return result;
    450         }
    451 
    452         public void put(String element, String value) {
    453             list.add(Row.of(element, (Item) new StringItem(value)));
    454         }
    455 
    456         @Override
    457         public int size() {
    458             return list.size();
    459         }
    460     }
    461 
    462     static class StringItem extends Item {
    463         private String value;
    464 
    465         public StringItem(String value2) {
    466             super(null);
    467             value = value2;
    468         }
    469 
    470         @Override
    471         public Appendable print(Appendable result, int i) {
    472             try {
    473                 return appendString(result, value, i + 1);
    474             } catch (IOException e) {
    475                 throw new IllegalArgumentException(e);
    476             }
    477         }
    478 
    479         @Override
    480         public Item makeSubItem(String element, Type ordered) {
    481             throw new UnsupportedOperationException();
    482         }
    483 
    484         @Override
    485         public void put(String element, String value) {
    486             throw new UnsupportedOperationException();
    487         }
    488 
    489         @Override
    490         public int size() {
    491             throw new UnsupportedOperationException();
    492         }
    493     }
    494 }
    495