Home | History | Annotate | Download | only in gennorm2
      1 /*
      2 *******************************************************************************
      3 *
      4 *   Copyright (C) 2009-2014, International Business Machines
      5 *   Corporation and others.  All Rights Reserved.
      6 *
      7 *******************************************************************************
      8 *   file name:  n2builder.cpp
      9 *   encoding:   US-ASCII
     10 *   tab size:   8 (not used)
     11 *   indentation:4
     12 *
     13 *   created on: 2009nov25
     14 *   created by: Markus W. Scherer
     15 *
     16 * Builds Normalizer2 data and writes a binary .nrm file.
     17 * For the file format see source/common/normalizer2impl.h.
     18 */
     19 
     20 #include "unicode/utypes.h"
     21 #include "n2builder.h"
     22 
     23 #include <stdio.h>
     24 #include <stdlib.h>
     25 #include <string.h>
     26 #if U_HAVE_STD_STRING
     27 #include <vector>
     28 #endif
     29 #include "unicode/errorcode.h"
     30 #include "unicode/localpointer.h"
     31 #include "unicode/putil.h"
     32 #include "unicode/udata.h"
     33 #include "unicode/uniset.h"
     34 #include "unicode/unistr.h"
     35 #include "unicode/ustring.h"
     36 #include "charstr.h"
     37 #include "hash.h"
     38 #include "normalizer2impl.h"
     39 #include "toolutil.h"
     40 #include "unewdata.h"
     41 #include "utrie2.h"
     42 #include "uvectr32.h"
     43 #include "writesrc.h"
     44 
     45 #if !UCONFIG_NO_NORMALIZATION
     46 
     47 /* UDataInfo cf. udata.h */
     48 static UDataInfo dataInfo={
     49     sizeof(UDataInfo),
     50     0,
     51 
     52     U_IS_BIG_ENDIAN,
     53     U_CHARSET_FAMILY,
     54     U_SIZEOF_UCHAR,
     55     0,
     56 
     57     { 0x4e, 0x72, 0x6d, 0x32 }, /* dataFormat="Nrm2" */
     58     { 2, 0, 0, 0 },             /* formatVersion */
     59     { 5, 2, 0, 0 }              /* dataVersion (Unicode version) */
     60 };
     61 
     62 U_NAMESPACE_BEGIN
     63 
     64 class HangulIterator {
     65 public:
     66     struct Range {
     67         UChar32 start, limit;
     68         uint16_t norm16;
     69     };
     70 
     71     HangulIterator() : rangeIndex(0) {}
     72     const Range *nextRange() {
     73         if(rangeIndex<UPRV_LENGTHOF(ranges)) {
     74             return ranges+rangeIndex++;
     75         } else {
     76             return NULL;
     77         }
     78     }
     79     void reset() { rangeIndex=0; }
     80 private:
     81     static const Range ranges[4];
     82     int32_t rangeIndex;
     83 };
     84 
     85 const HangulIterator::Range HangulIterator::ranges[4]={
     86     { Hangul::JAMO_L_BASE, Hangul::JAMO_L_BASE+Hangul::JAMO_L_COUNT, 1 },
     87     { Hangul::JAMO_V_BASE, Hangul::JAMO_V_BASE+Hangul::JAMO_V_COUNT, Normalizer2Impl::JAMO_VT },
     88     // JAMO_T_BASE+1: not U+11A7
     89     { Hangul::JAMO_T_BASE+1, Hangul::JAMO_T_BASE+Hangul::JAMO_T_COUNT, Normalizer2Impl::JAMO_VT },
     90     { Hangul::HANGUL_BASE, Hangul::HANGUL_BASE+Hangul::HANGUL_COUNT, 0 },  // will become minYesNo
     91 };
     92 
     93 struct CompositionPair {
     94     CompositionPair(UChar32 t, UChar32 c) : trail(t), composite(c) {}
     95     UChar32 trail, composite;
     96 };
     97 
     98 struct Norm {
     99     enum MappingType { NONE, REMOVED, ROUND_TRIP, ONE_WAY };
    100 
    101     UBool hasMapping() const { return mappingType>REMOVED; }
    102 
    103     // Requires hasMapping() and well-formed mapping.
    104     void setMappingCP() {
    105         UChar32 c;
    106         if(!mapping->isEmpty() && mapping->length()==U16_LENGTH(c=mapping->char32At(0))) {
    107             mappingCP=c;
    108         } else {
    109             mappingCP=U_SENTINEL;
    110         }
    111     }
    112 
    113     const CompositionPair *getCompositionPairs(int32_t &length) const {
    114         if(compositions==NULL) {
    115             length=0;
    116             return NULL;
    117         } else {
    118             length=compositions->size()/2;
    119             return reinterpret_cast<const CompositionPair *>(compositions->getBuffer());
    120         }
    121     }
    122 
    123     UnicodeString *mapping;
    124     UnicodeString *rawMapping;  // non-NULL if the mapping is further decomposed
    125     UChar32 mappingCP;  // >=0 if mapping to 1 code point
    126     int32_t mappingPhase;
    127     MappingType mappingType;
    128 
    129     UVector32 *compositions;  // (trail, composite) pairs
    130     uint8_t cc;
    131     UBool combinesBack;
    132     UBool hasNoCompBoundaryAfter;
    133 
    134     enum OffsetType {
    135         OFFSET_NONE,
    136         // Composition for back-combining character. Allowed, but not normally used.
    137         OFFSET_MAYBE_YES,
    138         // Composition for a starter that does not have a decomposition mapping.
    139         OFFSET_YES_YES,
    140         // Round-trip mapping & composition for a starter.
    141         OFFSET_YES_NO_MAPPING_AND_COMPOSITION,
    142         // Round-trip mapping for a starter that itself does not combine-forward.
    143         OFFSET_YES_NO_MAPPING_ONLY,
    144         // One-way mapping.
    145         OFFSET_NO_NO,
    146         // Delta for an algorithmic one-way mapping.
    147         OFFSET_DELTA
    148     };
    149     enum { OFFSET_SHIFT=4, OFFSET_MASK=(1<<OFFSET_SHIFT)-1 };
    150     int32_t offset;
    151 };
    152 
    153 class Normalizer2DBEnumerator {
    154 public:
    155     Normalizer2DBEnumerator(Normalizer2DataBuilder &b) : builder(b) {}
    156     virtual ~Normalizer2DBEnumerator() {}
    157     virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) = 0;
    158     Normalizer2DBEnumerator *ptr() { return this; }
    159 protected:
    160     Normalizer2DataBuilder &builder;
    161 };
    162 
    163 U_CDECL_BEGIN
    164 
    165 static UBool U_CALLCONV
    166 enumRangeHandler(const void *context, UChar32 start, UChar32 end, uint32_t value) {
    167     return ((Normalizer2DBEnumerator *)context)->rangeHandler(start, end, value);
    168 }
    169 
    170 U_CDECL_END
    171 
    172 Normalizer2DataBuilder::Normalizer2DataBuilder(UErrorCode &errorCode) :
    173         phase(0), overrideHandling(OVERRIDE_PREVIOUS), optimization(OPTIMIZE_NORMAL),
    174         norm16TrieLength(0) {
    175     memset(unicodeVersion, 0, sizeof(unicodeVersion));
    176     normTrie=utrie2_open(0, 0, &errorCode);
    177     normMem=utm_open("gennorm2 normalization structs", 10000, 0x110100, sizeof(Norm));
    178     norms=allocNorm();  // unused Norm struct at index 0
    179     memset(indexes, 0, sizeof(indexes));
    180     memset(smallFCD, 0, sizeof(smallFCD));
    181 }
    182 
    183 Normalizer2DataBuilder::~Normalizer2DataBuilder() {
    184     utrie2_close(normTrie);
    185     int32_t normsLength=utm_countItems(normMem);
    186     for(int32_t i=1; i<normsLength; ++i) {
    187         delete norms[i].mapping;
    188         delete norms[i].rawMapping;
    189         delete norms[i].compositions;
    190     }
    191     utm_close(normMem);
    192     utrie2_close(norm16Trie);
    193 }
    194 
    195 void
    196 Normalizer2DataBuilder::setUnicodeVersion(const char *v) {
    197     UVersionInfo nullVersion={ 0, 0, 0, 0 };
    198     UVersionInfo version;
    199     u_versionFromString(version, v);
    200     if( 0!=memcmp(version, unicodeVersion, U_MAX_VERSION_LENGTH) &&
    201         0!=memcmp(nullVersion, unicodeVersion, U_MAX_VERSION_LENGTH)
    202     ) {
    203         char buffer[U_MAX_VERSION_STRING_LENGTH];
    204         u_versionToString(unicodeVersion, buffer);
    205         fprintf(stderr, "gennorm2 error: multiple inconsistent Unicode version numbers %s vs. %s\n",
    206                 buffer, v);
    207         exit(U_ILLEGAL_ARGUMENT_ERROR);
    208     }
    209     memcpy(unicodeVersion, version, U_MAX_VERSION_LENGTH);
    210 }
    211 
    212 Norm *Normalizer2DataBuilder::allocNorm() {
    213     Norm *p=(Norm *)utm_alloc(normMem);
    214     norms=(Norm *)utm_getStart(normMem);  // in case it got reallocated
    215     return p;
    216 }
    217 
    218 /* get an existing Norm unit */
    219 Norm *Normalizer2DataBuilder::getNorm(UChar32 c) {
    220     uint32_t i=utrie2_get32(normTrie, c);
    221     if(i==0) {
    222         return NULL;
    223     }
    224     return norms+i;
    225 }
    226 
    227 const Norm &Normalizer2DataBuilder::getNormRef(UChar32 c) const {
    228     return norms[utrie2_get32(normTrie, c)];
    229 }
    230 
    231 /*
    232  * get or create a Norm unit;
    233  * get or create the intermediate trie entries for it as well
    234  */
    235 Norm *Normalizer2DataBuilder::createNorm(UChar32 c) {
    236     uint32_t i=utrie2_get32(normTrie, c);
    237     if(i!=0) {
    238         return norms+i;
    239     } else {
    240         /* allocate Norm */
    241         Norm *p=allocNorm();
    242         IcuToolErrorCode errorCode("gennorm2/createNorm()");
    243         utrie2_set32(normTrie, c, (uint32_t)(p-norms), errorCode);
    244         return p;
    245     }
    246 }
    247 
    248 Norm *Normalizer2DataBuilder::checkNormForMapping(Norm *p, UChar32 c) {
    249     if(p!=NULL) {
    250         if(p->mappingType!=Norm::NONE) {
    251             if( overrideHandling==OVERRIDE_NONE ||
    252                 (overrideHandling==OVERRIDE_PREVIOUS && p->mappingPhase==phase)
    253             ) {
    254                 fprintf(stderr,
    255                         "error in gennorm2 phase %d: "
    256                         "not permitted to override mapping for U+%04lX from phase %d\n",
    257                         (int)phase, (long)c, (int)p->mappingPhase);
    258                 exit(U_INVALID_FORMAT_ERROR);
    259             }
    260             delete p->mapping;
    261             p->mapping=NULL;
    262         }
    263         p->mappingPhase=phase;
    264     }
    265     return p;
    266 }
    267 
    268 void Normalizer2DataBuilder::setOverrideHandling(OverrideHandling oh) {
    269     overrideHandling=oh;
    270     ++phase;
    271 }
    272 
    273 void Normalizer2DataBuilder::setCC(UChar32 c, uint8_t cc) {
    274     createNorm(c)->cc=cc;
    275 }
    276 
    277 uint8_t Normalizer2DataBuilder::getCC(UChar32 c) const {
    278     return getNormRef(c).cc;
    279 }
    280 
    281 static UBool isWellFormed(const UnicodeString &s) {
    282     UErrorCode errorCode=U_ZERO_ERROR;
    283     u_strToUTF8(NULL, 0, NULL, s.getBuffer(), s.length(), &errorCode);
    284     return U_SUCCESS(errorCode) || errorCode==U_BUFFER_OVERFLOW_ERROR;
    285 }
    286 
    287 void Normalizer2DataBuilder::setOneWayMapping(UChar32 c, const UnicodeString &m) {
    288     if(!isWellFormed(m)) {
    289         fprintf(stderr,
    290                 "error in gennorm2 phase %d: "
    291                 "illegal one-way mapping from U+%04lX to malformed string\n",
    292                 (int)phase, (long)c);
    293         exit(U_INVALID_FORMAT_ERROR);
    294     }
    295     Norm *p=checkNormForMapping(createNorm(c), c);
    296     p->mapping=new UnicodeString(m);
    297     p->mappingType=Norm::ONE_WAY;
    298     p->setMappingCP();
    299 }
    300 
    301 void Normalizer2DataBuilder::setRoundTripMapping(UChar32 c, const UnicodeString &m) {
    302     if(U_IS_SURROGATE(c)) {
    303         fprintf(stderr,
    304                 "error in gennorm2 phase %d: "
    305                 "illegal round-trip mapping from surrogate code point U+%04lX\n",
    306                 (int)phase, (long)c);
    307         exit(U_INVALID_FORMAT_ERROR);
    308     }
    309     if(!isWellFormed(m)) {
    310         fprintf(stderr,
    311                 "error in gennorm2 phase %d: "
    312                 "illegal round-trip mapping from U+%04lX to malformed string\n",
    313                 (int)phase, (long)c);
    314         exit(U_INVALID_FORMAT_ERROR);
    315     }
    316     int32_t numCP=u_countChar32(m.getBuffer(), m.length());
    317     if(numCP!=2) {
    318         fprintf(stderr,
    319                 "error in gennorm2 phase %d: "
    320                 "illegal round-trip mapping from U+%04lX to %d!=2 code points\n",
    321                 (int)phase, (long)c, (int)numCP);
    322         exit(U_INVALID_FORMAT_ERROR);
    323     }
    324     Norm *p=checkNormForMapping(createNorm(c), c);
    325     p->mapping=new UnicodeString(m);
    326     p->mappingType=Norm::ROUND_TRIP;
    327     p->mappingCP=U_SENTINEL;
    328 }
    329 
    330 void Normalizer2DataBuilder::removeMapping(UChar32 c) {
    331     Norm *p=checkNormForMapping(getNorm(c), c);
    332     if(p!=NULL) {
    333         p->mappingType=Norm::REMOVED;
    334     }
    335 }
    336 
    337 class CompositionBuilder : public Normalizer2DBEnumerator {
    338 public:
    339     CompositionBuilder(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b) {}
    340     virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
    341         builder.addComposition(start, end, value);
    342         return TRUE;
    343     }
    344 };
    345 
    346 void
    347 Normalizer2DataBuilder::addComposition(UChar32 start, UChar32 end, uint32_t value) {
    348     if(norms[value].mappingType==Norm::ROUND_TRIP) {
    349         if(start!=end) {
    350             fprintf(stderr,
    351                     "gennorm2 error: same round-trip mapping for "
    352                     "more than 1 code point U+%04lX..U+%04lX\n",
    353                     (long)start, (long)end);
    354             exit(U_INVALID_FORMAT_ERROR);
    355         }
    356         if(norms[value].cc!=0) {
    357             fprintf(stderr,
    358                     "gennorm2 error: "
    359                     "U+%04lX has a round-trip mapping and ccc!=0, "
    360                     "not possible in Unicode normalization\n",
    361                     (long)start);
    362             exit(U_INVALID_FORMAT_ERROR);
    363         }
    364         // setRoundTripMapping() ensured that there are exactly two code points.
    365         const UnicodeString &m=*norms[value].mapping;
    366         UChar32 lead=m.char32At(0);
    367         UChar32 trail=m.char32At(m.length()-1);
    368         if(getCC(lead)!=0) {
    369             fprintf(stderr,
    370                     "gennorm2 error: "
    371                     "U+%04lX's round-trip mapping's starter U+%04lX has ccc!=0, "
    372                     "not possible in Unicode normalization\n",
    373                     (long)start, (long)lead);
    374             exit(U_INVALID_FORMAT_ERROR);
    375         }
    376         // Flag for trailing character.
    377         createNorm(trail)->combinesBack=TRUE;
    378         // Insert (trail, composite) pair into compositions list for the lead character.
    379         IcuToolErrorCode errorCode("gennorm2/addComposition()");
    380         Norm *leadNorm=createNorm(lead);
    381         UVector32 *compositions=leadNorm->compositions;
    382         int32_t i;
    383         if(compositions==NULL) {
    384             compositions=leadNorm->compositions=new UVector32(errorCode);
    385             i=0;  // "insert" the first pair at index 0
    386         } else {
    387             // Insertion sort, and check for duplicate trail characters.
    388             int32_t length;
    389             const CompositionPair *pairs=leadNorm->getCompositionPairs(length);
    390             for(i=0; i<length; ++i) {
    391                 if(trail==pairs[i].trail) {
    392                     fprintf(stderr,
    393                             "gennorm2 error: same round-trip mapping for "
    394                             "more than 1 code point (e.g., U+%04lX) to U+%04lX + U+%04lX\n",
    395                             (long)start, (long)lead, (long)trail);
    396                     exit(U_INVALID_FORMAT_ERROR);
    397                 }
    398                 if(trail<pairs[i].trail) {
    399                     break;
    400                 }
    401             }
    402         }
    403         compositions->insertElementAt(trail, 2*i, errorCode);
    404         compositions->insertElementAt(start, 2*i+1, errorCode);
    405     }
    406 }
    407 
    408 UBool Normalizer2DataBuilder::combinesWithCCBetween(const Norm &norm,
    409                                                     uint8_t lowCC, uint8_t highCC) const {
    410     if((highCC-lowCC)>=2) {
    411         int32_t length;
    412         const CompositionPair *pairs=norm.getCompositionPairs(length);
    413         for(int32_t i=0; i<length; ++i) {
    414             uint8_t trailCC=getCC(pairs[i].trail);
    415             if(lowCC<trailCC && trailCC<highCC) {
    416                 return TRUE;
    417             }
    418         }
    419     }
    420     return FALSE;
    421 }
    422 
    423 UChar32 Normalizer2DataBuilder::combine(const Norm &norm, UChar32 trail) const {
    424     int32_t length;
    425     const CompositionPair *pairs=norm.getCompositionPairs(length);
    426     for(int32_t i=0; i<length; ++i) {
    427         if(trail==pairs[i].trail) {
    428             return pairs[i].composite;
    429         }
    430         if(trail<pairs[i].trail) {
    431             break;
    432         }
    433     }
    434     return U_SENTINEL;
    435 }
    436 
    437 class Decomposer : public Normalizer2DBEnumerator {
    438 public:
    439     Decomposer(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b), didDecompose(FALSE) {}
    440     virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
    441         didDecompose|=builder.decompose(start, end, value);
    442         return TRUE;
    443     }
    444     UBool didDecompose;
    445 };
    446 
    447 UBool
    448 Normalizer2DataBuilder::decompose(UChar32 start, UChar32 end, uint32_t value) {
    449     if(norms[value].hasMapping()) {
    450         Norm &norm=norms[value];
    451         const UnicodeString &m=*norm.mapping;
    452         UnicodeString *decomposed=NULL;
    453         const UChar *s=m.getBuffer();
    454         int32_t length=m.length();
    455         int32_t prev, i=0;
    456         UChar32 c;
    457         while(i<length) {
    458             prev=i;
    459             U16_NEXT(s, i, length, c);
    460             if(start<=c && c<=end) {
    461                 fprintf(stderr,
    462                         "gennorm2 error: U+%04lX maps to itself directly or indirectly\n",
    463                         (long)c);
    464                 exit(U_INVALID_FORMAT_ERROR);
    465             }
    466             const Norm &cNorm=getNormRef(c);
    467             if(cNorm.hasMapping()) {
    468                 if(norm.mappingType==Norm::ROUND_TRIP) {
    469                     if(prev==0) {
    470                         if(cNorm.mappingType!=Norm::ROUND_TRIP) {
    471                             fprintf(stderr,
    472                                     "gennorm2 error: "
    473                                     "U+%04lX's round-trip mapping's starter "
    474                                     "U+%04lX one-way-decomposes, "
    475                                     "not possible in Unicode normalization\n",
    476                                     (long)start, (long)c);
    477                             exit(U_INVALID_FORMAT_ERROR);
    478                         }
    479                         uint8_t myTrailCC=getCC(m.char32At(i));
    480                         UChar32 cTrailChar=cNorm.mapping->char32At(cNorm.mapping->length()-1);
    481                         uint8_t cTrailCC=getCC(cTrailChar);
    482                         if(cTrailCC>myTrailCC) {
    483                             fprintf(stderr,
    484                                     "gennorm2 error: "
    485                                     "U+%04lX's round-trip mapping's starter "
    486                                     "U+%04lX decomposes and the "
    487                                     "inner/earlier tccc=%hu > outer/following tccc=%hu, "
    488                                     "not possible in Unicode normalization\n",
    489                                     (long)start, (long)c,
    490                                     (short)cTrailCC, (short)myTrailCC);
    491                             exit(U_INVALID_FORMAT_ERROR);
    492                         }
    493                     } else {
    494                         fprintf(stderr,
    495                                 "gennorm2 error: "
    496                                 "U+%04lX's round-trip mapping's non-starter "
    497                                 "U+%04lX decomposes, "
    498                                 "not possible in Unicode normalization\n",
    499                                 (long)start, (long)c);
    500                         exit(U_INVALID_FORMAT_ERROR);
    501                     }
    502                 }
    503                 if(decomposed==NULL) {
    504                     decomposed=new UnicodeString(m, 0, prev);
    505                 }
    506                 decomposed->append(*cNorm.mapping);
    507             } else if(Hangul::isHangul(c)) {
    508                 UChar buffer[3];
    509                 int32_t hangulLength=Hangul::decompose(c, buffer);
    510                 if(norm.mappingType==Norm::ROUND_TRIP && prev!=0) {
    511                     fprintf(stderr,
    512                             "gennorm2 error: "
    513                             "U+%04lX's round-trip mapping's non-starter "
    514                             "U+%04lX decomposes, "
    515                             "not possible in Unicode normalization\n",
    516                             (long)start, (long)c);
    517                     exit(U_INVALID_FORMAT_ERROR);
    518                 }
    519                 if(decomposed==NULL) {
    520                     decomposed=new UnicodeString(m, 0, prev);
    521                 }
    522                 decomposed->append(buffer, hangulLength);
    523             } else if(decomposed!=NULL) {
    524                 decomposed->append(m, prev, i-prev);
    525             }
    526         }
    527         if(decomposed!=NULL) {
    528             if(norm.rawMapping==NULL) {
    529                 // Remember the original mapping when decomposing recursively.
    530                 norm.rawMapping=norm.mapping;
    531             } else {
    532                 delete norm.mapping;
    533             }
    534             norm.mapping=decomposed;
    535             // Not  norm.setMappingCP();  because the original mapping
    536             // is most likely to be encodable as a delta.
    537             return TRUE;
    538         }
    539     }
    540     return FALSE;
    541 }
    542 
    543 class BuilderReorderingBuffer {
    544 public:
    545     BuilderReorderingBuffer() : fLength(0), fLastStarterIndex(-1), fDidReorder(FALSE) {}
    546     void reset() {
    547         fLength=0;
    548         fLastStarterIndex=-1;
    549         fDidReorder=FALSE;
    550     }
    551     int32_t length() const { return fLength; }
    552     UBool isEmpty() const { return fLength==0; }
    553     int32_t lastStarterIndex() const { return fLastStarterIndex; }
    554     UChar32 charAt(int32_t i) const { return fArray[i]>>8; }
    555     uint8_t ccAt(int32_t i) const { return (uint8_t)fArray[i]; }
    556     UBool didReorder() const { return fDidReorder; }
    557     void append(UChar32 c, uint8_t cc) {
    558         if(cc==0 || fLength==0 || ccAt(fLength-1)<=cc) {
    559             if(cc==0) {
    560                 fLastStarterIndex=fLength;
    561             }
    562             fArray[fLength++]=(c<<8)|cc;
    563             return;
    564         }
    565         // Let this character bubble back to its canonical order.
    566         int32_t i=fLength-1;
    567         while(i>fLastStarterIndex && ccAt(i)>cc) {
    568             --i;
    569         }
    570         ++i;  // after the last starter or prevCC<=cc
    571         // Move this and the following characters forward one to make space.
    572         for(int32_t j=fLength; i<j; --j) {
    573             fArray[j]=fArray[j-1];
    574         }
    575         fArray[i]=(c<<8)|cc;
    576         ++fLength;
    577         fDidReorder=TRUE;
    578     }
    579     void toString(UnicodeString &dest) {
    580         dest.remove();
    581         for(int32_t i=0; i<fLength; ++i) {
    582             dest.append(charAt(i));
    583         }
    584     }
    585     void setComposite(UChar32 composite, int32_t combMarkIndex) {
    586         fArray[fLastStarterIndex]=composite<<8;
    587         // Remove the combining mark that contributed to the composite.
    588         --fLength;
    589         while(combMarkIndex<fLength) {
    590             fArray[combMarkIndex]=fArray[combMarkIndex+1];
    591             ++combMarkIndex;
    592         }
    593     }
    594 private:
    595     int32_t fArray[Normalizer2Impl::MAPPING_LENGTH_MASK];
    596     int32_t fLength;
    597     int32_t fLastStarterIndex;
    598     UBool fDidReorder;
    599 };
    600 
    601 void
    602 Normalizer2DataBuilder::reorder(Norm *p, BuilderReorderingBuffer &buffer) {
    603     UnicodeString &m=*p->mapping;
    604     int32_t length=m.length();
    605     if(length>Normalizer2Impl::MAPPING_LENGTH_MASK) {
    606         return;  // writeMapping() will complain about it and print the code point.
    607     }
    608     const UChar *s=m.getBuffer();
    609     int32_t i=0;
    610     UChar32 c;
    611     while(i<length) {
    612         U16_NEXT(s, i, length, c);
    613         buffer.append(c, getCC(c));
    614     }
    615     if(buffer.didReorder()) {
    616         buffer.toString(m);
    617     }
    618 }
    619 
    620 /*
    621  * Computes the flag for the last code branch in Normalizer2Impl::hasCompBoundaryAfter().
    622  * A starter character with a mapping does not have a composition boundary after it
    623  * if the character itself combines-forward (which is tested by the caller of this function),
    624  * or it is deleted (mapped to the empty string),
    625  * or its mapping contains no starter,
    626  * or the last starter combines-forward.
    627  */
    628 UBool Normalizer2DataBuilder::hasNoCompBoundaryAfter(BuilderReorderingBuffer &buffer) {
    629     if(buffer.isEmpty()) {
    630         return TRUE;  // maps-to-empty-string is no boundary of any kind
    631     }
    632     int32_t lastStarterIndex=buffer.lastStarterIndex();
    633     if(lastStarterIndex<0) {
    634         return TRUE;  // no starter
    635     }
    636     UChar32 starter=buffer.charAt(lastStarterIndex);
    637     if( Hangul::isJamoL(starter) ||
    638         (Hangul::isJamoV(starter) &&
    639          0<lastStarterIndex && Hangul::isJamoL(buffer.charAt(lastStarterIndex-1)))
    640     ) {
    641         // A Jamo leading consonant or an LV pair combines-forward if it is at the end,
    642         // otherwise it is blocked.
    643         return lastStarterIndex==buffer.length()-1;
    644     }
    645     // Note: There can be no Hangul syllable in the fully decomposed mapping.
    646     const Norm *starterNorm=&getNormRef(starter);
    647     if(starterNorm->compositions==NULL) {
    648         return FALSE;  // the last starter does not combine forward
    649     }
    650     // Compose as far as possible, and see if further compositions are possible.
    651     uint8_t prevCC=0;
    652     for(int32_t combMarkIndex=lastStarterIndex+1; combMarkIndex<buffer.length();) {
    653         uint8_t cc=buffer.ccAt(combMarkIndex);  // !=0 because after last starter
    654         if(combinesWithCCBetween(*starterNorm, prevCC, cc)) {
    655             return TRUE;
    656         }
    657         if( prevCC<cc &&
    658             (starter=combine(*starterNorm, buffer.charAt(combMarkIndex)))>=0
    659         ) {
    660             buffer.setComposite(starter, combMarkIndex);
    661             starterNorm=&getNormRef(starter);
    662             if(starterNorm->compositions==NULL) {
    663                 return FALSE;  // the composite does not combine further
    664             }
    665         } else {
    666             prevCC=cc;
    667             ++combMarkIndex;
    668         }
    669     }
    670     // TRUE if the final, forward-combining starter is at the end.
    671     return prevCC==0;
    672 }
    673 
    674 // Requires p->hasMapping().
    675 // Returns the offset of the "first unit" from the beginning of the extraData for c.
    676 // That is the same as the length of the optional data for the raw mapping and the ccc/lccc word.
    677 int32_t Normalizer2DataBuilder::writeMapping(UChar32 c, const Norm *p, UnicodeString &dataString) {
    678     UnicodeString &m=*p->mapping;
    679     int32_t length=m.length();
    680     if(length>Normalizer2Impl::MAPPING_LENGTH_MASK) {
    681         fprintf(stderr,
    682                 "gennorm2 error: "
    683                 "mapping for U+%04lX longer than maximum of %d\n",
    684                 (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK);
    685         exit(U_INVALID_FORMAT_ERROR);
    686     }
    687     int32_t leadCC, trailCC;
    688     if(length==0) {
    689         leadCC=trailCC=0;
    690     } else {
    691         leadCC=getCC(m.char32At(0));
    692         trailCC=getCC(m.char32At(length-1));
    693     }
    694     if(c<Normalizer2Impl::MIN_CCC_LCCC_CP && (p->cc!=0 || leadCC!=0)) {
    695         fprintf(stderr,
    696                 "gennorm2 error: "
    697                 "U+%04lX below U+0300 has ccc!=0 or lccc!=0, not supported by ICU\n",
    698                 (long)c);
    699         exit(U_INVALID_FORMAT_ERROR);
    700     }
    701     // Write small-FCD data.
    702     if((leadCC|trailCC)!=0) {
    703         UChar32 lead= c<=0xffff ? c : U16_LEAD(c);
    704         smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7);
    705     }
    706     // Write the mapping & raw mapping extraData.
    707     int32_t firstUnit=length|(trailCC<<8);
    708     int32_t preMappingLength=0;
    709     if(p->rawMapping!=NULL) {
    710         UnicodeString &rm=*p->rawMapping;
    711         int32_t rmLength=rm.length();
    712         if(rmLength>Normalizer2Impl::MAPPING_LENGTH_MASK) {
    713             fprintf(stderr,
    714                     "gennorm2 error: "
    715                     "raw mapping for U+%04lX longer than maximum of %d\n",
    716                     (long)c, Normalizer2Impl::MAPPING_LENGTH_MASK);
    717             exit(U_INVALID_FORMAT_ERROR);
    718         }
    719         UChar rm0=rm.charAt(0);
    720         if( rmLength==length-1 &&
    721             // 99: overlong substring lengths get pinned to remainder lengths anyway
    722             0==rm.compare(1, 99, m, 2, 99) &&
    723             rm0>Normalizer2Impl::MAPPING_LENGTH_MASK
    724         ) {
    725             // Compression:
    726             // rawMapping=rm0+mapping.substring(2) -> store only rm0
    727             //
    728             // The raw mapping is the same as the final mapping after replacing
    729             // the final mapping's first two code units with the raw mapping's first one.
    730             // In this case, we store only that first unit, rm0.
    731             // This helps with a few hundred mappings.
    732             dataString.append(rm0);
    733             preMappingLength=1;
    734         } else {
    735             // Store the raw mapping with its length.
    736             dataString.append(rm);
    737             dataString.append((UChar)rmLength);
    738             preMappingLength=rmLength+1;
    739         }
    740         firstUnit|=Normalizer2Impl::MAPPING_HAS_RAW_MAPPING;
    741     }
    742     int32_t cccLccc=p->cc|(leadCC<<8);
    743     if(cccLccc!=0) {
    744         dataString.append((UChar)cccLccc);
    745         ++preMappingLength;
    746         firstUnit|=Normalizer2Impl::MAPPING_HAS_CCC_LCCC_WORD;
    747     }
    748     if(p->hasNoCompBoundaryAfter) {
    749         firstUnit|=Normalizer2Impl::MAPPING_NO_COMP_BOUNDARY_AFTER;
    750     }
    751     dataString.append((UChar)firstUnit);
    752     dataString.append(m);
    753     return preMappingLength;
    754 }
    755 
    756 // Requires p->compositions!=NULL.
    757 void Normalizer2DataBuilder::writeCompositions(UChar32 c, const Norm *p, UnicodeString &dataString) {
    758     if(p->cc!=0) {
    759         fprintf(stderr,
    760                 "gennorm2 error: "
    761                 "U+%04lX combines-forward and has ccc!=0, not possible in Unicode normalization\n",
    762                 (long)c);
    763         exit(U_INVALID_FORMAT_ERROR);
    764     }
    765     int32_t length;
    766     const CompositionPair *pairs=p->getCompositionPairs(length);
    767     for(int32_t i=0; i<length; ++i) {
    768         const CompositionPair &pair=pairs[i];
    769         // 22 bits for the composite character and whether it combines forward.
    770         UChar32 compositeAndFwd=pair.composite<<1;
    771         if(getNormRef(pair.composite).compositions!=NULL) {
    772             compositeAndFwd|=1;  // The composite character also combines-forward.
    773         }
    774         // Encode most pairs in two units and some in three.
    775         int32_t firstUnit, secondUnit, thirdUnit;
    776         if(pair.trail<Normalizer2Impl::COMP_1_TRAIL_LIMIT) {
    777             if(compositeAndFwd<=0xffff) {
    778                 firstUnit=pair.trail<<1;
    779                 secondUnit=compositeAndFwd;
    780                 thirdUnit=-1;
    781             } else {
    782                 firstUnit=(pair.trail<<1)|Normalizer2Impl::COMP_1_TRIPLE;
    783                 secondUnit=compositeAndFwd>>16;
    784                 thirdUnit=compositeAndFwd;
    785             }
    786         } else {
    787             firstUnit=(Normalizer2Impl::COMP_1_TRAIL_LIMIT+
    788                        (pair.trail>>Normalizer2Impl::COMP_1_TRAIL_SHIFT))|
    789                       Normalizer2Impl::COMP_1_TRIPLE;
    790             secondUnit=(pair.trail<<Normalizer2Impl::COMP_2_TRAIL_SHIFT)|
    791                        (compositeAndFwd>>16);
    792             thirdUnit=compositeAndFwd;
    793         }
    794         // Set the high bit of the first unit if this is the last composition pair.
    795         if(i==(length-1)) {
    796             firstUnit|=Normalizer2Impl::COMP_1_LAST_TUPLE;
    797         }
    798         dataString.append((UChar)firstUnit).append((UChar)secondUnit);
    799         if(thirdUnit>=0) {
    800             dataString.append((UChar)thirdUnit);
    801         }
    802     }
    803 }
    804 
    805 class ExtraDataWriter : public Normalizer2DBEnumerator {
    806 public:
    807     ExtraDataWriter(Normalizer2DataBuilder &b) :
    808         Normalizer2DBEnumerator(b),
    809         yesYesCompositions(1000, (UChar32)0xffff, 2),  // 0=inert, 1=Jamo L, 2=start of compositions
    810         yesNoMappingsAndCompositions(1000, (UChar32)0, 1) {}  // 0=Hangul, 1=start of normal data
    811     virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
    812         if(value!=0) {
    813             if(start!=end) {
    814                 fprintf(stderr,
    815                         "gennorm2 error: unexpected shared data for "
    816                         "multiple code points U+%04lX..U+%04lX\n",
    817                         (long)start, (long)end);
    818                 exit(U_INTERNAL_PROGRAM_ERROR);
    819             }
    820             builder.writeExtraData(start, value, *this);
    821         }
    822         return TRUE;
    823     }
    824     UnicodeString maybeYesCompositions;
    825     UnicodeString yesYesCompositions;
    826     UnicodeString yesNoMappingsAndCompositions;
    827     UnicodeString yesNoMappingsOnly;
    828     UnicodeString noNoMappings;
    829     Hashtable previousNoNoMappings;  // If constructed in runtime code, pass in UErrorCode.
    830 };
    831 
    832 void Normalizer2DataBuilder::writeExtraData(UChar32 c, uint32_t value, ExtraDataWriter &writer) {
    833     Norm *p=norms+value;
    834     if(!p->hasMapping()) {
    835         // Write small-FCD data.
    836         // There is similar code in writeMapping() for characters that do have a mapping.
    837         if(c<Normalizer2Impl::MIN_CCC_LCCC_CP && p->cc!=0) {
    838             fprintf(stderr,
    839                     "gennorm2 error: "
    840                     "U+%04lX below U+0300 has ccc!=0, not supported by ICU\n",
    841                     (long)c);
    842             exit(U_INVALID_FORMAT_ERROR);
    843         }
    844         if(p->cc!=0) {
    845             UChar32 lead= c<=0xffff ? c : U16_LEAD(c);
    846             smallFCD[lead>>8]|=(uint8_t)1<<((lead>>5)&7);
    847         }
    848     }
    849     if(p->combinesBack) {
    850         if(p->hasMapping()) {
    851             fprintf(stderr,
    852                     "gennorm2 error: "
    853                     "U+%04lX combines-back and decomposes, not possible in Unicode normalization\n",
    854                     (long)c);
    855             exit(U_INVALID_FORMAT_ERROR);
    856         }
    857         if(p->compositions!=NULL) {
    858             p->offset=
    859                 (writer.maybeYesCompositions.length()<<Norm::OFFSET_SHIFT)|
    860                 Norm::OFFSET_MAYBE_YES;
    861             writeCompositions(c, p, writer.maybeYesCompositions);
    862         }
    863     } else if(!p->hasMapping()) {
    864         if(p->compositions!=NULL) {
    865             p->offset=
    866                 (writer.yesYesCompositions.length()<<Norm::OFFSET_SHIFT)|
    867                 Norm::OFFSET_YES_YES;
    868             writeCompositions(c, p, writer.yesYesCompositions);
    869         }
    870     } else if(p->mappingType==Norm::ROUND_TRIP) {
    871         if(p->compositions!=NULL) {
    872             int32_t offset=writer.yesNoMappingsAndCompositions.length()+
    873                            writeMapping(c, p, writer.yesNoMappingsAndCompositions);
    874             p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_YES_NO_MAPPING_AND_COMPOSITION;
    875             writeCompositions(c, p, writer.yesNoMappingsAndCompositions);
    876         } else {
    877             int32_t offset=writer.yesNoMappingsOnly.length()+
    878                            writeMapping(c, p, writer.yesNoMappingsOnly);
    879             p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_YES_NO_MAPPING_ONLY;
    880         }
    881     } else /* one-way */ {
    882         if(p->compositions!=NULL) {
    883             fprintf(stderr,
    884                     "gennorm2 error: "
    885                     "U+%04lX combines-forward and has a one-way mapping, "
    886                     "not possible in Unicode normalization\n",
    887                     (long)c);
    888             exit(U_INVALID_FORMAT_ERROR);
    889         }
    890         if(p->cc==0 && optimization!=OPTIMIZE_FAST) {
    891             // Try a compact, algorithmic encoding.
    892             // Only for ccc=0, because we can't store additional information
    893             // and we do not recursively follow an algorithmic encoding for access to the ccc.
    894             //
    895             // Also, if hasNoCompBoundaryAfter is set, we can only use the algorithmic encoding
    896             // if the mappingCP decomposes further, to ensure that there is a place to store it.
    897             // We want to see that the final mapping does not have exactly 1 code point,
    898             // or else we would have to recursively ensure that the final mapping is stored
    899             // in normal extraData.
    900             if(p->mappingCP>=0 && (!p->hasNoCompBoundaryAfter || 1!=p->mapping->countChar32())) {
    901                 int32_t delta=p->mappingCP-c;
    902                 if(-Normalizer2Impl::MAX_DELTA<=delta && delta<=Normalizer2Impl::MAX_DELTA) {
    903                     p->offset=(delta<<Norm::OFFSET_SHIFT)|Norm::OFFSET_DELTA;
    904                 }
    905             }
    906         }
    907         if(p->offset==0) {
    908             int32_t oldNoNoLength=writer.noNoMappings.length();
    909             int32_t offset=oldNoNoLength+writeMapping(c, p, writer.noNoMappings);
    910             UnicodeString newMapping=writer.noNoMappings.tempSubString(oldNoNoLength);
    911             int32_t previousOffset=writer.previousNoNoMappings.geti(newMapping);
    912             if(previousOffset!=0) {
    913                 // Duplicate, remove the new units and point to the old ones.
    914                 writer.noNoMappings.truncate(oldNoNoLength);
    915                 p->offset=((previousOffset-1)<<Norm::OFFSET_SHIFT)|Norm::OFFSET_NO_NO;
    916             } else {
    917                 // Enter this new mapping into the hashtable, avoiding value 0 which is "not found".
    918                 IcuToolErrorCode errorCode("gennorm2/writeExtraData()/Hashtable.puti()");
    919                 writer.previousNoNoMappings.puti(newMapping, offset+1, errorCode);
    920                 p->offset=(offset<<Norm::OFFSET_SHIFT)|Norm::OFFSET_NO_NO;
    921             }
    922         }
    923     }
    924 }
    925 
    926 class Norm16Writer : public Normalizer2DBEnumerator {
    927 public:
    928     Norm16Writer(Normalizer2DataBuilder &b) : Normalizer2DBEnumerator(b) {}
    929     virtual UBool rangeHandler(UChar32 start, UChar32 end, uint32_t value) {
    930         builder.writeNorm16(start, end, value);
    931         return TRUE;
    932     }
    933 };
    934 
    935 void Normalizer2DataBuilder::writeNorm16(UChar32 start, UChar32 end, uint32_t value) {
    936     if(value!=0) {
    937         const Norm *p=norms+value;
    938         int32_t offset=p->offset>>Norm::OFFSET_SHIFT;
    939         int32_t norm16=0;
    940         UBool isDecompNo=FALSE;
    941         UBool isCompNoMaybe=FALSE;
    942         switch(p->offset&Norm::OFFSET_MASK) {
    943         case Norm::OFFSET_NONE:
    944             // No mapping, no compositions list.
    945             if(p->combinesBack) {
    946                 norm16=Normalizer2Impl::MIN_NORMAL_MAYBE_YES+p->cc;
    947                 isDecompNo=(UBool)(p->cc!=0);
    948                 isCompNoMaybe=TRUE;
    949             } else if(p->cc!=0) {
    950                 norm16=Normalizer2Impl::MIN_YES_YES_WITH_CC-1+p->cc;
    951                 isDecompNo=isCompNoMaybe=TRUE;
    952             }
    953             break;
    954         case Norm::OFFSET_MAYBE_YES:
    955             norm16=indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]+offset;
    956             isCompNoMaybe=TRUE;
    957             break;
    958         case Norm::OFFSET_YES_YES:
    959             norm16=offset;
    960             break;
    961         case Norm::OFFSET_YES_NO_MAPPING_AND_COMPOSITION:
    962             norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO]+offset;
    963             isDecompNo=TRUE;
    964             break;
    965         case Norm::OFFSET_YES_NO_MAPPING_ONLY:
    966             norm16=indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+offset;
    967             isDecompNo=TRUE;
    968             break;
    969         case Norm::OFFSET_NO_NO:
    970             norm16=indexes[Normalizer2Impl::IX_MIN_NO_NO]+offset;
    971             isDecompNo=isCompNoMaybe=TRUE;
    972             break;
    973         case Norm::OFFSET_DELTA:
    974             norm16=getCenterNoNoDelta()+offset;
    975             isDecompNo=isCompNoMaybe=TRUE;
    976             break;
    977         default:  // Should not occur.
    978             exit(U_INTERNAL_PROGRAM_ERROR);
    979         }
    980         IcuToolErrorCode errorCode("gennorm2/writeNorm16()");
    981         utrie2_setRange32(norm16Trie, start, end, (uint32_t)norm16, TRUE, errorCode);
    982         if(isDecompNo && start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
    983             indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=start;
    984         }
    985         if(isCompNoMaybe && start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) {
    986             indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=start;
    987         }
    988     }
    989 }
    990 
    991 void Normalizer2DataBuilder::setHangulData() {
    992     HangulIterator hi;
    993     const HangulIterator::Range *range;
    994     // Check that none of the Hangul/Jamo code points have data.
    995     while((range=hi.nextRange())!=NULL) {
    996         for(UChar32 c=range->start; c<range->limit; ++c) {
    997             if(utrie2_get32(norm16Trie, c)!=0) {
    998                 fprintf(stderr,
    999                         "gennorm2 error: "
   1000                         "illegal mapping/composition/ccc data for Hangul or Jamo U+%04lX\n",
   1001                         (long)c);
   1002                 exit(U_INVALID_FORMAT_ERROR);
   1003             }
   1004         }
   1005     }
   1006     // Set data for algorithmic runtime handling.
   1007     IcuToolErrorCode errorCode("gennorm2/setHangulData()");
   1008     hi.reset();
   1009     while((range=hi.nextRange())!=NULL) {
   1010         uint16_t norm16=range->norm16;
   1011         if(norm16==0) {
   1012             norm16=(uint16_t)indexes[Normalizer2Impl::IX_MIN_YES_NO];  // Hangul LV/LVT encoded as minYesNo
   1013             if(range->start<indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]) {
   1014                 indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=range->start;
   1015             }
   1016         } else {
   1017             if(range->start<indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]) {  // Jamo V/T are maybeYes
   1018                 indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=range->start;
   1019             }
   1020         }
   1021         utrie2_setRange32(norm16Trie, range->start, range->limit-1, norm16, TRUE, errorCode);
   1022         errorCode.assertSuccess();
   1023     }
   1024 }
   1025 
   1026 U_CDECL_BEGIN
   1027 
   1028 static UBool U_CALLCONV
   1029 enumRangeMaxValue(const void *context, UChar32 /*start*/, UChar32 /*end*/, uint32_t value) {
   1030     uint32_t *pMaxValue=(uint32_t *)context;
   1031     if(value>*pMaxValue) {
   1032         *pMaxValue=value;
   1033     }
   1034     return TRUE;
   1035 }
   1036 
   1037 U_CDECL_END
   1038 
   1039 void Normalizer2DataBuilder::processData() {
   1040     IcuToolErrorCode errorCode("gennorm2/processData()");
   1041     norm16Trie=utrie2_open(0, 0, errorCode);
   1042     errorCode.assertSuccess();
   1043 
   1044     utrie2_enum(normTrie, NULL, enumRangeHandler, CompositionBuilder(*this).ptr());
   1045 
   1046     Decomposer decomposer(*this);
   1047     do {
   1048         decomposer.didDecompose=FALSE;
   1049         utrie2_enum(normTrie, NULL, enumRangeHandler, &decomposer);
   1050     } while(decomposer.didDecompose);
   1051 
   1052     BuilderReorderingBuffer buffer;
   1053     int32_t normsLength=utm_countItems(normMem);
   1054     for(int32_t i=1; i<normsLength; ++i) {
   1055         // Set the hasNoCompBoundaryAfter flag for use by the last code branch
   1056         // in Normalizer2Impl::hasCompBoundaryAfter().
   1057         // For details see the comments on hasNoCompBoundaryAfter(buffer).
   1058         const Norm &norm=norms[i];
   1059         if(norm.hasMapping()) {
   1060             if(norm.compositions!=NULL) {
   1061                 norms[i].hasNoCompBoundaryAfter=TRUE;
   1062             } else {
   1063                 buffer.reset();
   1064                 reorder(norms+i, buffer);
   1065                 norms[i].hasNoCompBoundaryAfter=hasNoCompBoundaryAfter(buffer);
   1066             }
   1067         }
   1068     }
   1069 
   1070     indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=0x110000;
   1071     indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=0x110000;
   1072 
   1073     ExtraDataWriter extraDataWriter(*this);
   1074     utrie2_enum(normTrie, NULL, enumRangeHandler, &extraDataWriter);
   1075 
   1076     extraData=extraDataWriter.maybeYesCompositions;
   1077     extraData.append(extraDataWriter.yesYesCompositions).
   1078               append(extraDataWriter.yesNoMappingsAndCompositions).
   1079               append(extraDataWriter.yesNoMappingsOnly).
   1080               append(extraDataWriter.noNoMappings);
   1081     // Pad to even length for 4-byte alignment of following data.
   1082     if(extraData.length()&1) {
   1083         extraData.append((UChar)0);
   1084     }
   1085 
   1086     indexes[Normalizer2Impl::IX_MIN_YES_NO]=
   1087         extraDataWriter.yesYesCompositions.length();
   1088     indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]=
   1089         indexes[Normalizer2Impl::IX_MIN_YES_NO]+
   1090         extraDataWriter.yesNoMappingsAndCompositions.length();
   1091     indexes[Normalizer2Impl::IX_MIN_NO_NO]=
   1092         indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]+
   1093         extraDataWriter.yesNoMappingsOnly.length();
   1094     indexes[Normalizer2Impl::IX_LIMIT_NO_NO]=
   1095         indexes[Normalizer2Impl::IX_MIN_NO_NO]+
   1096         extraDataWriter.noNoMappings.length();
   1097     indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]=
   1098         Normalizer2Impl::MIN_NORMAL_MAYBE_YES-
   1099         extraDataWriter.maybeYesCompositions.length();
   1100 
   1101     int32_t minNoNoDelta=getCenterNoNoDelta()-Normalizer2Impl::MAX_DELTA;
   1102     if(indexes[Normalizer2Impl::IX_LIMIT_NO_NO]>minNoNoDelta) {
   1103         fprintf(stderr,
   1104                 "gennorm2 error: "
   1105                 "data structure overflow, too much mapping composition data\n");
   1106         exit(U_BUFFER_OVERFLOW_ERROR);
   1107     }
   1108 
   1109     utrie2_enum(normTrie, NULL, enumRangeHandler, Norm16Writer(*this).ptr());
   1110 
   1111     setHangulData();
   1112 
   1113     // Look for the "worst" norm16 value of any supplementary code point
   1114     // corresponding to a lead surrogate, and set it as that surrogate's value.
   1115     // Enables quick check inner loops to look at only code units.
   1116     //
   1117     // We could be more sophisticated:
   1118     // We could collect a bit set for whether there are values in the different
   1119     // norm16 ranges (yesNo, maybeYes, yesYesWithCC etc.)
   1120     // and select the best value that only breaks the composition and/or decomposition
   1121     // inner loops if necessary.
   1122     // However, that seems like overkill for an optimization for supplementary characters.
   1123     for(UChar lead=0xd800; lead<0xdc00; ++lead) {
   1124         uint32_t maxValue=utrie2_get32(norm16Trie, lead);
   1125         utrie2_enumForLeadSurrogate(norm16Trie, lead, NULL, enumRangeMaxValue, &maxValue);
   1126         if( maxValue>=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO] &&
   1127             maxValue>(uint32_t)indexes[Normalizer2Impl::IX_MIN_NO_NO]
   1128         ) {
   1129             // Set noNo ("worst" value) if it got into "less-bad" maybeYes or ccc!=0.
   1130             // Otherwise it might end up at something like JAMO_VT which stays in
   1131             // the inner decomposition quick check loop.
   1132             maxValue=(uint32_t)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]-1;
   1133         }
   1134         utrie2_set32ForLeadSurrogateCodeUnit(norm16Trie, lead, maxValue, errorCode);
   1135     }
   1136 
   1137     // Adjust supplementary minimum code points to break quick check loops at their lead surrogates.
   1138     // For an empty data file, minCP=0x110000 turns into 0xdc00 (first trail surrogate)
   1139     // which is harmless.
   1140     // As a result, the minimum code points are always BMP code points.
   1141     int32_t minCP=indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP];
   1142     if(minCP>=0x10000) {
   1143         indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]=U16_LEAD(minCP);
   1144     }
   1145     minCP=indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP];
   1146     if(minCP>=0x10000) {
   1147         indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]=U16_LEAD(minCP);
   1148     }
   1149 
   1150     utrie2_freeze(norm16Trie, UTRIE2_16_VALUE_BITS, errorCode);
   1151     norm16TrieLength=utrie2_serialize(norm16Trie, NULL, 0, errorCode);
   1152     if(errorCode.get()!=U_BUFFER_OVERFLOW_ERROR) {
   1153         fprintf(stderr, "gennorm2 error: unable to freeze/serialize the normalization trie - %s\n",
   1154                 errorCode.errorName());
   1155         exit(errorCode.reset());
   1156     }
   1157     errorCode.reset();
   1158 
   1159     int32_t offset=(int32_t)sizeof(indexes);
   1160     indexes[Normalizer2Impl::IX_NORM_TRIE_OFFSET]=offset;
   1161     offset+=norm16TrieLength;
   1162     indexes[Normalizer2Impl::IX_EXTRA_DATA_OFFSET]=offset;
   1163     offset+=extraData.length()*2;
   1164     indexes[Normalizer2Impl::IX_SMALL_FCD_OFFSET]=offset;
   1165     offset+=sizeof(smallFCD);
   1166     int32_t totalSize=offset;
   1167     for(int32_t i=Normalizer2Impl::IX_RESERVED3_OFFSET; i<=Normalizer2Impl::IX_TOTAL_SIZE; ++i) {
   1168         indexes[i]=totalSize;
   1169     }
   1170 
   1171     if(beVerbose) {
   1172         printf("size of normalization trie:         %5ld bytes\n", (long)norm16TrieLength);
   1173         printf("size of 16-bit extra data:          %5ld uint16_t\n", (long)extraData.length());
   1174         printf("size of small-FCD data:             %5ld bytes\n", (long)sizeof(smallFCD));
   1175         printf("size of binary data file contents:  %5ld bytes\n", (long)totalSize);
   1176         printf("minDecompNoCodePoint:              U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_DECOMP_NO_CP]);
   1177         printf("minCompNoMaybeCodePoint:           U+%04lX\n", (long)indexes[Normalizer2Impl::IX_MIN_COMP_NO_MAYBE_CP]);
   1178         printf("minYesNo:                          0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO]);
   1179         printf("minYesNoMappingsOnly:              0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_YES_NO_MAPPINGS_ONLY]);
   1180         printf("minNoNo:                           0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_NO_NO]);
   1181         printf("limitNoNo:                         0x%04x\n", (int)indexes[Normalizer2Impl::IX_LIMIT_NO_NO]);
   1182         printf("minMaybeYes:                       0x%04x\n", (int)indexes[Normalizer2Impl::IX_MIN_MAYBE_YES]);
   1183     }
   1184 
   1185     UVersionInfo nullVersion={ 0, 0, 0, 0 };
   1186     if(0==memcmp(nullVersion, unicodeVersion, 4)) {
   1187         u_versionFromString(unicodeVersion, U_UNICODE_VERSION);
   1188     }
   1189     memcpy(dataInfo.dataVersion, unicodeVersion, 4);
   1190 }
   1191 
   1192 void Normalizer2DataBuilder::writeBinaryFile(const char *filename) {
   1193     processData();
   1194 
   1195     IcuToolErrorCode errorCode("gennorm2/writeBinaryFile()");
   1196     LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]);
   1197     utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode);
   1198     errorCode.assertSuccess();
   1199 
   1200     UNewDataMemory *pData=
   1201         udata_create(NULL, NULL, filename, &dataInfo,
   1202                      haveCopyright ? U_COPYRIGHT_STRING : NULL, errorCode);
   1203     if(errorCode.isFailure()) {
   1204         fprintf(stderr, "gennorm2 error: unable to create the output file %s - %s\n",
   1205                 filename, errorCode.errorName());
   1206         exit(errorCode.reset());
   1207     }
   1208     udata_writeBlock(pData, indexes, sizeof(indexes));
   1209     udata_writeBlock(pData, norm16TrieBytes.getAlias(), norm16TrieLength);
   1210     udata_writeUString(pData, extraData.getBuffer(), extraData.length());
   1211     udata_writeBlock(pData, smallFCD, sizeof(smallFCD));
   1212     int32_t writtenSize=udata_finish(pData, errorCode);
   1213     if(errorCode.isFailure()) {
   1214         fprintf(stderr, "gennorm2: error %s writing the output file\n", errorCode.errorName());
   1215         exit(errorCode.reset());
   1216     }
   1217     int32_t totalSize=indexes[Normalizer2Impl::IX_TOTAL_SIZE];
   1218     if(writtenSize!=totalSize) {
   1219         fprintf(stderr, "gennorm2 error: written size %ld != calculated size %ld\n",
   1220             (long)writtenSize, (long)totalSize);
   1221         exit(U_INTERNAL_PROGRAM_ERROR);
   1222     }
   1223 }
   1224 
   1225 void
   1226 Normalizer2DataBuilder::writeCSourceFile(const char *filename) {
   1227     processData();
   1228 
   1229     IcuToolErrorCode errorCode("gennorm2/writeCSourceFile()");
   1230     const char *basename=findBasename(filename);
   1231     CharString path(filename, (int32_t)(basename-filename), errorCode);
   1232     CharString dataName(basename, errorCode);
   1233     const char *extension=strrchr(basename, '.');
   1234     if(extension!=NULL) {
   1235         dataName.truncate((int32_t)(extension-basename));
   1236     }
   1237     errorCode.assertSuccess();
   1238 
   1239     LocalArray<uint8_t> norm16TrieBytes(new uint8_t[norm16TrieLength]);
   1240     utrie2_serialize(norm16Trie, norm16TrieBytes.getAlias(), norm16TrieLength, errorCode);
   1241     errorCode.assertSuccess();
   1242 
   1243     FILE *f=usrc_create(path.data(), basename, "icu/source/tools/gennorm2/n2builder.cpp");
   1244     if(f==NULL) {
   1245         fprintf(stderr, "gennorm2/writeCSourceFile() error: unable to create the output file %s\n",
   1246                 filename);
   1247         exit(U_FILE_ACCESS_ERROR);
   1248         return;
   1249     }
   1250     char line[100];
   1251     sprintf(line, "static const UVersionInfo %s_formatVersion={", dataName.data());
   1252     usrc_writeArray(f, line, dataInfo.formatVersion, 8, 4, "};\n");
   1253     sprintf(line, "static const UVersionInfo %s_dataVersion={", dataName.data());
   1254     usrc_writeArray(f, line, dataInfo.dataVersion, 8, 4, "};\n\n");
   1255     sprintf(line, "static const int32_t %s_indexes[Normalizer2Impl::IX_COUNT]={\n",
   1256             dataName.data());
   1257     usrc_writeArray(f,
   1258         line,
   1259         indexes, 32, Normalizer2Impl::IX_COUNT,
   1260         "\n};\n\n");
   1261     sprintf(line, "static const uint16_t %s_trieIndex[%%ld]={\n", dataName.data());
   1262     usrc_writeUTrie2Arrays(f,
   1263         line, NULL,
   1264         norm16Trie,
   1265         "\n};\n\n");
   1266     sprintf(line, "static const uint16_t %s_extraData[%%ld]={\n", dataName.data());
   1267     usrc_writeArray(f,
   1268         line,
   1269         extraData.getBuffer(), 16, extraData.length(),
   1270         "\n};\n\n");
   1271     sprintf(line, "static const uint8_t %s_smallFCD[%%ld]={\n", dataName.data());
   1272     usrc_writeArray(f,
   1273         line,
   1274         smallFCD, 8, sizeof(smallFCD),
   1275         "\n};\n\n");
   1276     /*fputs(  // TODO
   1277         "static const UCaseProps %s_singleton={\n"
   1278         "  NULL,\n"
   1279         "  %s_indexes,\n"
   1280         "  %s_extraData,\n"
   1281         "  %s_smallFCD,\n",
   1282         f);*/
   1283     sprintf(line, "static const UTrie2 %s_trie={\n", dataName.data());
   1284     char line2[100];
   1285     sprintf(line2, "%s_trieIndex", dataName.data());
   1286     usrc_writeUTrie2Struct(f,
   1287         line,
   1288         norm16Trie, line2, NULL,
   1289         "};\n");
   1290     fclose(f);
   1291 }
   1292 
   1293 U_NAMESPACE_END
   1294 
   1295 #endif /* #if !UCONFIG_NO_NORMALIZATION */
   1296 
   1297 /*
   1298  * Hey, Emacs, please set the following:
   1299  *
   1300  * Local Variables:
   1301  * indent-tabs-mode: nil
   1302  * End:
   1303  */
   1304