Home | History | Annotate | Download | only in makeconv
      1 /*
      2 *******************************************************************************
      3 *
      4 *   Copyright (C) 2000-2010, International Business Machines
      5 *   Corporation and others.  All Rights Reserved.
      6 *
      7 *******************************************************************************
      8 *   file name:  genmbcs.c
      9 *   encoding:   US-ASCII
     10 *   tab size:   8 (not used)
     11 *   indentation:4
     12 *
     13 *   created on: 2000jul06
     14 *   created by: Markus W. Scherer
     15 */
     16 
     17 #include <stdio.h>
     18 #include "unicode/utypes.h"
     19 #include "cstring.h"
     20 #include "cmemory.h"
     21 #include "unewdata.h"
     22 #include "ucnv_cnv.h"
     23 #include "ucnvmbcs.h"
     24 #include "ucm.h"
     25 #include "makeconv.h"
     26 #include "genmbcs.h"
     27 
     28 /*
     29  * TODO: Split this file into toUnicode, SBCSFromUnicode and MBCSFromUnicode files.
     30  * Reduce tests for maxCharLength.
     31  */
     32 
     33 struct MBCSData {
     34     NewConverter newConverter;
     35 
     36     UCMFile *ucm;
     37 
     38     /* toUnicode (state table in ucm->states) */
     39     _MBCSToUFallback toUFallbacks[MBCS_MAX_FALLBACK_COUNT];
     40     int32_t countToUFallbacks;
     41     uint16_t *unicodeCodeUnits;
     42 
     43     /* fromUnicode */
     44     uint16_t stage1[MBCS_STAGE_1_SIZE];
     45     uint16_t stage2Single[MBCS_STAGE_2_SIZE]; /* stage 2 for single-byte codepages */
     46     uint32_t stage2[MBCS_STAGE_2_SIZE]; /* stage 2 for MBCS */
     47     uint8_t *fromUBytes;
     48     uint32_t stage2Top, stage3Top;
     49 
     50     /* fromUTF8 */
     51     uint16_t stageUTF8[0x10000>>MBCS_UTF8_STAGE_SHIFT];  /* allow for utf8Max=0xffff */
     52 
     53     /*
     54      * Maximum UTF-8-friendly code point.
     55      * 0 if !utf8Friendly, otherwise 0x01ff..0xffff in steps of 0x100.
     56      * If utf8Friendly, utf8Max is normally either MBCS_UTF8_MAX or 0xffff.
     57      */
     58     uint16_t utf8Max;
     59 
     60     UBool utf8Friendly;
     61     UBool omitFromU;
     62 };
     63 
     64 /* prototypes */
     65 static void
     66 MBCSClose(NewConverter *cnvData);
     67 
     68 static UBool
     69 MBCSStartMappings(MBCSData *mbcsData);
     70 
     71 static UBool
     72 MBCSAddToUnicode(MBCSData *mbcsData,
     73                  const uint8_t *bytes, int32_t length,
     74                  UChar32 c,
     75                  int8_t flag);
     76 
     77 static UBool
     78 MBCSIsValid(NewConverter *cnvData,
     79             const uint8_t *bytes, int32_t length);
     80 
     81 static UBool
     82 MBCSSingleAddFromUnicode(MBCSData *mbcsData,
     83                          const uint8_t *bytes, int32_t length,
     84                          UChar32 c,
     85                          int8_t flag);
     86 
     87 static UBool
     88 MBCSAddFromUnicode(MBCSData *mbcsData,
     89                    const uint8_t *bytes, int32_t length,
     90                    UChar32 c,
     91                    int8_t flag);
     92 
     93 static void
     94 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData *staticData);
     95 
     96 static UBool
     97 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData);
     98 
     99 static uint32_t
    100 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
    101           UNewDataMemory *pData, int32_t tableType);
    102 
    103 /* helper ------------------------------------------------------------------- */
    104 
    105 static U_INLINE char
    106 hexDigit(uint8_t digit) {
    107     return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit);
    108 }
    109 
    110 static U_INLINE char *
    111 printBytes(char *buffer, const uint8_t *bytes, int32_t length) {
    112     char *s=buffer;
    113     while(length>0) {
    114         *s++=hexDigit((uint8_t)(*bytes>>4));
    115         *s++=hexDigit((uint8_t)(*bytes&0xf));
    116         ++bytes;
    117         --length;
    118     }
    119 
    120     *s=0;
    121     return buffer;
    122 }
    123 
    124 /* implementation ----------------------------------------------------------- */
    125 
    126 static MBCSData gDummy;
    127 
    128 U_CFUNC const MBCSData *
    129 MBCSGetDummy() {
    130     uprv_memset(&gDummy, 0, sizeof(MBCSData));
    131 
    132     /*
    133      * Set "pessimistic" values which may sometimes move too many
    134      * mappings to the extension table (but never too few).
    135      * These values cause MBCSOkForBaseFromUnicode() to return FALSE for the
    136      * largest set of mappings.
    137      * Assume maxCharLength>1.
    138      */
    139     gDummy.utf8Friendly=TRUE;
    140     if(SMALL) {
    141         gDummy.utf8Max=0xffff;
    142         gDummy.omitFromU=TRUE;
    143     } else {
    144         gDummy.utf8Max=MBCS_UTF8_MAX;
    145     }
    146     return &gDummy;
    147 }
    148 
    149 static void
    150 MBCSInit(MBCSData *mbcsData, UCMFile *ucm) {
    151     uprv_memset(mbcsData, 0, sizeof(MBCSData));
    152 
    153     mbcsData->ucm=ucm; /* aliased, not owned */
    154 
    155     mbcsData->newConverter.close=MBCSClose;
    156     mbcsData->newConverter.isValid=MBCSIsValid;
    157     mbcsData->newConverter.addTable=MBCSAddTable;
    158     mbcsData->newConverter.write=MBCSWrite;
    159 }
    160 
    161 NewConverter *
    162 MBCSOpen(UCMFile *ucm) {
    163     MBCSData *mbcsData=(MBCSData *)uprv_malloc(sizeof(MBCSData));
    164     if(mbcsData==NULL) {
    165         printf("out of memory\n");
    166         exit(U_MEMORY_ALLOCATION_ERROR);
    167     }
    168 
    169     MBCSInit(mbcsData, ucm);
    170     return &mbcsData->newConverter;
    171 }
    172 
    173 static void
    174 MBCSDestruct(MBCSData *mbcsData) {
    175     uprv_free(mbcsData->unicodeCodeUnits);
    176     uprv_free(mbcsData->fromUBytes);
    177 }
    178 
    179 static void
    180 MBCSClose(NewConverter *cnvData) {
    181     MBCSData *mbcsData=(MBCSData *)cnvData;
    182     if(mbcsData!=NULL) {
    183         MBCSDestruct(mbcsData);
    184         uprv_free(mbcsData);
    185     }
    186 }
    187 
    188 static UBool
    189 MBCSStartMappings(MBCSData *mbcsData) {
    190     int32_t i, sum, maxCharLength,
    191             stage2NullLength, stage2AllocLength,
    192             stage3NullLength, stage3AllocLength;
    193 
    194     /* toUnicode */
    195 
    196     /* allocate the code unit array and prefill it with "unassigned" values */
    197     sum=mbcsData->ucm->states.countToUCodeUnits;
    198     if(VERBOSE) {
    199         printf("the total number of offsets is 0x%lx=%ld\n", (long)sum, (long)sum);
    200     }
    201 
    202     if(sum>0) {
    203         mbcsData->unicodeCodeUnits=(uint16_t *)uprv_malloc(sum*sizeof(uint16_t));
    204         if(mbcsData->unicodeCodeUnits==NULL) {
    205             fprintf(stderr, "error: out of memory allocating %ld 16-bit code units\n",
    206                 (long)sum);
    207             return FALSE;
    208         }
    209         for(i=0; i<sum; ++i) {
    210             mbcsData->unicodeCodeUnits[i]=0xfffe;
    211         }
    212     }
    213 
    214     /* fromUnicode */
    215     maxCharLength=mbcsData->ucm->states.maxCharLength;
    216 
    217     /* allocate the codepage mappings and preset the first 16 characters to 0 */
    218     if(maxCharLength==1) {
    219         /* allocate 64k 16-bit results for single-byte codepages */
    220         sum=0x20000;
    221     } else {
    222         /* allocate 1M * maxCharLength bytes for at most 1M mappings */
    223         sum=0x100000*maxCharLength;
    224     }
    225     mbcsData->fromUBytes=(uint8_t *)uprv_malloc(sum);
    226     if(mbcsData->fromUBytes==NULL) {
    227         fprintf(stderr, "error: out of memory allocating %ld B for target mappings\n", (long)sum);
    228         return FALSE;
    229     }
    230     uprv_memset(mbcsData->fromUBytes, 0, sum);
    231 
    232     /*
    233      * UTF-8-friendly fromUnicode tries: allocate multiple blocks at a time.
    234      * See ucnvmbcs.h for details.
    235      *
    236      * There is code, for example in ucnv_MBCSGetUnicodeSetForUnicode(), which
    237      * assumes that the initial stage 2/3 blocks are the all-unassigned ones.
    238      * Therefore, we refine the data structure while maintaining this placement
    239      * even though it would be convenient to allocate the ASCII block at the
    240      * beginning of stage 3, for example.
    241      *
    242      * UTF-8-friendly fromUnicode tries work from sorted tables and are built
    243      * pre-compacted, overlapping adjacent stage 2/3 blocks.
    244      * This is necessary because the block allocation and compaction changes
    245      * at SBCS_UTF8_MAX or MBCS_UTF8_MAX, and for MBCS tables the additional
    246      * stage table uses direct indexes into stage 3, without a multiplier and
    247      * thus with a smaller reach.
    248      *
    249      * Non-UTF-8-friendly fromUnicode tries work from unsorted tables
    250      * (because implicit precision is used), and are compacted
    251      * in post-processing.
    252      *
    253      * Preallocation for UTF-8-friendly fromUnicode tries:
    254      *
    255      * Stage 3:
    256      * 64-entry all-unassigned first block followed by ASCII (128 entries).
    257      *
    258      * Stage 2:
    259      * 64-entry all-unassigned first block followed by preallocated
    260      * 64-block for ASCII.
    261      */
    262 
    263     /* Preallocate ASCII as a linear 128-entry stage 3 block. */
    264     stage2NullLength=MBCS_STAGE_2_BLOCK_SIZE;
    265     stage2AllocLength=MBCS_STAGE_2_BLOCK_SIZE;
    266 
    267     stage3NullLength=MBCS_UTF8_STAGE_3_BLOCK_SIZE;
    268     stage3AllocLength=128; /* ASCII U+0000..U+007f */
    269 
    270     /* Initialize stage 1 for the preallocated blocks. */
    271     sum=stage2NullLength;
    272     for(i=0; i<(stage2AllocLength>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT); ++i) {
    273         mbcsData->stage1[i]=sum;
    274         sum+=MBCS_STAGE_2_BLOCK_SIZE;
    275     }
    276     mbcsData->stage2Top=stage2NullLength+stage2AllocLength; /* ==sum */
    277 
    278     /*
    279      * Stage 2 indexes count 16-blocks in stage 3 as follows:
    280      * SBCS: directly, indexes increment by 16
    281      * MBCS: indexes need to be multiplied by 16*maxCharLength, indexes increment by 1
    282      * MBCS UTF-8: directly, indexes increment by 16
    283      */
    284     if(maxCharLength==1) {
    285         sum=stage3NullLength;
    286         for(i=0; i<(stage3AllocLength/MBCS_STAGE_3_BLOCK_SIZE); ++i) {
    287             mbcsData->stage2Single[mbcsData->stage1[0]+i]=sum;
    288             sum+=MBCS_STAGE_3_BLOCK_SIZE;
    289         }
    290     } else {
    291         sum=stage3NullLength/MBCS_STAGE_3_GRANULARITY;
    292         for(i=0; i<(stage3AllocLength/MBCS_STAGE_3_BLOCK_SIZE); ++i) {
    293             mbcsData->stage2[mbcsData->stage1[0]+i]=sum;
    294             sum+=MBCS_STAGE_3_BLOCK_SIZE/MBCS_STAGE_3_GRANULARITY;
    295         }
    296     }
    297 
    298     sum=stage3NullLength;
    299     for(i=0; i<(stage3AllocLength/MBCS_UTF8_STAGE_3_BLOCK_SIZE); ++i) {
    300         mbcsData->stageUTF8[i]=sum;
    301         sum+=MBCS_UTF8_STAGE_3_BLOCK_SIZE;
    302     }
    303 
    304     /*
    305      * Allocate a 64-entry all-unassigned first stage 3 block,
    306      * for UTF-8-friendly lookup with a trail byte,
    307      * plus 128 entries for ASCII.
    308      */
    309     mbcsData->stage3Top=(stage3NullLength+stage3AllocLength)*maxCharLength; /* ==sum*maxCharLength */
    310 
    311     return TRUE;
    312 }
    313 
    314 /* return TRUE for success */
    315 static UBool
    316 setFallback(MBCSData *mbcsData, uint32_t offset, UChar32 c) {
    317     int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset);
    318     if(i>=0) {
    319         /* if there is already a fallback for this offset, then overwrite it */
    320         mbcsData->toUFallbacks[i].codePoint=c;
    321         return TRUE;
    322     } else {
    323         /* if there is no fallback for this offset, then add one */
    324         i=mbcsData->countToUFallbacks;
    325         if(i>=MBCS_MAX_FALLBACK_COUNT) {
    326             fprintf(stderr, "error: too many toUnicode fallbacks, currently at: U+%x\n", (int)c);
    327             return FALSE;
    328         } else {
    329             mbcsData->toUFallbacks[i].offset=offset;
    330             mbcsData->toUFallbacks[i].codePoint=c;
    331             mbcsData->countToUFallbacks=i+1;
    332             return TRUE;
    333         }
    334     }
    335 }
    336 
    337 /* remove fallback if there is one with this offset; return the code point if there was such a fallback, otherwise -1 */
    338 static int32_t
    339 removeFallback(MBCSData *mbcsData, uint32_t offset) {
    340     int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset);
    341     if(i>=0) {
    342         _MBCSToUFallback *toUFallbacks;
    343         int32_t limit, old;
    344 
    345         toUFallbacks=mbcsData->toUFallbacks;
    346         limit=mbcsData->countToUFallbacks;
    347         old=(int32_t)toUFallbacks[i].codePoint;
    348 
    349         /* copy the last fallback entry here to keep the list contiguous */
    350         toUFallbacks[i].offset=toUFallbacks[limit-1].offset;
    351         toUFallbacks[i].codePoint=toUFallbacks[limit-1].codePoint;
    352         mbcsData->countToUFallbacks=limit-1;
    353         return old;
    354     } else {
    355         return -1;
    356     }
    357 }
    358 
    359 /*
    360  * isFallback is almost a boolean:
    361  * 1 (TRUE)  this is a fallback mapping
    362  * 0 (FALSE) this is a precise mapping
    363  * -1        the precision of this mapping is not specified
    364  */
    365 static UBool
    366 MBCSAddToUnicode(MBCSData *mbcsData,
    367                  const uint8_t *bytes, int32_t length,
    368                  UChar32 c,
    369                  int8_t flag) {
    370     char buffer[10];
    371     uint32_t offset=0;
    372     int32_t i=0, entry, old;
    373     uint8_t state=0;
    374 
    375     if(mbcsData->ucm->states.countStates==0) {
    376         fprintf(stderr, "error: there is no state information!\n");
    377         return FALSE;
    378     }
    379 
    380     /* for SI/SO (like EBCDIC-stateful), double-byte sequences start in state 1 */
    381     if(length==2 && mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO) {
    382         state=1;
    383     }
    384 
    385     /*
    386      * Walk down the state table like in conversion,
    387      * much like getNextUChar().
    388      * We assume that c<=0x10ffff.
    389      */
    390     for(i=0;;) {
    391         entry=mbcsData->ucm->states.stateTable[state][bytes[i++]];
    392         if(MBCS_ENTRY_IS_TRANSITION(entry)) {
    393             if(i==length) {
    394                 fprintf(stderr, "error: byte sequence too short, ends in non-final state %hu: 0x%s (U+%x)\n",
    395                     (short)state, printBytes(buffer, bytes, length), (int)c);
    396                 return FALSE;
    397             }
    398             state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry);
    399             offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry);
    400         } else {
    401             if(i<length) {
    402                 fprintf(stderr, "error: byte sequence too long by %d bytes, final state %hu: 0x%s (U+%x)\n",
    403                     (int)(length-i), state, printBytes(buffer, bytes, length), (int)c);
    404                 return FALSE;
    405             }
    406             switch(MBCS_ENTRY_FINAL_ACTION(entry)) {
    407             case MBCS_STATE_ILLEGAL:
    408                 fprintf(stderr, "error: byte sequence ends in illegal state at U+%04x<->0x%s\n",
    409                     (int)c, printBytes(buffer, bytes, length));
    410                 return FALSE;
    411             case MBCS_STATE_CHANGE_ONLY:
    412                 fprintf(stderr, "error: byte sequence ends in state-change-only at U+%04x<->0x%s\n",
    413                     (int)c, printBytes(buffer, bytes, length));
    414                 return FALSE;
    415             case MBCS_STATE_UNASSIGNED:
    416                 fprintf(stderr, "error: byte sequence ends in unassigned state at U+%04x<->0x%s\n",
    417                     (int)c, printBytes(buffer, bytes, length));
    418                 return FALSE;
    419             case MBCS_STATE_FALLBACK_DIRECT_16:
    420             case MBCS_STATE_VALID_DIRECT_16:
    421             case MBCS_STATE_FALLBACK_DIRECT_20:
    422             case MBCS_STATE_VALID_DIRECT_20:
    423                 if(MBCS_ENTRY_SET_STATE(entry, 0)!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, 0xfffe)) {
    424                     /* the "direct" action's value is not "valid-direct-16-unassigned" any more */
    425                     if(MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_DIRECT_16 || MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_FALLBACK_DIRECT_16) {
    426                         old=MBCS_ENTRY_FINAL_VALUE(entry);
    427                     } else {
    428                         old=0x10000+MBCS_ENTRY_FINAL_VALUE(entry);
    429                     }
    430                     if(flag>=0) {
    431                         fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
    432                             (int)c, printBytes(buffer, bytes, length), (int)old);
    433                         return FALSE;
    434                     } else if(VERBOSE) {
    435                         fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
    436                             (int)c, printBytes(buffer, bytes, length), (int)old);
    437                     }
    438                     /*
    439                      * Continue after the above warning
    440                      * if the precision of the mapping is unspecified.
    441                      */
    442                 }
    443                 /* reassign the correct action code */
    444                 entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, (MBCS_STATE_VALID_DIRECT_16+(flag==3 ? 2 : 0)+(c>=0x10000 ? 1 : 0)));
    445 
    446                 /* put the code point into bits 22..7 for BMP, c-0x10000 into 26..7 for others */
    447                 if(c<=0xffff) {
    448                     entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c);
    449                 } else {
    450                     entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c-0x10000);
    451                 }
    452                 mbcsData->ucm->states.stateTable[state][bytes[i-1]]=entry;
    453                 break;
    454             case MBCS_STATE_VALID_16:
    455                 /* bits 26..16 are not used, 0 */
    456                 /* bits 15..7 contain the final offset delta to one 16-bit code unit */
    457                 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
    458                 /* check that this byte sequence is still unassigned */
    459                 if((old=mbcsData->unicodeCodeUnits[offset])!=0xfffe || (old=removeFallback(mbcsData, offset))!=-1) {
    460                     if(flag>=0) {
    461                         fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
    462                             (int)c, printBytes(buffer, bytes, length), (int)old);
    463                         return FALSE;
    464                     } else if(VERBOSE) {
    465                         fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
    466                             (int)c, printBytes(buffer, bytes, length), (int)old);
    467                     }
    468                 }
    469                 if(c>=0x10000) {
    470                     fprintf(stderr, "error: code point does not fit into valid-16-bit state at U+%04x<->0x%s\n",
    471                         (int)c, printBytes(buffer, bytes, length));
    472                     return FALSE;
    473                 }
    474                 if(flag>0) {
    475                     /* assign only if there is no precise mapping */
    476                     if(mbcsData->unicodeCodeUnits[offset]==0xfffe) {
    477                         return setFallback(mbcsData, offset, c);
    478                     }
    479                 } else {
    480                     mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
    481                 }
    482                 break;
    483             case MBCS_STATE_VALID_16_PAIR:
    484                 /* bits 26..16 are not used, 0 */
    485                 /* bits 15..7 contain the final offset delta to two 16-bit code units */
    486                 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry);
    487                 /* check that this byte sequence is still unassigned */
    488                 old=mbcsData->unicodeCodeUnits[offset];
    489                 if(old<0xfffe) {
    490                     int32_t real;
    491                     if(old<0xd800) {
    492                         real=old;
    493                     } else if(old<=0xdfff) {
    494                         real=0x10000+((old&0x3ff)<<10)+((mbcsData->unicodeCodeUnits[offset+1])&0x3ff);
    495                     } else /* old<=0xe001 */ {
    496                         real=mbcsData->unicodeCodeUnits[offset+1];
    497                     }
    498                     if(flag>=0) {
    499                         fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
    500                             (int)c, printBytes(buffer, bytes, length), (int)real);
    501                         return FALSE;
    502                     } else if(VERBOSE) {
    503                         fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n",
    504                             (int)c, printBytes(buffer, bytes, length), (int)real);
    505                     }
    506                 }
    507                 if(flag>0) {
    508                     /* assign only if there is no precise mapping */
    509                     if(old<=0xdbff || old==0xe000) {
    510                         /* do nothing */
    511                     } else if(c<=0xffff) {
    512                         /* set a BMP fallback code point as a pair with 0xe001 */
    513                         mbcsData->unicodeCodeUnits[offset++]=0xe001;
    514                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
    515                     } else {
    516                         /* set a fallback surrogate pair with two second surrogates */
    517                         mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xdbc0+(c>>10));
    518                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff));
    519                     }
    520                 } else {
    521                     if(c<0xd800) {
    522                         /* set a BMP code point */
    523                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
    524                     } else if(c<=0xffff) {
    525                         /* set a BMP code point above 0xd800 as a pair with 0xe000 */
    526                         mbcsData->unicodeCodeUnits[offset++]=0xe000;
    527                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)c;
    528                     } else {
    529                         /* set a surrogate pair */
    530                         mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xd7c0+(c>>10));
    531                         mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff));
    532                     }
    533                 }
    534                 break;
    535             default:
    536                 /* reserved, must never occur */
    537                 fprintf(stderr, "internal error: byte sequence reached reserved action code, entry 0x%02x: 0x%s (U+%x)\n",
    538                     (int)entry, printBytes(buffer, bytes, length), (int)c);
    539                 return FALSE;
    540             }
    541 
    542             return TRUE;
    543         }
    544     }
    545 }
    546 
    547 /* is this byte sequence valid? (this is almost the same as MBCSAddToUnicode()) */
    548 static UBool
    549 MBCSIsValid(NewConverter *cnvData,
    550             const uint8_t *bytes, int32_t length) {
    551     MBCSData *mbcsData=(MBCSData *)cnvData;
    552 
    553     return (UBool)(1==ucm_countChars(&mbcsData->ucm->states, bytes, length));
    554 }
    555 
    556 static UBool
    557 MBCSSingleAddFromUnicode(MBCSData *mbcsData,
    558                          const uint8_t *bytes, int32_t length,
    559                          UChar32 c,
    560                          int8_t flag) {
    561     uint16_t *stage3, *p;
    562     uint32_t idx;
    563     uint16_t old;
    564     uint8_t b;
    565 
    566     uint32_t blockSize, newTop, i, nextOffset, newBlock, min;
    567 
    568     /* ignore |2 SUB mappings */
    569     if(flag==2) {
    570         return TRUE;
    571     }
    572 
    573     /*
    574      * Walk down the triple-stage compact array ("trie") and
    575      * allocate parts as necessary.
    576      * Note that the first stage 2 and 3 blocks are reserved for all-unassigned mappings.
    577      * We assume that length<=maxCharLength and that c<=0x10ffff.
    578      */
    579     stage3=(uint16_t *)mbcsData->fromUBytes;
    580     b=*bytes;
    581 
    582     /* inspect stage 1 */
    583     idx=c>>MBCS_STAGE_1_SHIFT;
    584     if(mbcsData->utf8Friendly && c<=SBCS_UTF8_MAX) {
    585         nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK&~(MBCS_UTF8_STAGE_3_BLOCKS-1);
    586     } else {
    587         nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK;
    588     }
    589     if(mbcsData->stage1[idx]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) {
    590         /* allocate another block in stage 2 */
    591         newBlock=mbcsData->stage2Top;
    592         if(mbcsData->utf8Friendly) {
    593             min=newBlock-nextOffset; /* minimum block start with overlap */
    594             while(min<newBlock && mbcsData->stage2Single[newBlock-1]==0) {
    595                 --newBlock;
    596             }
    597         }
    598         newTop=newBlock+MBCS_STAGE_2_BLOCK_SIZE;
    599 
    600         if(newTop>MBCS_MAX_STAGE_2_TOP) {
    601             fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%02x\n", (int)c, b);
    602             return FALSE;
    603         }
    604 
    605         /*
    606          * each stage 2 block contains 64 16-bit words:
    607          * 6 code point bits 9..4 with 1 stage 3 index
    608          */
    609         mbcsData->stage1[idx]=(uint16_t)newBlock;
    610         mbcsData->stage2Top=newTop;
    611     }
    612 
    613     /* inspect stage 2 */
    614     idx=mbcsData->stage1[idx]+nextOffset;
    615     if(mbcsData->utf8Friendly && c<=SBCS_UTF8_MAX) {
    616         /* allocate 64-entry blocks for UTF-8-friendly lookup */
    617         blockSize=MBCS_UTF8_STAGE_3_BLOCK_SIZE;
    618         nextOffset=c&MBCS_UTF8_STAGE_3_BLOCK_MASK;
    619     } else {
    620         blockSize=MBCS_STAGE_3_BLOCK_SIZE;
    621         nextOffset=c&MBCS_STAGE_3_BLOCK_MASK;
    622     }
    623     if(mbcsData->stage2Single[idx]==0) {
    624         /* allocate another block in stage 3 */
    625         newBlock=mbcsData->stage3Top;
    626         if(mbcsData->utf8Friendly) {
    627             min=newBlock-nextOffset; /* minimum block start with overlap */
    628             while(min<newBlock && stage3[newBlock-1]==0) {
    629                 --newBlock;
    630             }
    631         }
    632         newTop=newBlock+blockSize;
    633 
    634         if(newTop>MBCS_STAGE_3_SBCS_SIZE) {
    635             fprintf(stderr, "error: too many code points at U+%04x<->0x%02x\n", (int)c, b);
    636             return FALSE;
    637         }
    638         /* each block has 16 uint16_t entries */
    639         i=idx;
    640         while(newBlock<newTop) {
    641             mbcsData->stage2Single[i++]=(uint16_t)newBlock;
    642             newBlock+=MBCS_STAGE_3_BLOCK_SIZE;
    643         }
    644         mbcsData->stage3Top=newTop; /* ==newBlock */
    645     }
    646 
    647     /* write the codepage entry into stage 3 and get the previous entry */
    648     p=stage3+mbcsData->stage2Single[idx]+nextOffset;
    649     old=*p;
    650     if(flag<=0) {
    651         *p=(uint16_t)(0xf00|b);
    652     } else if(IS_PRIVATE_USE(c)) {
    653         *p=(uint16_t)(0xc00|b);
    654     } else {
    655         *p=(uint16_t)(0x800|b);
    656     }
    657 
    658     /* check that this Unicode code point was still unassigned */
    659     if(old>=0x100) {
    660         if(flag>=0) {
    661             fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n",
    662                 (int)c, b, old&0xff);
    663             return FALSE;
    664         } else if(VERBOSE) {
    665             fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n",
    666                 (int)c, b, old&0xff);
    667         }
    668         /* continue after the above warning if the precision of the mapping is unspecified */
    669     }
    670 
    671     return TRUE;
    672 }
    673 
    674 static UBool
    675 MBCSAddFromUnicode(MBCSData *mbcsData,
    676                    const uint8_t *bytes, int32_t length,
    677                    UChar32 c,
    678                    int8_t flag) {
    679     char buffer[10];
    680     const uint8_t *pb;
    681     uint8_t *stage3, *p;
    682     uint32_t idx, b, old, stage3Index;
    683     int32_t maxCharLength;
    684 
    685     uint32_t blockSize, newTop, i, nextOffset, newBlock, min, overlap, maxOverlap;
    686 
    687     maxCharLength=mbcsData->ucm->states.maxCharLength;
    688 
    689     if( mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO &&
    690         (!IGNORE_SISO_CHECK && (*bytes==0xe || *bytes==0xf))
    691     ) {
    692         fprintf(stderr, "error: illegal mapping to SI or SO for SI/SO codepage: U+%04x<->0x%s\n",
    693             (int)c, printBytes(buffer, bytes, length));
    694         return FALSE;
    695     }
    696 
    697     if(flag==1 && length==1 && *bytes==0) {
    698         fprintf(stderr, "error: unable to encode a |1 fallback from U+%04x to 0x%02x\n",
    699             (int)c, *bytes);
    700         return FALSE;
    701     }
    702 
    703     /*
    704      * Walk down the triple-stage compact array ("trie") and
    705      * allocate parts as necessary.
    706      * Note that the first stage 2 and 3 blocks are reserved for
    707      * all-unassigned mappings.
    708      * We assume that length<=maxCharLength and that c<=0x10ffff.
    709      */
    710     stage3=mbcsData->fromUBytes;
    711 
    712     /* inspect stage 1 */
    713     idx=c>>MBCS_STAGE_1_SHIFT;
    714     if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) {
    715         nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK&~(MBCS_UTF8_STAGE_3_BLOCKS-1);
    716     } else {
    717         nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK;
    718     }
    719     if(mbcsData->stage1[idx]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) {
    720         /* allocate another block in stage 2 */
    721         newBlock=mbcsData->stage2Top;
    722         if(mbcsData->utf8Friendly) {
    723             min=newBlock-nextOffset; /* minimum block start with overlap */
    724             while(min<newBlock && mbcsData->stage2[newBlock-1]==0) {
    725                 --newBlock;
    726             }
    727         }
    728         newTop=newBlock+MBCS_STAGE_2_BLOCK_SIZE;
    729 
    730         if(newTop>MBCS_MAX_STAGE_2_TOP) {
    731             fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%s\n",
    732                 (int)c, printBytes(buffer, bytes, length));
    733             return FALSE;
    734         }
    735 
    736         /*
    737          * each stage 2 block contains 64 32-bit words:
    738          * 6 code point bits 9..4 with value with bits 31..16 "assigned" flags and bits 15..0 stage 3 index
    739          */
    740         i=idx;
    741         while(newBlock<newTop) {
    742             mbcsData->stage1[i++]=(uint16_t)newBlock;
    743             newBlock+=MBCS_STAGE_2_BLOCK_SIZE;
    744         }
    745         mbcsData->stage2Top=newTop; /* ==newBlock */
    746     }
    747 
    748     /* inspect stage 2 */
    749     idx=mbcsData->stage1[idx]+nextOffset;
    750     if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) {
    751         /* allocate 64-entry blocks for UTF-8-friendly lookup */
    752         blockSize=MBCS_UTF8_STAGE_3_BLOCK_SIZE*maxCharLength;
    753         nextOffset=c&MBCS_UTF8_STAGE_3_BLOCK_MASK;
    754     } else {
    755         blockSize=MBCS_STAGE_3_BLOCK_SIZE*maxCharLength;
    756         nextOffset=c&MBCS_STAGE_3_BLOCK_MASK;
    757     }
    758     if(mbcsData->stage2[idx]==0) {
    759         /* allocate another block in stage 3 */
    760         newBlock=mbcsData->stage3Top;
    761         if(mbcsData->utf8Friendly && nextOffset>=MBCS_STAGE_3_GRANULARITY) {
    762             /*
    763              * Overlap stage 3 blocks only in multiples of 16-entry blocks
    764              * because of the indexing granularity in stage 2.
    765              */
    766             maxOverlap=(nextOffset&~(MBCS_STAGE_3_GRANULARITY-1))*maxCharLength;
    767             for(overlap=0;
    768                 overlap<maxOverlap && stage3[newBlock-overlap-1]==0;
    769                 ++overlap) {}
    770 
    771             overlap=(overlap/MBCS_STAGE_3_GRANULARITY)/maxCharLength;
    772             overlap=(overlap*MBCS_STAGE_3_GRANULARITY)*maxCharLength;
    773 
    774             newBlock-=overlap;
    775         }
    776         newTop=newBlock+blockSize;
    777 
    778         if(newTop>MBCS_STAGE_3_MBCS_SIZE*(uint32_t)maxCharLength) {
    779             fprintf(stderr, "error: too many code points at U+%04x<->0x%s\n",
    780                 (int)c, printBytes(buffer, bytes, length));
    781             return FALSE;
    782         }
    783         /* each block has 16*maxCharLength bytes */
    784         i=idx;
    785         while(newBlock<newTop) {
    786             mbcsData->stage2[i++]=(newBlock/MBCS_STAGE_3_GRANULARITY)/maxCharLength;
    787             newBlock+=MBCS_STAGE_3_BLOCK_SIZE*maxCharLength;
    788         }
    789         mbcsData->stage3Top=newTop; /* ==newBlock */
    790     }
    791 
    792     stage3Index=MBCS_STAGE_3_GRANULARITY*(uint32_t)(uint16_t)mbcsData->stage2[idx];
    793 
    794     /* Build an alternate, UTF-8-friendly stage table as well. */
    795     if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) {
    796         /* Overflow for uint16_t entries in stageUTF8? */
    797         if(stage3Index>0xffff) {
    798             /*
    799              * This can occur only if the mapping table is nearly perfectly filled and if
    800              * utf8Max==0xffff.
    801              * (There is no known charset like this. GB 18030 does not map
    802              * surrogate code points and LMBCS does not map 256 PUA code points.)
    803              *
    804              * Otherwise, stage3Index<=MBCS_UTF8_LIMIT<0xffff
    805              * (stage3Index can at most reach exactly MBCS_UTF8_LIMIT)
    806              * because we have a sorted table and there are at most MBCS_UTF8_LIMIT
    807              * mappings with 0<=c<MBCS_UTF8_LIMIT, and there is only also
    808              * the initial all-unassigned block in stage3.
    809              *
    810              * Solution for the overflow: Reduce utf8Max to the next lower value, 0xfeff.
    811              *
    812              * (See svn revision 20866 of the markus/ucnvutf8 feature branch for
    813              * code that causes MBCSAddTable() to rebuild the table not utf8Friendly
    814              * in case of overflow. That code was not tested.)
    815              */
    816             mbcsData->utf8Max=0xfeff;
    817         } else {
    818             /*
    819              * The stage 3 block has been assigned for the regular trie.
    820              * Just copy its index into stageUTF8[], without the granularity.
    821              */
    822             mbcsData->stageUTF8[c>>MBCS_UTF8_STAGE_SHIFT]=(uint16_t)stage3Index;
    823         }
    824     }
    825 
    826     /* write the codepage bytes into stage 3 and get the previous bytes */
    827 
    828     /* assemble the bytes into a single integer */
    829     pb=bytes;
    830     b=0;
    831     switch(length) {
    832     case 4:
    833         b=*pb++;
    834     case 3:
    835         b=(b<<8)|*pb++;
    836     case 2:
    837         b=(b<<8)|*pb++;
    838     case 1:
    839     default:
    840         b=(b<<8)|*pb++;
    841         break;
    842     }
    843 
    844     old=0;
    845     p=stage3+(stage3Index+nextOffset)*maxCharLength;
    846     switch(maxCharLength) {
    847     case 2:
    848         old=*(uint16_t *)p;
    849         *(uint16_t *)p=(uint16_t)b;
    850         break;
    851     case 3:
    852         old=(uint32_t)*p<<16;
    853         *p++=(uint8_t)(b>>16);
    854         old|=(uint32_t)*p<<8;
    855         *p++=(uint8_t)(b>>8);
    856         old|=*p;
    857         *p=(uint8_t)b;
    858         break;
    859     case 4:
    860         old=*(uint32_t *)p;
    861         *(uint32_t *)p=b;
    862         break;
    863     default:
    864         /* will never occur */
    865         break;
    866     }
    867 
    868     /* check that this Unicode code point was still unassigned */
    869     if((mbcsData->stage2[idx+(nextOffset>>MBCS_STAGE_2_SHIFT)]&(1UL<<(16+(c&0xf))))!=0 || old!=0) {
    870         if(flag>=0) {
    871             fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n",
    872                 (int)c, printBytes(buffer, bytes, length), (int)old);
    873             return FALSE;
    874         } else if(VERBOSE) {
    875             fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n",
    876                 (int)c, printBytes(buffer, bytes, length), (int)old);
    877         }
    878         /* continue after the above warning if the precision of the mapping is
    879            unspecified */
    880     }
    881     if(flag<=0) {
    882         /* set the roundtrip flag */
    883         mbcsData->stage2[idx+(nextOffset>>4)]|=(1UL<<(16+(c&0xf)));
    884     }
    885 
    886     return TRUE;
    887 }
    888 
    889 U_CFUNC UBool
    890 MBCSOkForBaseFromUnicode(const MBCSData *mbcsData,
    891                          const uint8_t *bytes, int32_t length,
    892                          UChar32 c, int8_t flag) {
    893     /*
    894      * A 1:1 mapping does not fit into the MBCS base table's fromUnicode table under
    895      * the following conditions:
    896      *
    897      * - a |2 SUB mapping for <subchar1> (no base table data structure for them)
    898      * - a |1 fallback to 0x00 (result value 0, indistinguishable from unmappable entry)
    899      * - a multi-byte mapping with leading 0x00 bytes (no explicit length field)
    900      *
    901      * Some of these tests are redundant with ucm_mappingType().
    902      */
    903     if( (flag==2 && length==1) ||
    904         (flag==1 && bytes[0]==0) || /* testing length==1 would be redundant with the next test */
    905         (flag<=1 && length>1 && bytes[0]==0)
    906     ) {
    907         return FALSE;
    908     }
    909 
    910     /*
    911      * Additional restrictions for UTF-8-friendly fromUnicode tables,
    912      * for code points up to the maximum optimized one:
    913      *
    914      * - any mapping to 0x00 (result value 0, indistinguishable from unmappable entry)
    915      * - any |1 fallback (no roundtrip flags in the optimized table)
    916      */
    917     if(mbcsData->utf8Friendly && flag<=1 && c<=mbcsData->utf8Max && (bytes[0]==0 || flag==1)) {
    918         return FALSE;
    919     }
    920 
    921     /*
    922      * If we omit the fromUnicode data, we can only store roundtrips there
    923      * because only they are recoverable from the toUnicode data.
    924      * Fallbacks must go into the extension table.
    925      */
    926     if(mbcsData->omitFromU && flag!=0) {
    927         return FALSE;
    928     }
    929 
    930     /* All other mappings do fit into the base table. */
    931     return TRUE;
    932 }
    933 
    934 /* we can assume that the table only contains 1:1 mappings with <=4 bytes each */
    935 static UBool
    936 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData) {
    937     MBCSData *mbcsData;
    938     UCMapping *m;
    939     UChar32 c;
    940     int32_t i, maxCharLength;
    941     int8_t f;
    942     UBool isOK, utf8Friendly;
    943 
    944     staticData->unicodeMask=table->unicodeMask;
    945     if(staticData->unicodeMask==3) {
    946         fprintf(stderr, "error: contains mappings for both supplementary and surrogate code points\n");
    947         return FALSE;
    948     }
    949 
    950     staticData->conversionType=UCNV_MBCS;
    951 
    952     mbcsData=(MBCSData *)cnvData;
    953     maxCharLength=mbcsData->ucm->states.maxCharLength;
    954 
    955     /*
    956      * Generation of UTF-8-friendly data requires
    957      * a sorted table, which makeconv generates when explicit precision
    958      * indicators are used.
    959      */
    960     mbcsData->utf8Friendly=utf8Friendly=(UBool)((table->flagsType&UCM_FLAGS_EXPLICIT)!=0);
    961     if(utf8Friendly) {
    962         mbcsData->utf8Max=MBCS_UTF8_MAX;
    963         if(SMALL && maxCharLength>1) {
    964             mbcsData->omitFromU=TRUE;
    965         }
    966     } else {
    967         mbcsData->utf8Max=0;
    968         if(SMALL && maxCharLength>1) {
    969             fprintf(stderr,
    970                 "makeconv warning: --small not available for .ucm files without |0 etc.\n");
    971         }
    972     }
    973 
    974     if(!MBCSStartMappings(mbcsData)) {
    975         return FALSE;
    976     }
    977 
    978     staticData->hasFromUnicodeFallback=FALSE;
    979     staticData->hasToUnicodeFallback=FALSE;
    980 
    981     isOK=TRUE;
    982 
    983     m=table->mappings;
    984     for(i=0; i<table->mappingsLength; ++m, ++i) {
    985         c=m->u;
    986         f=m->f;
    987 
    988         /*
    989          * Small optimization for --small .cnv files:
    990          *
    991          * If there are fromUnicode mappings above MBCS_UTF8_MAX,
    992          * then the file size will be smaller if we make utf8Max larger
    993          * because the size increase in stageUTF8 will be more than balanced by
    994          * how much less of stage2 needs to be stored.
    995          *
    996          * There is no point in doing this incrementally because stageUTF8
    997          * uses so much less space per block than stage2,
    998          * so we immediately increase utf8Max to 0xffff.
    999          *
   1000          * Do not increase utf8Max if it is already at 0xfeff because MBCSAddFromUnicode()
   1001          * sets it to that value when stageUTF8 overflows.
   1002          */
   1003         if( mbcsData->omitFromU && f<=1 &&
   1004             mbcsData->utf8Max<c && c<=0xffff &&
   1005             mbcsData->utf8Max<0xfeff
   1006         ) {
   1007             mbcsData->utf8Max=0xffff;
   1008         }
   1009 
   1010         switch(f) {
   1011         case -1:
   1012             /* there was no precision/fallback indicator */
   1013             /* fall through to set the mappings */
   1014         case 0:
   1015             /* set roundtrip mappings */
   1016             isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, f);
   1017 
   1018             if(maxCharLength==1) {
   1019                 isOK&=MBCSSingleAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f);
   1020             } else if(MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f)) {
   1021                 isOK&=MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f);
   1022             } else {
   1023                 m->f|=MBCS_FROM_U_EXT_FLAG;
   1024                 m->moveFlag=UCM_MOVE_TO_EXT;
   1025             }
   1026             break;
   1027         case 1:
   1028             /* set only a fallback mapping from Unicode to codepage */
   1029             if(maxCharLength==1) {
   1030                 staticData->hasFromUnicodeFallback=TRUE;
   1031                 isOK&=MBCSSingleAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f);
   1032             } else if(MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f)) {
   1033                 staticData->hasFromUnicodeFallback=TRUE;
   1034                 isOK&=MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f);
   1035             } else {
   1036                 m->f|=MBCS_FROM_U_EXT_FLAG;
   1037                 m->moveFlag=UCM_MOVE_TO_EXT;
   1038             }
   1039             break;
   1040         case 2:
   1041             /* ignore |2 SUB mappings, except to move <subchar1> mappings to the extension table */
   1042             if(maxCharLength>1 && m->bLen==1) {
   1043                 m->f|=MBCS_FROM_U_EXT_FLAG;
   1044                 m->moveFlag=UCM_MOVE_TO_EXT;
   1045             }
   1046             break;
   1047         case 3:
   1048             /* set only a fallback mapping from codepage to Unicode */
   1049             staticData->hasToUnicodeFallback=TRUE;
   1050             isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, f);
   1051             break;
   1052         default:
   1053             /* will not occur because the parser checked it already */
   1054             fprintf(stderr, "error: illegal fallback indicator %d\n", f);
   1055             return FALSE;
   1056         }
   1057     }
   1058 
   1059     MBCSPostprocess(mbcsData, staticData);
   1060 
   1061     return isOK;
   1062 }
   1063 
   1064 static UBool
   1065 transformEUC(MBCSData *mbcsData) {
   1066     uint8_t *p8;
   1067     uint32_t i, value, oldLength, old3Top, new3Top;
   1068     uint8_t b;
   1069 
   1070     oldLength=mbcsData->ucm->states.maxCharLength;
   1071     if(oldLength<3) {
   1072         return FALSE;
   1073     }
   1074 
   1075     old3Top=mbcsData->stage3Top;
   1076 
   1077     /* careful: 2-byte and 4-byte codes are stored in platform endianness! */
   1078 
   1079     /* test if all first bytes are in {0, 0x8e, 0x8f} */
   1080     p8=mbcsData->fromUBytes;
   1081 
   1082 #if !U_IS_BIG_ENDIAN
   1083     if(oldLength==4) {
   1084         p8+=3;
   1085     }
   1086 #endif
   1087 
   1088     for(i=0; i<old3Top; i+=oldLength) {
   1089         b=p8[i];
   1090         if(b!=0 && b!=0x8e && b!=0x8f) {
   1091             /* some first byte does not fit the EUC pattern, nothing to be done */
   1092             return FALSE;
   1093         }
   1094     }
   1095     /* restore p if it was modified above */
   1096     p8=mbcsData->fromUBytes;
   1097 
   1098     /* modify outputType and adjust stage3Top */
   1099     mbcsData->ucm->states.outputType=(int8_t)(MBCS_OUTPUT_3_EUC+oldLength-3);
   1100     mbcsData->stage3Top=new3Top=(old3Top*(oldLength-1))/oldLength;
   1101 
   1102     /*
   1103      * EUC-encode all byte sequences;
   1104      * see "CJKV Information Processing" (1st ed. 1999) from Ken Lunde, O'Reilly,
   1105      * p. 161 in chapter 4 "Encoding Methods"
   1106      *
   1107      * This also must reverse the byte order if the platform is little-endian!
   1108      */
   1109     if(oldLength==3) {
   1110         uint16_t *q=(uint16_t *)p8;
   1111         for(i=0; i<old3Top; i+=oldLength) {
   1112             b=*p8;
   1113             if(b==0) {
   1114                 /* short sequences are stored directly */
   1115                 /* code set 0 or 1 */
   1116                 (*q++)=(uint16_t)((p8[1]<<8)|p8[2]);
   1117             } else if(b==0x8e) {
   1118                 /* code set 2 */
   1119                 (*q++)=(uint16_t)(((p8[1]&0x7f)<<8)|p8[2]);
   1120             } else /* b==0x8f */ {
   1121                 /* code set 3 */
   1122                 (*q++)=(uint16_t)((p8[1]<<8)|(p8[2]&0x7f));
   1123             }
   1124             p8+=3;
   1125         }
   1126     } else /* oldLength==4 */ {
   1127         uint8_t *q=p8;
   1128         uint32_t *p32=(uint32_t *)p8;
   1129         for(i=0; i<old3Top; i+=4) {
   1130             value=(*p32++);
   1131             if(value<=0xffffff) {
   1132                 /* short sequences are stored directly */
   1133                 /* code set 0 or 1 */
   1134                 (*q++)=(uint8_t)(value>>16);
   1135                 (*q++)=(uint8_t)(value>>8);
   1136                 (*q++)=(uint8_t)value;
   1137             } else if(value<=0x8effffff) {
   1138                 /* code set 2 */
   1139                 (*q++)=(uint8_t)((value>>16)&0x7f);
   1140                 (*q++)=(uint8_t)(value>>8);
   1141                 (*q++)=(uint8_t)value;
   1142             } else /* first byte is 0x8f */ {
   1143                 /* code set 3 */
   1144                 (*q++)=(uint8_t)(value>>16);
   1145                 (*q++)=(uint8_t)((value>>8)&0x7f);
   1146                 (*q++)=(uint8_t)value;
   1147             }
   1148         }
   1149     }
   1150 
   1151     return TRUE;
   1152 }
   1153 
   1154 /*
   1155  * Compact stage 2 for SBCS by overlapping adjacent stage 2 blocks as far
   1156  * as possible. Overlapping is done on unassigned head and tail
   1157  * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER.
   1158  * Stage 1 indexes need to be adjusted accordingly.
   1159  * This function is very similar to genprops/store.c/compactStage().
   1160  */
   1161 static void
   1162 singleCompactStage2(MBCSData *mbcsData) {
   1163     /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */
   1164     uint16_t map[MBCS_STAGE_2_MAX_BLOCKS];
   1165     uint16_t i, start, prevEnd, newStart;
   1166 
   1167     /* enter the all-unassigned first stage 2 block into the map */
   1168     map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX;
   1169 
   1170     /* begin with the first block after the all-unassigned one */
   1171     start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED;
   1172     while(start<mbcsData->stage2Top) {
   1173         prevEnd=(uint16_t)(newStart-1);
   1174 
   1175         /* find the size of the overlap */
   1176         for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2Single[start+i]==0 && mbcsData->stage2Single[prevEnd-i]==0; ++i) {}
   1177 
   1178         if(i>0) {
   1179             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i);
   1180 
   1181             /* move the non-overlapping indexes to their new positions */
   1182             start+=i;
   1183             for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) {
   1184                 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++];
   1185             }
   1186         } else if(newStart<start) {
   1187             /* move the indexes to their new positions */
   1188             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart;
   1189             for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) {
   1190                 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++];
   1191             }
   1192         } else /* no overlap && newStart==start */ {
   1193             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start;
   1194             start=newStart+=MBCS_STAGE_2_BLOCK_SIZE;
   1195         }
   1196     }
   1197 
   1198     /* adjust stage2Top */
   1199     if(VERBOSE && newStart<mbcsData->stage2Top) {
   1200         printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n",
   1201                 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart,
   1202                 (long)(mbcsData->stage2Top-newStart)*2);
   1203     }
   1204     mbcsData->stage2Top=newStart;
   1205 
   1206     /* now adjust stage 1 */
   1207     for(i=0; i<MBCS_STAGE_1_SIZE; ++i) {
   1208         mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT];
   1209     }
   1210 }
   1211 
   1212 /* Compact stage 3 for SBCS - same algorithm as above. */
   1213 static void
   1214 singleCompactStage3(MBCSData *mbcsData) {
   1215     uint16_t *stage3=(uint16_t *)mbcsData->fromUBytes;
   1216 
   1217     /* this array maps the ordinal number of a stage 3 block to its new stage 2 index */
   1218     uint16_t map[0x1000];
   1219     uint16_t i, start, prevEnd, newStart;
   1220 
   1221     /* enter the all-unassigned first stage 3 block into the map */
   1222     map[0]=0;
   1223 
   1224     /* begin with the first block after the all-unassigned one */
   1225     start=newStart=16;
   1226     while(start<mbcsData->stage3Top) {
   1227         prevEnd=(uint16_t)(newStart-1);
   1228 
   1229         /* find the size of the overlap */
   1230         for(i=0; i<16 && stage3[start+i]==0 && stage3[prevEnd-i]==0; ++i) {}
   1231 
   1232         if(i>0) {
   1233             map[start>>4]=(uint16_t)(newStart-i);
   1234 
   1235             /* move the non-overlapping indexes to their new positions */
   1236             start+=i;
   1237             for(i=(uint16_t)(16-i); i>0; --i) {
   1238                 stage3[newStart++]=stage3[start++];
   1239             }
   1240         } else if(newStart<start) {
   1241             /* move the indexes to their new positions */
   1242             map[start>>4]=newStart;
   1243             for(i=16; i>0; --i) {
   1244                 stage3[newStart++]=stage3[start++];
   1245             }
   1246         } else /* no overlap && newStart==start */ {
   1247             map[start>>4]=start;
   1248             start=newStart+=16;
   1249         }
   1250     }
   1251 
   1252     /* adjust stage3Top */
   1253     if(VERBOSE && newStart<mbcsData->stage3Top) {
   1254         printf("compacting stage 3 from stage3Top=0x%lx to 0x%lx, saving %ld bytes\n",
   1255                 (unsigned long)mbcsData->stage3Top, (unsigned long)newStart,
   1256                 (long)(mbcsData->stage3Top-newStart)*2);
   1257     }
   1258     mbcsData->stage3Top=newStart;
   1259 
   1260     /* now adjust stage 2 */
   1261     for(i=0; i<mbcsData->stage2Top; ++i) {
   1262         mbcsData->stage2Single[i]=map[mbcsData->stage2Single[i]>>4];
   1263     }
   1264 }
   1265 
   1266 /*
   1267  * Compact stage 2 by overlapping adjacent stage 2 blocks as far
   1268  * as possible. Overlapping is done on unassigned head and tail
   1269  * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER.
   1270  * Stage 1 indexes need to be adjusted accordingly.
   1271  * This function is very similar to genprops/store.c/compactStage().
   1272  */
   1273 static void
   1274 compactStage2(MBCSData *mbcsData) {
   1275     /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */
   1276     uint16_t map[MBCS_STAGE_2_MAX_BLOCKS];
   1277     uint16_t i, start, prevEnd, newStart;
   1278 
   1279     /* enter the all-unassigned first stage 2 block into the map */
   1280     map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX;
   1281 
   1282     /* begin with the first block after the all-unassigned one */
   1283     start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED;
   1284     while(start<mbcsData->stage2Top) {
   1285         prevEnd=(uint16_t)(newStart-1);
   1286 
   1287         /* find the size of the overlap */
   1288         for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2[start+i]==0 && mbcsData->stage2[prevEnd-i]==0; ++i) {}
   1289 
   1290         if(i>0) {
   1291             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i);
   1292 
   1293             /* move the non-overlapping indexes to their new positions */
   1294             start+=i;
   1295             for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) {
   1296                 mbcsData->stage2[newStart++]=mbcsData->stage2[start++];
   1297             }
   1298         } else if(newStart<start) {
   1299             /* move the indexes to their new positions */
   1300             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart;
   1301             for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) {
   1302                 mbcsData->stage2[newStart++]=mbcsData->stage2[start++];
   1303             }
   1304         } else /* no overlap && newStart==start */ {
   1305             map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start;
   1306             start=newStart+=MBCS_STAGE_2_BLOCK_SIZE;
   1307         }
   1308     }
   1309 
   1310     /* adjust stage2Top */
   1311     if(VERBOSE && newStart<mbcsData->stage2Top) {
   1312         printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n",
   1313                 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart,
   1314                 (long)(mbcsData->stage2Top-newStart)*4);
   1315     }
   1316     mbcsData->stage2Top=newStart;
   1317 
   1318     /* now adjust stage 1 */
   1319     for(i=0; i<MBCS_STAGE_1_SIZE; ++i) {
   1320         mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT];
   1321     }
   1322 }
   1323 
   1324 static void
   1325 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData *staticData) {
   1326     UCMStates *states;
   1327     int32_t maxCharLength, stage3Width;
   1328 
   1329     states=&mbcsData->ucm->states;
   1330     stage3Width=maxCharLength=states->maxCharLength;
   1331 
   1332     ucm_optimizeStates(states,
   1333                        &mbcsData->unicodeCodeUnits,
   1334                        mbcsData->toUFallbacks, mbcsData->countToUFallbacks,
   1335                        VERBOSE);
   1336 
   1337     /* try to compact the fromUnicode tables */
   1338     if(transformEUC(mbcsData)) {
   1339         --stage3Width;
   1340     }
   1341 
   1342     /*
   1343      * UTF-8-friendly tries are built precompacted, to cope with variable
   1344      * stage 3 allocation block sizes.
   1345      *
   1346      * Tables without precision indicators cannot be built that way,
   1347      * because if a block was overlapped with a previous one, then a smaller
   1348      * code point for the same block would not fit.
   1349      * Therefore, such tables are not marked UTF-8-friendly and must be
   1350      * compacted after all mappings are entered.
   1351      */
   1352     if(!mbcsData->utf8Friendly) {
   1353         if(maxCharLength==1) {
   1354             singleCompactStage3(mbcsData);
   1355             singleCompactStage2(mbcsData);
   1356         } else {
   1357             compactStage2(mbcsData);
   1358         }
   1359     }
   1360 
   1361     if(VERBOSE) {
   1362         /*uint32_t c, i1, i2, i2Limit, i3;*/
   1363 
   1364         printf("fromUnicode number of uint%s_t in stage 2: 0x%lx=%lu\n",
   1365                maxCharLength==1 ? "16" : "32",
   1366                (unsigned long)mbcsData->stage2Top,
   1367                (unsigned long)mbcsData->stage2Top);
   1368         printf("fromUnicode number of %d-byte stage 3 mapping entries: 0x%lx=%lu\n",
   1369                (int)stage3Width,
   1370                (unsigned long)mbcsData->stage3Top/stage3Width,
   1371                (unsigned long)mbcsData->stage3Top/stage3Width);
   1372 #if 0
   1373         c=0;
   1374         for(i1=0; i1<MBCS_STAGE_1_SIZE; ++i1) {
   1375             i2=mbcsData->stage1[i1];
   1376             if(i2==0) {
   1377                 c+=MBCS_STAGE_2_BLOCK_SIZE*MBCS_STAGE_3_BLOCK_SIZE;
   1378                 continue;
   1379             }
   1380             for(i2Limit=i2+MBCS_STAGE_2_BLOCK_SIZE; i2<i2Limit; ++i2) {
   1381                 if(maxCharLength==1) {
   1382                     i3=mbcsData->stage2Single[i2];
   1383                 } else {
   1384                     i3=(uint16_t)mbcsData->stage2[i2];
   1385                 }
   1386                 if(i3==0) {
   1387                     c+=MBCS_STAGE_3_BLOCK_SIZE;
   1388                     continue;
   1389                 }
   1390                 printf("U+%04lx i1=0x%02lx i2=0x%04lx i3=0x%04lx\n",
   1391                        (unsigned long)c,
   1392                        (unsigned long)i1,
   1393                        (unsigned long)i2,
   1394                        (unsigned long)i3);
   1395                 c+=MBCS_STAGE_3_BLOCK_SIZE;
   1396             }
   1397         }
   1398 #endif
   1399     }
   1400 }
   1401 
   1402 static uint32_t
   1403 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData,
   1404           UNewDataMemory *pData, int32_t tableType) {
   1405     MBCSData *mbcsData=(MBCSData *)cnvData;
   1406     uint32_t stage2Start, stage2Length;
   1407     uint32_t top, stageUTF8Length=0;
   1408     int32_t i, stage1Top;
   1409     uint32_t headerLength;
   1410 
   1411     _MBCSHeader header={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0 };
   1412 
   1413     stage2Length=mbcsData->stage2Top;
   1414     if(mbcsData->omitFromU) {
   1415         /* find how much of stage2 can be omitted */
   1416         int32_t utf8Limit=(int32_t)mbcsData->utf8Max+1;
   1417         uint32_t st2=0; /*initialized it to avoid compiler warnings */
   1418 
   1419         i=utf8Limit>>MBCS_STAGE_1_SHIFT;
   1420         if((utf8Limit&((1<<MBCS_STAGE_1_SHIFT)-1))!=0 && (st2=mbcsData->stage1[i])!=0) {
   1421             /* utf8Limit is in the middle of an existing stage 2 block */
   1422             stage2Start=st2+((utf8Limit>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK);
   1423         } else {
   1424             /* find the last stage2 block with mappings before utf8Limit */
   1425             while(i>0 && (st2=mbcsData->stage1[--i])==0) {}
   1426             /* stage2 up to the end of this block corresponds to stageUTF8 */
   1427             stage2Start=st2+MBCS_STAGE_2_BLOCK_SIZE;
   1428         }
   1429         header.options|=MBCS_OPT_NO_FROM_U;
   1430         header.fullStage2Length=stage2Length;
   1431         stage2Length-=stage2Start;
   1432         if(VERBOSE) {
   1433             printf("+ omitting %lu out of %lu stage2 entries and %lu fromUBytes\n",
   1434                     (unsigned long)stage2Start,
   1435                     (unsigned long)mbcsData->stage2Top,
   1436                     (unsigned long)mbcsData->stage3Top);
   1437             printf("+ total size savings: %lu bytes\n", (unsigned long)stage2Start*4+mbcsData->stage3Top);
   1438         }
   1439     } else {
   1440         stage2Start=0;
   1441     }
   1442 
   1443     if(staticData->unicodeMask&UCNV_HAS_SUPPLEMENTARY) {
   1444         stage1Top=MBCS_STAGE_1_SIZE; /* 0x440==1088 */
   1445     } else {
   1446         stage1Top=0x40; /* 0x40==64 */
   1447     }
   1448 
   1449     /* adjust stage 1 entries to include the size of stage 1 in the offsets to stage 2 */
   1450     if(mbcsData->ucm->states.maxCharLength==1) {
   1451         for(i=0; i<stage1Top; ++i) {
   1452             mbcsData->stage1[i]+=(uint16_t)stage1Top;
   1453         }
   1454 
   1455         /* stage2Top/Length have counted 16-bit results, now we need to count bytes */
   1456         /* also round up to a multiple of 4 bytes */
   1457         stage2Length=(stage2Length*2+1)&~1;
   1458 
   1459         /* stage3Top has counted 16-bit results, now we need to count bytes */
   1460         mbcsData->stage3Top*=2;
   1461 
   1462         if(mbcsData->utf8Friendly) {
   1463             header.version[2]=(uint8_t)(SBCS_UTF8_MAX>>8); /* store 0x1f for max==0x1fff */
   1464         }
   1465     } else {
   1466         for(i=0; i<stage1Top; ++i) {
   1467             mbcsData->stage1[i]+=(uint16_t)stage1Top/2; /* stage 2 contains 32-bit entries, stage 1 16-bit entries */
   1468         }
   1469 
   1470         /* stage2Top/Length have counted 32-bit results, now we need to count bytes */
   1471         stage2Length*=4;
   1472         /* leave stage2Start counting 32-bit units */
   1473 
   1474         if(mbcsData->utf8Friendly) {
   1475             stageUTF8Length=(mbcsData->utf8Max+1)>>MBCS_UTF8_STAGE_SHIFT;
   1476             header.version[2]=(uint8_t)(mbcsData->utf8Max>>8); /* store 0xd7 for max==0xd7ff */
   1477         }
   1478 
   1479         /* stage3Top has already counted bytes */
   1480     }
   1481 
   1482     /* round up stage3Top so that the sizes of all data blocks are multiples of 4 */
   1483     mbcsData->stage3Top=(mbcsData->stage3Top+3)&~3;
   1484 
   1485     /* fill the header */
   1486     if(header.options&MBCS_OPT_INCOMPATIBLE_MASK) {
   1487         header.version[0]=5;
   1488         if(header.options&MBCS_OPT_NO_FROM_U) {
   1489             headerLength=10;  /* include fullStage2Length */
   1490         } else {
   1491             headerLength=MBCS_HEADER_V5_MIN_LENGTH;  /* 9 */
   1492         }
   1493     } else {
   1494         header.version[0]=4;
   1495         headerLength=MBCS_HEADER_V4_LENGTH;  /* 8 */
   1496     }
   1497     header.version[1]=3;
   1498     /* header.version[2] set above for utf8Friendly data */
   1499 
   1500     header.options|=(uint32_t)headerLength;
   1501 
   1502     header.countStates=mbcsData->ucm->states.countStates;
   1503     header.countToUFallbacks=mbcsData->countToUFallbacks;
   1504 
   1505     header.offsetToUCodeUnits=
   1506         headerLength*4+
   1507         mbcsData->ucm->states.countStates*1024+
   1508         mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback);
   1509     header.offsetFromUTable=
   1510         header.offsetToUCodeUnits+
   1511         mbcsData->ucm->states.countToUCodeUnits*2;
   1512     header.offsetFromUBytes=
   1513         header.offsetFromUTable+
   1514         stage1Top*2+
   1515         stage2Length;
   1516     header.fromUBytesLength=mbcsData->stage3Top;
   1517 
   1518     top=header.offsetFromUBytes+stageUTF8Length*2;
   1519     if(!(header.options&MBCS_OPT_NO_FROM_U)) {
   1520         top+=header.fromUBytesLength;
   1521     }
   1522 
   1523     header.flags=(uint8_t)(mbcsData->ucm->states.outputType);
   1524 
   1525     if(tableType&TABLE_EXT) {
   1526         if(top>0xffffff) {
   1527             fprintf(stderr, "error: offset 0x%lx to extension table exceeds 0xffffff\n", (long)top);
   1528             return 0;
   1529         }
   1530 
   1531         header.flags|=top<<8;
   1532     }
   1533 
   1534     /* write the MBCS data */
   1535     udata_writeBlock(pData, &header, headerLength*4);
   1536     udata_writeBlock(pData, mbcsData->ucm->states.stateTable, header.countStates*1024);
   1537     udata_writeBlock(pData, mbcsData->toUFallbacks, mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback));
   1538     udata_writeBlock(pData, mbcsData->unicodeCodeUnits, mbcsData->ucm->states.countToUCodeUnits*2);
   1539     udata_writeBlock(pData, mbcsData->stage1, stage1Top*2);
   1540     if(mbcsData->ucm->states.maxCharLength==1) {
   1541         udata_writeBlock(pData, mbcsData->stage2Single+stage2Start, stage2Length);
   1542     } else {
   1543         udata_writeBlock(pData, mbcsData->stage2+stage2Start, stage2Length);
   1544     }
   1545     if(!(header.options&MBCS_OPT_NO_FROM_U)) {
   1546         udata_writeBlock(pData, mbcsData->fromUBytes, mbcsData->stage3Top);
   1547     }
   1548 
   1549     if(stageUTF8Length>0) {
   1550         udata_writeBlock(pData, mbcsData->stageUTF8, stageUTF8Length*2);
   1551     }
   1552 
   1553     /* return the number of bytes that should have been written */
   1554     return top;
   1555 }
   1556