1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ******************************************************************************* 5 * 6 * Copyright (C) 2000-2016, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ******************************************************************************* 10 * file name: genmbcs.cpp 11 * encoding: US-ASCII 12 * tab size: 8 (not used) 13 * indentation:4 14 * 15 * created on: 2000jul06 16 * created by: Markus W. Scherer 17 */ 18 19 #include <stdio.h> 20 #include "unicode/utypes.h" 21 #include "cstring.h" 22 #include "cmemory.h" 23 #include "unewdata.h" 24 #include "ucnv_cnv.h" 25 #include "ucnvmbcs.h" 26 #include "ucm.h" 27 #include "makeconv.h" 28 #include "genmbcs.h" 29 30 /* 31 * TODO: Split this file into toUnicode, SBCSFromUnicode and MBCSFromUnicode files. 32 * Reduce tests for maxCharLength. 33 */ 34 35 struct MBCSData { 36 NewConverter newConverter; 37 38 UCMFile *ucm; 39 40 /* toUnicode (state table in ucm->states) */ 41 _MBCSToUFallback toUFallbacks[MBCS_MAX_FALLBACK_COUNT]; 42 int32_t countToUFallbacks; 43 uint16_t *unicodeCodeUnits; 44 45 /* fromUnicode */ 46 uint16_t stage1[MBCS_STAGE_1_SIZE]; 47 uint16_t stage2Single[MBCS_STAGE_2_SIZE]; /* stage 2 for single-byte codepages */ 48 uint32_t stage2[MBCS_STAGE_2_SIZE]; /* stage 2 for MBCS */ 49 uint8_t *fromUBytes; 50 uint32_t stage2Top, stage3Top; 51 52 /* fromUTF8 */ 53 uint16_t stageUTF8[0x10000>>MBCS_UTF8_STAGE_SHIFT]; /* allow for utf8Max=0xffff */ 54 55 /* 56 * Maximum UTF-8-friendly code point. 57 * 0 if !utf8Friendly, otherwise 0x01ff..0xffff in steps of 0x100. 58 * If utf8Friendly, utf8Max is normally either MBCS_UTF8_MAX or 0xffff. 59 */ 60 uint16_t utf8Max; 61 62 UBool utf8Friendly; 63 UBool omitFromU; 64 }; 65 66 /* prototypes */ 67 static void 68 MBCSClose(NewConverter *cnvData); 69 70 static UBool 71 MBCSStartMappings(MBCSData *mbcsData); 72 73 static UBool 74 MBCSAddToUnicode(MBCSData *mbcsData, 75 const uint8_t *bytes, int32_t length, 76 UChar32 c, 77 int8_t flag); 78 79 static UBool 80 MBCSIsValid(NewConverter *cnvData, 81 const uint8_t *bytes, int32_t length); 82 83 static UBool 84 MBCSSingleAddFromUnicode(MBCSData *mbcsData, 85 const uint8_t *bytes, int32_t length, 86 UChar32 c, 87 int8_t flag); 88 89 static UBool 90 MBCSAddFromUnicode(MBCSData *mbcsData, 91 const uint8_t *bytes, int32_t length, 92 UChar32 c, 93 int8_t flag); 94 95 static void 96 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData *staticData); 97 98 static UBool 99 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData); 100 101 static uint32_t 102 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData, 103 UNewDataMemory *pData, int32_t tableType); 104 105 /* helper ------------------------------------------------------------------- */ 106 107 static inline char 108 hexDigit(uint8_t digit) { 109 return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit); 110 } 111 112 static inline char * 113 printBytes(char *buffer, const uint8_t *bytes, int32_t length) { 114 char *s=buffer; 115 while(length>0) { 116 *s++=hexDigit((uint8_t)(*bytes>>4)); 117 *s++=hexDigit((uint8_t)(*bytes&0xf)); 118 ++bytes; 119 --length; 120 } 121 122 *s=0; 123 return buffer; 124 } 125 126 /* implementation ----------------------------------------------------------- */ 127 128 static MBCSData gDummy; 129 130 U_CFUNC const MBCSData * 131 MBCSGetDummy() { 132 uprv_memset(&gDummy, 0, sizeof(MBCSData)); 133 134 /* 135 * Set "pessimistic" values which may sometimes move too many 136 * mappings to the extension table (but never too few). 137 * These values cause MBCSOkForBaseFromUnicode() to return FALSE for the 138 * largest set of mappings. 139 * Assume maxCharLength>1. 140 */ 141 gDummy.utf8Friendly=TRUE; 142 if(SMALL) { 143 gDummy.utf8Max=0xffff; 144 gDummy.omitFromU=TRUE; 145 } else { 146 gDummy.utf8Max=MBCS_UTF8_MAX; 147 } 148 return &gDummy; 149 } 150 151 static void 152 MBCSInit(MBCSData *mbcsData, UCMFile *ucm) { 153 uprv_memset(mbcsData, 0, sizeof(MBCSData)); 154 155 mbcsData->ucm=ucm; /* aliased, not owned */ 156 157 mbcsData->newConverter.close=MBCSClose; 158 mbcsData->newConverter.isValid=MBCSIsValid; 159 mbcsData->newConverter.addTable=MBCSAddTable; 160 mbcsData->newConverter.write=MBCSWrite; 161 } 162 163 NewConverter * 164 MBCSOpen(UCMFile *ucm) { 165 MBCSData *mbcsData=(MBCSData *)uprv_malloc(sizeof(MBCSData)); 166 if(mbcsData==NULL) { 167 printf("out of memory\n"); 168 exit(U_MEMORY_ALLOCATION_ERROR); 169 } 170 171 MBCSInit(mbcsData, ucm); 172 return &mbcsData->newConverter; 173 } 174 175 static void 176 MBCSDestruct(MBCSData *mbcsData) { 177 uprv_free(mbcsData->unicodeCodeUnits); 178 uprv_free(mbcsData->fromUBytes); 179 } 180 181 static void 182 MBCSClose(NewConverter *cnvData) { 183 MBCSData *mbcsData=(MBCSData *)cnvData; 184 if(mbcsData!=NULL) { 185 MBCSDestruct(mbcsData); 186 uprv_free(mbcsData); 187 } 188 } 189 190 static UBool 191 MBCSStartMappings(MBCSData *mbcsData) { 192 int32_t i, sum, maxCharLength, 193 stage2NullLength, stage2AllocLength, 194 stage3NullLength, stage3AllocLength; 195 196 /* toUnicode */ 197 198 /* allocate the code unit array and prefill it with "unassigned" values */ 199 sum=mbcsData->ucm->states.countToUCodeUnits; 200 if(VERBOSE) { 201 printf("the total number of offsets is 0x%lx=%ld\n", (long)sum, (long)sum); 202 } 203 204 if(sum>0) { 205 mbcsData->unicodeCodeUnits=(uint16_t *)uprv_malloc(sum*sizeof(uint16_t)); 206 if(mbcsData->unicodeCodeUnits==NULL) { 207 fprintf(stderr, "error: out of memory allocating %ld 16-bit code units\n", 208 (long)sum); 209 return FALSE; 210 } 211 for(i=0; i<sum; ++i) { 212 mbcsData->unicodeCodeUnits[i]=0xfffe; 213 } 214 } 215 216 /* fromUnicode */ 217 maxCharLength=mbcsData->ucm->states.maxCharLength; 218 219 /* allocate the codepage mappings and preset the first 16 characters to 0 */ 220 if(maxCharLength==1) { 221 /* allocate 64k 16-bit results for single-byte codepages */ 222 sum=0x20000; 223 } else { 224 /* allocate 1M * maxCharLength bytes for at most 1M mappings */ 225 sum=0x100000*maxCharLength; 226 } 227 mbcsData->fromUBytes=(uint8_t *)uprv_malloc(sum); 228 if(mbcsData->fromUBytes==NULL) { 229 fprintf(stderr, "error: out of memory allocating %ld B for target mappings\n", (long)sum); 230 return FALSE; 231 } 232 uprv_memset(mbcsData->fromUBytes, 0, sum); 233 234 /* 235 * UTF-8-friendly fromUnicode tries: allocate multiple blocks at a time. 236 * See ucnvmbcs.h for details. 237 * 238 * There is code, for example in ucnv_MBCSGetUnicodeSetForUnicode(), which 239 * assumes that the initial stage 2/3 blocks are the all-unassigned ones. 240 * Therefore, we refine the data structure while maintaining this placement 241 * even though it would be convenient to allocate the ASCII block at the 242 * beginning of stage 3, for example. 243 * 244 * UTF-8-friendly fromUnicode tries work from sorted tables and are built 245 * pre-compacted, overlapping adjacent stage 2/3 blocks. 246 * This is necessary because the block allocation and compaction changes 247 * at SBCS_UTF8_MAX or MBCS_UTF8_MAX, and for MBCS tables the additional 248 * stage table uses direct indexes into stage 3, without a multiplier and 249 * thus with a smaller reach. 250 * 251 * Non-UTF-8-friendly fromUnicode tries work from unsorted tables 252 * (because implicit precision is used), and are compacted 253 * in post-processing. 254 * 255 * Preallocation for UTF-8-friendly fromUnicode tries: 256 * 257 * Stage 3: 258 * 64-entry all-unassigned first block followed by ASCII (128 entries). 259 * 260 * Stage 2: 261 * 64-entry all-unassigned first block followed by preallocated 262 * 64-block for ASCII. 263 */ 264 265 /* Preallocate ASCII as a linear 128-entry stage 3 block. */ 266 stage2NullLength=MBCS_STAGE_2_BLOCK_SIZE; 267 stage2AllocLength=MBCS_STAGE_2_BLOCK_SIZE; 268 269 stage3NullLength=MBCS_UTF8_STAGE_3_BLOCK_SIZE; 270 stage3AllocLength=128; /* ASCII U+0000..U+007f */ 271 272 /* Initialize stage 1 for the preallocated blocks. */ 273 sum=stage2NullLength; 274 for(i=0; i<(stage2AllocLength>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT); ++i) { 275 mbcsData->stage1[i]=sum; 276 sum+=MBCS_STAGE_2_BLOCK_SIZE; 277 } 278 mbcsData->stage2Top=stage2NullLength+stage2AllocLength; /* ==sum */ 279 280 /* 281 * Stage 2 indexes count 16-blocks in stage 3 as follows: 282 * SBCS: directly, indexes increment by 16 283 * MBCS: indexes need to be multiplied by 16*maxCharLength, indexes increment by 1 284 * MBCS UTF-8: directly, indexes increment by 16 285 */ 286 if(maxCharLength==1) { 287 sum=stage3NullLength; 288 for(i=0; i<(stage3AllocLength/MBCS_STAGE_3_BLOCK_SIZE); ++i) { 289 mbcsData->stage2Single[mbcsData->stage1[0]+i]=sum; 290 sum+=MBCS_STAGE_3_BLOCK_SIZE; 291 } 292 } else { 293 sum=stage3NullLength/MBCS_STAGE_3_GRANULARITY; 294 for(i=0; i<(stage3AllocLength/MBCS_STAGE_3_BLOCK_SIZE); ++i) { 295 mbcsData->stage2[mbcsData->stage1[0]+i]=sum; 296 sum+=MBCS_STAGE_3_BLOCK_SIZE/MBCS_STAGE_3_GRANULARITY; 297 } 298 } 299 300 sum=stage3NullLength; 301 for(i=0; i<(stage3AllocLength/MBCS_UTF8_STAGE_3_BLOCK_SIZE); ++i) { 302 mbcsData->stageUTF8[i]=sum; 303 sum+=MBCS_UTF8_STAGE_3_BLOCK_SIZE; 304 } 305 306 /* 307 * Allocate a 64-entry all-unassigned first stage 3 block, 308 * for UTF-8-friendly lookup with a trail byte, 309 * plus 128 entries for ASCII. 310 */ 311 mbcsData->stage3Top=(stage3NullLength+stage3AllocLength)*maxCharLength; /* ==sum*maxCharLength */ 312 313 return TRUE; 314 } 315 316 /* return TRUE for success */ 317 static UBool 318 setFallback(MBCSData *mbcsData, uint32_t offset, UChar32 c) { 319 int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset); 320 if(i>=0) { 321 /* if there is already a fallback for this offset, then overwrite it */ 322 mbcsData->toUFallbacks[i].codePoint=c; 323 return TRUE; 324 } else { 325 /* if there is no fallback for this offset, then add one */ 326 i=mbcsData->countToUFallbacks; 327 if(i>=MBCS_MAX_FALLBACK_COUNT) { 328 fprintf(stderr, "error: too many toUnicode fallbacks, currently at: U+%x\n", (int)c); 329 return FALSE; 330 } else { 331 mbcsData->toUFallbacks[i].offset=offset; 332 mbcsData->toUFallbacks[i].codePoint=c; 333 mbcsData->countToUFallbacks=i+1; 334 return TRUE; 335 } 336 } 337 } 338 339 /* remove fallback if there is one with this offset; return the code point if there was such a fallback, otherwise -1 */ 340 static int32_t 341 removeFallback(MBCSData *mbcsData, uint32_t offset) { 342 int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset); 343 if(i>=0) { 344 _MBCSToUFallback *toUFallbacks; 345 int32_t limit, old; 346 347 toUFallbacks=mbcsData->toUFallbacks; 348 limit=mbcsData->countToUFallbacks; 349 old=(int32_t)toUFallbacks[i].codePoint; 350 351 /* copy the last fallback entry here to keep the list contiguous */ 352 toUFallbacks[i].offset=toUFallbacks[limit-1].offset; 353 toUFallbacks[i].codePoint=toUFallbacks[limit-1].codePoint; 354 mbcsData->countToUFallbacks=limit-1; 355 return old; 356 } else { 357 return -1; 358 } 359 } 360 361 /* 362 * isFallback is almost a boolean: 363 * 1 (TRUE) this is a fallback mapping 364 * 0 (FALSE) this is a precise mapping 365 * -1 the precision of this mapping is not specified 366 */ 367 static UBool 368 MBCSAddToUnicode(MBCSData *mbcsData, 369 const uint8_t *bytes, int32_t length, 370 UChar32 c, 371 int8_t flag) { 372 char buffer[10]; 373 uint32_t offset=0; 374 int32_t i=0, entry, old; 375 uint8_t state=0; 376 377 if(mbcsData->ucm->states.countStates==0) { 378 fprintf(stderr, "error: there is no state information!\n"); 379 return FALSE; 380 } 381 382 /* for SI/SO (like EBCDIC-stateful), double-byte sequences start in state 1 */ 383 if(length==2 && mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO) { 384 state=1; 385 } 386 387 /* 388 * Walk down the state table like in conversion, 389 * much like getNextUChar(). 390 * We assume that c<=0x10ffff. 391 */ 392 for(i=0;;) { 393 entry=mbcsData->ucm->states.stateTable[state][bytes[i++]]; 394 if(MBCS_ENTRY_IS_TRANSITION(entry)) { 395 if(i==length) { 396 fprintf(stderr, "error: byte sequence too short, ends in non-final state %hu: 0x%s (U+%x)\n", 397 (short)state, printBytes(buffer, bytes, length), (int)c); 398 return FALSE; 399 } 400 state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); 401 offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); 402 } else { 403 if(i<length) { 404 fprintf(stderr, "error: byte sequence too long by %d bytes, final state %u: 0x%s (U+%x)\n", 405 (int)(length-i), state, printBytes(buffer, bytes, length), (int)c); 406 return FALSE; 407 } 408 switch(MBCS_ENTRY_FINAL_ACTION(entry)) { 409 case MBCS_STATE_ILLEGAL: 410 fprintf(stderr, "error: byte sequence ends in illegal state at U+%04x<->0x%s\n", 411 (int)c, printBytes(buffer, bytes, length)); 412 return FALSE; 413 case MBCS_STATE_CHANGE_ONLY: 414 fprintf(stderr, "error: byte sequence ends in state-change-only at U+%04x<->0x%s\n", 415 (int)c, printBytes(buffer, bytes, length)); 416 return FALSE; 417 case MBCS_STATE_UNASSIGNED: 418 fprintf(stderr, "error: byte sequence ends in unassigned state at U+%04x<->0x%s\n", 419 (int)c, printBytes(buffer, bytes, length)); 420 return FALSE; 421 case MBCS_STATE_FALLBACK_DIRECT_16: 422 case MBCS_STATE_VALID_DIRECT_16: 423 case MBCS_STATE_FALLBACK_DIRECT_20: 424 case MBCS_STATE_VALID_DIRECT_20: 425 if(MBCS_ENTRY_SET_STATE(entry, 0)!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, 0xfffe)) { 426 /* the "direct" action's value is not "valid-direct-16-unassigned" any more */ 427 if(MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_DIRECT_16 || MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_FALLBACK_DIRECT_16) { 428 old=MBCS_ENTRY_FINAL_VALUE(entry); 429 } else { 430 old=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); 431 } 432 if(flag>=0) { 433 fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 434 (int)c, printBytes(buffer, bytes, length), (int)old); 435 return FALSE; 436 } else if(VERBOSE) { 437 fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 438 (int)c, printBytes(buffer, bytes, length), (int)old); 439 } 440 /* 441 * Continue after the above warning 442 * if the precision of the mapping is unspecified. 443 */ 444 } 445 /* reassign the correct action code */ 446 entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, (MBCS_STATE_VALID_DIRECT_16+(flag==3 ? 2 : 0)+(c>=0x10000 ? 1 : 0))); 447 448 /* put the code point into bits 22..7 for BMP, c-0x10000 into 26..7 for others */ 449 if(c<=0xffff) { 450 entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c); 451 } else { 452 entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c-0x10000); 453 } 454 mbcsData->ucm->states.stateTable[state][bytes[i-1]]=entry; 455 break; 456 case MBCS_STATE_VALID_16: 457 /* bits 26..16 are not used, 0 */ 458 /* bits 15..7 contain the final offset delta to one 16-bit code unit */ 459 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); 460 /* check that this byte sequence is still unassigned */ 461 if((old=mbcsData->unicodeCodeUnits[offset])!=0xfffe || (old=removeFallback(mbcsData, offset))!=-1) { 462 if(flag>=0) { 463 fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 464 (int)c, printBytes(buffer, bytes, length), (int)old); 465 return FALSE; 466 } else if(VERBOSE) { 467 fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 468 (int)c, printBytes(buffer, bytes, length), (int)old); 469 } 470 } 471 if(c>=0x10000) { 472 fprintf(stderr, "error: code point does not fit into valid-16-bit state at U+%04x<->0x%s\n", 473 (int)c, printBytes(buffer, bytes, length)); 474 return FALSE; 475 } 476 if(flag>0) { 477 /* assign only if there is no precise mapping */ 478 if(mbcsData->unicodeCodeUnits[offset]==0xfffe) { 479 return setFallback(mbcsData, offset, c); 480 } 481 } else { 482 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 483 } 484 break; 485 case MBCS_STATE_VALID_16_PAIR: 486 /* bits 26..16 are not used, 0 */ 487 /* bits 15..7 contain the final offset delta to two 16-bit code units */ 488 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); 489 /* check that this byte sequence is still unassigned */ 490 old=mbcsData->unicodeCodeUnits[offset]; 491 if(old<0xfffe) { 492 int32_t real; 493 if(old<0xd800) { 494 real=old; 495 } else if(old<=0xdfff) { 496 real=0x10000+((old&0x3ff)<<10)+((mbcsData->unicodeCodeUnits[offset+1])&0x3ff); 497 } else /* old<=0xe001 */ { 498 real=mbcsData->unicodeCodeUnits[offset+1]; 499 } 500 if(flag>=0) { 501 fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 502 (int)c, printBytes(buffer, bytes, length), (int)real); 503 return FALSE; 504 } else if(VERBOSE) { 505 fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 506 (int)c, printBytes(buffer, bytes, length), (int)real); 507 } 508 } 509 if(flag>0) { 510 /* assign only if there is no precise mapping */ 511 if(old<=0xdbff || old==0xe000) { 512 /* do nothing */ 513 } else if(c<=0xffff) { 514 /* set a BMP fallback code point as a pair with 0xe001 */ 515 mbcsData->unicodeCodeUnits[offset++]=0xe001; 516 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 517 } else { 518 /* set a fallback surrogate pair with two second surrogates */ 519 mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xdbc0+(c>>10)); 520 mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff)); 521 } 522 } else { 523 if(c<0xd800) { 524 /* set a BMP code point */ 525 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 526 } else if(c<=0xffff) { 527 /* set a BMP code point above 0xd800 as a pair with 0xe000 */ 528 mbcsData->unicodeCodeUnits[offset++]=0xe000; 529 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 530 } else { 531 /* set a surrogate pair */ 532 mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xd7c0+(c>>10)); 533 mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff)); 534 } 535 } 536 break; 537 default: 538 /* reserved, must never occur */ 539 fprintf(stderr, "internal error: byte sequence reached reserved action code, entry 0x%02x: 0x%s (U+%x)\n", 540 (int)entry, printBytes(buffer, bytes, length), (int)c); 541 return FALSE; 542 } 543 544 return TRUE; 545 } 546 } 547 } 548 549 /* is this byte sequence valid? (this is almost the same as MBCSAddToUnicode()) */ 550 static UBool 551 MBCSIsValid(NewConverter *cnvData, 552 const uint8_t *bytes, int32_t length) { 553 MBCSData *mbcsData=(MBCSData *)cnvData; 554 555 return (UBool)(1==ucm_countChars(&mbcsData->ucm->states, bytes, length)); 556 } 557 558 static UBool 559 MBCSSingleAddFromUnicode(MBCSData *mbcsData, 560 const uint8_t *bytes, int32_t /*length*/, 561 UChar32 c, 562 int8_t flag) { 563 uint16_t *stage3, *p; 564 uint32_t idx; 565 uint16_t old; 566 uint8_t b; 567 568 uint32_t blockSize, newTop, i, nextOffset, newBlock, min; 569 570 /* ignore |2 SUB mappings */ 571 if(flag==2) { 572 return TRUE; 573 } 574 575 /* 576 * Walk down the triple-stage compact array ("trie") and 577 * allocate parts as necessary. 578 * Note that the first stage 2 and 3 blocks are reserved for all-unassigned mappings. 579 * We assume that length<=maxCharLength and that c<=0x10ffff. 580 */ 581 stage3=(uint16_t *)mbcsData->fromUBytes; 582 b=*bytes; 583 584 /* inspect stage 1 */ 585 idx=c>>MBCS_STAGE_1_SHIFT; 586 if(mbcsData->utf8Friendly && c<=SBCS_UTF8_MAX) { 587 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK&~(MBCS_UTF8_STAGE_3_BLOCKS-1); 588 } else { 589 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK; 590 } 591 if(mbcsData->stage1[idx]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) { 592 /* allocate another block in stage 2 */ 593 newBlock=mbcsData->stage2Top; 594 if(mbcsData->utf8Friendly) { 595 min=newBlock-nextOffset; /* minimum block start with overlap */ 596 while(min<newBlock && mbcsData->stage2Single[newBlock-1]==0) { 597 --newBlock; 598 } 599 } 600 newTop=newBlock+MBCS_STAGE_2_BLOCK_SIZE; 601 602 if(newTop>MBCS_MAX_STAGE_2_TOP) { 603 fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%02x\n", (int)c, b); 604 return FALSE; 605 } 606 607 /* 608 * each stage 2 block contains 64 16-bit words: 609 * 6 code point bits 9..4 with 1 stage 3 index 610 */ 611 mbcsData->stage1[idx]=(uint16_t)newBlock; 612 mbcsData->stage2Top=newTop; 613 } 614 615 /* inspect stage 2 */ 616 idx=mbcsData->stage1[idx]+nextOffset; 617 if(mbcsData->utf8Friendly && c<=SBCS_UTF8_MAX) { 618 /* allocate 64-entry blocks for UTF-8-friendly lookup */ 619 blockSize=MBCS_UTF8_STAGE_3_BLOCK_SIZE; 620 nextOffset=c&MBCS_UTF8_STAGE_3_BLOCK_MASK; 621 } else { 622 blockSize=MBCS_STAGE_3_BLOCK_SIZE; 623 nextOffset=c&MBCS_STAGE_3_BLOCK_MASK; 624 } 625 if(mbcsData->stage2Single[idx]==0) { 626 /* allocate another block in stage 3 */ 627 newBlock=mbcsData->stage3Top; 628 if(mbcsData->utf8Friendly) { 629 min=newBlock-nextOffset; /* minimum block start with overlap */ 630 while(min<newBlock && stage3[newBlock-1]==0) { 631 --newBlock; 632 } 633 } 634 newTop=newBlock+blockSize; 635 636 if(newTop>MBCS_STAGE_3_SBCS_SIZE) { 637 fprintf(stderr, "error: too many code points at U+%04x<->0x%02x\n", (int)c, b); 638 return FALSE; 639 } 640 /* each block has 16 uint16_t entries */ 641 i=idx; 642 while(newBlock<newTop) { 643 mbcsData->stage2Single[i++]=(uint16_t)newBlock; 644 newBlock+=MBCS_STAGE_3_BLOCK_SIZE; 645 } 646 mbcsData->stage3Top=newTop; /* ==newBlock */ 647 } 648 649 /* write the codepage entry into stage 3 and get the previous entry */ 650 p=stage3+mbcsData->stage2Single[idx]+nextOffset; 651 old=*p; 652 if(flag<=0) { 653 *p=(uint16_t)(0xf00|b); 654 } else if(IS_PRIVATE_USE(c)) { 655 *p=(uint16_t)(0xc00|b); 656 } else { 657 *p=(uint16_t)(0x800|b); 658 } 659 660 /* check that this Unicode code point was still unassigned */ 661 if(old>=0x100) { 662 if(flag>=0) { 663 fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n", 664 (int)c, b, old&0xff); 665 return FALSE; 666 } else if(VERBOSE) { 667 fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n", 668 (int)c, b, old&0xff); 669 } 670 /* continue after the above warning if the precision of the mapping is unspecified */ 671 } 672 673 return TRUE; 674 } 675 676 static UBool 677 MBCSAddFromUnicode(MBCSData *mbcsData, 678 const uint8_t *bytes, int32_t length, 679 UChar32 c, 680 int8_t flag) { 681 char buffer[10]; 682 const uint8_t *pb; 683 uint8_t *stage3, *p; 684 uint32_t idx, b, old, stage3Index; 685 int32_t maxCharLength; 686 687 uint32_t blockSize, newTop, i, nextOffset, newBlock, min, overlap, maxOverlap; 688 689 maxCharLength=mbcsData->ucm->states.maxCharLength; 690 691 if( mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO && 692 (!IGNORE_SISO_CHECK && (*bytes==0xe || *bytes==0xf)) 693 ) { 694 fprintf(stderr, "error: illegal mapping to SI or SO for SI/SO codepage: U+%04x<->0x%s\n", 695 (int)c, printBytes(buffer, bytes, length)); 696 return FALSE; 697 } 698 699 if(flag==1 && length==1 && *bytes==0) { 700 fprintf(stderr, "error: unable to encode a |1 fallback from U+%04x to 0x%02x\n", 701 (int)c, *bytes); 702 return FALSE; 703 } 704 705 /* 706 * Walk down the triple-stage compact array ("trie") and 707 * allocate parts as necessary. 708 * Note that the first stage 2 and 3 blocks are reserved for 709 * all-unassigned mappings. 710 * We assume that length<=maxCharLength and that c<=0x10ffff. 711 */ 712 stage3=mbcsData->fromUBytes; 713 714 /* inspect stage 1 */ 715 idx=c>>MBCS_STAGE_1_SHIFT; 716 if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) { 717 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK&~(MBCS_UTF8_STAGE_3_BLOCKS-1); 718 } else { 719 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK; 720 } 721 if(mbcsData->stage1[idx]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) { 722 /* allocate another block in stage 2 */ 723 newBlock=mbcsData->stage2Top; 724 if(mbcsData->utf8Friendly) { 725 min=newBlock-nextOffset; /* minimum block start with overlap */ 726 while(min<newBlock && mbcsData->stage2[newBlock-1]==0) { 727 --newBlock; 728 } 729 } 730 newTop=newBlock+MBCS_STAGE_2_BLOCK_SIZE; 731 732 if(newTop>MBCS_MAX_STAGE_2_TOP) { 733 fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%s\n", 734 (int)c, printBytes(buffer, bytes, length)); 735 return FALSE; 736 } 737 738 /* 739 * each stage 2 block contains 64 32-bit words: 740 * 6 code point bits 9..4 with value with bits 31..16 "assigned" flags and bits 15..0 stage 3 index 741 */ 742 i=idx; 743 while(newBlock<newTop) { 744 mbcsData->stage1[i++]=(uint16_t)newBlock; 745 newBlock+=MBCS_STAGE_2_BLOCK_SIZE; 746 } 747 mbcsData->stage2Top=newTop; /* ==newBlock */ 748 } 749 750 /* inspect stage 2 */ 751 idx=mbcsData->stage1[idx]+nextOffset; 752 if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) { 753 /* allocate 64-entry blocks for UTF-8-friendly lookup */ 754 blockSize=MBCS_UTF8_STAGE_3_BLOCK_SIZE*maxCharLength; 755 nextOffset=c&MBCS_UTF8_STAGE_3_BLOCK_MASK; 756 } else { 757 blockSize=MBCS_STAGE_3_BLOCK_SIZE*maxCharLength; 758 nextOffset=c&MBCS_STAGE_3_BLOCK_MASK; 759 } 760 if(mbcsData->stage2[idx]==0) { 761 /* allocate another block in stage 3 */ 762 newBlock=mbcsData->stage3Top; 763 if(mbcsData->utf8Friendly && nextOffset>=MBCS_STAGE_3_GRANULARITY) { 764 /* 765 * Overlap stage 3 blocks only in multiples of 16-entry blocks 766 * because of the indexing granularity in stage 2. 767 */ 768 maxOverlap=(nextOffset&~(MBCS_STAGE_3_GRANULARITY-1))*maxCharLength; 769 for(overlap=0; 770 overlap<maxOverlap && stage3[newBlock-overlap-1]==0; 771 ++overlap) {} 772 773 overlap=(overlap/MBCS_STAGE_3_GRANULARITY)/maxCharLength; 774 overlap=(overlap*MBCS_STAGE_3_GRANULARITY)*maxCharLength; 775 776 newBlock-=overlap; 777 } 778 newTop=newBlock+blockSize; 779 780 if(newTop>MBCS_STAGE_3_MBCS_SIZE*(uint32_t)maxCharLength) { 781 fprintf(stderr, "error: too many code points at U+%04x<->0x%s\n", 782 (int)c, printBytes(buffer, bytes, length)); 783 return FALSE; 784 } 785 /* each block has 16*maxCharLength bytes */ 786 i=idx; 787 while(newBlock<newTop) { 788 mbcsData->stage2[i++]=(newBlock/MBCS_STAGE_3_GRANULARITY)/maxCharLength; 789 newBlock+=MBCS_STAGE_3_BLOCK_SIZE*maxCharLength; 790 } 791 mbcsData->stage3Top=newTop; /* ==newBlock */ 792 } 793 794 stage3Index=MBCS_STAGE_3_GRANULARITY*(uint32_t)(uint16_t)mbcsData->stage2[idx]; 795 796 /* Build an alternate, UTF-8-friendly stage table as well. */ 797 if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) { 798 /* Overflow for uint16_t entries in stageUTF8? */ 799 if(stage3Index>0xffff) { 800 /* 801 * This can occur only if the mapping table is nearly perfectly filled and if 802 * utf8Max==0xffff. 803 * (There is no known charset like this. GB 18030 does not map 804 * surrogate code points and LMBCS does not map 256 PUA code points.) 805 * 806 * Otherwise, stage3Index<=MBCS_UTF8_LIMIT<0xffff 807 * (stage3Index can at most reach exactly MBCS_UTF8_LIMIT) 808 * because we have a sorted table and there are at most MBCS_UTF8_LIMIT 809 * mappings with 0<=c<MBCS_UTF8_LIMIT, and there is only also 810 * the initial all-unassigned block in stage3. 811 * 812 * Solution for the overflow: Reduce utf8Max to the next lower value, 0xfeff. 813 * 814 * (See svn revision 20866 of the markus/ucnvutf8 feature branch for 815 * code that causes MBCSAddTable() to rebuild the table not utf8Friendly 816 * in case of overflow. That code was not tested.) 817 */ 818 mbcsData->utf8Max=0xfeff; 819 } else { 820 /* 821 * The stage 3 block has been assigned for the regular trie. 822 * Just copy its index into stageUTF8[], without the granularity. 823 */ 824 mbcsData->stageUTF8[c>>MBCS_UTF8_STAGE_SHIFT]=(uint16_t)stage3Index; 825 } 826 } 827 828 /* write the codepage bytes into stage 3 and get the previous bytes */ 829 830 /* assemble the bytes into a single integer */ 831 pb=bytes; 832 b=0; 833 switch(length) { 834 case 4: 835 b=*pb++; 836 U_FALLTHROUGH; 837 case 3: 838 b=(b<<8)|*pb++; 839 U_FALLTHROUGH; 840 case 2: 841 b=(b<<8)|*pb++; 842 U_FALLTHROUGH; 843 case 1: 844 default: 845 b=(b<<8)|*pb++; 846 break; 847 } 848 849 old=0; 850 p=stage3+(stage3Index+nextOffset)*maxCharLength; 851 switch(maxCharLength) { 852 case 2: 853 old=*(uint16_t *)p; 854 *(uint16_t *)p=(uint16_t)b; 855 break; 856 case 3: 857 old=(uint32_t)*p<<16; 858 *p++=(uint8_t)(b>>16); 859 old|=(uint32_t)*p<<8; 860 *p++=(uint8_t)(b>>8); 861 old|=*p; 862 *p=(uint8_t)b; 863 break; 864 case 4: 865 old=*(uint32_t *)p; 866 *(uint32_t *)p=b; 867 break; 868 default: 869 /* will never occur */ 870 break; 871 } 872 873 /* check that this Unicode code point was still unassigned */ 874 if((mbcsData->stage2[idx+(nextOffset>>MBCS_STAGE_2_SHIFT)]&(1UL<<(16+(c&0xf))))!=0 || old!=0) { 875 if(flag>=0) { 876 fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n", 877 (int)c, printBytes(buffer, bytes, length), (int)old); 878 return FALSE; 879 } else if(VERBOSE) { 880 fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n", 881 (int)c, printBytes(buffer, bytes, length), (int)old); 882 } 883 /* continue after the above warning if the precision of the mapping is 884 unspecified */ 885 } 886 if(flag<=0) { 887 /* set the roundtrip flag */ 888 mbcsData->stage2[idx+(nextOffset>>4)]|=(1UL<<(16+(c&0xf))); 889 } 890 891 return TRUE; 892 } 893 894 U_CFUNC UBool 895 MBCSOkForBaseFromUnicode(const MBCSData *mbcsData, 896 const uint8_t *bytes, int32_t length, 897 UChar32 c, int8_t flag) { 898 /* 899 * A 1:1 mapping does not fit into the MBCS base table's fromUnicode table under 900 * the following conditions: 901 * 902 * - a |2 SUB mapping for <subchar1> (no base table data structure for them) 903 * - a |1 fallback to 0x00 (result value 0, indistinguishable from unmappable entry) 904 * - a multi-byte mapping with leading 0x00 bytes (no explicit length field) 905 * 906 * Some of these tests are redundant with ucm_mappingType(). 907 */ 908 if( (flag==2 && length==1) || 909 (flag==1 && bytes[0]==0) || /* testing length==1 would be redundant with the next test */ 910 (flag<=1 && length>1 && bytes[0]==0) 911 ) { 912 return FALSE; 913 } 914 915 /* 916 * Additional restrictions for UTF-8-friendly fromUnicode tables, 917 * for code points up to the maximum optimized one: 918 * 919 * - any mapping to 0x00 (result value 0, indistinguishable from unmappable entry) 920 * - any |1 fallback (no roundtrip flags in the optimized table) 921 */ 922 if(mbcsData->utf8Friendly && flag<=1 && c<=mbcsData->utf8Max && (bytes[0]==0 || flag==1)) { 923 return FALSE; 924 } 925 926 /* 927 * If we omit the fromUnicode data, we can only store roundtrips there 928 * because only they are recoverable from the toUnicode data. 929 * Fallbacks must go into the extension table. 930 */ 931 if(mbcsData->omitFromU && flag!=0) { 932 return FALSE; 933 } 934 935 /* All other mappings do fit into the base table. */ 936 return TRUE; 937 } 938 939 /* we can assume that the table only contains 1:1 mappings with <=4 bytes each */ 940 static UBool 941 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData) { 942 MBCSData *mbcsData; 943 UCMapping *m; 944 UChar32 c; 945 int32_t i, maxCharLength; 946 int8_t f; 947 UBool isOK, utf8Friendly; 948 949 staticData->unicodeMask=table->unicodeMask; 950 if(staticData->unicodeMask==3) { 951 fprintf(stderr, "error: contains mappings for both supplementary and surrogate code points\n"); 952 return FALSE; 953 } 954 955 staticData->conversionType=UCNV_MBCS; 956 957 mbcsData=(MBCSData *)cnvData; 958 maxCharLength=mbcsData->ucm->states.maxCharLength; 959 960 /* 961 * Generation of UTF-8-friendly data requires 962 * a sorted table, which makeconv generates when explicit precision 963 * indicators are used. 964 */ 965 mbcsData->utf8Friendly=utf8Friendly=(UBool)((table->flagsType&UCM_FLAGS_EXPLICIT)!=0); 966 if(utf8Friendly) { 967 mbcsData->utf8Max=MBCS_UTF8_MAX; 968 if(SMALL && maxCharLength>1) { 969 mbcsData->omitFromU=TRUE; 970 } 971 } else { 972 mbcsData->utf8Max=0; 973 if(SMALL && maxCharLength>1) { 974 fprintf(stderr, 975 "makeconv warning: --small not available for .ucm files without |0 etc.\n"); 976 } 977 } 978 979 if(!MBCSStartMappings(mbcsData)) { 980 return FALSE; 981 } 982 983 staticData->hasFromUnicodeFallback=FALSE; 984 staticData->hasToUnicodeFallback=FALSE; 985 986 isOK=TRUE; 987 988 m=table->mappings; 989 for(i=0; i<table->mappingsLength; ++m, ++i) { 990 c=m->u; 991 f=m->f; 992 993 /* 994 * Small optimization for --small .cnv files: 995 * 996 * If there are fromUnicode mappings above MBCS_UTF8_MAX, 997 * then the file size will be smaller if we make utf8Max larger 998 * because the size increase in stageUTF8 will be more than balanced by 999 * how much less of stage2 needs to be stored. 1000 * 1001 * There is no point in doing this incrementally because stageUTF8 1002 * uses so much less space per block than stage2, 1003 * so we immediately increase utf8Max to 0xffff. 1004 * 1005 * Do not increase utf8Max if it is already at 0xfeff because MBCSAddFromUnicode() 1006 * sets it to that value when stageUTF8 overflows. 1007 */ 1008 if( mbcsData->omitFromU && f<=1 && 1009 mbcsData->utf8Max<c && c<=0xffff && 1010 mbcsData->utf8Max<0xfeff 1011 ) { 1012 mbcsData->utf8Max=0xffff; 1013 } 1014 1015 switch(f) { 1016 case -1: 1017 /* there was no precision/fallback indicator */ 1018 /* fall through to set the mappings */ 1019 U_FALLTHROUGH; 1020 case 0: 1021 /* set roundtrip mappings */ 1022 isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1023 1024 if(maxCharLength==1) { 1025 isOK&=MBCSSingleAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1026 } else if(MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f)) { 1027 isOK&=MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1028 } else { 1029 m->f|=MBCS_FROM_U_EXT_FLAG; 1030 m->moveFlag=UCM_MOVE_TO_EXT; 1031 } 1032 break; 1033 case 1: 1034 /* set only a fallback mapping from Unicode to codepage */ 1035 if(maxCharLength==1) { 1036 staticData->hasFromUnicodeFallback=TRUE; 1037 isOK&=MBCSSingleAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1038 } else if(MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f)) { 1039 staticData->hasFromUnicodeFallback=TRUE; 1040 isOK&=MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1041 } else { 1042 m->f|=MBCS_FROM_U_EXT_FLAG; 1043 m->moveFlag=UCM_MOVE_TO_EXT; 1044 } 1045 break; 1046 case 2: 1047 /* ignore |2 SUB mappings, except to move <subchar1> mappings to the extension table */ 1048 if(maxCharLength>1 && m->bLen==1) { 1049 m->f|=MBCS_FROM_U_EXT_FLAG; 1050 m->moveFlag=UCM_MOVE_TO_EXT; 1051 } 1052 break; 1053 case 3: 1054 /* set only a fallback mapping from codepage to Unicode */ 1055 staticData->hasToUnicodeFallback=TRUE; 1056 isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1057 break; 1058 case 4: 1059 /* move "good one-way" mappings to the extension table */ 1060 m->f|=MBCS_FROM_U_EXT_FLAG; 1061 m->moveFlag=UCM_MOVE_TO_EXT; 1062 break; 1063 default: 1064 /* will not occur because the parser checked it already */ 1065 fprintf(stderr, "error: illegal fallback indicator %d\n", f); 1066 return FALSE; 1067 } 1068 } 1069 1070 MBCSPostprocess(mbcsData, staticData); 1071 1072 return isOK; 1073 } 1074 1075 static UBool 1076 transformEUC(MBCSData *mbcsData) { 1077 uint8_t *p8; 1078 uint32_t i, value, oldLength, old3Top; 1079 uint8_t b; 1080 1081 oldLength=mbcsData->ucm->states.maxCharLength; 1082 if(oldLength<3) { 1083 return FALSE; 1084 } 1085 1086 old3Top=mbcsData->stage3Top; 1087 1088 /* careful: 2-byte and 4-byte codes are stored in platform endianness! */ 1089 1090 /* test if all first bytes are in {0, 0x8e, 0x8f} */ 1091 p8=mbcsData->fromUBytes; 1092 1093 #if !U_IS_BIG_ENDIAN 1094 if(oldLength==4) { 1095 p8+=3; 1096 } 1097 #endif 1098 1099 for(i=0; i<old3Top; i+=oldLength) { 1100 b=p8[i]; 1101 if(b!=0 && b!=0x8e && b!=0x8f) { 1102 /* some first byte does not fit the EUC pattern, nothing to be done */ 1103 return FALSE; 1104 } 1105 } 1106 /* restore p if it was modified above */ 1107 p8=mbcsData->fromUBytes; 1108 1109 /* modify outputType and adjust stage3Top */ 1110 mbcsData->ucm->states.outputType=(int8_t)(MBCS_OUTPUT_3_EUC+oldLength-3); 1111 mbcsData->stage3Top=(old3Top*(oldLength-1))/oldLength; 1112 1113 /* 1114 * EUC-encode all byte sequences; 1115 * see "CJKV Information Processing" (1st ed. 1999) from Ken Lunde, O'Reilly, 1116 * p. 161 in chapter 4 "Encoding Methods" 1117 * 1118 * This also must reverse the byte order if the platform is little-endian! 1119 */ 1120 if(oldLength==3) { 1121 uint16_t *q=(uint16_t *)p8; 1122 for(i=0; i<old3Top; i+=oldLength) { 1123 b=*p8; 1124 if(b==0) { 1125 /* short sequences are stored directly */ 1126 /* code set 0 or 1 */ 1127 (*q++)=(uint16_t)((p8[1]<<8)|p8[2]); 1128 } else if(b==0x8e) { 1129 /* code set 2 */ 1130 (*q++)=(uint16_t)(((p8[1]&0x7f)<<8)|p8[2]); 1131 } else /* b==0x8f */ { 1132 /* code set 3 */ 1133 (*q++)=(uint16_t)((p8[1]<<8)|(p8[2]&0x7f)); 1134 } 1135 p8+=3; 1136 } 1137 } else /* oldLength==4 */ { 1138 uint8_t *q=p8; 1139 uint32_t *p32=(uint32_t *)p8; 1140 for(i=0; i<old3Top; i+=4) { 1141 value=(*p32++); 1142 if(value<=0xffffff) { 1143 /* short sequences are stored directly */ 1144 /* code set 0 or 1 */ 1145 (*q++)=(uint8_t)(value>>16); 1146 (*q++)=(uint8_t)(value>>8); 1147 (*q++)=(uint8_t)value; 1148 } else if(value<=0x8effffff) { 1149 /* code set 2 */ 1150 (*q++)=(uint8_t)((value>>16)&0x7f); 1151 (*q++)=(uint8_t)(value>>8); 1152 (*q++)=(uint8_t)value; 1153 } else /* first byte is 0x8f */ { 1154 /* code set 3 */ 1155 (*q++)=(uint8_t)(value>>16); 1156 (*q++)=(uint8_t)((value>>8)&0x7f); 1157 (*q++)=(uint8_t)value; 1158 } 1159 } 1160 } 1161 1162 return TRUE; 1163 } 1164 1165 /* 1166 * Compact stage 2 for SBCS by overlapping adjacent stage 2 blocks as far 1167 * as possible. Overlapping is done on unassigned head and tail 1168 * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER. 1169 * Stage 1 indexes need to be adjusted accordingly. 1170 * This function is very similar to genprops/store.c/compactStage(). 1171 */ 1172 static void 1173 singleCompactStage2(MBCSData *mbcsData) { 1174 /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */ 1175 uint16_t map[MBCS_STAGE_2_MAX_BLOCKS]; 1176 uint16_t i, start, prevEnd, newStart; 1177 1178 /* enter the all-unassigned first stage 2 block into the map */ 1179 map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX; 1180 1181 /* begin with the first block after the all-unassigned one */ 1182 start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED; 1183 while(start<mbcsData->stage2Top) { 1184 prevEnd=(uint16_t)(newStart-1); 1185 1186 /* find the size of the overlap */ 1187 for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2Single[start+i]==0 && mbcsData->stage2Single[prevEnd-i]==0; ++i) {} 1188 1189 if(i>0) { 1190 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i); 1191 1192 /* move the non-overlapping indexes to their new positions */ 1193 start+=i; 1194 for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) { 1195 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++]; 1196 } 1197 } else if(newStart<start) { 1198 /* move the indexes to their new positions */ 1199 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart; 1200 for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) { 1201 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++]; 1202 } 1203 } else /* no overlap && newStart==start */ { 1204 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start; 1205 start=newStart+=MBCS_STAGE_2_BLOCK_SIZE; 1206 } 1207 } 1208 1209 /* adjust stage2Top */ 1210 if(VERBOSE && newStart<mbcsData->stage2Top) { 1211 printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n", 1212 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart, 1213 (long)(mbcsData->stage2Top-newStart)*2); 1214 } 1215 mbcsData->stage2Top=newStart; 1216 1217 /* now adjust stage 1 */ 1218 for(i=0; i<MBCS_STAGE_1_SIZE; ++i) { 1219 mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]; 1220 } 1221 } 1222 1223 /* Compact stage 3 for SBCS - same algorithm as above. */ 1224 static void 1225 singleCompactStage3(MBCSData *mbcsData) { 1226 uint16_t *stage3=(uint16_t *)mbcsData->fromUBytes; 1227 1228 /* this array maps the ordinal number of a stage 3 block to its new stage 2 index */ 1229 uint16_t map[0x1000]; 1230 uint16_t i, start, prevEnd, newStart; 1231 1232 /* enter the all-unassigned first stage 3 block into the map */ 1233 map[0]=0; 1234 1235 /* begin with the first block after the all-unassigned one */ 1236 start=newStart=16; 1237 while(start<mbcsData->stage3Top) { 1238 prevEnd=(uint16_t)(newStart-1); 1239 1240 /* find the size of the overlap */ 1241 for(i=0; i<16 && stage3[start+i]==0 && stage3[prevEnd-i]==0; ++i) {} 1242 1243 if(i>0) { 1244 map[start>>4]=(uint16_t)(newStart-i); 1245 1246 /* move the non-overlapping indexes to their new positions */ 1247 start+=i; 1248 for(i=(uint16_t)(16-i); i>0; --i) { 1249 stage3[newStart++]=stage3[start++]; 1250 } 1251 } else if(newStart<start) { 1252 /* move the indexes to their new positions */ 1253 map[start>>4]=newStart; 1254 for(i=16; i>0; --i) { 1255 stage3[newStart++]=stage3[start++]; 1256 } 1257 } else /* no overlap && newStart==start */ { 1258 map[start>>4]=start; 1259 start=newStart+=16; 1260 } 1261 } 1262 1263 /* adjust stage3Top */ 1264 if(VERBOSE && newStart<mbcsData->stage3Top) { 1265 printf("compacting stage 3 from stage3Top=0x%lx to 0x%lx, saving %ld bytes\n", 1266 (unsigned long)mbcsData->stage3Top, (unsigned long)newStart, 1267 (long)(mbcsData->stage3Top-newStart)*2); 1268 } 1269 mbcsData->stage3Top=newStart; 1270 1271 /* now adjust stage 2 */ 1272 for(i=0; i<mbcsData->stage2Top; ++i) { 1273 mbcsData->stage2Single[i]=map[mbcsData->stage2Single[i]>>4]; 1274 } 1275 } 1276 1277 /* 1278 * Compact stage 2 by overlapping adjacent stage 2 blocks as far 1279 * as possible. Overlapping is done on unassigned head and tail 1280 * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER. 1281 * Stage 1 indexes need to be adjusted accordingly. 1282 * This function is very similar to genprops/store.c/compactStage(). 1283 */ 1284 static void 1285 compactStage2(MBCSData *mbcsData) { 1286 /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */ 1287 uint16_t map[MBCS_STAGE_2_MAX_BLOCKS]; 1288 uint16_t i, start, prevEnd, newStart; 1289 1290 /* enter the all-unassigned first stage 2 block into the map */ 1291 map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX; 1292 1293 /* begin with the first block after the all-unassigned one */ 1294 start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED; 1295 while(start<mbcsData->stage2Top) { 1296 prevEnd=(uint16_t)(newStart-1); 1297 1298 /* find the size of the overlap */ 1299 for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2[start+i]==0 && mbcsData->stage2[prevEnd-i]==0; ++i) {} 1300 1301 if(i>0) { 1302 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i); 1303 1304 /* move the non-overlapping indexes to their new positions */ 1305 start+=i; 1306 for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) { 1307 mbcsData->stage2[newStart++]=mbcsData->stage2[start++]; 1308 } 1309 } else if(newStart<start) { 1310 /* move the indexes to their new positions */ 1311 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart; 1312 for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) { 1313 mbcsData->stage2[newStart++]=mbcsData->stage2[start++]; 1314 } 1315 } else /* no overlap && newStart==start */ { 1316 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start; 1317 start=newStart+=MBCS_STAGE_2_BLOCK_SIZE; 1318 } 1319 } 1320 1321 /* adjust stage2Top */ 1322 if(VERBOSE && newStart<mbcsData->stage2Top) { 1323 printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n", 1324 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart, 1325 (long)(mbcsData->stage2Top-newStart)*4); 1326 } 1327 mbcsData->stage2Top=newStart; 1328 1329 /* now adjust stage 1 */ 1330 for(i=0; i<MBCS_STAGE_1_SIZE; ++i) { 1331 mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]; 1332 } 1333 } 1334 1335 static void 1336 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData * /*staticData*/) { 1337 UCMStates *states; 1338 int32_t maxCharLength, stage3Width; 1339 1340 states=&mbcsData->ucm->states; 1341 stage3Width=maxCharLength=states->maxCharLength; 1342 1343 ucm_optimizeStates(states, 1344 &mbcsData->unicodeCodeUnits, 1345 mbcsData->toUFallbacks, mbcsData->countToUFallbacks, 1346 VERBOSE); 1347 1348 /* try to compact the fromUnicode tables */ 1349 if(transformEUC(mbcsData)) { 1350 --stage3Width; 1351 } 1352 1353 /* 1354 * UTF-8-friendly tries are built precompacted, to cope with variable 1355 * stage 3 allocation block sizes. 1356 * 1357 * Tables without precision indicators cannot be built that way, 1358 * because if a block was overlapped with a previous one, then a smaller 1359 * code point for the same block would not fit. 1360 * Therefore, such tables are not marked UTF-8-friendly and must be 1361 * compacted after all mappings are entered. 1362 */ 1363 if(!mbcsData->utf8Friendly) { 1364 if(maxCharLength==1) { 1365 singleCompactStage3(mbcsData); 1366 singleCompactStage2(mbcsData); 1367 } else { 1368 compactStage2(mbcsData); 1369 } 1370 } 1371 1372 if(VERBOSE) { 1373 /*uint32_t c, i1, i2, i2Limit, i3;*/ 1374 1375 printf("fromUnicode number of uint%s_t in stage 2: 0x%lx=%lu\n", 1376 maxCharLength==1 ? "16" : "32", 1377 (unsigned long)mbcsData->stage2Top, 1378 (unsigned long)mbcsData->stage2Top); 1379 printf("fromUnicode number of %d-byte stage 3 mapping entries: 0x%lx=%lu\n", 1380 (int)stage3Width, 1381 (unsigned long)mbcsData->stage3Top/stage3Width, 1382 (unsigned long)mbcsData->stage3Top/stage3Width); 1383 #if 0 1384 c=0; 1385 for(i1=0; i1<MBCS_STAGE_1_SIZE; ++i1) { 1386 i2=mbcsData->stage1[i1]; 1387 if(i2==0) { 1388 c+=MBCS_STAGE_2_BLOCK_SIZE*MBCS_STAGE_3_BLOCK_SIZE; 1389 continue; 1390 } 1391 for(i2Limit=i2+MBCS_STAGE_2_BLOCK_SIZE; i2<i2Limit; ++i2) { 1392 if(maxCharLength==1) { 1393 i3=mbcsData->stage2Single[i2]; 1394 } else { 1395 i3=(uint16_t)mbcsData->stage2[i2]; 1396 } 1397 if(i3==0) { 1398 c+=MBCS_STAGE_3_BLOCK_SIZE; 1399 continue; 1400 } 1401 printf("U+%04lx i1=0x%02lx i2=0x%04lx i3=0x%04lx\n", 1402 (unsigned long)c, 1403 (unsigned long)i1, 1404 (unsigned long)i2, 1405 (unsigned long)i3); 1406 c+=MBCS_STAGE_3_BLOCK_SIZE; 1407 } 1408 } 1409 #endif 1410 } 1411 } 1412 1413 static uint32_t 1414 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData, 1415 UNewDataMemory *pData, int32_t tableType) { 1416 MBCSData *mbcsData=(MBCSData *)cnvData; 1417 uint32_t stage2Start, stage2Length; 1418 uint32_t top, stageUTF8Length=0; 1419 int32_t i, stage1Top; 1420 uint32_t headerLength; 1421 1422 _MBCSHeader header=UCNV_MBCS_HEADER_INITIALIZER; 1423 1424 stage2Length=mbcsData->stage2Top; 1425 if(mbcsData->omitFromU) { 1426 /* find how much of stage2 can be omitted */ 1427 int32_t utf8Limit=(int32_t)mbcsData->utf8Max+1; 1428 uint32_t st2=0; /*initialized it to avoid compiler warnings */ 1429 1430 i=utf8Limit>>MBCS_STAGE_1_SHIFT; 1431 if((utf8Limit&((1<<MBCS_STAGE_1_SHIFT)-1))!=0 && (st2=mbcsData->stage1[i])!=0) { 1432 /* utf8Limit is in the middle of an existing stage 2 block */ 1433 stage2Start=st2+((utf8Limit>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK); 1434 } else { 1435 /* find the last stage2 block with mappings before utf8Limit */ 1436 while(i>0 && (st2=mbcsData->stage1[--i])==0) {} 1437 /* stage2 up to the end of this block corresponds to stageUTF8 */ 1438 stage2Start=st2+MBCS_STAGE_2_BLOCK_SIZE; 1439 } 1440 header.options|=MBCS_OPT_NO_FROM_U; 1441 header.fullStage2Length=stage2Length; 1442 stage2Length-=stage2Start; 1443 if(VERBOSE) { 1444 printf("+ omitting %lu out of %lu stage2 entries and %lu fromUBytes\n", 1445 (unsigned long)stage2Start, 1446 (unsigned long)mbcsData->stage2Top, 1447 (unsigned long)mbcsData->stage3Top); 1448 printf("+ total size savings: %lu bytes\n", (unsigned long)stage2Start*4+mbcsData->stage3Top); 1449 } 1450 } else { 1451 stage2Start=0; 1452 } 1453 1454 if(staticData->unicodeMask&UCNV_HAS_SUPPLEMENTARY) { 1455 stage1Top=MBCS_STAGE_1_SIZE; /* 0x440==1088 */ 1456 } else { 1457 stage1Top=0x40; /* 0x40==64 */ 1458 } 1459 1460 /* adjust stage 1 entries to include the size of stage 1 in the offsets to stage 2 */ 1461 if(mbcsData->ucm->states.maxCharLength==1) { 1462 for(i=0; i<stage1Top; ++i) { 1463 mbcsData->stage1[i]+=(uint16_t)stage1Top; 1464 } 1465 1466 /* stage2Top/Length have counted 16-bit results, now we need to count bytes */ 1467 /* also round up to a multiple of 4 bytes */ 1468 stage2Length=(stage2Length*2+1)&~1; 1469 1470 /* stage3Top has counted 16-bit results, now we need to count bytes */ 1471 mbcsData->stage3Top*=2; 1472 1473 if(mbcsData->utf8Friendly) { 1474 header.version[2]=(uint8_t)(SBCS_UTF8_MAX>>8); /* store 0x1f for max==0x1fff */ 1475 } 1476 } else { 1477 for(i=0; i<stage1Top; ++i) { 1478 mbcsData->stage1[i]+=(uint16_t)stage1Top/2; /* stage 2 contains 32-bit entries, stage 1 16-bit entries */ 1479 } 1480 1481 /* stage2Top/Length have counted 32-bit results, now we need to count bytes */ 1482 stage2Length*=4; 1483 /* leave stage2Start counting 32-bit units */ 1484 1485 if(mbcsData->utf8Friendly) { 1486 stageUTF8Length=(mbcsData->utf8Max+1)>>MBCS_UTF8_STAGE_SHIFT; 1487 header.version[2]=(uint8_t)(mbcsData->utf8Max>>8); /* store 0xd7 for max==0xd7ff */ 1488 } 1489 1490 /* stage3Top has already counted bytes */ 1491 } 1492 1493 /* round up stage3Top so that the sizes of all data blocks are multiples of 4 */ 1494 mbcsData->stage3Top=(mbcsData->stage3Top+3)&~3; 1495 1496 /* fill the header */ 1497 if(header.options&MBCS_OPT_INCOMPATIBLE_MASK) { 1498 header.version[0]=5; 1499 if(header.options&MBCS_OPT_NO_FROM_U) { 1500 headerLength=10; /* include fullStage2Length */ 1501 } else { 1502 headerLength=MBCS_HEADER_V5_MIN_LENGTH; /* 9 */ 1503 } 1504 } else { 1505 header.version[0]=4; 1506 headerLength=MBCS_HEADER_V4_LENGTH; /* 8 */ 1507 } 1508 header.version[1]=4; 1509 /* header.version[2] set above for utf8Friendly data */ 1510 1511 header.options|=(uint32_t)headerLength; 1512 1513 header.countStates=mbcsData->ucm->states.countStates; 1514 header.countToUFallbacks=mbcsData->countToUFallbacks; 1515 1516 header.offsetToUCodeUnits= 1517 headerLength*4+ 1518 mbcsData->ucm->states.countStates*1024+ 1519 mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback); 1520 header.offsetFromUTable= 1521 header.offsetToUCodeUnits+ 1522 mbcsData->ucm->states.countToUCodeUnits*2; 1523 header.offsetFromUBytes= 1524 header.offsetFromUTable+ 1525 stage1Top*2+ 1526 stage2Length; 1527 header.fromUBytesLength=mbcsData->stage3Top; 1528 1529 top=header.offsetFromUBytes+stageUTF8Length*2; 1530 if(!(header.options&MBCS_OPT_NO_FROM_U)) { 1531 top+=header.fromUBytesLength; 1532 } 1533 1534 header.flags=(uint8_t)(mbcsData->ucm->states.outputType); 1535 1536 if(tableType&TABLE_EXT) { 1537 if(top>0xffffff) { 1538 fprintf(stderr, "error: offset 0x%lx to extension table exceeds 0xffffff\n", (long)top); 1539 return 0; 1540 } 1541 1542 header.flags|=top<<8; 1543 } 1544 1545 /* write the MBCS data */ 1546 udata_writeBlock(pData, &header, headerLength*4); 1547 udata_writeBlock(pData, mbcsData->ucm->states.stateTable, header.countStates*1024); 1548 udata_writeBlock(pData, mbcsData->toUFallbacks, mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback)); 1549 udata_writeBlock(pData, mbcsData->unicodeCodeUnits, mbcsData->ucm->states.countToUCodeUnits*2); 1550 udata_writeBlock(pData, mbcsData->stage1, stage1Top*2); 1551 if(mbcsData->ucm->states.maxCharLength==1) { 1552 udata_writeBlock(pData, mbcsData->stage2Single+stage2Start, stage2Length); 1553 } else { 1554 udata_writeBlock(pData, mbcsData->stage2+stage2Start, stage2Length); 1555 } 1556 if(!(header.options&MBCS_OPT_NO_FROM_U)) { 1557 udata_writeBlock(pData, mbcsData->fromUBytes, mbcsData->stage3Top); 1558 } 1559 1560 if(stageUTF8Length>0) { 1561 udata_writeBlock(pData, mbcsData->stageUTF8, stageUTF8Length*2); 1562 } 1563 1564 /* return the number of bytes that should have been written */ 1565 return top; 1566 } 1567
