1 /* 2 ******************************************************************************* 3 * 4 * Copyright (C) 2000-2009, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ******************************************************************************* 8 * file name: genmbcs.c 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2000jul06 14 * created by: Markus W. Scherer 15 */ 16 17 #include <stdio.h> 18 #include "unicode/utypes.h" 19 #include "cstring.h" 20 #include "cmemory.h" 21 #include "unewdata.h" 22 #include "ucnv_cnv.h" 23 #include "ucnvmbcs.h" 24 #include "ucm.h" 25 #include "makeconv.h" 26 #include "genmbcs.h" 27 28 /* 29 * TODO: Split this file into toUnicode, SBCSFromUnicode and MBCSFromUnicode files. 30 * Reduce tests for maxCharLength. 31 */ 32 33 struct MBCSData { 34 NewConverter newConverter; 35 36 UCMFile *ucm; 37 38 /* toUnicode (state table in ucm->states) */ 39 _MBCSToUFallback toUFallbacks[MBCS_MAX_FALLBACK_COUNT]; 40 int32_t countToUFallbacks; 41 uint16_t *unicodeCodeUnits; 42 43 /* fromUnicode */ 44 uint16_t stage1[MBCS_STAGE_1_SIZE]; 45 uint16_t stage2Single[MBCS_STAGE_2_SIZE]; /* stage 2 for single-byte codepages */ 46 uint32_t stage2[MBCS_STAGE_2_SIZE]; /* stage 2 for MBCS */ 47 uint8_t *fromUBytes; 48 uint32_t stage2Top, stage3Top; 49 50 /* fromUTF8 */ 51 uint16_t stageUTF8[0x10000>>MBCS_UTF8_STAGE_SHIFT]; /* allow for utf8Max=0xffff */ 52 53 /* 54 * Maximum UTF-8-friendly code point. 55 * 0 if !utf8Friendly, otherwise 0x01ff..0xffff in steps of 0x100. 56 * If utf8Friendly, utf8Max is normally either MBCS_UTF8_MAX or 0xffff. 57 */ 58 uint16_t utf8Max; 59 60 UBool utf8Friendly; 61 UBool omitFromU; 62 }; 63 64 /* prototypes */ 65 static void 66 MBCSClose(NewConverter *cnvData); 67 68 static UBool 69 MBCSStartMappings(MBCSData *mbcsData); 70 71 static UBool 72 MBCSAddToUnicode(MBCSData *mbcsData, 73 const uint8_t *bytes, int32_t length, 74 UChar32 c, 75 int8_t flag); 76 77 static UBool 78 MBCSIsValid(NewConverter *cnvData, 79 const uint8_t *bytes, int32_t length); 80 81 static UBool 82 MBCSSingleAddFromUnicode(MBCSData *mbcsData, 83 const uint8_t *bytes, int32_t length, 84 UChar32 c, 85 int8_t flag); 86 87 static UBool 88 MBCSAddFromUnicode(MBCSData *mbcsData, 89 const uint8_t *bytes, int32_t length, 90 UChar32 c, 91 int8_t flag); 92 93 static void 94 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData *staticData); 95 96 static UBool 97 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData); 98 99 static uint32_t 100 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData, 101 UNewDataMemory *pData, int32_t tableType); 102 103 /* helper ------------------------------------------------------------------- */ 104 105 static U_INLINE char 106 hexDigit(uint8_t digit) { 107 return digit<=9 ? (char)('0'+digit) : (char)('a'-10+digit); 108 } 109 110 static U_INLINE char * 111 printBytes(char *buffer, const uint8_t *bytes, int32_t length) { 112 char *s=buffer; 113 while(length>0) { 114 *s++=hexDigit((uint8_t)(*bytes>>4)); 115 *s++=hexDigit((uint8_t)(*bytes&0xf)); 116 ++bytes; 117 --length; 118 } 119 120 *s=0; 121 return buffer; 122 } 123 124 /* implementation ----------------------------------------------------------- */ 125 126 static MBCSData gDummy; 127 128 U_CFUNC const MBCSData * 129 MBCSGetDummy() { 130 uprv_memset(&gDummy, 0, sizeof(MBCSData)); 131 132 /* 133 * Set "pessimistic" values which may sometimes move too many 134 * mappings to the extension table (but never too few). 135 * These values cause MBCSOkForBaseFromUnicode() to return FALSE for the 136 * largest set of mappings. 137 * Assume maxCharLength>1. 138 */ 139 gDummy.utf8Friendly=TRUE; 140 if(SMALL) { 141 gDummy.utf8Max=0xffff; 142 gDummy.omitFromU=TRUE; 143 } else { 144 gDummy.utf8Max=MBCS_UTF8_MAX; 145 } 146 return &gDummy; 147 } 148 149 static void 150 MBCSInit(MBCSData *mbcsData, UCMFile *ucm) { 151 uprv_memset(mbcsData, 0, sizeof(MBCSData)); 152 153 mbcsData->ucm=ucm; /* aliased, not owned */ 154 155 mbcsData->newConverter.close=MBCSClose; 156 mbcsData->newConverter.isValid=MBCSIsValid; 157 mbcsData->newConverter.addTable=MBCSAddTable; 158 mbcsData->newConverter.write=MBCSWrite; 159 } 160 161 NewConverter * 162 MBCSOpen(UCMFile *ucm) { 163 MBCSData *mbcsData=(MBCSData *)uprv_malloc(sizeof(MBCSData)); 164 if(mbcsData==NULL) { 165 printf("out of memory\n"); 166 exit(U_MEMORY_ALLOCATION_ERROR); 167 } 168 169 MBCSInit(mbcsData, ucm); 170 return &mbcsData->newConverter; 171 } 172 173 static void 174 MBCSDestruct(MBCSData *mbcsData) { 175 uprv_free(mbcsData->unicodeCodeUnits); 176 uprv_free(mbcsData->fromUBytes); 177 } 178 179 static void 180 MBCSClose(NewConverter *cnvData) { 181 MBCSData *mbcsData=(MBCSData *)cnvData; 182 if(mbcsData!=NULL) { 183 MBCSDestruct(mbcsData); 184 uprv_free(mbcsData); 185 } 186 } 187 188 static UBool 189 MBCSStartMappings(MBCSData *mbcsData) { 190 int32_t i, sum, maxCharLength, 191 stage2NullLength, stage2AllocLength, 192 stage3NullLength, stage3AllocLength; 193 194 /* toUnicode */ 195 196 /* allocate the code unit array and prefill it with "unassigned" values */ 197 sum=mbcsData->ucm->states.countToUCodeUnits; 198 if(VERBOSE) { 199 printf("the total number of offsets is 0x%lx=%ld\n", (long)sum, (long)sum); 200 } 201 202 if(sum>0) { 203 mbcsData->unicodeCodeUnits=(uint16_t *)uprv_malloc(sum*sizeof(uint16_t)); 204 if(mbcsData->unicodeCodeUnits==NULL) { 205 fprintf(stderr, "error: out of memory allocating %ld 16-bit code units\n", 206 (long)sum); 207 return FALSE; 208 } 209 for(i=0; i<sum; ++i) { 210 mbcsData->unicodeCodeUnits[i]=0xfffe; 211 } 212 } 213 214 /* fromUnicode */ 215 maxCharLength=mbcsData->ucm->states.maxCharLength; 216 217 /* allocate the codepage mappings and preset the first 16 characters to 0 */ 218 if(maxCharLength==1) { 219 /* allocate 64k 16-bit results for single-byte codepages */ 220 sum=0x20000; 221 } else { 222 /* allocate 1M * maxCharLength bytes for at most 1M mappings */ 223 sum=0x100000*maxCharLength; 224 } 225 mbcsData->fromUBytes=(uint8_t *)uprv_malloc(sum); 226 if(mbcsData->fromUBytes==NULL) { 227 fprintf(stderr, "error: out of memory allocating %ld B for target mappings\n", (long)sum); 228 return FALSE; 229 } 230 uprv_memset(mbcsData->fromUBytes, 0, sum); 231 232 /* 233 * UTF-8-friendly fromUnicode tries: allocate multiple blocks at a time. 234 * See ucnvmbcs.h for details. 235 * 236 * There is code, for example in ucnv_MBCSGetUnicodeSetForUnicode(), which 237 * assumes that the initial stage 2/3 blocks are the all-unassigned ones. 238 * Therefore, we refine the data structure while maintaining this placement 239 * even though it would be convenient to allocate the ASCII block at the 240 * beginning of stage 3, for example. 241 * 242 * UTF-8-friendly fromUnicode tries work from sorted tables and are built 243 * pre-compacted, overlapping adjacent stage 2/3 blocks. 244 * This is necessary because the block allocation and compaction changes 245 * at SBCS_UTF8_MAX or MBCS_UTF8_MAX, and for MBCS tables the additional 246 * stage table uses direct indexes into stage 3, without a multiplier and 247 * thus with a smaller reach. 248 * 249 * Non-UTF-8-friendly fromUnicode tries work from unsorted tables 250 * (because implicit precision is used), and are compacted 251 * in post-processing. 252 * 253 * Preallocation for UTF-8-friendly fromUnicode tries: 254 * 255 * Stage 3: 256 * 64-entry all-unassigned first block followed by ASCII (128 entries). 257 * 258 * Stage 2: 259 * 64-entry all-unassigned first block followed by preallocated 260 * 64-block for ASCII. 261 */ 262 263 /* Preallocate ASCII as a linear 128-entry stage 3 block. */ 264 stage2NullLength=MBCS_STAGE_2_BLOCK_SIZE; 265 stage2AllocLength=MBCS_STAGE_2_BLOCK_SIZE; 266 267 stage3NullLength=MBCS_UTF8_STAGE_3_BLOCK_SIZE; 268 stage3AllocLength=128; /* ASCII U+0000..U+007f */ 269 270 /* Initialize stage 1 for the preallocated blocks. */ 271 sum=stage2NullLength; 272 for(i=0; i<(stage2AllocLength>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT); ++i) { 273 mbcsData->stage1[i]=sum; 274 sum+=MBCS_STAGE_2_BLOCK_SIZE; 275 } 276 mbcsData->stage2Top=stage2NullLength+stage2AllocLength; /* ==sum */ 277 278 /* 279 * Stage 2 indexes count 16-blocks in stage 3 as follows: 280 * SBCS: directly, indexes increment by 16 281 * MBCS: indexes need to be multiplied by 16*maxCharLength, indexes increment by 1 282 * MBCS UTF-8: directly, indexes increment by 16 283 */ 284 if(maxCharLength==1) { 285 sum=stage3NullLength; 286 for(i=0; i<(stage3AllocLength/MBCS_STAGE_3_BLOCK_SIZE); ++i) { 287 mbcsData->stage2Single[mbcsData->stage1[0]+i]=sum; 288 sum+=MBCS_STAGE_3_BLOCK_SIZE; 289 } 290 } else { 291 sum=stage3NullLength/MBCS_STAGE_3_GRANULARITY; 292 for(i=0; i<(stage3AllocLength/MBCS_STAGE_3_BLOCK_SIZE); ++i) { 293 mbcsData->stage2[mbcsData->stage1[0]+i]=sum; 294 sum+=MBCS_STAGE_3_BLOCK_SIZE/MBCS_STAGE_3_GRANULARITY; 295 } 296 } 297 298 sum=stage3NullLength; 299 for(i=0; i<(stage3AllocLength/MBCS_UTF8_STAGE_3_BLOCK_SIZE); ++i) { 300 mbcsData->stageUTF8[i]=sum; 301 sum+=MBCS_UTF8_STAGE_3_BLOCK_SIZE; 302 } 303 304 /* 305 * Allocate a 64-entry all-unassigned first stage 3 block, 306 * for UTF-8-friendly lookup with a trail byte, 307 * plus 128 entries for ASCII. 308 */ 309 mbcsData->stage3Top=(stage3NullLength+stage3AllocLength)*maxCharLength; /* ==sum*maxCharLength */ 310 311 return TRUE; 312 } 313 314 /* return TRUE for success */ 315 static UBool 316 setFallback(MBCSData *mbcsData, uint32_t offset, UChar32 c) { 317 int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset); 318 if(i>=0) { 319 /* if there is already a fallback for this offset, then overwrite it */ 320 mbcsData->toUFallbacks[i].codePoint=c; 321 return TRUE; 322 } else { 323 /* if there is no fallback for this offset, then add one */ 324 i=mbcsData->countToUFallbacks; 325 if(i>=MBCS_MAX_FALLBACK_COUNT) { 326 fprintf(stderr, "error: too many toUnicode fallbacks, currently at: U+%x\n", (int)c); 327 return FALSE; 328 } else { 329 mbcsData->toUFallbacks[i].offset=offset; 330 mbcsData->toUFallbacks[i].codePoint=c; 331 mbcsData->countToUFallbacks=i+1; 332 return TRUE; 333 } 334 } 335 } 336 337 /* remove fallback if there is one with this offset; return the code point if there was such a fallback, otherwise -1 */ 338 static int32_t 339 removeFallback(MBCSData *mbcsData, uint32_t offset) { 340 int32_t i=ucm_findFallback(mbcsData->toUFallbacks, mbcsData->countToUFallbacks, offset); 341 if(i>=0) { 342 _MBCSToUFallback *toUFallbacks; 343 int32_t limit, old; 344 345 toUFallbacks=mbcsData->toUFallbacks; 346 limit=mbcsData->countToUFallbacks; 347 old=(int32_t)toUFallbacks[i].codePoint; 348 349 /* copy the last fallback entry here to keep the list contiguous */ 350 toUFallbacks[i].offset=toUFallbacks[limit-1].offset; 351 toUFallbacks[i].codePoint=toUFallbacks[limit-1].codePoint; 352 mbcsData->countToUFallbacks=limit-1; 353 return old; 354 } else { 355 return -1; 356 } 357 } 358 359 /* 360 * isFallback is almost a boolean: 361 * 1 (TRUE) this is a fallback mapping 362 * 0 (FALSE) this is a precise mapping 363 * -1 the precision of this mapping is not specified 364 */ 365 static UBool 366 MBCSAddToUnicode(MBCSData *mbcsData, 367 const uint8_t *bytes, int32_t length, 368 UChar32 c, 369 int8_t flag) { 370 char buffer[10]; 371 uint32_t offset=0; 372 int32_t i=0, entry, old; 373 uint8_t state=0; 374 375 if(mbcsData->ucm->states.countStates==0) { 376 fprintf(stderr, "error: there is no state information!\n"); 377 return FALSE; 378 } 379 380 /* for SI/SO (like EBCDIC-stateful), double-byte sequences start in state 1 */ 381 if(length==2 && mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO) { 382 state=1; 383 } 384 385 /* 386 * Walk down the state table like in conversion, 387 * much like getNextUChar(). 388 * We assume that c<=0x10ffff. 389 */ 390 for(i=0;;) { 391 entry=mbcsData->ucm->states.stateTable[state][bytes[i++]]; 392 if(MBCS_ENTRY_IS_TRANSITION(entry)) { 393 if(i==length) { 394 fprintf(stderr, "error: byte sequence too short, ends in non-final state %hu: 0x%s (U+%x)\n", 395 (short)state, printBytes(buffer, bytes, length), (int)c); 396 return FALSE; 397 } 398 state=(uint8_t)MBCS_ENTRY_TRANSITION_STATE(entry); 399 offset+=MBCS_ENTRY_TRANSITION_OFFSET(entry); 400 } else { 401 if(i<length) { 402 fprintf(stderr, "error: byte sequence too long by %d bytes, final state %hu: 0x%s (U+%x)\n", 403 (int)(length-i), state, printBytes(buffer, bytes, length), (int)c); 404 return FALSE; 405 } 406 switch(MBCS_ENTRY_FINAL_ACTION(entry)) { 407 case MBCS_STATE_ILLEGAL: 408 fprintf(stderr, "error: byte sequence ends in illegal state at U+%04x<->0x%s\n", 409 (int)c, printBytes(buffer, bytes, length)); 410 return FALSE; 411 case MBCS_STATE_CHANGE_ONLY: 412 fprintf(stderr, "error: byte sequence ends in state-change-only at U+%04x<->0x%s\n", 413 (int)c, printBytes(buffer, bytes, length)); 414 return FALSE; 415 case MBCS_STATE_UNASSIGNED: 416 fprintf(stderr, "error: byte sequence ends in unassigned state at U+%04x<->0x%s\n", 417 (int)c, printBytes(buffer, bytes, length)); 418 return FALSE; 419 case MBCS_STATE_FALLBACK_DIRECT_16: 420 case MBCS_STATE_VALID_DIRECT_16: 421 case MBCS_STATE_FALLBACK_DIRECT_20: 422 case MBCS_STATE_VALID_DIRECT_20: 423 if(MBCS_ENTRY_SET_STATE(entry, 0)!=MBCS_ENTRY_FINAL(0, MBCS_STATE_VALID_DIRECT_16, 0xfffe)) { 424 /* the "direct" action's value is not "valid-direct-16-unassigned" any more */ 425 if(MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_VALID_DIRECT_16 || MBCS_ENTRY_FINAL_ACTION(entry)==MBCS_STATE_FALLBACK_DIRECT_16) { 426 old=MBCS_ENTRY_FINAL_VALUE(entry); 427 } else { 428 old=0x10000+MBCS_ENTRY_FINAL_VALUE(entry); 429 } 430 if(flag>=0) { 431 fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 432 (int)c, printBytes(buffer, bytes, length), (int)old); 433 return FALSE; 434 } else if(VERBOSE) { 435 fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 436 (int)c, printBytes(buffer, bytes, length), (int)old); 437 } 438 /* 439 * Continue after the above warning 440 * if the precision of the mapping is unspecified. 441 */ 442 } 443 /* reassign the correct action code */ 444 entry=MBCS_ENTRY_FINAL_SET_ACTION(entry, (MBCS_STATE_VALID_DIRECT_16+(flag==3 ? 2 : 0)+(c>=0x10000 ? 1 : 0))); 445 446 /* put the code point into bits 22..7 for BMP, c-0x10000 into 26..7 for others */ 447 if(c<=0xffff) { 448 entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c); 449 } else { 450 entry=MBCS_ENTRY_FINAL_SET_VALUE(entry, c-0x10000); 451 } 452 mbcsData->ucm->states.stateTable[state][bytes[i-1]]=entry; 453 break; 454 case MBCS_STATE_VALID_16: 455 /* bits 26..16 are not used, 0 */ 456 /* bits 15..7 contain the final offset delta to one 16-bit code unit */ 457 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); 458 /* check that this byte sequence is still unassigned */ 459 if((old=mbcsData->unicodeCodeUnits[offset])!=0xfffe || (old=removeFallback(mbcsData, offset))!=-1) { 460 if(flag>=0) { 461 fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 462 (int)c, printBytes(buffer, bytes, length), (int)old); 463 return FALSE; 464 } else if(VERBOSE) { 465 fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 466 (int)c, printBytes(buffer, bytes, length), (int)old); 467 } 468 } 469 if(c>=0x10000) { 470 fprintf(stderr, "error: code point does not fit into valid-16-bit state at U+%04x<->0x%s\n", 471 (int)c, printBytes(buffer, bytes, length)); 472 return FALSE; 473 } 474 if(flag>0) { 475 /* assign only if there is no precise mapping */ 476 if(mbcsData->unicodeCodeUnits[offset]==0xfffe) { 477 return setFallback(mbcsData, offset, c); 478 } 479 } else { 480 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 481 } 482 break; 483 case MBCS_STATE_VALID_16_PAIR: 484 /* bits 26..16 are not used, 0 */ 485 /* bits 15..7 contain the final offset delta to two 16-bit code units */ 486 offset+=MBCS_ENTRY_FINAL_VALUE_16(entry); 487 /* check that this byte sequence is still unassigned */ 488 old=mbcsData->unicodeCodeUnits[offset]; 489 if(old<0xfffe) { 490 int32_t real; 491 if(old<0xd800) { 492 real=old; 493 } else if(old<=0xdfff) { 494 real=0x10000+((old&0x3ff)<<10)+((mbcsData->unicodeCodeUnits[offset+1])&0x3ff); 495 } else /* old<=0xe001 */ { 496 real=mbcsData->unicodeCodeUnits[offset+1]; 497 } 498 if(flag>=0) { 499 fprintf(stderr, "error: duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 500 (int)c, printBytes(buffer, bytes, length), (int)real); 501 return FALSE; 502 } else if(VERBOSE) { 503 fprintf(stderr, "duplicate codepage byte sequence at U+%04x<->0x%s see U+%04x\n", 504 (int)c, printBytes(buffer, bytes, length), (int)real); 505 } 506 } 507 if(flag>0) { 508 /* assign only if there is no precise mapping */ 509 if(old<=0xdbff || old==0xe000) { 510 /* do nothing */ 511 } else if(c<=0xffff) { 512 /* set a BMP fallback code point as a pair with 0xe001 */ 513 mbcsData->unicodeCodeUnits[offset++]=0xe001; 514 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 515 } else { 516 /* set a fallback surrogate pair with two second surrogates */ 517 mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xdbc0+(c>>10)); 518 mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff)); 519 } 520 } else { 521 if(c<0xd800) { 522 /* set a BMP code point */ 523 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 524 } else if(c<=0xffff) { 525 /* set a BMP code point above 0xd800 as a pair with 0xe000 */ 526 mbcsData->unicodeCodeUnits[offset++]=0xe000; 527 mbcsData->unicodeCodeUnits[offset]=(uint16_t)c; 528 } else { 529 /* set a surrogate pair */ 530 mbcsData->unicodeCodeUnits[offset++]=(uint16_t)(0xd7c0+(c>>10)); 531 mbcsData->unicodeCodeUnits[offset]=(uint16_t)(0xdc00+(c&0x3ff)); 532 } 533 } 534 break; 535 default: 536 /* reserved, must never occur */ 537 fprintf(stderr, "internal error: byte sequence reached reserved action code, entry 0x%02x: 0x%s (U+%x)\n", 538 (int)entry, printBytes(buffer, bytes, length), (int)c); 539 return FALSE; 540 } 541 542 return TRUE; 543 } 544 } 545 } 546 547 /* is this byte sequence valid? (this is almost the same as MBCSAddToUnicode()) */ 548 static UBool 549 MBCSIsValid(NewConverter *cnvData, 550 const uint8_t *bytes, int32_t length) { 551 MBCSData *mbcsData=(MBCSData *)cnvData; 552 553 return (UBool)(1==ucm_countChars(&mbcsData->ucm->states, bytes, length)); 554 } 555 556 static UBool 557 MBCSSingleAddFromUnicode(MBCSData *mbcsData, 558 const uint8_t *bytes, int32_t length, 559 UChar32 c, 560 int8_t flag) { 561 uint16_t *stage3, *p; 562 uint32_t idx; 563 uint16_t old; 564 uint8_t b; 565 566 uint32_t blockSize, newTop, i, nextOffset, newBlock, min; 567 568 /* ignore |2 SUB mappings */ 569 if(flag==2) { 570 return TRUE; 571 } 572 573 /* 574 * Walk down the triple-stage compact array ("trie") and 575 * allocate parts as necessary. 576 * Note that the first stage 2 and 3 blocks are reserved for all-unassigned mappings. 577 * We assume that length<=maxCharLength and that c<=0x10ffff. 578 */ 579 stage3=(uint16_t *)mbcsData->fromUBytes; 580 b=*bytes; 581 582 /* inspect stage 1 */ 583 idx=c>>MBCS_STAGE_1_SHIFT; 584 if(mbcsData->utf8Friendly && c<=SBCS_UTF8_MAX) { 585 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK&~(MBCS_UTF8_STAGE_3_BLOCKS-1); 586 } else { 587 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK; 588 } 589 if(mbcsData->stage1[idx]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) { 590 /* allocate another block in stage 2 */ 591 newBlock=mbcsData->stage2Top; 592 if(mbcsData->utf8Friendly) { 593 min=newBlock-nextOffset; /* minimum block start with overlap */ 594 while(min<newBlock && mbcsData->stage2Single[newBlock-1]==0) { 595 --newBlock; 596 } 597 } 598 newTop=newBlock+MBCS_STAGE_2_BLOCK_SIZE; 599 600 if(newTop>MBCS_MAX_STAGE_2_TOP) { 601 fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%02x\n", (int)c, b); 602 return FALSE; 603 } 604 605 /* 606 * each stage 2 block contains 64 16-bit words: 607 * 6 code point bits 9..4 with 1 stage 3 index 608 */ 609 mbcsData->stage1[idx]=(uint16_t)newBlock; 610 mbcsData->stage2Top=newTop; 611 } 612 613 /* inspect stage 2 */ 614 idx=mbcsData->stage1[idx]+nextOffset; 615 if(mbcsData->utf8Friendly && c<=SBCS_UTF8_MAX) { 616 /* allocate 64-entry blocks for UTF-8-friendly lookup */ 617 blockSize=MBCS_UTF8_STAGE_3_BLOCK_SIZE; 618 nextOffset=c&MBCS_UTF8_STAGE_3_BLOCK_MASK; 619 } else { 620 blockSize=MBCS_STAGE_3_BLOCK_SIZE; 621 nextOffset=c&MBCS_STAGE_3_BLOCK_MASK; 622 } 623 if(mbcsData->stage2Single[idx]==0) { 624 /* allocate another block in stage 3 */ 625 newBlock=mbcsData->stage3Top; 626 if(mbcsData->utf8Friendly) { 627 min=newBlock-nextOffset; /* minimum block start with overlap */ 628 while(min<newBlock && stage3[newBlock-1]==0) { 629 --newBlock; 630 } 631 } 632 newTop=newBlock+blockSize; 633 634 if(newTop>MBCS_STAGE_3_SBCS_SIZE) { 635 fprintf(stderr, "error: too many code points at U+%04x<->0x%02x\n", (int)c, b); 636 return FALSE; 637 } 638 /* each block has 16 uint16_t entries */ 639 i=idx; 640 while(newBlock<newTop) { 641 mbcsData->stage2Single[i++]=(uint16_t)newBlock; 642 newBlock+=MBCS_STAGE_3_BLOCK_SIZE; 643 } 644 mbcsData->stage3Top=newTop; /* ==newBlock */ 645 } 646 647 /* write the codepage entry into stage 3 and get the previous entry */ 648 p=stage3+mbcsData->stage2Single[idx]+nextOffset; 649 old=*p; 650 if(flag<=0) { 651 *p=(uint16_t)(0xf00|b); 652 } else if(IS_PRIVATE_USE(c)) { 653 *p=(uint16_t)(0xc00|b); 654 } else { 655 *p=(uint16_t)(0x800|b); 656 } 657 658 /* check that this Unicode code point was still unassigned */ 659 if(old>=0x100) { 660 if(flag>=0) { 661 fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n", 662 (int)c, b, old&0xff); 663 return FALSE; 664 } else if(VERBOSE) { 665 fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%02x see 0x%02x\n", 666 (int)c, b, old&0xff); 667 } 668 /* continue after the above warning if the precision of the mapping is unspecified */ 669 } 670 671 return TRUE; 672 } 673 674 static UBool 675 MBCSAddFromUnicode(MBCSData *mbcsData, 676 const uint8_t *bytes, int32_t length, 677 UChar32 c, 678 int8_t flag) { 679 char buffer[10]; 680 const uint8_t *pb; 681 uint8_t *stage3, *p; 682 uint32_t idx, b, old, stage3Index; 683 int32_t maxCharLength; 684 685 uint32_t blockSize, newTop, i, nextOffset, newBlock, min, overlap, maxOverlap; 686 687 maxCharLength=mbcsData->ucm->states.maxCharLength; 688 689 if( mbcsData->ucm->states.outputType==MBCS_OUTPUT_2_SISO && 690 (*bytes==0xe || *bytes==0xf) 691 ) { 692 fprintf(stderr, "error: illegal mapping to SI or SO for SI/SO codepage: U+%04x<->0x%s\n", 693 (int)c, printBytes(buffer, bytes, length)); 694 return FALSE; 695 } 696 697 if(flag==1 && length==1 && *bytes==0) { 698 fprintf(stderr, "error: unable to encode a |1 fallback from U+%04x to 0x%02x\n", 699 (int)c, *bytes); 700 return FALSE; 701 } 702 703 /* 704 * Walk down the triple-stage compact array ("trie") and 705 * allocate parts as necessary. 706 * Note that the first stage 2 and 3 blocks are reserved for 707 * all-unassigned mappings. 708 * We assume that length<=maxCharLength and that c<=0x10ffff. 709 */ 710 stage3=mbcsData->fromUBytes; 711 712 /* inspect stage 1 */ 713 idx=c>>MBCS_STAGE_1_SHIFT; 714 if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) { 715 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK&~(MBCS_UTF8_STAGE_3_BLOCKS-1); 716 } else { 717 nextOffset=(c>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK; 718 } 719 if(mbcsData->stage1[idx]==MBCS_STAGE_2_ALL_UNASSIGNED_INDEX) { 720 /* allocate another block in stage 2 */ 721 newBlock=mbcsData->stage2Top; 722 if(mbcsData->utf8Friendly) { 723 min=newBlock-nextOffset; /* minimum block start with overlap */ 724 while(min<newBlock && mbcsData->stage2[newBlock-1]==0) { 725 --newBlock; 726 } 727 } 728 newTop=newBlock+MBCS_STAGE_2_BLOCK_SIZE; 729 730 if(newTop>MBCS_MAX_STAGE_2_TOP) { 731 fprintf(stderr, "error: too many stage 2 entries at U+%04x<->0x%s\n", 732 (int)c, printBytes(buffer, bytes, length)); 733 return FALSE; 734 } 735 736 /* 737 * each stage 2 block contains 64 32-bit words: 738 * 6 code point bits 9..4 with value with bits 31..16 "assigned" flags and bits 15..0 stage 3 index 739 */ 740 i=idx; 741 while(newBlock<newTop) { 742 mbcsData->stage1[i++]=(uint16_t)newBlock; 743 newBlock+=MBCS_STAGE_2_BLOCK_SIZE; 744 } 745 mbcsData->stage2Top=newTop; /* ==newBlock */ 746 } 747 748 /* inspect stage 2 */ 749 idx=mbcsData->stage1[idx]+nextOffset; 750 if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) { 751 /* allocate 64-entry blocks for UTF-8-friendly lookup */ 752 blockSize=MBCS_UTF8_STAGE_3_BLOCK_SIZE*maxCharLength; 753 nextOffset=c&MBCS_UTF8_STAGE_3_BLOCK_MASK; 754 } else { 755 blockSize=MBCS_STAGE_3_BLOCK_SIZE*maxCharLength; 756 nextOffset=c&MBCS_STAGE_3_BLOCK_MASK; 757 } 758 if(mbcsData->stage2[idx]==0) { 759 /* allocate another block in stage 3 */ 760 newBlock=mbcsData->stage3Top; 761 if(mbcsData->utf8Friendly && nextOffset>=MBCS_STAGE_3_GRANULARITY) { 762 /* 763 * Overlap stage 3 blocks only in multiples of 16-entry blocks 764 * because of the indexing granularity in stage 2. 765 */ 766 maxOverlap=(nextOffset&~(MBCS_STAGE_3_GRANULARITY-1))*maxCharLength; 767 for(overlap=0; 768 overlap<maxOverlap && stage3[newBlock-overlap-1]==0; 769 ++overlap) {} 770 771 overlap=(overlap/MBCS_STAGE_3_GRANULARITY)/maxCharLength; 772 overlap=(overlap*MBCS_STAGE_3_GRANULARITY)*maxCharLength; 773 774 newBlock-=overlap; 775 } 776 newTop=newBlock+blockSize; 777 778 if(newTop>MBCS_STAGE_3_MBCS_SIZE*(uint32_t)maxCharLength) { 779 fprintf(stderr, "error: too many code points at U+%04x<->0x%s\n", 780 (int)c, printBytes(buffer, bytes, length)); 781 return FALSE; 782 } 783 /* each block has 16*maxCharLength bytes */ 784 i=idx; 785 while(newBlock<newTop) { 786 mbcsData->stage2[i++]=(newBlock/MBCS_STAGE_3_GRANULARITY)/maxCharLength; 787 newBlock+=MBCS_STAGE_3_BLOCK_SIZE*maxCharLength; 788 } 789 mbcsData->stage3Top=newTop; /* ==newBlock */ 790 } 791 792 stage3Index=MBCS_STAGE_3_GRANULARITY*(uint32_t)(uint16_t)mbcsData->stage2[idx]; 793 794 /* Build an alternate, UTF-8-friendly stage table as well. */ 795 if(mbcsData->utf8Friendly && c<=mbcsData->utf8Max) { 796 /* Overflow for uint16_t entries in stageUTF8? */ 797 if(stage3Index>0xffff) { 798 /* 799 * This can occur only if the mapping table is nearly perfectly filled and if 800 * utf8Max==0xffff. 801 * (There is no known charset like this. GB 18030 does not map 802 * surrogate code points and LMBCS does not map 256 PUA code points.) 803 * 804 * Otherwise, stage3Index<=MBCS_UTF8_LIMIT<0xffff 805 * (stage3Index can at most reach exactly MBCS_UTF8_LIMIT) 806 * because we have a sorted table and there are at most MBCS_UTF8_LIMIT 807 * mappings with 0<=c<MBCS_UTF8_LIMIT, and there is only also 808 * the initial all-unassigned block in stage3. 809 * 810 * Solution for the overflow: Reduce utf8Max to the next lower value, 0xfeff. 811 * 812 * (See svn revision 20866 of the markus/ucnvutf8 feature branch for 813 * code that causes MBCSAddTable() to rebuild the table not utf8Friendly 814 * in case of overflow. That code was not tested.) 815 */ 816 mbcsData->utf8Max=0xfeff; 817 } else { 818 /* 819 * The stage 3 block has been assigned for the regular trie. 820 * Just copy its index into stageUTF8[], without the granularity. 821 */ 822 mbcsData->stageUTF8[c>>MBCS_UTF8_STAGE_SHIFT]=(uint16_t)stage3Index; 823 } 824 } 825 826 /* write the codepage bytes into stage 3 and get the previous bytes */ 827 828 /* assemble the bytes into a single integer */ 829 pb=bytes; 830 b=0; 831 switch(length) { 832 case 4: 833 b=*pb++; 834 case 3: 835 b=(b<<8)|*pb++; 836 case 2: 837 b=(b<<8)|*pb++; 838 case 1: 839 default: 840 b=(b<<8)|*pb++; 841 break; 842 } 843 844 old=0; 845 p=stage3+(stage3Index+nextOffset)*maxCharLength; 846 switch(maxCharLength) { 847 case 2: 848 old=*(uint16_t *)p; 849 *(uint16_t *)p=(uint16_t)b; 850 break; 851 case 3: 852 old=(uint32_t)*p<<16; 853 *p++=(uint8_t)(b>>16); 854 old|=(uint32_t)*p<<8; 855 *p++=(uint8_t)(b>>8); 856 old|=*p; 857 *p=(uint8_t)b; 858 break; 859 case 4: 860 old=*(uint32_t *)p; 861 *(uint32_t *)p=b; 862 break; 863 default: 864 /* will never occur */ 865 break; 866 } 867 868 /* check that this Unicode code point was still unassigned */ 869 if((mbcsData->stage2[idx+(nextOffset>>MBCS_STAGE_2_SHIFT)]&(1UL<<(16+(c&0xf))))!=0 || old!=0) { 870 if(flag>=0) { 871 fprintf(stderr, "error: duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n", 872 (int)c, printBytes(buffer, bytes, length), (int)old); 873 return FALSE; 874 } else if(VERBOSE) { 875 fprintf(stderr, "duplicate Unicode code point at U+%04x<->0x%s see 0x%02x\n", 876 (int)c, printBytes(buffer, bytes, length), (int)old); 877 } 878 /* continue after the above warning if the precision of the mapping is 879 unspecified */ 880 } 881 if(flag<=0) { 882 /* set the roundtrip flag */ 883 mbcsData->stage2[idx+(nextOffset>>4)]|=(1UL<<(16+(c&0xf))); 884 } 885 886 return TRUE; 887 } 888 889 U_CFUNC UBool 890 MBCSOkForBaseFromUnicode(const MBCSData *mbcsData, 891 const uint8_t *bytes, int32_t length, 892 UChar32 c, int8_t flag) { 893 /* 894 * A 1:1 mapping does not fit into the MBCS base table's fromUnicode table under 895 * the following conditions: 896 * 897 * - a |2 SUB mapping for <subchar1> (no base table data structure for them) 898 * - a |1 fallback to 0x00 (result value 0, indistinguishable from unmappable entry) 899 * - a multi-byte mapping with leading 0x00 bytes (no explicit length field) 900 * 901 * Some of these tests are redundant with ucm_mappingType(). 902 */ 903 if( (flag==2 && length==1) || 904 (flag==1 && bytes[0]==0) || /* testing length==1 would be redundant with the next test */ 905 (flag<=1 && length>1 && bytes[0]==0) 906 ) { 907 return FALSE; 908 } 909 910 /* 911 * Additional restrictions for UTF-8-friendly fromUnicode tables, 912 * for code points up to the maximum optimized one: 913 * 914 * - any mapping to 0x00 (result value 0, indistinguishable from unmappable entry) 915 * - any |1 fallback (no roundtrip flags in the optimized table) 916 */ 917 if(mbcsData->utf8Friendly && flag<=1 && c<=mbcsData->utf8Max && (bytes[0]==0 || flag==1)) { 918 return FALSE; 919 } 920 921 /* 922 * If we omit the fromUnicode data, we can only store roundtrips there 923 * because only they are recoverable from the toUnicode data. 924 * Fallbacks must go into the extension table. 925 */ 926 if(mbcsData->omitFromU && flag!=0) { 927 return FALSE; 928 } 929 930 /* All other mappings do fit into the base table. */ 931 return TRUE; 932 } 933 934 /* we can assume that the table only contains 1:1 mappings with <=4 bytes each */ 935 static UBool 936 MBCSAddTable(NewConverter *cnvData, UCMTable *table, UConverterStaticData *staticData) { 937 MBCSData *mbcsData; 938 UCMapping *m; 939 UChar32 c; 940 int32_t i, maxCharLength; 941 int8_t f; 942 UBool isOK, utf8Friendly; 943 944 staticData->unicodeMask=table->unicodeMask; 945 if(staticData->unicodeMask==3) { 946 fprintf(stderr, "error: contains mappings for both supplementary and surrogate code points\n"); 947 return FALSE; 948 } 949 950 staticData->conversionType=UCNV_MBCS; 951 952 mbcsData=(MBCSData *)cnvData; 953 maxCharLength=mbcsData->ucm->states.maxCharLength; 954 955 /* 956 * Generation of UTF-8-friendly data requires 957 * a sorted table, which makeconv generates when explicit precision 958 * indicators are used. 959 */ 960 mbcsData->utf8Friendly=utf8Friendly=(UBool)((table->flagsType&UCM_FLAGS_EXPLICIT)!=0); 961 if(utf8Friendly) { 962 mbcsData->utf8Max=MBCS_UTF8_MAX; 963 if(SMALL && maxCharLength>1) { 964 mbcsData->omitFromU=TRUE; 965 } 966 } else { 967 mbcsData->utf8Max=0; 968 if(SMALL && maxCharLength>1) { 969 fprintf(stderr, 970 "makeconv warning: --small not available for .ucm files without |0 etc.\n"); 971 } 972 } 973 974 if(!MBCSStartMappings(mbcsData)) { 975 return FALSE; 976 } 977 978 staticData->hasFromUnicodeFallback=FALSE; 979 staticData->hasToUnicodeFallback=FALSE; 980 981 isOK=TRUE; 982 983 m=table->mappings; 984 for(i=0; i<table->mappingsLength; ++m, ++i) { 985 c=m->u; 986 f=m->f; 987 988 /* 989 * Small optimization for --small .cnv files: 990 * 991 * If there are fromUnicode mappings above MBCS_UTF8_MAX, 992 * then the file size will be smaller if we make utf8Max larger 993 * because the size increase in stageUTF8 will be more than balanced by 994 * how much less of stage2 needs to be stored. 995 * 996 * There is no point in doing this incrementally because stageUTF8 997 * uses so much less space per block than stage2, 998 * so we immediately increase utf8Max to 0xffff. 999 * 1000 * Do not increase utf8Max if it is already at 0xfeff because MBCSAddFromUnicode() 1001 * sets it to that value when stageUTF8 overflows. 1002 */ 1003 if( mbcsData->omitFromU && f<=1 && 1004 mbcsData->utf8Max<c && c<=0xffff && 1005 mbcsData->utf8Max<0xfeff 1006 ) { 1007 mbcsData->utf8Max=0xffff; 1008 } 1009 1010 switch(f) { 1011 case -1: 1012 /* there was no precision/fallback indicator */ 1013 /* fall through to set the mappings */ 1014 case 0: 1015 /* set roundtrip mappings */ 1016 isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1017 1018 if(maxCharLength==1) { 1019 isOK&=MBCSSingleAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1020 } else if(MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f)) { 1021 isOK&=MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1022 } else { 1023 m->f|=MBCS_FROM_U_EXT_FLAG; 1024 m->moveFlag=UCM_MOVE_TO_EXT; 1025 } 1026 break; 1027 case 1: 1028 /* set only a fallback mapping from Unicode to codepage */ 1029 if(maxCharLength==1) { 1030 staticData->hasFromUnicodeFallback=TRUE; 1031 isOK&=MBCSSingleAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1032 } else if(MBCSOkForBaseFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f)) { 1033 staticData->hasFromUnicodeFallback=TRUE; 1034 isOK&=MBCSAddFromUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1035 } else { 1036 m->f|=MBCS_FROM_U_EXT_FLAG; 1037 m->moveFlag=UCM_MOVE_TO_EXT; 1038 } 1039 break; 1040 case 2: 1041 /* ignore |2 SUB mappings, except to move <subchar1> mappings to the extension table */ 1042 if(maxCharLength>1 && m->bLen==1) { 1043 m->f|=MBCS_FROM_U_EXT_FLAG; 1044 m->moveFlag=UCM_MOVE_TO_EXT; 1045 } 1046 break; 1047 case 3: 1048 /* set only a fallback mapping from codepage to Unicode */ 1049 staticData->hasToUnicodeFallback=TRUE; 1050 isOK&=MBCSAddToUnicode(mbcsData, m->b.bytes, m->bLen, c, f); 1051 break; 1052 default: 1053 /* will not occur because the parser checked it already */ 1054 fprintf(stderr, "error: illegal fallback indicator %d\n", f); 1055 return FALSE; 1056 } 1057 } 1058 1059 MBCSPostprocess(mbcsData, staticData); 1060 1061 return isOK; 1062 } 1063 1064 static UBool 1065 transformEUC(MBCSData *mbcsData) { 1066 uint8_t *p8; 1067 uint32_t i, value, oldLength, old3Top, new3Top; 1068 uint8_t b; 1069 1070 oldLength=mbcsData->ucm->states.maxCharLength; 1071 if(oldLength<3) { 1072 return FALSE; 1073 } 1074 1075 old3Top=mbcsData->stage3Top; 1076 1077 /* careful: 2-byte and 4-byte codes are stored in platform endianness! */ 1078 1079 /* test if all first bytes are in {0, 0x8e, 0x8f} */ 1080 p8=mbcsData->fromUBytes; 1081 1082 #if !U_IS_BIG_ENDIAN 1083 if(oldLength==4) { 1084 p8+=3; 1085 } 1086 #endif 1087 1088 for(i=0; i<old3Top; i+=oldLength) { 1089 b=p8[i]; 1090 if(b!=0 && b!=0x8e && b!=0x8f) { 1091 /* some first byte does not fit the EUC pattern, nothing to be done */ 1092 return FALSE; 1093 } 1094 } 1095 /* restore p if it was modified above */ 1096 p8=mbcsData->fromUBytes; 1097 1098 /* modify outputType and adjust stage3Top */ 1099 mbcsData->ucm->states.outputType=(int8_t)(MBCS_OUTPUT_3_EUC+oldLength-3); 1100 mbcsData->stage3Top=new3Top=(old3Top*(oldLength-1))/oldLength; 1101 1102 /* 1103 * EUC-encode all byte sequences; 1104 * see "CJKV Information Processing" (1st ed. 1999) from Ken Lunde, O'Reilly, 1105 * p. 161 in chapter 4 "Encoding Methods" 1106 * 1107 * This also must reverse the byte order if the platform is little-endian! 1108 */ 1109 if(oldLength==3) { 1110 uint16_t *q=(uint16_t *)p8; 1111 for(i=0; i<old3Top; i+=oldLength) { 1112 b=*p8; 1113 if(b==0) { 1114 /* short sequences are stored directly */ 1115 /* code set 0 or 1 */ 1116 (*q++)=(uint16_t)((p8[1]<<8)|p8[2]); 1117 } else if(b==0x8e) { 1118 /* code set 2 */ 1119 (*q++)=(uint16_t)(((p8[1]&0x7f)<<8)|p8[2]); 1120 } else /* b==0x8f */ { 1121 /* code set 3 */ 1122 (*q++)=(uint16_t)((p8[1]<<8)|(p8[2]&0x7f)); 1123 } 1124 p8+=3; 1125 } 1126 } else /* oldLength==4 */ { 1127 uint8_t *q=p8; 1128 uint32_t *p32=(uint32_t *)p8; 1129 for(i=0; i<old3Top; i+=4) { 1130 value=(*p32++); 1131 if(value<=0xffffff) { 1132 /* short sequences are stored directly */ 1133 /* code set 0 or 1 */ 1134 (*q++)=(uint8_t)(value>>16); 1135 (*q++)=(uint8_t)(value>>8); 1136 (*q++)=(uint8_t)value; 1137 } else if(value<=0x8effffff) { 1138 /* code set 2 */ 1139 (*q++)=(uint8_t)((value>>16)&0x7f); 1140 (*q++)=(uint8_t)(value>>8); 1141 (*q++)=(uint8_t)value; 1142 } else /* first byte is 0x8f */ { 1143 /* code set 3 */ 1144 (*q++)=(uint8_t)(value>>16); 1145 (*q++)=(uint8_t)((value>>8)&0x7f); 1146 (*q++)=(uint8_t)value; 1147 } 1148 } 1149 } 1150 1151 return TRUE; 1152 } 1153 1154 /* 1155 * Compact stage 2 for SBCS by overlapping adjacent stage 2 blocks as far 1156 * as possible. Overlapping is done on unassigned head and tail 1157 * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER. 1158 * Stage 1 indexes need to be adjusted accordingly. 1159 * This function is very similar to genprops/store.c/compactStage(). 1160 */ 1161 static void 1162 singleCompactStage2(MBCSData *mbcsData) { 1163 /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */ 1164 uint16_t map[MBCS_STAGE_2_MAX_BLOCKS]; 1165 uint16_t i, start, prevEnd, newStart; 1166 1167 /* enter the all-unassigned first stage 2 block into the map */ 1168 map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX; 1169 1170 /* begin with the first block after the all-unassigned one */ 1171 start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED; 1172 while(start<mbcsData->stage2Top) { 1173 prevEnd=(uint16_t)(newStart-1); 1174 1175 /* find the size of the overlap */ 1176 for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2Single[start+i]==0 && mbcsData->stage2Single[prevEnd-i]==0; ++i) {} 1177 1178 if(i>0) { 1179 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i); 1180 1181 /* move the non-overlapping indexes to their new positions */ 1182 start+=i; 1183 for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) { 1184 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++]; 1185 } 1186 } else if(newStart<start) { 1187 /* move the indexes to their new positions */ 1188 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart; 1189 for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) { 1190 mbcsData->stage2Single[newStart++]=mbcsData->stage2Single[start++]; 1191 } 1192 } else /* no overlap && newStart==start */ { 1193 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start; 1194 start=newStart+=MBCS_STAGE_2_BLOCK_SIZE; 1195 } 1196 } 1197 1198 /* adjust stage2Top */ 1199 if(VERBOSE && newStart<mbcsData->stage2Top) { 1200 printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n", 1201 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart, 1202 (long)(mbcsData->stage2Top-newStart)*2); 1203 } 1204 mbcsData->stage2Top=newStart; 1205 1206 /* now adjust stage 1 */ 1207 for(i=0; i<MBCS_STAGE_1_SIZE; ++i) { 1208 mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]; 1209 } 1210 } 1211 1212 /* Compact stage 3 for SBCS - same algorithm as above. */ 1213 static void 1214 singleCompactStage3(MBCSData *mbcsData) { 1215 uint16_t *stage3=(uint16_t *)mbcsData->fromUBytes; 1216 1217 /* this array maps the ordinal number of a stage 3 block to its new stage 2 index */ 1218 uint16_t map[0x1000]; 1219 uint16_t i, start, prevEnd, newStart; 1220 1221 /* enter the all-unassigned first stage 3 block into the map */ 1222 map[0]=0; 1223 1224 /* begin with the first block after the all-unassigned one */ 1225 start=newStart=16; 1226 while(start<mbcsData->stage3Top) { 1227 prevEnd=(uint16_t)(newStart-1); 1228 1229 /* find the size of the overlap */ 1230 for(i=0; i<16 && stage3[start+i]==0 && stage3[prevEnd-i]==0; ++i) {} 1231 1232 if(i>0) { 1233 map[start>>4]=(uint16_t)(newStart-i); 1234 1235 /* move the non-overlapping indexes to their new positions */ 1236 start+=i; 1237 for(i=(uint16_t)(16-i); i>0; --i) { 1238 stage3[newStart++]=stage3[start++]; 1239 } 1240 } else if(newStart<start) { 1241 /* move the indexes to their new positions */ 1242 map[start>>4]=newStart; 1243 for(i=16; i>0; --i) { 1244 stage3[newStart++]=stage3[start++]; 1245 } 1246 } else /* no overlap && newStart==start */ { 1247 map[start>>4]=start; 1248 start=newStart+=16; 1249 } 1250 } 1251 1252 /* adjust stage3Top */ 1253 if(VERBOSE && newStart<mbcsData->stage3Top) { 1254 printf("compacting stage 3 from stage3Top=0x%lx to 0x%lx, saving %ld bytes\n", 1255 (unsigned long)mbcsData->stage3Top, (unsigned long)newStart, 1256 (long)(mbcsData->stage3Top-newStart)*2); 1257 } 1258 mbcsData->stage3Top=newStart; 1259 1260 /* now adjust stage 2 */ 1261 for(i=0; i<mbcsData->stage2Top; ++i) { 1262 mbcsData->stage2Single[i]=map[mbcsData->stage2Single[i]>>4]; 1263 } 1264 } 1265 1266 /* 1267 * Compact stage 2 by overlapping adjacent stage 2 blocks as far 1268 * as possible. Overlapping is done on unassigned head and tail 1269 * parts of blocks in steps of MBCS_STAGE_2_MULTIPLIER. 1270 * Stage 1 indexes need to be adjusted accordingly. 1271 * This function is very similar to genprops/store.c/compactStage(). 1272 */ 1273 static void 1274 compactStage2(MBCSData *mbcsData) { 1275 /* this array maps the ordinal number of a stage 2 block to its new stage 1 index */ 1276 uint16_t map[MBCS_STAGE_2_MAX_BLOCKS]; 1277 uint16_t i, start, prevEnd, newStart; 1278 1279 /* enter the all-unassigned first stage 2 block into the map */ 1280 map[0]=MBCS_STAGE_2_ALL_UNASSIGNED_INDEX; 1281 1282 /* begin with the first block after the all-unassigned one */ 1283 start=newStart=MBCS_STAGE_2_FIRST_ASSIGNED; 1284 while(start<mbcsData->stage2Top) { 1285 prevEnd=(uint16_t)(newStart-1); 1286 1287 /* find the size of the overlap */ 1288 for(i=0; i<MBCS_STAGE_2_BLOCK_SIZE && mbcsData->stage2[start+i]==0 && mbcsData->stage2[prevEnd-i]==0; ++i) {} 1289 1290 if(i>0) { 1291 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=(uint16_t)(newStart-i); 1292 1293 /* move the non-overlapping indexes to their new positions */ 1294 start+=i; 1295 for(i=(uint16_t)(MBCS_STAGE_2_BLOCK_SIZE-i); i>0; --i) { 1296 mbcsData->stage2[newStart++]=mbcsData->stage2[start++]; 1297 } 1298 } else if(newStart<start) { 1299 /* move the indexes to their new positions */ 1300 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=newStart; 1301 for(i=MBCS_STAGE_2_BLOCK_SIZE; i>0; --i) { 1302 mbcsData->stage2[newStart++]=mbcsData->stage2[start++]; 1303 } 1304 } else /* no overlap && newStart==start */ { 1305 map[start>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]=start; 1306 start=newStart+=MBCS_STAGE_2_BLOCK_SIZE; 1307 } 1308 } 1309 1310 /* adjust stage2Top */ 1311 if(VERBOSE && newStart<mbcsData->stage2Top) { 1312 printf("compacting stage 2 from stage2Top=0x%lx to 0x%lx, saving %ld bytes\n", 1313 (unsigned long)mbcsData->stage2Top, (unsigned long)newStart, 1314 (long)(mbcsData->stage2Top-newStart)*4); 1315 } 1316 mbcsData->stage2Top=newStart; 1317 1318 /* now adjust stage 1 */ 1319 for(i=0; i<MBCS_STAGE_1_SIZE; ++i) { 1320 mbcsData->stage1[i]=map[mbcsData->stage1[i]>>MBCS_STAGE_2_BLOCK_SIZE_SHIFT]; 1321 } 1322 } 1323 1324 static void 1325 MBCSPostprocess(MBCSData *mbcsData, const UConverterStaticData *staticData) { 1326 UCMStates *states; 1327 int32_t maxCharLength, stage3Width; 1328 1329 states=&mbcsData->ucm->states; 1330 stage3Width=maxCharLength=states->maxCharLength; 1331 1332 ucm_optimizeStates(states, 1333 &mbcsData->unicodeCodeUnits, 1334 mbcsData->toUFallbacks, mbcsData->countToUFallbacks, 1335 VERBOSE); 1336 1337 /* try to compact the fromUnicode tables */ 1338 if(transformEUC(mbcsData)) { 1339 --stage3Width; 1340 } 1341 1342 /* 1343 * UTF-8-friendly tries are built precompacted, to cope with variable 1344 * stage 3 allocation block sizes. 1345 * 1346 * Tables without precision indicators cannot be built that way, 1347 * because if a block was overlapped with a previous one, then a smaller 1348 * code point for the same block would not fit. 1349 * Therefore, such tables are not marked UTF-8-friendly and must be 1350 * compacted after all mappings are entered. 1351 */ 1352 if(!mbcsData->utf8Friendly) { 1353 if(maxCharLength==1) { 1354 singleCompactStage3(mbcsData); 1355 singleCompactStage2(mbcsData); 1356 } else { 1357 compactStage2(mbcsData); 1358 } 1359 } 1360 1361 if(VERBOSE) { 1362 /*uint32_t c, i1, i2, i2Limit, i3;*/ 1363 1364 printf("fromUnicode number of uint%s_t in stage 2: 0x%lx=%lu\n", 1365 maxCharLength==1 ? "16" : "32", 1366 (unsigned long)mbcsData->stage2Top, 1367 (unsigned long)mbcsData->stage2Top); 1368 printf("fromUnicode number of %d-byte stage 3 mapping entries: 0x%lx=%lu\n", 1369 (int)stage3Width, 1370 (unsigned long)mbcsData->stage3Top/stage3Width, 1371 (unsigned long)mbcsData->stage3Top/stage3Width); 1372 #if 0 1373 c=0; 1374 for(i1=0; i1<MBCS_STAGE_1_SIZE; ++i1) { 1375 i2=mbcsData->stage1[i1]; 1376 if(i2==0) { 1377 c+=MBCS_STAGE_2_BLOCK_SIZE*MBCS_STAGE_3_BLOCK_SIZE; 1378 continue; 1379 } 1380 for(i2Limit=i2+MBCS_STAGE_2_BLOCK_SIZE; i2<i2Limit; ++i2) { 1381 if(maxCharLength==1) { 1382 i3=mbcsData->stage2Single[i2]; 1383 } else { 1384 i3=(uint16_t)mbcsData->stage2[i2]; 1385 } 1386 if(i3==0) { 1387 c+=MBCS_STAGE_3_BLOCK_SIZE; 1388 continue; 1389 } 1390 printf("U+%04lx i1=0x%02lx i2=0x%04lx i3=0x%04lx\n", 1391 (unsigned long)c, 1392 (unsigned long)i1, 1393 (unsigned long)i2, 1394 (unsigned long)i3); 1395 c+=MBCS_STAGE_3_BLOCK_SIZE; 1396 } 1397 } 1398 #endif 1399 } 1400 } 1401 1402 static uint32_t 1403 MBCSWrite(NewConverter *cnvData, const UConverterStaticData *staticData, 1404 UNewDataMemory *pData, int32_t tableType) { 1405 MBCSData *mbcsData=(MBCSData *)cnvData; 1406 uint32_t stage2Start, stage2Length; 1407 uint32_t top, stageUTF8Length=0; 1408 int32_t i, stage1Top; 1409 uint32_t headerLength; 1410 1411 _MBCSHeader header={ { 0, 0, 0, 0 }, 0, 0, 0, 0, 0, 0, 0 }; 1412 1413 stage2Length=mbcsData->stage2Top; 1414 if(mbcsData->omitFromU) { 1415 /* find how much of stage2 can be omitted */ 1416 int32_t utf8Limit=(int32_t)mbcsData->utf8Max+1; 1417 uint32_t st2=0; /*initialized it to avoid compiler warnings */ 1418 1419 i=utf8Limit>>MBCS_STAGE_1_SHIFT; 1420 if((utf8Limit&((1<<MBCS_STAGE_1_SHIFT)-1))!=0 && (st2=mbcsData->stage1[i])!=0) { 1421 /* utf8Limit is in the middle of an existing stage 2 block */ 1422 stage2Start=st2+((utf8Limit>>MBCS_STAGE_2_SHIFT)&MBCS_STAGE_2_BLOCK_MASK); 1423 } else { 1424 /* find the last stage2 block with mappings before utf8Limit */ 1425 while(i>0 && (st2=mbcsData->stage1[--i])==0) {} 1426 /* stage2 up to the end of this block corresponds to stageUTF8 */ 1427 stage2Start=st2+MBCS_STAGE_2_BLOCK_SIZE; 1428 } 1429 header.options|=MBCS_OPT_NO_FROM_U; 1430 header.fullStage2Length=stage2Length; 1431 stage2Length-=stage2Start; 1432 if(VERBOSE) { 1433 printf("+ omitting %lu out of %lu stage2 entries and %lu fromUBytes\n", 1434 (unsigned long)stage2Start, 1435 (unsigned long)mbcsData->stage2Top, 1436 (unsigned long)mbcsData->stage3Top); 1437 printf("+ total size savings: %lu bytes\n", (unsigned long)stage2Start*4+mbcsData->stage3Top); 1438 } 1439 } else { 1440 stage2Start=0; 1441 } 1442 1443 if(staticData->unicodeMask&UCNV_HAS_SUPPLEMENTARY) { 1444 stage1Top=MBCS_STAGE_1_SIZE; /* 0x440==1088 */ 1445 } else { 1446 stage1Top=0x40; /* 0x40==64 */ 1447 } 1448 1449 /* adjust stage 1 entries to include the size of stage 1 in the offsets to stage 2 */ 1450 if(mbcsData->ucm->states.maxCharLength==1) { 1451 for(i=0; i<stage1Top; ++i) { 1452 mbcsData->stage1[i]+=(uint16_t)stage1Top; 1453 } 1454 1455 /* stage2Top/Length have counted 16-bit results, now we need to count bytes */ 1456 /* also round up to a multiple of 4 bytes */ 1457 stage2Length=(stage2Length*2+1)&~1; 1458 1459 /* stage3Top has counted 16-bit results, now we need to count bytes */ 1460 mbcsData->stage3Top*=2; 1461 1462 if(mbcsData->utf8Friendly) { 1463 header.version[2]=(uint8_t)(SBCS_UTF8_MAX>>8); /* store 0x1f for max==0x1fff */ 1464 } 1465 } else { 1466 for(i=0; i<stage1Top; ++i) { 1467 mbcsData->stage1[i]+=(uint16_t)stage1Top/2; /* stage 2 contains 32-bit entries, stage 1 16-bit entries */ 1468 } 1469 1470 /* stage2Top/Length have counted 32-bit results, now we need to count bytes */ 1471 stage2Length*=4; 1472 /* leave stage2Start counting 32-bit units */ 1473 1474 if(mbcsData->utf8Friendly) { 1475 stageUTF8Length=(mbcsData->utf8Max+1)>>MBCS_UTF8_STAGE_SHIFT; 1476 header.version[2]=(uint8_t)(mbcsData->utf8Max>>8); /* store 0xd7 for max==0xd7ff */ 1477 } 1478 1479 /* stage3Top has already counted bytes */ 1480 } 1481 1482 /* round up stage3Top so that the sizes of all data blocks are multiples of 4 */ 1483 mbcsData->stage3Top=(mbcsData->stage3Top+3)&~3; 1484 1485 /* fill the header */ 1486 if(header.options&MBCS_OPT_INCOMPATIBLE_MASK) { 1487 header.version[0]=5; 1488 if(header.options&MBCS_OPT_NO_FROM_U) { 1489 headerLength=10; /* include fullStage2Length */ 1490 } else { 1491 headerLength=MBCS_HEADER_V5_MIN_LENGTH; /* 9 */ 1492 } 1493 } else { 1494 header.version[0]=4; 1495 headerLength=MBCS_HEADER_V4_LENGTH; /* 8 */ 1496 } 1497 header.version[1]=3; 1498 /* header.version[2] set above for utf8Friendly data */ 1499 1500 header.options|=(uint32_t)headerLength; 1501 1502 header.countStates=mbcsData->ucm->states.countStates; 1503 header.countToUFallbacks=mbcsData->countToUFallbacks; 1504 1505 header.offsetToUCodeUnits= 1506 headerLength*4+ 1507 mbcsData->ucm->states.countStates*1024+ 1508 mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback); 1509 header.offsetFromUTable= 1510 header.offsetToUCodeUnits+ 1511 mbcsData->ucm->states.countToUCodeUnits*2; 1512 header.offsetFromUBytes= 1513 header.offsetFromUTable+ 1514 stage1Top*2+ 1515 stage2Length; 1516 header.fromUBytesLength=mbcsData->stage3Top; 1517 1518 top=header.offsetFromUBytes+stageUTF8Length*2; 1519 if(!(header.options&MBCS_OPT_NO_FROM_U)) { 1520 top+=header.fromUBytesLength; 1521 } 1522 1523 header.flags=(uint8_t)(mbcsData->ucm->states.outputType); 1524 1525 if(tableType&TABLE_EXT) { 1526 if(top>0xffffff) { 1527 fprintf(stderr, "error: offset 0x%lx to extension table exceeds 0xffffff\n", (long)top); 1528 return 0; 1529 } 1530 1531 header.flags|=top<<8; 1532 } 1533 1534 /* write the MBCS data */ 1535 udata_writeBlock(pData, &header, headerLength*4); 1536 udata_writeBlock(pData, mbcsData->ucm->states.stateTable, header.countStates*1024); 1537 udata_writeBlock(pData, mbcsData->toUFallbacks, mbcsData->countToUFallbacks*sizeof(_MBCSToUFallback)); 1538 udata_writeBlock(pData, mbcsData->unicodeCodeUnits, mbcsData->ucm->states.countToUCodeUnits*2); 1539 udata_writeBlock(pData, mbcsData->stage1, stage1Top*2); 1540 if(mbcsData->ucm->states.maxCharLength==1) { 1541 udata_writeBlock(pData, mbcsData->stage2Single+stage2Start, stage2Length); 1542 } else { 1543 udata_writeBlock(pData, mbcsData->stage2+stage2Start, stage2Length); 1544 } 1545 if(!(header.options&MBCS_OPT_NO_FROM_U)) { 1546 udata_writeBlock(pData, mbcsData->fromUBytes, mbcsData->stage3Top); 1547 } 1548 1549 if(stageUTF8Length>0) { 1550 udata_writeBlock(pData, mbcsData->stageUTF8, stageUTF8Length*2); 1551 } 1552 1553 /* return the number of bytes that should have been written */ 1554 return top; 1555 } 1556
