1 /* 2 ****************************************************************************** 3 * 4 * Copyright (C) 2003-2011, International Business Machines 5 * Corporation and others. All Rights Reserved. 6 * 7 ****************************************************************************** 8 * file name: ucnv_ext.cpp 9 * encoding: US-ASCII 10 * tab size: 8 (not used) 11 * indentation:4 12 * 13 * created on: 2003jun13 14 * created by: Markus W. Scherer 15 * 16 * Conversion extensions 17 */ 18 19 #include "unicode/utypes.h" 20 21 #if !UCONFIG_NO_CONVERSION && !UCONFIG_NO_LEGACY_CONVERSION 22 23 #include "unicode/uset.h" 24 #include "ucnv_bld.h" 25 #include "ucnv_cnv.h" 26 #include "ucnv_ext.h" 27 #include "cmemory.h" 28 #include "uassert.h" 29 30 /* to Unicode --------------------------------------------------------------- */ 31 32 /* 33 * @return lookup value for the byte, if found; else 0 34 */ 35 static inline uint32_t 36 ucnv_extFindToU(const uint32_t *toUSection, int32_t length, uint8_t byte) { 37 uint32_t word0, word; 38 int32_t i, start, limit; 39 40 /* check the input byte against the lowest and highest section bytes */ 41 start=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[0]); 42 limit=(int32_t)UCNV_EXT_TO_U_GET_BYTE(toUSection[length-1]); 43 if(byte<start || limit<byte) { 44 return 0; /* the byte is out of range */ 45 } 46 47 if(length==((limit-start)+1)) { 48 /* direct access on a linear array */ 49 return UCNV_EXT_TO_U_GET_VALUE(toUSection[byte-start]); /* could be 0 */ 50 } 51 52 /* word0 is suitable for <=toUSection[] comparison, word for <toUSection[] */ 53 word0=UCNV_EXT_TO_U_MAKE_WORD(byte, 0); 54 55 /* 56 * Shift byte once instead of each section word and add 0xffffff. 57 * We will compare the shifted/added byte (bbffffff) against 58 * section words which have byte values in the same bit position. 59 * If and only if byte bb < section byte ss then bbffffff<ssvvvvvv 60 * for all v=0..f 61 * so we need not mask off the lower 24 bits of each section word. 62 */ 63 word=word0|UCNV_EXT_TO_U_VALUE_MASK; 64 65 /* binary search */ 66 start=0; 67 limit=length; 68 for(;;) { 69 i=limit-start; 70 if(i<=1) { 71 break; /* done */ 72 } 73 /* start<limit-1 */ 74 75 if(i<=4) { 76 /* linear search for the last part */ 77 if(word0<=toUSection[start]) { 78 break; 79 } 80 if(++start<limit && word0<=toUSection[start]) { 81 break; 82 } 83 if(++start<limit && word0<=toUSection[start]) { 84 break; 85 } 86 /* always break at start==limit-1 */ 87 ++start; 88 break; 89 } 90 91 i=(start+limit)/2; 92 if(word<toUSection[i]) { 93 limit=i; 94 } else { 95 start=i; 96 } 97 } 98 99 /* did we really find it? */ 100 if(start<limit && byte==UCNV_EXT_TO_U_GET_BYTE(word=toUSection[start])) { 101 return UCNV_EXT_TO_U_GET_VALUE(word); /* never 0 */ 102 } else { 103 return 0; /* not found */ 104 } 105 } 106 107 /* 108 * TRUE if not an SI/SO stateful converter, 109 * or if the match length fits with the current converter state 110 */ 111 #define UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, match) \ 112 ((sisoState)<0 || ((sisoState)==0) == (match==1)) 113 114 /* 115 * this works like ucnv_extMatchFromU() except 116 * - the first character is in pre 117 * - no trie is used 118 * - the returned matchLength is not offset by 2 119 */ 120 static int32_t 121 ucnv_extMatchToU(const int32_t *cx, int8_t sisoState, 122 const char *pre, int32_t preLength, 123 const char *src, int32_t srcLength, 124 uint32_t *pMatchValue, 125 UBool /*useFallback*/, UBool flush) { 126 const uint32_t *toUTable, *toUSection; 127 128 uint32_t value, matchValue; 129 int32_t i, j, idx, length, matchLength; 130 uint8_t b; 131 132 if(cx==NULL || cx[UCNV_EXT_TO_U_LENGTH]<=0) { 133 return 0; /* no extension data, no match */ 134 } 135 136 /* initialize */ 137 toUTable=UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_INDEX, uint32_t); 138 idx=0; 139 140 matchValue=0; 141 i=j=matchLength=0; 142 143 if(sisoState==0) { 144 /* SBCS state of an SI/SO stateful converter, look at only exactly 1 byte */ 145 if(preLength>1) { 146 return 0; /* no match of a DBCS sequence in SBCS mode */ 147 } else if(preLength==1) { 148 srcLength=0; 149 } else /* preLength==0 */ { 150 if(srcLength>1) { 151 srcLength=1; 152 } 153 } 154 flush=TRUE; 155 } 156 157 /* we must not remember fallback matches when not using fallbacks */ 158 159 /* match input units until there is a full match or the input is consumed */ 160 for(;;) { 161 /* go to the next section */ 162 toUSection=toUTable+idx; 163 164 /* read first pair of the section */ 165 value=*toUSection++; 166 length=UCNV_EXT_TO_U_GET_BYTE(value); 167 value=UCNV_EXT_TO_U_GET_VALUE(value); 168 if( value!=0 && 169 (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) || 170 TO_U_USE_FALLBACK(useFallback)) && 171 UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j) 172 ) { 173 /* remember longest match so far */ 174 matchValue=value; 175 matchLength=i+j; 176 } 177 178 /* match pre[] then src[] */ 179 if(i<preLength) { 180 b=(uint8_t)pre[i++]; 181 } else if(j<srcLength) { 182 b=(uint8_t)src[j++]; 183 } else { 184 /* all input consumed, partial match */ 185 if(flush || (length=(i+j))>UCNV_EXT_MAX_BYTES) { 186 /* 187 * end of the entire input stream, stop with the longest match so far 188 * or: partial match must not be longer than UCNV_EXT_MAX_BYTES 189 * because it must fit into state buffers 190 */ 191 break; 192 } else { 193 /* continue with more input next time */ 194 return -length; 195 } 196 } 197 198 /* search for the current UChar */ 199 value=ucnv_extFindToU(toUSection, length, b); 200 if(value==0) { 201 /* no match here, stop with the longest match so far */ 202 break; 203 } else { 204 if(UCNV_EXT_TO_U_IS_PARTIAL(value)) { 205 /* partial match, continue */ 206 idx=(int32_t)UCNV_EXT_TO_U_GET_PARTIAL_INDEX(value); 207 } else { 208 if( (UCNV_EXT_TO_U_IS_ROUNDTRIP(value) || 209 TO_U_USE_FALLBACK(useFallback)) && 210 UCNV_EXT_TO_U_VERIFY_SISO_MATCH(sisoState, i+j) 211 ) { 212 /* full match, stop with result */ 213 matchValue=value; 214 matchLength=i+j; 215 } else { 216 /* full match on fallback not taken, stop with the longest match so far */ 217 } 218 break; 219 } 220 } 221 } 222 223 if(matchLength==0) { 224 /* no match at all */ 225 return 0; 226 } 227 228 /* return result */ 229 *pMatchValue=UCNV_EXT_TO_U_MASK_ROUNDTRIP(matchValue); 230 return matchLength; 231 } 232 233 static inline void 234 ucnv_extWriteToU(UConverter *cnv, const int32_t *cx, 235 uint32_t value, 236 UChar **target, const UChar *targetLimit, 237 int32_t **offsets, int32_t srcIndex, 238 UErrorCode *pErrorCode) { 239 /* output the result */ 240 if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) { 241 /* output a single code point */ 242 ucnv_toUWriteCodePoint( 243 cnv, UCNV_EXT_TO_U_GET_CODE_POINT(value), 244 target, targetLimit, 245 offsets, srcIndex, 246 pErrorCode); 247 } else { 248 /* output a string - with correct data we have resultLength>0 */ 249 ucnv_toUWriteUChars( 250 cnv, 251 UCNV_EXT_ARRAY(cx, UCNV_EXT_TO_U_UCHARS_INDEX, UChar)+ 252 UCNV_EXT_TO_U_GET_INDEX(value), 253 UCNV_EXT_TO_U_GET_LENGTH(value), 254 target, targetLimit, 255 offsets, srcIndex, 256 pErrorCode); 257 } 258 } 259 260 /* 261 * get the SI/SO toU state (state 0 is for SBCS, 1 for DBCS), 262 * or 1 for DBCS-only, 263 * or -1 if the converter is not SI/SO stateful 264 * 265 * Note: For SI/SO stateful converters getting here, 266 * cnv->mode==0 is equivalent to firstLength==1. 267 */ 268 #define UCNV_SISO_STATE(cnv) \ 269 ((cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_2_SISO ? (int8_t)(cnv)->mode : \ 270 (cnv)->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY ? 1 : -1) 271 272 /* 273 * target<targetLimit; set error code for overflow 274 */ 275 U_CFUNC UBool 276 ucnv_extInitialMatchToU(UConverter *cnv, const int32_t *cx, 277 int32_t firstLength, 278 const char **src, const char *srcLimit, 279 UChar **target, const UChar *targetLimit, 280 int32_t **offsets, int32_t srcIndex, 281 UBool flush, 282 UErrorCode *pErrorCode) { 283 uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */ 284 int32_t match; 285 286 /* try to match */ 287 match=ucnv_extMatchToU(cx, (int8_t)UCNV_SISO_STATE(cnv), 288 (const char *)cnv->toUBytes, firstLength, 289 *src, (int32_t)(srcLimit-*src), 290 &value, 291 cnv->useFallback, flush); 292 if(match>0) { 293 /* advance src pointer for the consumed input */ 294 *src+=match-firstLength; 295 296 /* write result to target */ 297 ucnv_extWriteToU(cnv, cx, 298 value, 299 target, targetLimit, 300 offsets, srcIndex, 301 pErrorCode); 302 return TRUE; 303 } else if(match<0) { 304 /* save state for partial match */ 305 const char *s; 306 int32_t j; 307 308 /* copy the first code point */ 309 s=(const char *)cnv->toUBytes; 310 cnv->preToUFirstLength=(int8_t)firstLength; 311 for(j=0; j<firstLength; ++j) { 312 cnv->preToU[j]=*s++; 313 } 314 315 /* now copy the newly consumed input */ 316 s=*src; 317 match=-match; 318 for(; j<match; ++j) { 319 cnv->preToU[j]=*s++; 320 } 321 *src=s; /* same as *src=srcLimit; because we reached the end of input */ 322 cnv->preToULength=(int8_t)match; 323 return TRUE; 324 } else /* match==0 no match */ { 325 return FALSE; 326 } 327 } 328 329 U_CFUNC UChar32 330 ucnv_extSimpleMatchToU(const int32_t *cx, 331 const char *source, int32_t length, 332 UBool useFallback) { 333 uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */ 334 int32_t match; 335 336 if(length<=0) { 337 return 0xffff; 338 } 339 340 /* try to match */ 341 match=ucnv_extMatchToU(cx, -1, 342 source, length, 343 NULL, 0, 344 &value, 345 useFallback, TRUE); 346 if(match==length) { 347 /* write result for simple, single-character conversion */ 348 if(UCNV_EXT_TO_U_IS_CODE_POINT(value)) { 349 return UCNV_EXT_TO_U_GET_CODE_POINT(value); 350 } 351 } 352 353 /* 354 * return no match because 355 * - match>0 && value points to string: simple conversion cannot handle multiple code points 356 * - match>0 && match!=length: not all input consumed, forbidden for this function 357 * - match==0: no match found in the first place 358 * - match<0: partial match, not supported for simple conversion (and flush==TRUE) 359 */ 360 return 0xfffe; 361 } 362 363 /* 364 * continue partial match with new input 365 * never called for simple, single-character conversion 366 */ 367 U_CFUNC void 368 ucnv_extContinueMatchToU(UConverter *cnv, 369 UConverterToUnicodeArgs *pArgs, int32_t srcIndex, 370 UErrorCode *pErrorCode) { 371 uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */ 372 int32_t match, length; 373 374 match=ucnv_extMatchToU(cnv->sharedData->mbcs.extIndexes, (int8_t)UCNV_SISO_STATE(cnv), 375 cnv->preToU, cnv->preToULength, 376 pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source), 377 &value, 378 cnv->useFallback, pArgs->flush); 379 if(match>0) { 380 if(match>=cnv->preToULength) { 381 /* advance src pointer for the consumed input */ 382 pArgs->source+=match-cnv->preToULength; 383 cnv->preToULength=0; 384 } else { 385 /* the match did not use all of preToU[] - keep the rest for replay */ 386 length=cnv->preToULength-match; 387 uprv_memmove(cnv->preToU, cnv->preToU+match, length); 388 cnv->preToULength=(int8_t)-length; 389 } 390 391 /* write result */ 392 ucnv_extWriteToU(cnv, cnv->sharedData->mbcs.extIndexes, 393 value, 394 &pArgs->target, pArgs->targetLimit, 395 &pArgs->offsets, srcIndex, 396 pErrorCode); 397 } else if(match<0) { 398 /* save state for partial match */ 399 const char *s; 400 int32_t j; 401 402 /* just _append_ the newly consumed input to preToU[] */ 403 s=pArgs->source; 404 match=-match; 405 for(j=cnv->preToULength; j<match; ++j) { 406 cnv->preToU[j]=*s++; 407 } 408 pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */ 409 cnv->preToULength=(int8_t)match; 410 } else /* match==0 */ { 411 /* 412 * no match 413 * 414 * We need to split the previous input into two parts: 415 * 416 * 1. The first codepage character is unmappable - that's how we got into 417 * trying the extension data in the first place. 418 * We need to move it from the preToU buffer 419 * to the error buffer, set an error code, 420 * and prepare the rest of the previous input for 2. 421 * 422 * 2. The rest of the previous input must be converted once we 423 * come back from the callback for the first character. 424 * At that time, we have to try again from scratch to convert 425 * these input characters. 426 * The replay will be handled by the ucnv.c conversion code. 427 */ 428 429 /* move the first codepage character to the error field */ 430 uprv_memcpy(cnv->toUBytes, cnv->preToU, cnv->preToUFirstLength); 431 cnv->toULength=cnv->preToUFirstLength; 432 433 /* move the rest up inside the buffer */ 434 length=cnv->preToULength-cnv->preToUFirstLength; 435 if(length>0) { 436 uprv_memmove(cnv->preToU, cnv->preToU+cnv->preToUFirstLength, length); 437 } 438 439 /* mark preToU for replay */ 440 cnv->preToULength=(int8_t)-length; 441 442 /* set the error code for unassigned */ 443 *pErrorCode=U_INVALID_CHAR_FOUND; 444 } 445 } 446 447 /* from Unicode ------------------------------------------------------------- */ 448 449 /* 450 * @return index of the UChar, if found; else <0 451 */ 452 static inline int32_t 453 ucnv_extFindFromU(const UChar *fromUSection, int32_t length, UChar u) { 454 int32_t i, start, limit; 455 456 /* binary search */ 457 start=0; 458 limit=length; 459 for(;;) { 460 i=limit-start; 461 if(i<=1) { 462 break; /* done */ 463 } 464 /* start<limit-1 */ 465 466 if(i<=4) { 467 /* linear search for the last part */ 468 if(u<=fromUSection[start]) { 469 break; 470 } 471 if(++start<limit && u<=fromUSection[start]) { 472 break; 473 } 474 if(++start<limit && u<=fromUSection[start]) { 475 break; 476 } 477 /* always break at start==limit-1 */ 478 ++start; 479 break; 480 } 481 482 i=(start+limit)/2; 483 if(u<fromUSection[i]) { 484 limit=i; 485 } else { 486 start=i; 487 } 488 } 489 490 /* did we really find it? */ 491 if(start<limit && u==fromUSection[start]) { 492 return start; 493 } else { 494 return -1; /* not found */ 495 } 496 } 497 498 /* 499 * @param cx pointer to extension data; if NULL, returns 0 500 * @param firstCP the first code point before all the other UChars 501 * @param pre UChars that must match; !initialMatch: partial match with them 502 * @param preLength length of pre, >=0 503 * @param src UChars that can be used to complete a match 504 * @param srcLength length of src, >=0 505 * @param pMatchValue [out] output result value for the match from the data structure 506 * @param useFallback "use fallback" flag, usually from cnv->useFallback 507 * @param flush TRUE if the end of the input stream is reached 508 * @return >1: matched, return value=total match length (number of input units matched) 509 * 1: matched, no mapping but request for <subchar1> 510 * (only for the first code point) 511 * 0: no match 512 * <0: partial match, return value=negative total match length 513 * (partial matches are never returned for flush==TRUE) 514 * (partial matches are never returned as being longer than UCNV_EXT_MAX_UCHARS) 515 * the matchLength is 2 if only firstCP matched, and >2 if firstCP and 516 * further code units matched 517 */ 518 static int32_t 519 ucnv_extMatchFromU(const int32_t *cx, 520 UChar32 firstCP, 521 const UChar *pre, int32_t preLength, 522 const UChar *src, int32_t srcLength, 523 uint32_t *pMatchValue, 524 UBool useFallback, UBool flush) { 525 const uint16_t *stage12, *stage3; 526 const uint32_t *stage3b; 527 528 const UChar *fromUTableUChars, *fromUSectionUChars; 529 const uint32_t *fromUTableValues, *fromUSectionValues; 530 531 uint32_t value, matchValue; 532 int32_t i, j, idx, length, matchLength; 533 UChar c; 534 535 if(cx==NULL) { 536 return 0; /* no extension data, no match */ 537 } 538 539 /* trie lookup of firstCP */ 540 idx=firstCP>>10; /* stage 1 index */ 541 if(idx>=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH]) { 542 return 0; /* the first code point is outside the trie */ 543 } 544 545 stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t); 546 stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t); 547 idx=UCNV_EXT_FROM_U(stage12, stage3, idx, firstCP); 548 549 stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t); 550 value=stage3b[idx]; 551 if(value==0) { 552 return 0; 553 } 554 555 /* 556 * Tests for (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0: 557 * Do not interpret values with reserved bits used, for forward compatibility, 558 * and do not even remember intermediate results with reserved bits used. 559 */ 560 561 if(UCNV_EXT_TO_U_IS_PARTIAL(value)) { 562 /* partial match, enter the loop below */ 563 idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value); 564 565 /* initialize */ 566 fromUTableUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar); 567 fromUTableValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t); 568 569 matchValue=0; 570 i=j=matchLength=0; 571 572 /* we must not remember fallback matches when not using fallbacks */ 573 574 /* match input units until there is a full match or the input is consumed */ 575 for(;;) { 576 /* go to the next section */ 577 fromUSectionUChars=fromUTableUChars+idx; 578 fromUSectionValues=fromUTableValues+idx; 579 580 /* read first pair of the section */ 581 length=*fromUSectionUChars++; 582 value=*fromUSectionValues++; 583 if( value!=0 && 584 (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) || 585 FROM_U_USE_FALLBACK(useFallback, firstCP)) && 586 (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 587 ) { 588 /* remember longest match so far */ 589 matchValue=value; 590 matchLength=2+i+j; 591 } 592 593 /* match pre[] then src[] */ 594 if(i<preLength) { 595 c=pre[i++]; 596 } else if(j<srcLength) { 597 c=src[j++]; 598 } else { 599 /* all input consumed, partial match */ 600 if(flush || (length=(i+j))>UCNV_EXT_MAX_UCHARS) { 601 /* 602 * end of the entire input stream, stop with the longest match so far 603 * or: partial match must not be longer than UCNV_EXT_MAX_UCHARS 604 * because it must fit into state buffers 605 */ 606 break; 607 } else { 608 /* continue with more input next time */ 609 return -(2+length); 610 } 611 } 612 613 /* search for the current UChar */ 614 idx=ucnv_extFindFromU(fromUSectionUChars, length, c); 615 if(idx<0) { 616 /* no match here, stop with the longest match so far */ 617 break; 618 } else { 619 value=fromUSectionValues[idx]; 620 if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) { 621 /* partial match, continue */ 622 idx=(int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value); 623 } else { 624 if( (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) || 625 FROM_U_USE_FALLBACK(useFallback, firstCP)) && 626 (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 627 ) { 628 /* full match, stop with result */ 629 matchValue=value; 630 matchLength=2+i+j; 631 } else { 632 /* full match on fallback not taken, stop with the longest match so far */ 633 } 634 break; 635 } 636 } 637 } 638 639 if(matchLength==0) { 640 /* no match at all */ 641 return 0; 642 } 643 } else /* result from firstCP trie lookup */ { 644 if( (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) || 645 FROM_U_USE_FALLBACK(useFallback, firstCP)) && 646 (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 647 ) { 648 /* full match, stop with result */ 649 matchValue=value; 650 matchLength=2; 651 } else { 652 /* fallback not taken */ 653 return 0; 654 } 655 } 656 657 /* return result */ 658 if(matchValue==UCNV_EXT_FROM_U_SUBCHAR1) { 659 return 1; /* assert matchLength==2 */ 660 } 661 662 *pMatchValue=matchValue; 663 return matchLength; 664 } 665 666 /* 667 * @param value fromUnicode mapping table value; ignores roundtrip and reserved bits 668 */ 669 static inline void 670 ucnv_extWriteFromU(UConverter *cnv, const int32_t *cx, 671 uint32_t value, 672 char **target, const char *targetLimit, 673 int32_t **offsets, int32_t srcIndex, 674 UErrorCode *pErrorCode) { 675 uint8_t buffer[1+UCNV_EXT_MAX_BYTES]; 676 const uint8_t *result; 677 int32_t length, prevLength; 678 679 length=UCNV_EXT_FROM_U_GET_LENGTH(value); 680 value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value); 681 682 /* output the result */ 683 if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) { 684 /* 685 * Generate a byte array and then write it below. 686 * This is not the fastest possible way, but it should be ok for 687 * extension mappings, and it is much simpler. 688 * Offset and overflow handling are only done once this way. 689 */ 690 uint8_t *p=buffer+1; /* reserve buffer[0] for shiftByte below */ 691 switch(length) { 692 case 3: 693 *p++=(uint8_t)(value>>16); 694 case 2: /*fall through*/ 695 *p++=(uint8_t)(value>>8); 696 case 1: /*fall through*/ 697 *p++=(uint8_t)value; 698 default: 699 break; /* will never occur */ 700 } 701 result=buffer+1; 702 } else { 703 result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value; 704 } 705 706 /* with correct data we have length>0 */ 707 708 if((prevLength=cnv->fromUnicodeStatus)!=0) { 709 /* handle SI/SO stateful output */ 710 uint8_t shiftByte; 711 712 if(prevLength>1 && length==1) { 713 /* change from double-byte mode to single-byte */ 714 shiftByte=(uint8_t)UCNV_SI; 715 cnv->fromUnicodeStatus=1; 716 } else if(prevLength==1 && length>1) { 717 /* change from single-byte mode to double-byte */ 718 shiftByte=(uint8_t)UCNV_SO; 719 cnv->fromUnicodeStatus=2; 720 } else { 721 shiftByte=0; 722 } 723 724 if(shiftByte!=0) { 725 /* prepend the shift byte to the result bytes */ 726 buffer[0]=shiftByte; 727 if(result!=buffer+1) { 728 uprv_memcpy(buffer+1, result, length); 729 } 730 result=buffer; 731 ++length; 732 } 733 } 734 735 ucnv_fromUWriteBytes(cnv, (const char *)result, length, 736 target, targetLimit, 737 offsets, srcIndex, 738 pErrorCode); 739 } 740 741 /* 742 * target<targetLimit; set error code for overflow 743 */ 744 U_CFUNC UBool 745 ucnv_extInitialMatchFromU(UConverter *cnv, const int32_t *cx, 746 UChar32 cp, 747 const UChar **src, const UChar *srcLimit, 748 char **target, const char *targetLimit, 749 int32_t **offsets, int32_t srcIndex, 750 UBool flush, 751 UErrorCode *pErrorCode) { 752 uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */ 753 int32_t match; 754 755 /* try to match */ 756 match=ucnv_extMatchFromU(cx, cp, 757 NULL, 0, 758 *src, (int32_t)(srcLimit-*src), 759 &value, 760 cnv->useFallback, flush); 761 762 /* reject a match if the result is a single byte for DBCS-only */ 763 if( match>=2 && 764 !(UCNV_EXT_FROM_U_GET_LENGTH(value)==1 && 765 cnv->sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY) 766 ) { 767 /* advance src pointer for the consumed input */ 768 *src+=match-2; /* remove 2 for the initial code point */ 769 770 /* write result to target */ 771 ucnv_extWriteFromU(cnv, cx, 772 value, 773 target, targetLimit, 774 offsets, srcIndex, 775 pErrorCode); 776 return TRUE; 777 } else if(match<0) { 778 /* save state for partial match */ 779 const UChar *s; 780 int32_t j; 781 782 /* copy the first code point */ 783 cnv->preFromUFirstCP=cp; 784 785 /* now copy the newly consumed input */ 786 s=*src; 787 match=-match-2; /* remove 2 for the initial code point */ 788 for(j=0; j<match; ++j) { 789 cnv->preFromU[j]=*s++; 790 } 791 *src=s; /* same as *src=srcLimit; because we reached the end of input */ 792 cnv->preFromULength=(int8_t)match; 793 return TRUE; 794 } else if(match==1) { 795 /* matched, no mapping but request for <subchar1> */ 796 cnv->useSubChar1=TRUE; 797 return FALSE; 798 } else /* match==0 no match */ { 799 return FALSE; 800 } 801 } 802 803 /* 804 * Used by ISO 2022 implementation. 805 * @return number of bytes in *pValue; negative number if fallback; 0 for no mapping 806 */ 807 U_CFUNC int32_t 808 ucnv_extSimpleMatchFromU(const int32_t *cx, 809 UChar32 cp, uint32_t *pValue, 810 UBool useFallback) { 811 uint32_t value; 812 int32_t match; 813 814 /* try to match */ 815 match=ucnv_extMatchFromU(cx, 816 cp, 817 NULL, 0, 818 NULL, 0, 819 &value, 820 useFallback, TRUE); 821 if(match>=2) { 822 /* write result for simple, single-character conversion */ 823 int32_t length; 824 int isRoundtrip; 825 826 isRoundtrip=UCNV_EXT_FROM_U_IS_ROUNDTRIP(value); 827 length=UCNV_EXT_FROM_U_GET_LENGTH(value); 828 value=(uint32_t)UCNV_EXT_FROM_U_GET_DATA(value); 829 830 if(length<=UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH) { 831 *pValue=value; 832 return isRoundtrip ? length : -length; 833 #if 0 /* not currently used */ 834 } else if(length==4) { 835 /* de-serialize a 4-byte result */ 836 const uint8_t *result=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_BYTES_INDEX, uint8_t)+value; 837 *pValue= 838 ((uint32_t)result[0]<<24)| 839 ((uint32_t)result[1]<<16)| 840 ((uint32_t)result[2]<<8)| 841 result[3]; 842 return isRoundtrip ? 4 : -4; 843 #endif 844 } 845 } 846 847 /* 848 * return no match because 849 * - match>1 && resultLength>4: result too long for simple conversion 850 * - match==1: no match found, <subchar1> preferred 851 * - match==0: no match found in the first place 852 * - match<0: partial match, not supported for simple conversion (and flush==TRUE) 853 */ 854 return 0; 855 } 856 857 /* 858 * continue partial match with new input, requires cnv->preFromUFirstCP>=0 859 * never called for simple, single-character conversion 860 */ 861 U_CFUNC void 862 ucnv_extContinueMatchFromU(UConverter *cnv, 863 UConverterFromUnicodeArgs *pArgs, int32_t srcIndex, 864 UErrorCode *pErrorCode) { 865 uint32_t value = 0; /* initialize output-only param to 0 to silence gcc */ 866 int32_t match; 867 868 match=ucnv_extMatchFromU(cnv->sharedData->mbcs.extIndexes, 869 cnv->preFromUFirstCP, 870 cnv->preFromU, cnv->preFromULength, 871 pArgs->source, (int32_t)(pArgs->sourceLimit-pArgs->source), 872 &value, 873 cnv->useFallback, pArgs->flush); 874 if(match>=2) { 875 match-=2; /* remove 2 for the initial code point */ 876 877 if(match>=cnv->preFromULength) { 878 /* advance src pointer for the consumed input */ 879 pArgs->source+=match-cnv->preFromULength; 880 cnv->preFromULength=0; 881 } else { 882 /* the match did not use all of preFromU[] - keep the rest for replay */ 883 int32_t length=cnv->preFromULength-match; 884 uprv_memmove(cnv->preFromU, cnv->preFromU+match, length*U_SIZEOF_UCHAR); 885 cnv->preFromULength=(int8_t)-length; 886 } 887 888 /* finish the partial match */ 889 cnv->preFromUFirstCP=U_SENTINEL; 890 891 /* write result */ 892 ucnv_extWriteFromU(cnv, cnv->sharedData->mbcs.extIndexes, 893 value, 894 &pArgs->target, pArgs->targetLimit, 895 &pArgs->offsets, srcIndex, 896 pErrorCode); 897 } else if(match<0) { 898 /* save state for partial match */ 899 const UChar *s; 900 int32_t j; 901 902 /* just _append_ the newly consumed input to preFromU[] */ 903 s=pArgs->source; 904 match=-match-2; /* remove 2 for the initial code point */ 905 for(j=cnv->preFromULength; j<match; ++j) { 906 U_ASSERT(j>=0); 907 cnv->preFromU[j]=*s++; 908 } 909 pArgs->source=s; /* same as *src=srcLimit; because we reached the end of input */ 910 cnv->preFromULength=(int8_t)match; 911 } else /* match==0 or 1 */ { 912 /* 913 * no match 914 * 915 * We need to split the previous input into two parts: 916 * 917 * 1. The first code point is unmappable - that's how we got into 918 * trying the extension data in the first place. 919 * We need to move it from the preFromU buffer 920 * to the error buffer, set an error code, 921 * and prepare the rest of the previous input for 2. 922 * 923 * 2. The rest of the previous input must be converted once we 924 * come back from the callback for the first code point. 925 * At that time, we have to try again from scratch to convert 926 * these input characters. 927 * The replay will be handled by the ucnv.c conversion code. 928 */ 929 930 if(match==1) { 931 /* matched, no mapping but request for <subchar1> */ 932 cnv->useSubChar1=TRUE; 933 } 934 935 /* move the first code point to the error field */ 936 cnv->fromUChar32=cnv->preFromUFirstCP; 937 cnv->preFromUFirstCP=U_SENTINEL; 938 939 /* mark preFromU for replay */ 940 cnv->preFromULength=-cnv->preFromULength; 941 942 /* set the error code for unassigned */ 943 *pErrorCode=U_INVALID_CHAR_FOUND; 944 } 945 } 946 947 static void 948 ucnv_extGetUnicodeSetString(const UConverterSharedData *sharedData, 949 const int32_t *cx, 950 const USetAdder *sa, 951 UBool useFallback, 952 int32_t minLength, 953 UChar32 c, 954 UChar s[UCNV_EXT_MAX_UCHARS], int32_t length, 955 int32_t sectionIndex, 956 UErrorCode *pErrorCode) { 957 const UChar *fromUSectionUChars; 958 const uint32_t *fromUSectionValues; 959 960 uint32_t value; 961 int32_t i, count; 962 963 fromUSectionUChars=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_UCHARS_INDEX, UChar)+sectionIndex; 964 fromUSectionValues=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_VALUES_INDEX, uint32_t)+sectionIndex; 965 966 /* read first pair of the section */ 967 count=*fromUSectionUChars++; 968 value=*fromUSectionValues++; 969 970 if( value!=0 && 971 (UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) || useFallback) && 972 UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength 973 ) { 974 if(c>=0) { 975 /* add the initial code point */ 976 sa->add(sa->set, c); 977 } else { 978 /* add the string so far */ 979 sa->addString(sa->set, s, length); 980 } 981 } 982 983 for(i=0; i<count; ++i) { 984 /* append this code unit and recurse or add the string */ 985 s[length]=fromUSectionUChars[i]; 986 value=fromUSectionValues[i]; 987 988 if(value==0) { 989 /* no mapping, do nothing */ 990 } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) { 991 ucnv_extGetUnicodeSetString( 992 sharedData, cx, sa, useFallback, minLength, 993 U_SENTINEL, s, length+1, 994 (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value), 995 pErrorCode); 996 } else if((useFallback ? 997 (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 : 998 ((value&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG|UCNV_EXT_FROM_U_RESERVED_MASK))== 999 UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)) && 1000 UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength 1001 ) { 1002 sa->addString(sa->set, s, length+1); 1003 } 1004 } 1005 } 1006 1007 U_CFUNC void 1008 ucnv_extGetUnicodeSet(const UConverterSharedData *sharedData, 1009 const USetAdder *sa, 1010 UConverterUnicodeSet which, 1011 UConverterSetFilter filter, 1012 UErrorCode *pErrorCode) { 1013 const int32_t *cx; 1014 const uint16_t *stage12, *stage3, *ps2, *ps3; 1015 const uint32_t *stage3b; 1016 1017 uint32_t value; 1018 int32_t st1, stage1Length, st2, st3, minLength; 1019 UBool useFallback; 1020 1021 UChar s[UCNV_EXT_MAX_UCHARS]; 1022 UChar32 c; 1023 int32_t length; 1024 1025 cx=sharedData->mbcs.extIndexes; 1026 if(cx==NULL) { 1027 return; 1028 } 1029 1030 stage12=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_12_INDEX, uint16_t); 1031 stage3=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3_INDEX, uint16_t); 1032 stage3b=UCNV_EXT_ARRAY(cx, UCNV_EXT_FROM_U_STAGE_3B_INDEX, uint32_t); 1033 1034 stage1Length=cx[UCNV_EXT_FROM_U_STAGE_1_LENGTH]; 1035 1036 useFallback=(UBool)(which==UCNV_ROUNDTRIP_AND_FALLBACK_SET); 1037 1038 /* enumerate the from-Unicode trie table */ 1039 c=0; /* keep track of the current code point while enumerating */ 1040 1041 if(filter==UCNV_SET_FILTER_2022_CN) { 1042 minLength=3; 1043 } else if( sharedData->mbcs.outputType==MBCS_OUTPUT_DBCS_ONLY || 1044 filter!=UCNV_SET_FILTER_NONE 1045 ) { 1046 /* DBCS-only, ignore single-byte results */ 1047 minLength=2; 1048 } else { 1049 minLength=1; 1050 } 1051 1052 /* 1053 * the trie enumeration is almost the same as 1054 * in MBCSGetUnicodeSet() for MBCS_OUTPUT_1 1055 */ 1056 for(st1=0; st1<stage1Length; ++st1) { 1057 st2=stage12[st1]; 1058 if(st2>stage1Length) { 1059 ps2=stage12+st2; 1060 for(st2=0; st2<64; ++st2) { 1061 if((st3=(int32_t)ps2[st2]<<UCNV_EXT_STAGE_2_LEFT_SHIFT)!=0) { 1062 /* read the stage 3 block */ 1063 ps3=stage3+st3; 1064 1065 /* 1066 * Add code points for which the roundtrip flag is set. 1067 * Do not add <subchar1> entries or other (future?) pseudo-entries 1068 * with an output length of 0, or entries with reserved bits set. 1069 * Recurse for partial results. 1070 */ 1071 do { 1072 value=stage3b[*ps3++]; 1073 if(value==0) { 1074 /* no mapping, do nothing */ 1075 } else if(UCNV_EXT_FROM_U_IS_PARTIAL(value)) { 1076 length=0; 1077 U16_APPEND_UNSAFE(s, length, c); 1078 ucnv_extGetUnicodeSetString( 1079 sharedData, cx, sa, useFallback, minLength, 1080 c, s, length, 1081 (int32_t)UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value), 1082 pErrorCode); 1083 } else if((useFallback ? 1084 (value&UCNV_EXT_FROM_U_RESERVED_MASK)==0 : 1085 ((value&(UCNV_EXT_FROM_U_ROUNDTRIP_FLAG|UCNV_EXT_FROM_U_RESERVED_MASK))== 1086 UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)) && 1087 UCNV_EXT_FROM_U_GET_LENGTH(value)>=minLength 1088 ) { 1089 switch(filter) { 1090 case UCNV_SET_FILTER_2022_CN: 1091 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==3 && UCNV_EXT_FROM_U_GET_DATA(value)<=0x82ffff)) { 1092 continue; 1093 } 1094 break; 1095 case UCNV_SET_FILTER_SJIS: 1096 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 && (value=UCNV_EXT_FROM_U_GET_DATA(value))>=0x8140 && value<=0xeffc)) { 1097 continue; 1098 } 1099 break; 1100 case UCNV_SET_FILTER_GR94DBCS: 1101 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 && 1102 (uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfefe - 0xa1a1) && 1103 (uint8_t)(value-0xa1)<=(0xfe - 0xa1))) { 1104 continue; 1105 } 1106 break; 1107 case UCNV_SET_FILTER_HZ: 1108 if(!(UCNV_EXT_FROM_U_GET_LENGTH(value)==2 && 1109 (uint16_t)((value=UCNV_EXT_FROM_U_GET_DATA(value))-0xa1a1)<=(0xfdfe - 0xa1a1) && 1110 (uint8_t)(value-0xa1)<=(0xfe - 0xa1))) { 1111 continue; 1112 } 1113 break; 1114 default: 1115 /* 1116 * UCNV_SET_FILTER_NONE, 1117 * or UCNV_SET_FILTER_DBCS_ONLY which is handled via minLength 1118 */ 1119 break; 1120 } 1121 sa->add(sa->set, c); 1122 } 1123 } while((++c&0xf)!=0); 1124 } else { 1125 c+=16; /* empty stage 3 block */ 1126 } 1127 } 1128 } else { 1129 c+=1024; /* empty stage 2 block */ 1130 } 1131 } 1132 } 1133 1134 #endif /* #if !UCONFIG_NO_LEGACY_CONVERSION */ 1135
