1 // Copyright (C) 2016 and later: Unicode, Inc. and others. 2 // License & terms of use: http://www.unicode.org/copyright.html 3 /* 4 ****************************************************************************** 5 * 6 * Copyright (C) 2001-2016, International Business Machines 7 * Corporation and others. All Rights Reserved. 8 * 9 ****************************************************************************** 10 * 11 * File ustrtrns.cpp 12 * 13 * Modification History: 14 * 15 * Date Name Description 16 * 9/10/2001 Ram Creation. 17 ****************************************************************************** 18 */ 19 20 /******************************************************************************* 21 * 22 * u_strTo* and u_strFrom* APIs 23 * WCS functions moved to ustr_wcs.c for better modularization 24 * 25 ******************************************************************************* 26 */ 27 28 29 #include "unicode/putil.h" 30 #include "unicode/ustring.h" 31 #include "unicode/utf.h" 32 #include "unicode/utf8.h" 33 #include "unicode/utf16.h" 34 #include "cstring.h" 35 #include "cmemory.h" 36 #include "ustr_imp.h" 37 #include "uassert.h" 38 39 U_CAPI UChar* U_EXPORT2 40 u_strFromUTF32WithSub(UChar *dest, 41 int32_t destCapacity, 42 int32_t *pDestLength, 43 const UChar32 *src, 44 int32_t srcLength, 45 UChar32 subchar, int32_t *pNumSubstitutions, 46 UErrorCode *pErrorCode) { 47 const UChar32 *srcLimit; 48 UChar32 ch; 49 UChar *destLimit; 50 UChar *pDest; 51 int32_t reqLength; 52 int32_t numSubstitutions; 53 54 /* args check */ 55 if(U_FAILURE(*pErrorCode)){ 56 return NULL; 57 } 58 if( (src==NULL && srcLength!=0) || srcLength < -1 || 59 (destCapacity<0) || (dest == NULL && destCapacity > 0) || 60 subchar > 0x10ffff || U_IS_SURROGATE(subchar) 61 ) { 62 *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; 63 return NULL; 64 } 65 66 if(pNumSubstitutions != NULL) { 67 *pNumSubstitutions = 0; 68 } 69 70 pDest = dest; 71 destLimit = (dest!=NULL)?(dest + destCapacity):NULL; 72 reqLength = 0; 73 numSubstitutions = 0; 74 75 if(srcLength < 0) { 76 /* simple loop for conversion of a NUL-terminated BMP string */ 77 while((ch=*src) != 0 && 78 ((uint32_t)ch < 0xd800 || (0xe000 <= ch && ch <= 0xffff))) { 79 ++src; 80 if(pDest < destLimit) { 81 *pDest++ = (UChar)ch; 82 } else { 83 ++reqLength; 84 } 85 } 86 srcLimit = src; 87 if(ch != 0) { 88 /* "complicated" case, find the end of the remaining string */ 89 while(*++srcLimit != 0) {} 90 } 91 } else { 92 srcLimit = (src!=NULL)?(src + srcLength):NULL; 93 } 94 95 /* convert with length */ 96 while(src < srcLimit) { 97 ch = *src++; 98 do { 99 /* usually "loops" once; twice only for writing subchar */ 100 if((uint32_t)ch < 0xd800 || (0xe000 <= ch && ch <= 0xffff)) { 101 if(pDest < destLimit) { 102 *pDest++ = (UChar)ch; 103 } else { 104 ++reqLength; 105 } 106 break; 107 } else if(0x10000 <= ch && ch <= 0x10ffff) { 108 if(pDest!=NULL && ((pDest + 2) <= destLimit)) { 109 *pDest++ = U16_LEAD(ch); 110 *pDest++ = U16_TRAIL(ch); 111 } else { 112 reqLength += 2; 113 } 114 break; 115 } else if((ch = subchar) < 0) { 116 /* surrogate code point, or not a Unicode code point at all */ 117 *pErrorCode = U_INVALID_CHAR_FOUND; 118 return NULL; 119 } else { 120 ++numSubstitutions; 121 } 122 } while(TRUE); 123 } 124 125 reqLength += (int32_t)(pDest - dest); 126 if(pDestLength) { 127 *pDestLength = reqLength; 128 } 129 if(pNumSubstitutions != NULL) { 130 *pNumSubstitutions = numSubstitutions; 131 } 132 133 /* Terminate the buffer */ 134 u_terminateUChars(dest, destCapacity, reqLength, pErrorCode); 135 136 return dest; 137 } 138 139 U_CAPI UChar* U_EXPORT2 140 u_strFromUTF32(UChar *dest, 141 int32_t destCapacity, 142 int32_t *pDestLength, 143 const UChar32 *src, 144 int32_t srcLength, 145 UErrorCode *pErrorCode) { 146 return u_strFromUTF32WithSub( 147 dest, destCapacity, pDestLength, 148 src, srcLength, 149 U_SENTINEL, NULL, 150 pErrorCode); 151 } 152 153 U_CAPI UChar32* U_EXPORT2 154 u_strToUTF32WithSub(UChar32 *dest, 155 int32_t destCapacity, 156 int32_t *pDestLength, 157 const UChar *src, 158 int32_t srcLength, 159 UChar32 subchar, int32_t *pNumSubstitutions, 160 UErrorCode *pErrorCode) { 161 const UChar *srcLimit; 162 UChar32 ch; 163 UChar ch2; 164 UChar32 *destLimit; 165 UChar32 *pDest; 166 int32_t reqLength; 167 int32_t numSubstitutions; 168 169 /* args check */ 170 if(U_FAILURE(*pErrorCode)){ 171 return NULL; 172 } 173 if( (src==NULL && srcLength!=0) || srcLength < -1 || 174 (destCapacity<0) || (dest == NULL && destCapacity > 0) || 175 subchar > 0x10ffff || U_IS_SURROGATE(subchar) 176 ) { 177 *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; 178 return NULL; 179 } 180 181 if(pNumSubstitutions != NULL) { 182 *pNumSubstitutions = 0; 183 } 184 185 pDest = dest; 186 destLimit = (dest!=NULL)?(dest + destCapacity):NULL; 187 reqLength = 0; 188 numSubstitutions = 0; 189 190 if(srcLength < 0) { 191 /* simple loop for conversion of a NUL-terminated BMP string */ 192 while((ch=*src) != 0 && !U16_IS_SURROGATE(ch)) { 193 ++src; 194 if(pDest < destLimit) { 195 *pDest++ = ch; 196 } else { 197 ++reqLength; 198 } 199 } 200 srcLimit = src; 201 if(ch != 0) { 202 /* "complicated" case, find the end of the remaining string */ 203 while(*++srcLimit != 0) {} 204 } 205 } else { 206 srcLimit = (src!=NULL)?(src + srcLength):NULL; 207 } 208 209 /* convert with length */ 210 while(src < srcLimit) { 211 ch = *src++; 212 if(!U16_IS_SURROGATE(ch)) { 213 /* write or count ch below */ 214 } else if(U16_IS_SURROGATE_LEAD(ch) && src < srcLimit && U16_IS_TRAIL(ch2 = *src)) { 215 ++src; 216 ch = U16_GET_SUPPLEMENTARY(ch, ch2); 217 } else if((ch = subchar) < 0) { 218 /* unpaired surrogate */ 219 *pErrorCode = U_INVALID_CHAR_FOUND; 220 return NULL; 221 } else { 222 ++numSubstitutions; 223 } 224 if(pDest < destLimit) { 225 *pDest++ = ch; 226 } else { 227 ++reqLength; 228 } 229 } 230 231 reqLength += (int32_t)(pDest - dest); 232 if(pDestLength) { 233 *pDestLength = reqLength; 234 } 235 if(pNumSubstitutions != NULL) { 236 *pNumSubstitutions = numSubstitutions; 237 } 238 239 /* Terminate the buffer */ 240 u_terminateUChar32s(dest, destCapacity, reqLength, pErrorCode); 241 242 return dest; 243 } 244 245 U_CAPI UChar32* U_EXPORT2 246 u_strToUTF32(UChar32 *dest, 247 int32_t destCapacity, 248 int32_t *pDestLength, 249 const UChar *src, 250 int32_t srcLength, 251 UErrorCode *pErrorCode) { 252 return u_strToUTF32WithSub( 253 dest, destCapacity, pDestLength, 254 src, srcLength, 255 U_SENTINEL, NULL, 256 pErrorCode); 257 } 258 259 /* for utf8_nextCharSafeBodyTerminated() */ 260 static const UChar32 261 utf8_minLegal[4]={ 0, 0x80, 0x800, 0x10000 }; 262 263 /* 264 * Version of utf8_nextCharSafeBody() with the following differences: 265 * - checks for NUL termination instead of length 266 * - works with pointers instead of indexes 267 * - always strict (strict==-1) 268 * 269 * *ps points to after the lead byte and will be moved to after the last trail byte. 270 * c is the lead byte. 271 * @return the code point, or U_SENTINEL 272 */ 273 static UChar32 274 utf8_nextCharSafeBodyTerminated(const uint8_t **ps, UChar32 c) { 275 const uint8_t *s=*ps; 276 uint8_t trail, illegal=0; 277 uint8_t count=U8_COUNT_TRAIL_BYTES(c); 278 U_ASSERT(count<6); 279 U8_MASK_LEAD_BYTE((c), count); 280 /* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */ 281 switch(count) { 282 /* each branch falls through to the next one */ 283 case 5: 284 case 4: 285 /* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */ 286 illegal=1; 287 break; 288 case 3: 289 trail=(uint8_t)(*s++ - 0x80); 290 c=(c<<6)|trail; 291 if(trail>0x3f || c>=0x110) { 292 /* not a trail byte, or code point>0x10ffff (outside Unicode) */ 293 illegal=1; 294 break; 295 } 296 U_FALLTHROUGH; 297 case 2: 298 trail=(uint8_t)(*s++ - 0x80); 299 if(trail>0x3f) { 300 /* not a trail byte */ 301 illegal=1; 302 break; 303 } 304 c=(c<<6)|trail; 305 U_FALLTHROUGH; 306 case 1: 307 trail=(uint8_t)(*s++ - 0x80); 308 if(trail>0x3f) { 309 /* not a trail byte */ 310 illegal=1; 311 } 312 c=(c<<6)|trail; 313 break; 314 case 0: 315 return U_SENTINEL; 316 /* no default branch to optimize switch() - all values are covered */ 317 } 318 319 /* correct sequence - all trail bytes have (b7..b6)==(10)? */ 320 /* illegal is also set if count>=4 */ 321 if(illegal || c<utf8_minLegal[count] || U_IS_SURROGATE(c)) { 322 /* error handling */ 323 /* don't go beyond this sequence */ 324 s=*ps; 325 while(count>0 && U8_IS_TRAIL(*s)) { 326 ++s; 327 --count; 328 } 329 c=U_SENTINEL; 330 } 331 *ps=s; 332 return c; 333 } 334 335 /* 336 * Version of utf8_nextCharSafeBody() with the following differences: 337 * - works with pointers instead of indexes 338 * - always strict (strict==-1) 339 * 340 * *ps points to after the lead byte and will be moved to after the last trail byte. 341 * c is the lead byte. 342 * @return the code point, or U_SENTINEL 343 */ 344 static UChar32 345 utf8_nextCharSafeBodyPointer(const uint8_t **ps, const uint8_t *limit, UChar32 c) { 346 const uint8_t *s=*ps; 347 uint8_t trail, illegal=0; 348 uint8_t count=U8_COUNT_TRAIL_BYTES(c); 349 if((limit-s)>=count) { 350 U8_MASK_LEAD_BYTE((c), count); 351 /* count==0 for illegally leading trail bytes and the illegal bytes 0xfe and 0xff */ 352 switch(count) { 353 /* each branch falls through to the next one */ 354 case 5: 355 case 4: 356 /* count>=4 is always illegal: no more than 3 trail bytes in Unicode's UTF-8 */ 357 illegal=1; 358 break; 359 case 3: 360 trail=*s++; 361 c=(c<<6)|(trail&0x3f); 362 if(c<0x110) { 363 illegal|=(trail&0xc0)^0x80; 364 } else { 365 /* code point>0x10ffff, outside Unicode */ 366 illegal=1; 367 break; 368 } 369 U_FALLTHROUGH; 370 case 2: 371 trail=*s++; 372 c=(c<<6)|(trail&0x3f); 373 illegal|=(trail&0xc0)^0x80; 374 U_FALLTHROUGH; 375 case 1: 376 trail=*s++; 377 c=(c<<6)|(trail&0x3f); 378 illegal|=(trail&0xc0)^0x80; 379 break; 380 case 0: 381 return U_SENTINEL; 382 /* no default branch to optimize switch() - all values are covered */ 383 } 384 } else { 385 illegal=1; /* too few bytes left */ 386 } 387 388 /* correct sequence - all trail bytes have (b7..b6)==(10)? */ 389 /* illegal is also set if count>=4 */ 390 U_ASSERT(illegal || count<UPRV_LENGTHOF(utf8_minLegal)); 391 if(illegal || c<utf8_minLegal[count] || U_IS_SURROGATE(c)) { 392 /* error handling */ 393 /* don't go beyond this sequence */ 394 s=*ps; 395 while(count>0 && s<limit && U8_IS_TRAIL(*s)) { 396 ++s; 397 --count; 398 } 399 c=U_SENTINEL; 400 } 401 *ps=s; 402 return c; 403 } 404 405 U_CAPI UChar* U_EXPORT2 406 u_strFromUTF8WithSub(UChar *dest, 407 int32_t destCapacity, 408 int32_t *pDestLength, 409 const char* src, 410 int32_t srcLength, 411 UChar32 subchar, int32_t *pNumSubstitutions, 412 UErrorCode *pErrorCode){ 413 UChar *pDest = dest; 414 UChar *pDestLimit = dest+destCapacity; 415 UChar32 ch; 416 int32_t reqLength = 0; 417 const uint8_t* pSrc = (const uint8_t*) src; 418 uint8_t t1, t2; /* trail bytes */ 419 int32_t numSubstitutions; 420 421 /* args check */ 422 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ 423 return NULL; 424 } 425 426 if( (src==NULL && srcLength!=0) || srcLength < -1 || 427 (destCapacity<0) || (dest == NULL && destCapacity > 0) || 428 subchar > 0x10ffff || U_IS_SURROGATE(subchar) 429 ) { 430 *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; 431 return NULL; 432 } 433 434 if(pNumSubstitutions!=NULL) { 435 *pNumSubstitutions=0; 436 } 437 numSubstitutions=0; 438 439 /* 440 * Inline processing of UTF-8 byte sequences: 441 * 442 * Byte sequences for the most common characters are handled inline in 443 * the conversion loops. In order to reduce the path lengths for those 444 * characters, the tests are arranged in a kind of binary search. 445 * ASCII (<=0x7f) is checked first, followed by the dividing point 446 * between 2- and 3-byte sequences (0xe0). 447 * The 3-byte branch is tested first to speed up CJK text. 448 * The compiler should combine the subtractions for the two tests for 0xe0. 449 * Each branch then tests for the other end of its range. 450 */ 451 452 if(srcLength < 0){ 453 /* 454 * Transform a NUL-terminated string. 455 * The code explicitly checks for NULs only in the lead byte position. 456 * A NUL byte in the trail byte position fails the trail byte range check anyway. 457 */ 458 while(((ch = *pSrc) != 0) && (pDest < pDestLimit)) { 459 if(ch <= 0x7f){ 460 *pDest++=(UChar)ch; 461 ++pSrc; 462 } else { 463 if(ch > 0xe0) { 464 if( /* handle U+1000..U+CFFF inline */ 465 ch <= 0xec && 466 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && 467 (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f 468 ) { 469 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 470 *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); 471 pSrc += 3; 472 continue; 473 } 474 } else if(ch < 0xe0) { 475 if( /* handle U+0080..U+07FF inline */ 476 ch >= 0xc2 && 477 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f 478 ) { 479 *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); 480 pSrc += 2; 481 continue; 482 } 483 } 484 485 /* function call for "complicated" and error cases */ 486 ++pSrc; /* continue after the lead byte */ 487 ch=utf8_nextCharSafeBodyTerminated(&pSrc, ch); 488 if(ch<0 && (++numSubstitutions, ch = subchar) < 0) { 489 *pErrorCode = U_INVALID_CHAR_FOUND; 490 return NULL; 491 } else if(ch<=0xFFFF) { 492 *(pDest++)=(UChar)ch; 493 } else { 494 *(pDest++)=U16_LEAD(ch); 495 if(pDest<pDestLimit) { 496 *(pDest++)=U16_TRAIL(ch); 497 } else { 498 reqLength++; 499 break; 500 } 501 } 502 } 503 } 504 505 /* Pre-flight the rest of the string. */ 506 while((ch = *pSrc) != 0) { 507 if(ch <= 0x7f){ 508 ++reqLength; 509 ++pSrc; 510 } else { 511 if(ch > 0xe0) { 512 if( /* handle U+1000..U+CFFF inline */ 513 ch <= 0xec && 514 (uint8_t)(pSrc[1] - 0x80) <= 0x3f && 515 (uint8_t)(pSrc[2] - 0x80) <= 0x3f 516 ) { 517 ++reqLength; 518 pSrc += 3; 519 continue; 520 } 521 } else if(ch < 0xe0) { 522 if( /* handle U+0080..U+07FF inline */ 523 ch >= 0xc2 && 524 (uint8_t)(pSrc[1] - 0x80) <= 0x3f 525 ) { 526 ++reqLength; 527 pSrc += 2; 528 continue; 529 } 530 } 531 532 /* function call for "complicated" and error cases */ 533 ++pSrc; /* continue after the lead byte */ 534 ch=utf8_nextCharSafeBodyTerminated(&pSrc, ch); 535 if(ch<0 && (++numSubstitutions, ch = subchar) < 0) { 536 *pErrorCode = U_INVALID_CHAR_FOUND; 537 return NULL; 538 } 539 reqLength += U16_LENGTH(ch); 540 } 541 } 542 } else /* srcLength >= 0 */ { 543 const uint8_t *pSrcLimit = pSrc + srcLength; 544 int32_t count; 545 546 /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ 547 for(;;) { 548 /* 549 * Each iteration of the inner loop progresses by at most 3 UTF-8 550 * bytes and one UChar, for most characters. 551 * For supplementary code points (4 & 2), which are rare, 552 * there is an additional adjustment. 553 */ 554 count = (int32_t)(pDestLimit - pDest); 555 srcLength = (int32_t)((pSrcLimit - pSrc) / 3); 556 if(count > srcLength) { 557 count = srcLength; /* min(remaining dest, remaining src/3) */ 558 } 559 if(count < 3) { 560 /* 561 * Too much overhead if we get near the end of the string, 562 * continue with the next loop. 563 */ 564 break; 565 } 566 567 do { 568 ch = *pSrc; 569 if(ch <= 0x7f){ 570 *pDest++=(UChar)ch; 571 ++pSrc; 572 } else { 573 if(ch > 0xe0) { 574 if( /* handle U+1000..U+CFFF inline */ 575 ch <= 0xec && 576 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && 577 (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f 578 ) { 579 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 580 *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); 581 pSrc += 3; 582 continue; 583 } 584 } else if(ch < 0xe0) { 585 if( /* handle U+0080..U+07FF inline */ 586 ch >= 0xc2 && 587 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f 588 ) { 589 *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); 590 pSrc += 2; 591 continue; 592 } 593 } 594 595 if(ch >= 0xf0 || subchar > 0xffff) { 596 /* 597 * We may read up to six bytes and write up to two UChars, 598 * which we didn't account for with computing count, 599 * so we adjust it here. 600 */ 601 if(--count == 0) { 602 break; 603 } 604 } 605 606 /* function call for "complicated" and error cases */ 607 ++pSrc; /* continue after the lead byte */ 608 ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); 609 if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ 610 *pErrorCode = U_INVALID_CHAR_FOUND; 611 return NULL; 612 }else if(ch<=0xFFFF){ 613 *(pDest++)=(UChar)ch; 614 }else{ 615 *(pDest++)=U16_LEAD(ch); 616 *(pDest++)=U16_TRAIL(ch); 617 } 618 } 619 } while(--count > 0); 620 } 621 622 while((pSrc<pSrcLimit) && (pDest<pDestLimit)) { 623 ch = *pSrc; 624 if(ch <= 0x7f){ 625 *pDest++=(UChar)ch; 626 ++pSrc; 627 } else { 628 if(ch > 0xe0) { 629 if( /* handle U+1000..U+CFFF inline */ 630 ch <= 0xec && 631 ((pSrcLimit - pSrc) >= 3) && 632 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && 633 (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f 634 ) { 635 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 636 *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); 637 pSrc += 3; 638 continue; 639 } 640 } else if(ch < 0xe0) { 641 if( /* handle U+0080..U+07FF inline */ 642 ch >= 0xc2 && 643 ((pSrcLimit - pSrc) >= 2) && 644 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f 645 ) { 646 *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); 647 pSrc += 2; 648 continue; 649 } 650 } 651 652 /* function call for "complicated" and error cases */ 653 ++pSrc; /* continue after the lead byte */ 654 ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); 655 if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ 656 *pErrorCode = U_INVALID_CHAR_FOUND; 657 return NULL; 658 }else if(ch<=0xFFFF){ 659 *(pDest++)=(UChar)ch; 660 }else{ 661 *(pDest++)=U16_LEAD(ch); 662 if(pDest<pDestLimit){ 663 *(pDest++)=U16_TRAIL(ch); 664 }else{ 665 reqLength++; 666 break; 667 } 668 } 669 } 670 } 671 /* do not fill the dest buffer just count the UChars needed */ 672 while(pSrc < pSrcLimit){ 673 ch = *pSrc; 674 if(ch <= 0x7f){ 675 reqLength++; 676 ++pSrc; 677 } else { 678 if(ch > 0xe0) { 679 if( /* handle U+1000..U+CFFF inline */ 680 ch <= 0xec && 681 ((pSrcLimit - pSrc) >= 3) && 682 (uint8_t)(pSrc[1] - 0x80) <= 0x3f && 683 (uint8_t)(pSrc[2] - 0x80) <= 0x3f 684 ) { 685 reqLength++; 686 pSrc += 3; 687 continue; 688 } 689 } else if(ch < 0xe0) { 690 if( /* handle U+0080..U+07FF inline */ 691 ch >= 0xc2 && 692 ((pSrcLimit - pSrc) >= 2) && 693 (uint8_t)(pSrc[1] - 0x80) <= 0x3f 694 ) { 695 reqLength++; 696 pSrc += 2; 697 continue; 698 } 699 } 700 701 /* function call for "complicated" and error cases */ 702 ++pSrc; /* continue after the lead byte */ 703 ch=utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); 704 if(ch<0 && (++numSubstitutions, ch = subchar) < 0){ 705 *pErrorCode = U_INVALID_CHAR_FOUND; 706 return NULL; 707 } 708 reqLength+=U16_LENGTH(ch); 709 } 710 } 711 } 712 713 reqLength+=(int32_t)(pDest - dest); 714 715 if(pNumSubstitutions!=NULL) { 716 *pNumSubstitutions=numSubstitutions; 717 } 718 719 if(pDestLength){ 720 *pDestLength = reqLength; 721 } 722 723 /* Terminate the buffer */ 724 u_terminateUChars(dest,destCapacity,reqLength,pErrorCode); 725 726 return dest; 727 } 728 729 U_CAPI UChar* U_EXPORT2 730 u_strFromUTF8(UChar *dest, 731 int32_t destCapacity, 732 int32_t *pDestLength, 733 const char* src, 734 int32_t srcLength, 735 UErrorCode *pErrorCode){ 736 return u_strFromUTF8WithSub( 737 dest, destCapacity, pDestLength, 738 src, srcLength, 739 U_SENTINEL, NULL, 740 pErrorCode); 741 } 742 743 U_CAPI UChar * U_EXPORT2 744 u_strFromUTF8Lenient(UChar *dest, 745 int32_t destCapacity, 746 int32_t *pDestLength, 747 const char *src, 748 int32_t srcLength, 749 UErrorCode *pErrorCode) { 750 UChar *pDest = dest; 751 UChar32 ch; 752 int32_t reqLength = 0; 753 uint8_t* pSrc = (uint8_t*) src; 754 755 /* args check */ 756 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ 757 return NULL; 758 } 759 760 if( (src==NULL && srcLength!=0) || srcLength < -1 || 761 (destCapacity<0) || (dest == NULL && destCapacity > 0) 762 ) { 763 *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; 764 return NULL; 765 } 766 767 if(srcLength < 0) { 768 /* Transform a NUL-terminated string. */ 769 UChar *pDestLimit = (dest!=NULL)?(dest+destCapacity):NULL; 770 uint8_t t1, t2, t3; /* trail bytes */ 771 772 while(((ch = *pSrc) != 0) && (pDest < pDestLimit)) { 773 if(ch < 0xc0) { 774 /* 775 * ASCII, or a trail byte in lead position which is treated like 776 * a single-byte sequence for better character boundary 777 * resynchronization after illegal sequences. 778 */ 779 *pDest++=(UChar)ch; 780 ++pSrc; 781 continue; 782 } else if(ch < 0xe0) { /* U+0080..U+07FF */ 783 if((t1 = pSrc[1]) != 0) { 784 /* 0x3080 = (0xc0 << 6) + 0x80 */ 785 *pDest++ = (UChar)((ch << 6) + t1 - 0x3080); 786 pSrc += 2; 787 continue; 788 } 789 } else if(ch < 0xf0) { /* U+0800..U+FFFF */ 790 if((t1 = pSrc[1]) != 0 && (t2 = pSrc[2]) != 0) { 791 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 792 /* 0x2080 = (0x80 << 6) + 0x80 */ 793 *pDest++ = (UChar)((ch << 12) + (t1 << 6) + t2 - 0x2080); 794 pSrc += 3; 795 continue; 796 } 797 } else /* f0..f4 */ { /* U+10000..U+10FFFF */ 798 if((t1 = pSrc[1]) != 0 && (t2 = pSrc[2]) != 0 && (t3 = pSrc[3]) != 0) { 799 pSrc += 4; 800 /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ 801 ch = (ch << 18) + (t1 << 12) + (t2 << 6) + t3 - 0x3c82080; 802 *(pDest++) = U16_LEAD(ch); 803 if(pDest < pDestLimit) { 804 *(pDest++) = U16_TRAIL(ch); 805 } else { 806 reqLength = 1; 807 break; 808 } 809 continue; 810 } 811 } 812 813 /* truncated character at the end */ 814 *pDest++ = 0xfffd; 815 while(*++pSrc != 0) {} 816 break; 817 } 818 819 /* Pre-flight the rest of the string. */ 820 while((ch = *pSrc) != 0) { 821 if(ch < 0xc0) { 822 /* 823 * ASCII, or a trail byte in lead position which is treated like 824 * a single-byte sequence for better character boundary 825 * resynchronization after illegal sequences. 826 */ 827 ++reqLength; 828 ++pSrc; 829 continue; 830 } else if(ch < 0xe0) { /* U+0080..U+07FF */ 831 if(pSrc[1] != 0) { 832 ++reqLength; 833 pSrc += 2; 834 continue; 835 } 836 } else if(ch < 0xf0) { /* U+0800..U+FFFF */ 837 if(pSrc[1] != 0 && pSrc[2] != 0) { 838 ++reqLength; 839 pSrc += 3; 840 continue; 841 } 842 } else /* f0..f4 */ { /* U+10000..U+10FFFF */ 843 if(pSrc[1] != 0 && pSrc[2] != 0 && pSrc[3] != 0) { 844 reqLength += 2; 845 pSrc += 4; 846 continue; 847 } 848 } 849 850 /* truncated character at the end */ 851 ++reqLength; 852 break; 853 } 854 } else /* srcLength >= 0 */ { 855 const uint8_t *pSrcLimit = (pSrc!=NULL)?(pSrc + srcLength):NULL; 856 857 /* 858 * This function requires that if srcLength is given, then it must be 859 * destCapatity >= srcLength so that we need not check for 860 * destination buffer overflow in the loop. 861 */ 862 if(destCapacity < srcLength) { 863 if(pDestLength != NULL) { 864 *pDestLength = srcLength; /* this likely overestimates the true destLength! */ 865 } 866 *pErrorCode = U_BUFFER_OVERFLOW_ERROR; 867 return NULL; 868 } 869 870 if((pSrcLimit - pSrc) >= 4) { 871 pSrcLimit -= 3; /* temporarily reduce pSrcLimit */ 872 873 /* in this loop, we can always access at least 4 bytes, up to pSrc+3 */ 874 do { 875 ch = *pSrc++; 876 if(ch < 0xc0) { 877 /* 878 * ASCII, or a trail byte in lead position which is treated like 879 * a single-byte sequence for better character boundary 880 * resynchronization after illegal sequences. 881 */ 882 *pDest++=(UChar)ch; 883 } else if(ch < 0xe0) { /* U+0080..U+07FF */ 884 /* 0x3080 = (0xc0 << 6) + 0x80 */ 885 *pDest++ = (UChar)((ch << 6) + *pSrc++ - 0x3080); 886 } else if(ch < 0xf0) { /* U+0800..U+FFFF */ 887 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 888 /* 0x2080 = (0x80 << 6) + 0x80 */ 889 ch = (ch << 12) + (*pSrc++ << 6); 890 *pDest++ = (UChar)(ch + *pSrc++ - 0x2080); 891 } else /* f0..f4 */ { /* U+10000..U+10FFFF */ 892 /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ 893 ch = (ch << 18) + (*pSrc++ << 12); 894 ch += *pSrc++ << 6; 895 ch += *pSrc++ - 0x3c82080; 896 *(pDest++) = U16_LEAD(ch); 897 *(pDest++) = U16_TRAIL(ch); 898 } 899 } while(pSrc < pSrcLimit); 900 901 pSrcLimit += 3; /* restore original pSrcLimit */ 902 } 903 904 while(pSrc < pSrcLimit) { 905 ch = *pSrc++; 906 if(ch < 0xc0) { 907 /* 908 * ASCII, or a trail byte in lead position which is treated like 909 * a single-byte sequence for better character boundary 910 * resynchronization after illegal sequences. 911 */ 912 *pDest++=(UChar)ch; 913 continue; 914 } else if(ch < 0xe0) { /* U+0080..U+07FF */ 915 if(pSrc < pSrcLimit) { 916 /* 0x3080 = (0xc0 << 6) + 0x80 */ 917 *pDest++ = (UChar)((ch << 6) + *pSrc++ - 0x3080); 918 continue; 919 } 920 } else if(ch < 0xf0) { /* U+0800..U+FFFF */ 921 if((pSrcLimit - pSrc) >= 2) { 922 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 923 /* 0x2080 = (0x80 << 6) + 0x80 */ 924 ch = (ch << 12) + (*pSrc++ << 6); 925 *pDest++ = (UChar)(ch + *pSrc++ - 0x2080); 926 pSrc += 3; 927 continue; 928 } 929 } else /* f0..f4 */ { /* U+10000..U+10FFFF */ 930 if((pSrcLimit - pSrc) >= 3) { 931 /* 0x3c82080 = (0xf0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80 */ 932 ch = (ch << 18) + (*pSrc++ << 12); 933 ch += *pSrc++ << 6; 934 ch += *pSrc++ - 0x3c82080; 935 *(pDest++) = U16_LEAD(ch); 936 *(pDest++) = U16_TRAIL(ch); 937 pSrc += 4; 938 continue; 939 } 940 } 941 942 /* truncated character at the end */ 943 *pDest++ = 0xfffd; 944 break; 945 } 946 } 947 948 reqLength+=(int32_t)(pDest - dest); 949 950 if(pDestLength){ 951 *pDestLength = reqLength; 952 } 953 954 /* Terminate the buffer */ 955 u_terminateUChars(dest,destCapacity,reqLength,pErrorCode); 956 957 return dest; 958 } 959 960 static inline uint8_t * 961 _appendUTF8(uint8_t *pDest, UChar32 c) { 962 /* it is 0<=c<=0x10ffff and not a surrogate if called by a validating function */ 963 if((c)<=0x7f) { 964 *pDest++=(uint8_t)c; 965 } else if(c<=0x7ff) { 966 *pDest++=(uint8_t)((c>>6)|0xc0); 967 *pDest++=(uint8_t)((c&0x3f)|0x80); 968 } else if(c<=0xffff) { 969 *pDest++=(uint8_t)((c>>12)|0xe0); 970 *pDest++=(uint8_t)(((c>>6)&0x3f)|0x80); 971 *pDest++=(uint8_t)(((c)&0x3f)|0x80); 972 } else /* if((uint32_t)(c)<=0x10ffff) */ { 973 *pDest++=(uint8_t)(((c)>>18)|0xf0); 974 *pDest++=(uint8_t)((((c)>>12)&0x3f)|0x80); 975 *pDest++=(uint8_t)((((c)>>6)&0x3f)|0x80); 976 *pDest++=(uint8_t)(((c)&0x3f)|0x80); 977 } 978 return pDest; 979 } 980 981 982 U_CAPI char* U_EXPORT2 983 u_strToUTF8WithSub(char *dest, 984 int32_t destCapacity, 985 int32_t *pDestLength, 986 const UChar *pSrc, 987 int32_t srcLength, 988 UChar32 subchar, int32_t *pNumSubstitutions, 989 UErrorCode *pErrorCode){ 990 int32_t reqLength=0; 991 uint32_t ch=0,ch2=0; 992 uint8_t *pDest = (uint8_t *)dest; 993 uint8_t *pDestLimit = (pDest!=NULL)?(pDest + destCapacity):NULL; 994 int32_t numSubstitutions; 995 996 /* args check */ 997 if(pErrorCode==NULL || U_FAILURE(*pErrorCode)){ 998 return NULL; 999 } 1000 1001 if( (pSrc==NULL && srcLength!=0) || srcLength < -1 || 1002 (destCapacity<0) || (dest == NULL && destCapacity > 0) || 1003 subchar > 0x10ffff || U_IS_SURROGATE(subchar) 1004 ) { 1005 *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; 1006 return NULL; 1007 } 1008 1009 if(pNumSubstitutions!=NULL) { 1010 *pNumSubstitutions=0; 1011 } 1012 numSubstitutions=0; 1013 1014 if(srcLength==-1) { 1015 while((ch=*pSrc)!=0) { 1016 ++pSrc; 1017 if(ch <= 0x7f) { 1018 if(pDest<pDestLimit) { 1019 *pDest++ = (uint8_t)ch; 1020 } else { 1021 reqLength = 1; 1022 break; 1023 } 1024 } else if(ch <= 0x7ff) { 1025 if((pDestLimit - pDest) >= 2) { 1026 *pDest++=(uint8_t)((ch>>6)|0xc0); 1027 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1028 } else { 1029 reqLength = 2; 1030 break; 1031 } 1032 } else if(ch <= 0xd7ff || ch >= 0xe000) { 1033 if((pDestLimit - pDest) >= 3) { 1034 *pDest++=(uint8_t)((ch>>12)|0xe0); 1035 *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); 1036 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1037 } else { 1038 reqLength = 3; 1039 break; 1040 } 1041 } else /* ch is a surrogate */ { 1042 int32_t length; 1043 1044 /*need not check for NUL because NUL fails U16_IS_TRAIL() anyway*/ 1045 if(U16_IS_SURROGATE_LEAD(ch) && U16_IS_TRAIL(ch2=*pSrc)) { 1046 ++pSrc; 1047 ch=U16_GET_SUPPLEMENTARY(ch, ch2); 1048 } else if(subchar>=0) { 1049 ch=subchar; 1050 ++numSubstitutions; 1051 } else { 1052 /* Unicode 3.2 forbids surrogate code points in UTF-8 */ 1053 *pErrorCode = U_INVALID_CHAR_FOUND; 1054 return NULL; 1055 } 1056 1057 length = U8_LENGTH(ch); 1058 if((pDestLimit - pDest) >= length) { 1059 /* convert and append*/ 1060 pDest=_appendUTF8(pDest, ch); 1061 } else { 1062 reqLength = length; 1063 break; 1064 } 1065 } 1066 } 1067 while((ch=*pSrc++)!=0) { 1068 if(ch<=0x7f) { 1069 ++reqLength; 1070 } else if(ch<=0x7ff) { 1071 reqLength+=2; 1072 } else if(!U16_IS_SURROGATE(ch)) { 1073 reqLength+=3; 1074 } else if(U16_IS_SURROGATE_LEAD(ch) && U16_IS_TRAIL(ch2=*pSrc)) { 1075 ++pSrc; 1076 reqLength+=4; 1077 } else if(subchar>=0) { 1078 reqLength+=U8_LENGTH(subchar); 1079 ++numSubstitutions; 1080 } else { 1081 /* Unicode 3.2 forbids surrogate code points in UTF-8 */ 1082 *pErrorCode = U_INVALID_CHAR_FOUND; 1083 return NULL; 1084 } 1085 } 1086 } else { 1087 const UChar *pSrcLimit = (pSrc!=NULL)?(pSrc+srcLength):NULL; 1088 int32_t count; 1089 1090 /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ 1091 for(;;) { 1092 /* 1093 * Each iteration of the inner loop progresses by at most 3 UTF-8 1094 * bytes and one UChar, for most characters. 1095 * For supplementary code points (4 & 2), which are rare, 1096 * there is an additional adjustment. 1097 */ 1098 count = (int32_t)((pDestLimit - pDest) / 3); 1099 srcLength = (int32_t)(pSrcLimit - pSrc); 1100 if(count > srcLength) { 1101 count = srcLength; /* min(remaining dest/3, remaining src) */ 1102 } 1103 if(count < 3) { 1104 /* 1105 * Too much overhead if we get near the end of the string, 1106 * continue with the next loop. 1107 */ 1108 break; 1109 } 1110 do { 1111 ch=*pSrc++; 1112 if(ch <= 0x7f) { 1113 *pDest++ = (uint8_t)ch; 1114 } else if(ch <= 0x7ff) { 1115 *pDest++=(uint8_t)((ch>>6)|0xc0); 1116 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1117 } else if(ch <= 0xd7ff || ch >= 0xe000) { 1118 *pDest++=(uint8_t)((ch>>12)|0xe0); 1119 *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); 1120 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1121 } else /* ch is a surrogate */ { 1122 /* 1123 * We will read two UChars and probably output four bytes, 1124 * which we didn't account for with computing count, 1125 * so we adjust it here. 1126 */ 1127 if(--count == 0) { 1128 --pSrc; /* undo ch=*pSrc++ for the lead surrogate */ 1129 break; /* recompute count */ 1130 } 1131 1132 if(U16_IS_SURROGATE_LEAD(ch) && U16_IS_TRAIL(ch2=*pSrc)) { 1133 ++pSrc; 1134 ch=U16_GET_SUPPLEMENTARY(ch, ch2); 1135 1136 /* writing 4 bytes per 2 UChars is ok */ 1137 *pDest++=(uint8_t)((ch>>18)|0xf0); 1138 *pDest++=(uint8_t)(((ch>>12)&0x3f)|0x80); 1139 *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); 1140 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1141 } else { 1142 /* Unicode 3.2 forbids surrogate code points in UTF-8 */ 1143 if(subchar>=0) { 1144 ch=subchar; 1145 ++numSubstitutions; 1146 } else { 1147 *pErrorCode = U_INVALID_CHAR_FOUND; 1148 return NULL; 1149 } 1150 1151 /* convert and append*/ 1152 pDest=_appendUTF8(pDest, ch); 1153 } 1154 } 1155 } while(--count > 0); 1156 } 1157 1158 while(pSrc<pSrcLimit) { 1159 ch=*pSrc++; 1160 if(ch <= 0x7f) { 1161 if(pDest<pDestLimit) { 1162 *pDest++ = (uint8_t)ch; 1163 } else { 1164 reqLength = 1; 1165 break; 1166 } 1167 } else if(ch <= 0x7ff) { 1168 if((pDestLimit - pDest) >= 2) { 1169 *pDest++=(uint8_t)((ch>>6)|0xc0); 1170 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1171 } else { 1172 reqLength = 2; 1173 break; 1174 } 1175 } else if(ch <= 0xd7ff || ch >= 0xe000) { 1176 if((pDestLimit - pDest) >= 3) { 1177 *pDest++=(uint8_t)((ch>>12)|0xe0); 1178 *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); 1179 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1180 } else { 1181 reqLength = 3; 1182 break; 1183 } 1184 } else /* ch is a surrogate */ { 1185 int32_t length; 1186 1187 if(U16_IS_SURROGATE_LEAD(ch) && pSrc<pSrcLimit && U16_IS_TRAIL(ch2=*pSrc)) { 1188 ++pSrc; 1189 ch=U16_GET_SUPPLEMENTARY(ch, ch2); 1190 } else if(subchar>=0) { 1191 ch=subchar; 1192 ++numSubstitutions; 1193 } else { 1194 /* Unicode 3.2 forbids surrogate code points in UTF-8 */ 1195 *pErrorCode = U_INVALID_CHAR_FOUND; 1196 return NULL; 1197 } 1198 1199 length = U8_LENGTH(ch); 1200 if((pDestLimit - pDest) >= length) { 1201 /* convert and append*/ 1202 pDest=_appendUTF8(pDest, ch); 1203 } else { 1204 reqLength = length; 1205 break; 1206 } 1207 } 1208 } 1209 while(pSrc<pSrcLimit) { 1210 ch=*pSrc++; 1211 if(ch<=0x7f) { 1212 ++reqLength; 1213 } else if(ch<=0x7ff) { 1214 reqLength+=2; 1215 } else if(!U16_IS_SURROGATE(ch)) { 1216 reqLength+=3; 1217 } else if(U16_IS_SURROGATE_LEAD(ch) && pSrc<pSrcLimit && U16_IS_TRAIL(ch2=*pSrc)) { 1218 ++pSrc; 1219 reqLength+=4; 1220 } else if(subchar>=0) { 1221 reqLength+=U8_LENGTH(subchar); 1222 ++numSubstitutions; 1223 } else { 1224 /* Unicode 3.2 forbids surrogate code points in UTF-8 */ 1225 *pErrorCode = U_INVALID_CHAR_FOUND; 1226 return NULL; 1227 } 1228 } 1229 } 1230 1231 reqLength+=(int32_t)(pDest - (uint8_t *)dest); 1232 1233 if(pNumSubstitutions!=NULL) { 1234 *pNumSubstitutions=numSubstitutions; 1235 } 1236 1237 if(pDestLength){ 1238 *pDestLength = reqLength; 1239 } 1240 1241 /* Terminate the buffer */ 1242 u_terminateChars(dest, destCapacity, reqLength, pErrorCode); 1243 return dest; 1244 } 1245 1246 U_CAPI char* U_EXPORT2 1247 u_strToUTF8(char *dest, 1248 int32_t destCapacity, 1249 int32_t *pDestLength, 1250 const UChar *pSrc, 1251 int32_t srcLength, 1252 UErrorCode *pErrorCode){ 1253 return u_strToUTF8WithSub( 1254 dest, destCapacity, pDestLength, 1255 pSrc, srcLength, 1256 U_SENTINEL, NULL, 1257 pErrorCode); 1258 } 1259 1260 U_CAPI UChar* U_EXPORT2 1261 u_strFromJavaModifiedUTF8WithSub( 1262 UChar *dest, 1263 int32_t destCapacity, 1264 int32_t *pDestLength, 1265 const char *src, 1266 int32_t srcLength, 1267 UChar32 subchar, int32_t *pNumSubstitutions, 1268 UErrorCode *pErrorCode) { 1269 UChar *pDest = dest; 1270 UChar *pDestLimit = dest+destCapacity; 1271 UChar32 ch; 1272 int32_t reqLength = 0; 1273 const uint8_t* pSrc = (const uint8_t*) src; 1274 const uint8_t *pSrcLimit; 1275 int32_t count; 1276 uint8_t t1, t2; /* trail bytes */ 1277 int32_t numSubstitutions; 1278 1279 /* args check */ 1280 if(U_FAILURE(*pErrorCode)){ 1281 return NULL; 1282 } 1283 if( (src==NULL && srcLength!=0) || srcLength < -1 || 1284 (dest==NULL && destCapacity!=0) || destCapacity<0 || 1285 subchar > 0x10ffff || U_IS_SURROGATE(subchar) 1286 ) { 1287 *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; 1288 return NULL; 1289 } 1290 1291 if(pNumSubstitutions!=NULL) { 1292 *pNumSubstitutions=0; 1293 } 1294 numSubstitutions=0; 1295 1296 if(srcLength < 0) { 1297 /* 1298 * Transform a NUL-terminated ASCII string. 1299 * Handle non-ASCII strings with slower code. 1300 */ 1301 while(((ch = *pSrc) != 0) && ch <= 0x7f && (pDest < pDestLimit)) { 1302 *pDest++=(UChar)ch; 1303 ++pSrc; 1304 } 1305 if(ch == 0) { 1306 reqLength=(int32_t)(pDest - dest); 1307 if(pDestLength) { 1308 *pDestLength = reqLength; 1309 } 1310 1311 /* Terminate the buffer */ 1312 u_terminateUChars(dest, destCapacity, reqLength, pErrorCode); 1313 return dest; 1314 } 1315 srcLength = uprv_strlen((const char *)pSrc); 1316 } 1317 1318 /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ 1319 pSrcLimit = (pSrc == NULL) ? NULL : pSrc + srcLength; 1320 for(;;) { 1321 count = (int32_t)(pDestLimit - pDest); 1322 srcLength = (int32_t)(pSrcLimit - pSrc); 1323 if(count >= srcLength && srcLength > 0 && *pSrc <= 0x7f) { 1324 /* fast ASCII loop */ 1325 const uint8_t *prevSrc = pSrc; 1326 int32_t delta; 1327 while(pSrc < pSrcLimit && (ch = *pSrc) <= 0x7f) { 1328 *pDest++=(UChar)ch; 1329 ++pSrc; 1330 } 1331 delta = (int32_t)(pSrc - prevSrc); 1332 count -= delta; 1333 srcLength -= delta; 1334 } 1335 /* 1336 * Each iteration of the inner loop progresses by at most 3 UTF-8 1337 * bytes and one UChar. 1338 */ 1339 srcLength /= 3; 1340 if(count > srcLength) { 1341 count = srcLength; /* min(remaining dest, remaining src/3) */ 1342 } 1343 if(count < 3) { 1344 /* 1345 * Too much overhead if we get near the end of the string, 1346 * continue with the next loop. 1347 */ 1348 break; 1349 } 1350 do { 1351 ch = *pSrc; 1352 if(ch <= 0x7f){ 1353 *pDest++=(UChar)ch; 1354 ++pSrc; 1355 } else { 1356 if(ch >= 0xe0) { 1357 if( /* handle U+0000..U+FFFF inline */ 1358 ch <= 0xef && 1359 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && 1360 (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f 1361 ) { 1362 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 1363 *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); 1364 pSrc += 3; 1365 continue; 1366 } 1367 } else { 1368 if( /* handle U+0000..U+07FF inline */ 1369 ch >= 0xc0 && 1370 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f 1371 ) { 1372 *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); 1373 pSrc += 2; 1374 continue; 1375 } 1376 } 1377 1378 if(subchar < 0) { 1379 *pErrorCode = U_INVALID_CHAR_FOUND; 1380 return NULL; 1381 } else if(subchar > 0xffff && --count == 0) { 1382 /* 1383 * We need to write two UChars, adjusted count for that, 1384 * and ran out of space. 1385 */ 1386 break; 1387 } else { 1388 /* function call for error cases */ 1389 ++pSrc; /* continue after the lead byte */ 1390 utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); 1391 ++numSubstitutions; 1392 if(subchar<=0xFFFF) { 1393 *(pDest++)=(UChar)subchar; 1394 } else { 1395 *(pDest++)=U16_LEAD(subchar); 1396 *(pDest++)=U16_TRAIL(subchar); 1397 } 1398 } 1399 } 1400 } while(--count > 0); 1401 } 1402 1403 while((pSrc<pSrcLimit) && (pDest<pDestLimit)) { 1404 ch = *pSrc; 1405 if(ch <= 0x7f){ 1406 *pDest++=(UChar)ch; 1407 ++pSrc; 1408 } else { 1409 if(ch >= 0xe0) { 1410 if( /* handle U+0000..U+FFFF inline */ 1411 ch <= 0xef && 1412 ((pSrcLimit - pSrc) >= 3) && 1413 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f && 1414 (t2 = (uint8_t)(pSrc[2] - 0x80)) <= 0x3f 1415 ) { 1416 /* no need for (ch & 0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ 1417 *pDest++ = (UChar)((ch << 12) | (t1 << 6) | t2); 1418 pSrc += 3; 1419 continue; 1420 } 1421 } else { 1422 if( /* handle U+0000..U+07FF inline */ 1423 ch >= 0xc0 && 1424 ((pSrcLimit - pSrc) >= 2) && 1425 (t1 = (uint8_t)(pSrc[1] - 0x80)) <= 0x3f 1426 ) { 1427 *pDest++ = (UChar)(((ch & 0x1f) << 6) | t1); 1428 pSrc += 2; 1429 continue; 1430 } 1431 } 1432 1433 if(subchar < 0) { 1434 *pErrorCode = U_INVALID_CHAR_FOUND; 1435 return NULL; 1436 } else { 1437 /* function call for error cases */ 1438 ++pSrc; /* continue after the lead byte */ 1439 utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); 1440 ++numSubstitutions; 1441 if(subchar<=0xFFFF) { 1442 *(pDest++)=(UChar)subchar; 1443 } else { 1444 *(pDest++)=U16_LEAD(subchar); 1445 if(pDest<pDestLimit) { 1446 *(pDest++)=U16_TRAIL(subchar); 1447 } else { 1448 reqLength++; 1449 break; 1450 } 1451 } 1452 } 1453 } 1454 } 1455 1456 /* do not fill the dest buffer just count the UChars needed */ 1457 while(pSrc < pSrcLimit){ 1458 ch = *pSrc; 1459 if(ch <= 0x7f) { 1460 reqLength++; 1461 ++pSrc; 1462 } else { 1463 if(ch >= 0xe0) { 1464 if( /* handle U+0000..U+FFFF inline */ 1465 ch <= 0xef && 1466 ((pSrcLimit - pSrc) >= 3) && 1467 (uint8_t)(pSrc[1] - 0x80) <= 0x3f && 1468 (uint8_t)(pSrc[2] - 0x80) <= 0x3f 1469 ) { 1470 reqLength++; 1471 pSrc += 3; 1472 continue; 1473 } 1474 } else { 1475 if( /* handle U+0000..U+07FF inline */ 1476 ch >= 0xc0 && 1477 ((pSrcLimit - pSrc) >= 2) && 1478 (uint8_t)(pSrc[1] - 0x80) <= 0x3f 1479 ) { 1480 reqLength++; 1481 pSrc += 2; 1482 continue; 1483 } 1484 } 1485 1486 if(subchar < 0) { 1487 *pErrorCode = U_INVALID_CHAR_FOUND; 1488 return NULL; 1489 } else { 1490 /* function call for error cases */ 1491 ++pSrc; /* continue after the lead byte */ 1492 utf8_nextCharSafeBodyPointer(&pSrc, pSrcLimit, ch); 1493 ++numSubstitutions; 1494 reqLength+=U16_LENGTH(ch); 1495 } 1496 } 1497 } 1498 1499 if(pNumSubstitutions!=NULL) { 1500 *pNumSubstitutions=numSubstitutions; 1501 } 1502 1503 reqLength+=(int32_t)(pDest - dest); 1504 if(pDestLength) { 1505 *pDestLength = reqLength; 1506 } 1507 1508 /* Terminate the buffer */ 1509 u_terminateUChars(dest, destCapacity, reqLength, pErrorCode); 1510 return dest; 1511 } 1512 1513 U_CAPI char* U_EXPORT2 1514 u_strToJavaModifiedUTF8( 1515 char *dest, 1516 int32_t destCapacity, 1517 int32_t *pDestLength, 1518 const UChar *src, 1519 int32_t srcLength, 1520 UErrorCode *pErrorCode) { 1521 int32_t reqLength=0; 1522 uint32_t ch=0; 1523 uint8_t *pDest = (uint8_t *)dest; 1524 uint8_t *pDestLimit = pDest + destCapacity; 1525 const UChar *pSrcLimit; 1526 int32_t count; 1527 1528 /* args check */ 1529 if(U_FAILURE(*pErrorCode)){ 1530 return NULL; 1531 } 1532 if( (src==NULL && srcLength!=0) || srcLength < -1 || 1533 (dest==NULL && destCapacity!=0) || destCapacity<0 1534 ) { 1535 *pErrorCode = U_ILLEGAL_ARGUMENT_ERROR; 1536 return NULL; 1537 } 1538 1539 if(srcLength==-1) { 1540 /* Convert NUL-terminated ASCII, then find the string length. */ 1541 while((ch=*src)<=0x7f && ch != 0 && pDest<pDestLimit) { 1542 *pDest++ = (uint8_t)ch; 1543 ++src; 1544 } 1545 if(ch == 0) { 1546 reqLength=(int32_t)(pDest - (uint8_t *)dest); 1547 if(pDestLength) { 1548 *pDestLength = reqLength; 1549 } 1550 1551 /* Terminate the buffer */ 1552 u_terminateChars(dest, destCapacity, reqLength, pErrorCode); 1553 return dest; 1554 } 1555 srcLength = u_strlen(src); 1556 } 1557 1558 /* Faster loop without ongoing checking for pSrcLimit and pDestLimit. */ 1559 pSrcLimit = (src!=NULL)?(src+srcLength):NULL; 1560 for(;;) { 1561 count = (int32_t)(pDestLimit - pDest); 1562 srcLength = (int32_t)(pSrcLimit - src); 1563 if(count >= srcLength && srcLength > 0 && *src <= 0x7f) { 1564 /* fast ASCII loop */ 1565 const UChar *prevSrc = src; 1566 int32_t delta; 1567 while(src < pSrcLimit && (ch = *src) <= 0x7f && ch != 0) { 1568 *pDest++=(uint8_t)ch; 1569 ++src; 1570 } 1571 delta = (int32_t)(src - prevSrc); 1572 count -= delta; 1573 srcLength -= delta; 1574 } 1575 /* 1576 * Each iteration of the inner loop progresses by at most 3 UTF-8 1577 * bytes and one UChar. 1578 */ 1579 count /= 3; 1580 if(count > srcLength) { 1581 count = srcLength; /* min(remaining dest/3, remaining src) */ 1582 } 1583 if(count < 3) { 1584 /* 1585 * Too much overhead if we get near the end of the string, 1586 * continue with the next loop. 1587 */ 1588 break; 1589 } 1590 do { 1591 ch=*src++; 1592 if(ch <= 0x7f && ch != 0) { 1593 *pDest++ = (uint8_t)ch; 1594 } else if(ch <= 0x7ff) { 1595 *pDest++=(uint8_t)((ch>>6)|0xc0); 1596 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1597 } else { 1598 *pDest++=(uint8_t)((ch>>12)|0xe0); 1599 *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); 1600 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1601 } 1602 } while(--count > 0); 1603 } 1604 1605 while(src<pSrcLimit) { 1606 ch=*src++; 1607 if(ch <= 0x7f && ch != 0) { 1608 if(pDest<pDestLimit) { 1609 *pDest++ = (uint8_t)ch; 1610 } else { 1611 reqLength = 1; 1612 break; 1613 } 1614 } else if(ch <= 0x7ff) { 1615 if((pDestLimit - pDest) >= 2) { 1616 *pDest++=(uint8_t)((ch>>6)|0xc0); 1617 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1618 } else { 1619 reqLength = 2; 1620 break; 1621 } 1622 } else { 1623 if((pDestLimit - pDest) >= 3) { 1624 *pDest++=(uint8_t)((ch>>12)|0xe0); 1625 *pDest++=(uint8_t)(((ch>>6)&0x3f)|0x80); 1626 *pDest++=(uint8_t)((ch&0x3f)|0x80); 1627 } else { 1628 reqLength = 3; 1629 break; 1630 } 1631 } 1632 } 1633 while(src<pSrcLimit) { 1634 ch=*src++; 1635 if(ch <= 0x7f && ch != 0) { 1636 ++reqLength; 1637 } else if(ch<=0x7ff) { 1638 reqLength+=2; 1639 } else { 1640 reqLength+=3; 1641 } 1642 } 1643 1644 reqLength+=(int32_t)(pDest - (uint8_t *)dest); 1645 if(pDestLength){ 1646 *pDestLength = reqLength; 1647 } 1648 1649 /* Terminate the buffer */ 1650 u_terminateChars(dest, destCapacity, reqLength, pErrorCode); 1651 return dest; 1652 } 1653
