1 /*===--- ConvertUTF.c - Universal Character Names conversions ---------------=== 2 * 3 * The LLVM Compiler Infrastructure 4 * 5 * This file is distributed under the University of Illinois Open Source 6 * License. See LICENSE.TXT for details. 7 * 8 *===------------------------------------------------------------------------=*/ 9 /* 10 * Copyright 2001-2004 Unicode, Inc. 11 * 12 * Disclaimer 13 * 14 * This source code is provided as is by Unicode, Inc. No claims are 15 * made as to fitness for any particular purpose. No warranties of any 16 * kind are expressed or implied. The recipient agrees to determine 17 * applicability of information provided. If this file has been 18 * purchased on magnetic or optical media from Unicode, Inc., the 19 * sole remedy for any claim will be exchange of defective media 20 * within 90 days of receipt. 21 * 22 * Limitations on Rights to Redistribute This Code 23 * 24 * Unicode, Inc. hereby grants the right to freely use the information 25 * supplied in this file in the creation of products supporting the 26 * Unicode Standard, and to make copies of this file in any form 27 * for internal or external distribution as long as this notice 28 * remains attached. 29 */ 30 31 /* --------------------------------------------------------------------- 32 33 Conversions between UTF32, UTF-16, and UTF-8. Source code file. 34 Author: Mark E. Davis, 1994. 35 Rev History: Rick McGowan, fixes & updates May 2001. 36 Sept 2001: fixed const & error conditions per 37 mods suggested by S. Parent & A. Lillich. 38 June 2002: Tim Dodd added detection and handling of incomplete 39 source sequences, enhanced error detection, added casts 40 to eliminate compiler warnings. 41 July 2003: slight mods to back out aggressive FFFE detection. 42 Jan 2004: updated switches in from-UTF8 conversions. 43 Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. 44 45 See the header file "ConvertUTF.h" for complete documentation. 46 47 ------------------------------------------------------------------------ */ 48 49 50 #include "llvm/Support/ConvertUTF.h" 51 #ifdef CVTUTF_DEBUG 52 #include <stdio.h> 53 #endif 54 #include <assert.h> 55 56 namespace llvm { 57 58 static const int halfShift = 10; /* used for shifting by 10 bits */ 59 60 static const UTF32 halfBase = 0x0010000UL; 61 static const UTF32 halfMask = 0x3FFUL; 62 63 #define UNI_SUR_HIGH_START (UTF32)0xD800 64 #define UNI_SUR_HIGH_END (UTF32)0xDBFF 65 #define UNI_SUR_LOW_START (UTF32)0xDC00 66 #define UNI_SUR_LOW_END (UTF32)0xDFFF 67 68 /* --------------------------------------------------------------------- */ 69 70 /* 71 * Index into the table below with the first byte of a UTF-8 sequence to 72 * get the number of trailing bytes that are supposed to follow it. 73 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is 74 * left as-is for anyone who may want to do such conversion, which was 75 * allowed in earlier algorithms. 76 */ 77 static const char trailingBytesForUTF8[256] = { 78 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 79 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 80 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 81 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 82 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 83 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 84 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 85 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 86 }; 87 88 /* 89 * Magic values subtracted from a buffer value during UTF8 conversion. 90 * This table contains as many values as there might be trailing bytes 91 * in a UTF-8 sequence. 92 */ 93 static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 94 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; 95 96 /* 97 * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed 98 * into the first byte, depending on how many bytes follow. There are 99 * as many entries in this table as there are UTF-8 sequence types. 100 * (I.e., one byte sequence, two byte... etc.). Remember that sequencs 101 * for *legal* UTF-8 will be 4 or fewer bytes total. 102 */ 103 static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; 104 105 /* --------------------------------------------------------------------- */ 106 107 /* The interface converts a whole buffer to avoid function-call overhead. 108 * Constants have been gathered. Loops & conditionals have been removed as 109 * much as possible for efficiency, in favor of drop-through switches. 110 * (See "Note A" at the bottom of the file for equivalent code.) 111 * If your compiler supports it, the "isLegalUTF8" call can be turned 112 * into an inline function. 113 */ 114 115 116 /* --------------------------------------------------------------------- */ 117 118 ConversionResult ConvertUTF32toUTF16 ( 119 const UTF32** sourceStart, const UTF32* sourceEnd, 120 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { 121 ConversionResult result = conversionOK; 122 const UTF32* source = *sourceStart; 123 UTF16* target = *targetStart; 124 while (source < sourceEnd) { 125 UTF32 ch; 126 if (target >= targetEnd) { 127 result = targetExhausted; break; 128 } 129 ch = *source++; 130 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ 131 /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ 132 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 133 if (flags == strictConversion) { 134 --source; /* return to the illegal value itself */ 135 result = sourceIllegal; 136 break; 137 } else { 138 *target++ = UNI_REPLACEMENT_CHAR; 139 } 140 } else { 141 *target++ = (UTF16)ch; /* normal case */ 142 } 143 } else if (ch > UNI_MAX_LEGAL_UTF32) { 144 if (flags == strictConversion) { 145 result = sourceIllegal; 146 } else { 147 *target++ = UNI_REPLACEMENT_CHAR; 148 } 149 } else { 150 /* target is a character in range 0xFFFF - 0x10FFFF. */ 151 if (target + 1 >= targetEnd) { 152 --source; /* Back up source pointer! */ 153 result = targetExhausted; break; 154 } 155 ch -= halfBase; 156 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); 157 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); 158 } 159 } 160 *sourceStart = source; 161 *targetStart = target; 162 return result; 163 } 164 165 /* --------------------------------------------------------------------- */ 166 167 ConversionResult ConvertUTF16toUTF32 ( 168 const UTF16** sourceStart, const UTF16* sourceEnd, 169 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { 170 ConversionResult result = conversionOK; 171 const UTF16* source = *sourceStart; 172 UTF32* target = *targetStart; 173 UTF32 ch, ch2; 174 while (source < sourceEnd) { 175 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ 176 ch = *source++; 177 /* If we have a surrogate pair, convert to UTF32 first. */ 178 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { 179 /* If the 16 bits following the high surrogate are in the source buffer... */ 180 if (source < sourceEnd) { 181 ch2 = *source; 182 /* If it's a low surrogate, convert to UTF32. */ 183 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { 184 ch = ((ch - UNI_SUR_HIGH_START) << halfShift) 185 + (ch2 - UNI_SUR_LOW_START) + halfBase; 186 ++source; 187 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ 188 --source; /* return to the illegal value itself */ 189 result = sourceIllegal; 190 break; 191 } 192 } else { /* We don't have the 16 bits following the high surrogate. */ 193 --source; /* return to the high surrogate */ 194 result = sourceExhausted; 195 break; 196 } 197 } else if (flags == strictConversion) { 198 /* UTF-16 surrogate values are illegal in UTF-32 */ 199 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { 200 --source; /* return to the illegal value itself */ 201 result = sourceIllegal; 202 break; 203 } 204 } 205 if (target >= targetEnd) { 206 source = oldSource; /* Back up source pointer! */ 207 result = targetExhausted; break; 208 } 209 *target++ = ch; 210 } 211 *sourceStart = source; 212 *targetStart = target; 213 #ifdef CVTUTF_DEBUG 214 if (result == sourceIllegal) { 215 fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); 216 fflush(stderr); 217 } 218 #endif 219 return result; 220 } 221 ConversionResult ConvertUTF16toUTF8 ( 222 const UTF16** sourceStart, const UTF16* sourceEnd, 223 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { 224 ConversionResult result = conversionOK; 225 const UTF16* source = *sourceStart; 226 UTF8* target = *targetStart; 227 while (source < sourceEnd) { 228 UTF32 ch; 229 unsigned short bytesToWrite = 0; 230 const UTF32 byteMask = 0xBF; 231 const UTF32 byteMark = 0x80; 232 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ 233 ch = *source++; 234 /* If we have a surrogate pair, convert to UTF32 first. */ 235 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { 236 /* If the 16 bits following the high surrogate are in the source buffer... */ 237 if (source < sourceEnd) { 238 UTF32 ch2 = *source; 239 /* If it's a low surrogate, convert to UTF32. */ 240 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { 241 ch = ((ch - UNI_SUR_HIGH_START) << halfShift) 242 + (ch2 - UNI_SUR_LOW_START) + halfBase; 243 ++source; 244 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ 245 --source; /* return to the illegal value itself */ 246 result = sourceIllegal; 247 break; 248 } 249 } else { /* We don't have the 16 bits following the high surrogate. */ 250 --source; /* return to the high surrogate */ 251 result = sourceExhausted; 252 break; 253 } 254 } else if (flags == strictConversion) { 255 /* UTF-16 surrogate values are illegal in UTF-32 */ 256 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { 257 --source; /* return to the illegal value itself */ 258 result = sourceIllegal; 259 break; 260 } 261 } 262 /* Figure out how many bytes the result will require */ 263 if (ch < (UTF32)0x80) { bytesToWrite = 1; 264 } else if (ch < (UTF32)0x800) { bytesToWrite = 2; 265 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; 266 } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; 267 } else { bytesToWrite = 3; 268 ch = UNI_REPLACEMENT_CHAR; 269 } 270 271 target += bytesToWrite; 272 if (target > targetEnd) { 273 source = oldSource; /* Back up source pointer! */ 274 target -= bytesToWrite; result = targetExhausted; break; 275 } 276 switch (bytesToWrite) { /* note: everything falls through. */ 277 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 278 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 279 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 280 case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); 281 } 282 target += bytesToWrite; 283 } 284 *sourceStart = source; 285 *targetStart = target; 286 return result; 287 } 288 289 /* --------------------------------------------------------------------- */ 290 291 ConversionResult ConvertUTF32toUTF8 ( 292 const UTF32** sourceStart, const UTF32* sourceEnd, 293 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { 294 ConversionResult result = conversionOK; 295 const UTF32* source = *sourceStart; 296 UTF8* target = *targetStart; 297 while (source < sourceEnd) { 298 UTF32 ch; 299 unsigned short bytesToWrite = 0; 300 const UTF32 byteMask = 0xBF; 301 const UTF32 byteMark = 0x80; 302 ch = *source++; 303 if (flags == strictConversion ) { 304 /* UTF-16 surrogate values are illegal in UTF-32 */ 305 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 306 --source; /* return to the illegal value itself */ 307 result = sourceIllegal; 308 break; 309 } 310 } 311 /* 312 * Figure out how many bytes the result will require. Turn any 313 * illegally large UTF32 things (> Plane 17) into replacement chars. 314 */ 315 if (ch < (UTF32)0x80) { bytesToWrite = 1; 316 } else if (ch < (UTF32)0x800) { bytesToWrite = 2; 317 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; 318 } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; 319 } else { bytesToWrite = 3; 320 ch = UNI_REPLACEMENT_CHAR; 321 result = sourceIllegal; 322 } 323 324 target += bytesToWrite; 325 if (target > targetEnd) { 326 --source; /* Back up source pointer! */ 327 target -= bytesToWrite; result = targetExhausted; break; 328 } 329 switch (bytesToWrite) { /* note: everything falls through. */ 330 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 331 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 332 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; 333 case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); 334 } 335 target += bytesToWrite; 336 } 337 *sourceStart = source; 338 *targetStart = target; 339 return result; 340 } 341 342 /* --------------------------------------------------------------------- */ 343 344 /* 345 * Utility routine to tell whether a sequence of bytes is legal UTF-8. 346 * This must be called with the length pre-determined by the first byte. 347 * If not calling this from ConvertUTF8to*, then the length can be set by: 348 * length = trailingBytesForUTF8[*source]+1; 349 * and the sequence is illegal right away if there aren't that many bytes 350 * available. 351 * If presented with a length > 4, this returns false. The Unicode 352 * definition of UTF-8 goes up to 4-byte sequences. 353 */ 354 355 static Boolean isLegalUTF8(const UTF8 *source, int length) { 356 UTF8 a; 357 const UTF8 *srcptr = source+length; 358 switch (length) { 359 default: return false; 360 /* Everything else falls through when "true"... */ 361 case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 362 case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 363 case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 364 365 switch (*source) { 366 /* no fall-through in this inner switch */ 367 case 0xE0: if (a < 0xA0) return false; break; 368 case 0xED: if (a > 0x9F) return false; break; 369 case 0xF0: if (a < 0x90) return false; break; 370 case 0xF4: if (a > 0x8F) return false; break; 371 default: if (a < 0x80) return false; 372 } 373 374 case 1: if (*source >= 0x80 && *source < 0xC2) return false; 375 } 376 if (*source > 0xF4) return false; 377 return true; 378 } 379 380 /* --------------------------------------------------------------------- */ 381 382 /* 383 * Exported function to return whether a UTF-8 sequence is legal or not. 384 * This is not used here; it's just exported. 385 */ 386 Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { 387 int length = trailingBytesForUTF8[*source]+1; 388 if (length > sourceEnd - source) { 389 return false; 390 } 391 return isLegalUTF8(source, length); 392 } 393 394 /* --------------------------------------------------------------------- */ 395 396 static unsigned 397 findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source, 398 const UTF8 *sourceEnd) { 399 UTF8 b1, b2, b3; 400 401 assert(!isLegalUTF8Sequence(source, sourceEnd)); 402 403 /* 404 * Unicode 6.3.0, D93b: 405 * 406 * Maximal subpart of an ill-formed subsequence: The longest code unit 407 * subsequence starting at an unconvertible offset that is either: 408 * a. the initial subsequence of a well-formed code unit sequence, or 409 * b. a subsequence of length one. 410 */ 411 412 if (source == sourceEnd) 413 return 0; 414 415 /* 416 * Perform case analysis. See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8 417 * Byte Sequences. 418 */ 419 420 b1 = *source; 421 ++source; 422 if (b1 >= 0xC2 && b1 <= 0xDF) { 423 /* 424 * First byte is valid, but we know that this code unit sequence is 425 * invalid, so the maximal subpart has to end after the first byte. 426 */ 427 return 1; 428 } 429 430 if (source == sourceEnd) 431 return 1; 432 433 b2 = *source; 434 ++source; 435 436 if (b1 == 0xE0) { 437 return (b2 >= 0xA0 && b2 <= 0xBF) ? 2 : 1; 438 } 439 if (b1 >= 0xE1 && b1 <= 0xEC) { 440 return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1; 441 } 442 if (b1 == 0xED) { 443 return (b2 >= 0x80 && b2 <= 0x9F) ? 2 : 1; 444 } 445 if (b1 >= 0xEE && b1 <= 0xEF) { 446 return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1; 447 } 448 if (b1 == 0xF0) { 449 if (b2 >= 0x90 && b2 <= 0xBF) { 450 if (source == sourceEnd) 451 return 2; 452 453 b3 = *source; 454 return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; 455 } 456 return 1; 457 } 458 if (b1 >= 0xF1 && b1 <= 0xF3) { 459 if (b2 >= 0x80 && b2 <= 0xBF) { 460 if (source == sourceEnd) 461 return 2; 462 463 b3 = *source; 464 return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; 465 } 466 return 1; 467 } 468 if (b1 == 0xF4) { 469 if (b2 >= 0x80 && b2 <= 0x8F) { 470 if (source == sourceEnd) 471 return 2; 472 473 b3 = *source; 474 return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2; 475 } 476 return 1; 477 } 478 479 assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5); 480 /* 481 * There are no valid sequences that start with these bytes. Maximal subpart 482 * is defined to have length 1 in these cases. 483 */ 484 return 1; 485 } 486 487 /* --------------------------------------------------------------------- */ 488 489 /* 490 * Exported function to return the total number of bytes in a codepoint 491 * represented in UTF-8, given the value of the first byte. 492 */ 493 unsigned getNumBytesForUTF8(UTF8 first) { 494 return trailingBytesForUTF8[first] + 1; 495 } 496 497 /* --------------------------------------------------------------------- */ 498 499 /* 500 * Exported function to return whether a UTF-8 string is legal or not. 501 * This is not used here; it's just exported. 502 */ 503 Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) { 504 while (*source != sourceEnd) { 505 int length = trailingBytesForUTF8[**source] + 1; 506 if (length > sourceEnd - *source || !isLegalUTF8(*source, length)) 507 return false; 508 *source += length; 509 } 510 return true; 511 } 512 513 /* --------------------------------------------------------------------- */ 514 515 ConversionResult ConvertUTF8toUTF16 ( 516 const UTF8** sourceStart, const UTF8* sourceEnd, 517 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { 518 ConversionResult result = conversionOK; 519 const UTF8* source = *sourceStart; 520 UTF16* target = *targetStart; 521 while (source < sourceEnd) { 522 UTF32 ch = 0; 523 unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; 524 if (extraBytesToRead >= sourceEnd - source) { 525 result = sourceExhausted; break; 526 } 527 /* Do this check whether lenient or strict */ 528 if (!isLegalUTF8(source, extraBytesToRead+1)) { 529 result = sourceIllegal; 530 break; 531 } 532 /* 533 * The cases all fall through. See "Note A" below. 534 */ 535 switch (extraBytesToRead) { 536 case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ 537 case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ 538 case 3: ch += *source++; ch <<= 6; 539 case 2: ch += *source++; ch <<= 6; 540 case 1: ch += *source++; ch <<= 6; 541 case 0: ch += *source++; 542 } 543 ch -= offsetsFromUTF8[extraBytesToRead]; 544 545 if (target >= targetEnd) { 546 source -= (extraBytesToRead+1); /* Back up source pointer! */ 547 result = targetExhausted; break; 548 } 549 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ 550 /* UTF-16 surrogate values are illegal in UTF-32 */ 551 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 552 if (flags == strictConversion) { 553 source -= (extraBytesToRead+1); /* return to the illegal value itself */ 554 result = sourceIllegal; 555 break; 556 } else { 557 *target++ = UNI_REPLACEMENT_CHAR; 558 } 559 } else { 560 *target++ = (UTF16)ch; /* normal case */ 561 } 562 } else if (ch > UNI_MAX_UTF16) { 563 if (flags == strictConversion) { 564 result = sourceIllegal; 565 source -= (extraBytesToRead+1); /* return to the start */ 566 break; /* Bail out; shouldn't continue */ 567 } else { 568 *target++ = UNI_REPLACEMENT_CHAR; 569 } 570 } else { 571 /* target is a character in range 0xFFFF - 0x10FFFF. */ 572 if (target + 1 >= targetEnd) { 573 source -= (extraBytesToRead+1); /* Back up source pointer! */ 574 result = targetExhausted; break; 575 } 576 ch -= halfBase; 577 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); 578 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); 579 } 580 } 581 *sourceStart = source; 582 *targetStart = target; 583 return result; 584 } 585 586 /* --------------------------------------------------------------------- */ 587 588 static ConversionResult ConvertUTF8toUTF32Impl( 589 const UTF8** sourceStart, const UTF8* sourceEnd, 590 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags, 591 Boolean InputIsPartial) { 592 ConversionResult result = conversionOK; 593 const UTF8* source = *sourceStart; 594 UTF32* target = *targetStart; 595 while (source < sourceEnd) { 596 UTF32 ch = 0; 597 unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; 598 if (extraBytesToRead >= sourceEnd - source) { 599 if (flags == strictConversion || InputIsPartial) { 600 result = sourceExhausted; 601 break; 602 } else { 603 result = sourceIllegal; 604 605 /* 606 * Replace the maximal subpart of ill-formed sequence with 607 * replacement character. 608 */ 609 source += findMaximalSubpartOfIllFormedUTF8Sequence(source, 610 sourceEnd); 611 *target++ = UNI_REPLACEMENT_CHAR; 612 continue; 613 } 614 } 615 if (target >= targetEnd) { 616 result = targetExhausted; break; 617 } 618 619 /* Do this check whether lenient or strict */ 620 if (!isLegalUTF8(source, extraBytesToRead+1)) { 621 result = sourceIllegal; 622 if (flags == strictConversion) { 623 /* Abort conversion. */ 624 break; 625 } else { 626 /* 627 * Replace the maximal subpart of ill-formed sequence with 628 * replacement character. 629 */ 630 source += findMaximalSubpartOfIllFormedUTF8Sequence(source, 631 sourceEnd); 632 *target++ = UNI_REPLACEMENT_CHAR; 633 continue; 634 } 635 } 636 /* 637 * The cases all fall through. See "Note A" below. 638 */ 639 switch (extraBytesToRead) { 640 case 5: ch += *source++; ch <<= 6; 641 case 4: ch += *source++; ch <<= 6; 642 case 3: ch += *source++; ch <<= 6; 643 case 2: ch += *source++; ch <<= 6; 644 case 1: ch += *source++; ch <<= 6; 645 case 0: ch += *source++; 646 } 647 ch -= offsetsFromUTF8[extraBytesToRead]; 648 649 if (ch <= UNI_MAX_LEGAL_UTF32) { 650 /* 651 * UTF-16 surrogate values are illegal in UTF-32, and anything 652 * over Plane 17 (> 0x10FFFF) is illegal. 653 */ 654 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { 655 if (flags == strictConversion) { 656 source -= (extraBytesToRead+1); /* return to the illegal value itself */ 657 result = sourceIllegal; 658 break; 659 } else { 660 *target++ = UNI_REPLACEMENT_CHAR; 661 } 662 } else { 663 *target++ = ch; 664 } 665 } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ 666 result = sourceIllegal; 667 *target++ = UNI_REPLACEMENT_CHAR; 668 } 669 } 670 *sourceStart = source; 671 *targetStart = target; 672 return result; 673 } 674 675 ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart, 676 const UTF8 *sourceEnd, 677 UTF32 **targetStart, 678 UTF32 *targetEnd, 679 ConversionFlags flags) { 680 return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd, 681 flags, /*InputIsPartial=*/true); 682 } 683 684 ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart, 685 const UTF8 *sourceEnd, UTF32 **targetStart, 686 UTF32 *targetEnd, ConversionFlags flags) { 687 return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd, 688 flags, /*InputIsPartial=*/false); 689 } 690 691 /* --------------------------------------------------------------------- 692 693 Note A. 694 The fall-through switches in UTF-8 reading code save a 695 temp variable, some decrements & conditionals. The switches 696 are equivalent to the following loop: 697 { 698 int tmpBytesToRead = extraBytesToRead+1; 699 do { 700 ch += *source++; 701 --tmpBytesToRead; 702 if (tmpBytesToRead) ch <<= 6; 703 } while (tmpBytesToRead > 0); 704 } 705 In UTF-8 writing code, the switches on "bytesToWrite" are 706 similarly unrolled loops. 707 708 --------------------------------------------------------------------- */ 709 710 } // namespace llvm 711
