Home | History | Annotate | Download | only in Support
      1 /*===--- ConvertUTF.c - Universal Character Names conversions ---------------===
      2  *
      3  *                     The LLVM Compiler Infrastructure
      4  *
      5  * This file is distributed under the University of Illinois Open Source
      6  * License. See LICENSE.TXT for details.
      7  *
      8  *===------------------------------------------------------------------------=*/
      9 /*
     10  * Copyright 2001-2004 Unicode, Inc.
     11  *
     12  * Disclaimer
     13  *
     14  * This source code is provided as is by Unicode, Inc. No claims are
     15  * made as to fitness for any particular purpose. No warranties of any
     16  * kind are expressed or implied. The recipient agrees to determine
     17  * applicability of information provided. If this file has been
     18  * purchased on magnetic or optical media from Unicode, Inc., the
     19  * sole remedy for any claim will be exchange of defective media
     20  * within 90 days of receipt.
     21  *
     22  * Limitations on Rights to Redistribute This Code
     23  *
     24  * Unicode, Inc. hereby grants the right to freely use the information
     25  * supplied in this file in the creation of products supporting the
     26  * Unicode Standard, and to make copies of this file in any form
     27  * for internal or external distribution as long as this notice
     28  * remains attached.
     29  */
     30 
     31 /* ---------------------------------------------------------------------
     32 
     33     Conversions between UTF32, UTF-16, and UTF-8. Source code file.
     34     Author: Mark E. Davis, 1994.
     35     Rev History: Rick McGowan, fixes & updates May 2001.
     36     Sept 2001: fixed const & error conditions per
     37         mods suggested by S. Parent & A. Lillich.
     38     June 2002: Tim Dodd added detection and handling of incomplete
     39         source sequences, enhanced error detection, added casts
     40         to eliminate compiler warnings.
     41     July 2003: slight mods to back out aggressive FFFE detection.
     42     Jan 2004: updated switches in from-UTF8 conversions.
     43     Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
     44 
     45     See the header file "ConvertUTF.h" for complete documentation.
     46 
     47 ------------------------------------------------------------------------ */
     48 
     49 
     50 #include "llvm/Support/ConvertUTF.h"
     51 #ifdef CVTUTF_DEBUG
     52 #include <stdio.h>
     53 #endif
     54 #include <assert.h>
     55 
     56 namespace llvm {
     57 
     58 static const int halfShift  = 10; /* used for shifting by 10 bits */
     59 
     60 static const UTF32 halfBase = 0x0010000UL;
     61 static const UTF32 halfMask = 0x3FFUL;
     62 
     63 #define UNI_SUR_HIGH_START  (UTF32)0xD800
     64 #define UNI_SUR_HIGH_END    (UTF32)0xDBFF
     65 #define UNI_SUR_LOW_START   (UTF32)0xDC00
     66 #define UNI_SUR_LOW_END     (UTF32)0xDFFF
     67 
     68 /* --------------------------------------------------------------------- */
     69 
     70 /*
     71  * Index into the table below with the first byte of a UTF-8 sequence to
     72  * get the number of trailing bytes that are supposed to follow it.
     73  * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
     74  * left as-is for anyone who may want to do such conversion, which was
     75  * allowed in earlier algorithms.
     76  */
     77 static const char trailingBytesForUTF8[256] = {
     78     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     79     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     80     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     81     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     82     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     83     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
     84     1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
     85     2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
     86 };
     87 
     88 /*
     89  * Magic values subtracted from a buffer value during UTF8 conversion.
     90  * This table contains as many values as there might be trailing bytes
     91  * in a UTF-8 sequence.
     92  */
     93 static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
     94                      0x03C82080UL, 0xFA082080UL, 0x82082080UL };
     95 
     96 /*
     97  * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
     98  * into the first byte, depending on how many bytes follow.  There are
     99  * as many entries in this table as there are UTF-8 sequence types.
    100  * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
    101  * for *legal* UTF-8 will be 4 or fewer bytes total.
    102  */
    103 static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
    104 
    105 /* --------------------------------------------------------------------- */
    106 
    107 /* The interface converts a whole buffer to avoid function-call overhead.
    108  * Constants have been gathered. Loops & conditionals have been removed as
    109  * much as possible for efficiency, in favor of drop-through switches.
    110  * (See "Note A" at the bottom of the file for equivalent code.)
    111  * If your compiler supports it, the "isLegalUTF8" call can be turned
    112  * into an inline function.
    113  */
    114 
    115 
    116 /* --------------------------------------------------------------------- */
    117 
    118 ConversionResult ConvertUTF32toUTF16 (
    119         const UTF32** sourceStart, const UTF32* sourceEnd,
    120         UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
    121     ConversionResult result = conversionOK;
    122     const UTF32* source = *sourceStart;
    123     UTF16* target = *targetStart;
    124     while (source < sourceEnd) {
    125         UTF32 ch;
    126         if (target >= targetEnd) {
    127             result = targetExhausted; break;
    128         }
    129         ch = *source++;
    130         if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
    131             /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
    132             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
    133                 if (flags == strictConversion) {
    134                     --source; /* return to the illegal value itself */
    135                     result = sourceIllegal;
    136                     break;
    137                 } else {
    138                     *target++ = UNI_REPLACEMENT_CHAR;
    139                 }
    140             } else {
    141                 *target++ = (UTF16)ch; /* normal case */
    142             }
    143         } else if (ch > UNI_MAX_LEGAL_UTF32) {
    144             if (flags == strictConversion) {
    145                 result = sourceIllegal;
    146             } else {
    147                 *target++ = UNI_REPLACEMENT_CHAR;
    148             }
    149         } else {
    150             /* target is a character in range 0xFFFF - 0x10FFFF. */
    151             if (target + 1 >= targetEnd) {
    152                 --source; /* Back up source pointer! */
    153                 result = targetExhausted; break;
    154             }
    155             ch -= halfBase;
    156             *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
    157             *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
    158         }
    159     }
    160     *sourceStart = source;
    161     *targetStart = target;
    162     return result;
    163 }
    164 
    165 /* --------------------------------------------------------------------- */
    166 
    167 ConversionResult ConvertUTF16toUTF32 (
    168         const UTF16** sourceStart, const UTF16* sourceEnd,
    169         UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
    170     ConversionResult result = conversionOK;
    171     const UTF16* source = *sourceStart;
    172     UTF32* target = *targetStart;
    173     UTF32 ch, ch2;
    174     while (source < sourceEnd) {
    175         const UTF16* oldSource = source; /*  In case we have to back up because of target overflow. */
    176         ch = *source++;
    177         /* If we have a surrogate pair, convert to UTF32 first. */
    178         if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
    179             /* If the 16 bits following the high surrogate are in the source buffer... */
    180             if (source < sourceEnd) {
    181                 ch2 = *source;
    182                 /* If it's a low surrogate, convert to UTF32. */
    183                 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
    184                     ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
    185                         + (ch2 - UNI_SUR_LOW_START) + halfBase;
    186                     ++source;
    187                 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
    188                     --source; /* return to the illegal value itself */
    189                     result = sourceIllegal;
    190                     break;
    191                 }
    192             } else { /* We don't have the 16 bits following the high surrogate. */
    193                 --source; /* return to the high surrogate */
    194                 result = sourceExhausted;
    195                 break;
    196             }
    197         } else if (flags == strictConversion) {
    198             /* UTF-16 surrogate values are illegal in UTF-32 */
    199             if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
    200                 --source; /* return to the illegal value itself */
    201                 result = sourceIllegal;
    202                 break;
    203             }
    204         }
    205         if (target >= targetEnd) {
    206             source = oldSource; /* Back up source pointer! */
    207             result = targetExhausted; break;
    208         }
    209         *target++ = ch;
    210     }
    211     *sourceStart = source;
    212     *targetStart = target;
    213 #ifdef CVTUTF_DEBUG
    214 if (result == sourceIllegal) {
    215     fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
    216     fflush(stderr);
    217 }
    218 #endif
    219     return result;
    220 }
    221 ConversionResult ConvertUTF16toUTF8 (
    222         const UTF16** sourceStart, const UTF16* sourceEnd,
    223         UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
    224     ConversionResult result = conversionOK;
    225     const UTF16* source = *sourceStart;
    226     UTF8* target = *targetStart;
    227     while (source < sourceEnd) {
    228         UTF32 ch;
    229         unsigned short bytesToWrite = 0;
    230         const UTF32 byteMask = 0xBF;
    231         const UTF32 byteMark = 0x80;
    232         const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
    233         ch = *source++;
    234         /* If we have a surrogate pair, convert to UTF32 first. */
    235         if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
    236             /* If the 16 bits following the high surrogate are in the source buffer... */
    237             if (source < sourceEnd) {
    238                 UTF32 ch2 = *source;
    239                 /* If it's a low surrogate, convert to UTF32. */
    240                 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
    241                     ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
    242                         + (ch2 - UNI_SUR_LOW_START) + halfBase;
    243                     ++source;
    244                 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
    245                     --source; /* return to the illegal value itself */
    246                     result = sourceIllegal;
    247                     break;
    248                 }
    249             } else { /* We don't have the 16 bits following the high surrogate. */
    250                 --source; /* return to the high surrogate */
    251                 result = sourceExhausted;
    252                 break;
    253             }
    254         } else if (flags == strictConversion) {
    255             /* UTF-16 surrogate values are illegal in UTF-32 */
    256             if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
    257                 --source; /* return to the illegal value itself */
    258                 result = sourceIllegal;
    259                 break;
    260             }
    261         }
    262         /* Figure out how many bytes the result will require */
    263         if (ch < (UTF32)0x80) {      bytesToWrite = 1;
    264         } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
    265         } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
    266         } else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
    267         } else {                            bytesToWrite = 3;
    268                                             ch = UNI_REPLACEMENT_CHAR;
    269         }
    270 
    271         target += bytesToWrite;
    272         if (target > targetEnd) {
    273             source = oldSource; /* Back up source pointer! */
    274             target -= bytesToWrite; result = targetExhausted; break;
    275         }
    276         switch (bytesToWrite) { /* note: everything falls through. */
    277             case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
    278             case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
    279             case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
    280             case 1: *--target =  (UTF8)(ch | firstByteMark[bytesToWrite]);
    281         }
    282         target += bytesToWrite;
    283     }
    284     *sourceStart = source;
    285     *targetStart = target;
    286     return result;
    287 }
    288 
    289 /* --------------------------------------------------------------------- */
    290 
    291 ConversionResult ConvertUTF32toUTF8 (
    292         const UTF32** sourceStart, const UTF32* sourceEnd,
    293         UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
    294     ConversionResult result = conversionOK;
    295     const UTF32* source = *sourceStart;
    296     UTF8* target = *targetStart;
    297     while (source < sourceEnd) {
    298         UTF32 ch;
    299         unsigned short bytesToWrite = 0;
    300         const UTF32 byteMask = 0xBF;
    301         const UTF32 byteMark = 0x80;
    302         ch = *source++;
    303         if (flags == strictConversion ) {
    304             /* UTF-16 surrogate values are illegal in UTF-32 */
    305             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
    306                 --source; /* return to the illegal value itself */
    307                 result = sourceIllegal;
    308                 break;
    309             }
    310         }
    311         /*
    312          * Figure out how many bytes the result will require. Turn any
    313          * illegally large UTF32 things (> Plane 17) into replacement chars.
    314          */
    315         if (ch < (UTF32)0x80) {      bytesToWrite = 1;
    316         } else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
    317         } else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
    318         } else if (ch <= UNI_MAX_LEGAL_UTF32) {  bytesToWrite = 4;
    319         } else {                            bytesToWrite = 3;
    320                                             ch = UNI_REPLACEMENT_CHAR;
    321                                             result = sourceIllegal;
    322         }
    323 
    324         target += bytesToWrite;
    325         if (target > targetEnd) {
    326             --source; /* Back up source pointer! */
    327             target -= bytesToWrite; result = targetExhausted; break;
    328         }
    329         switch (bytesToWrite) { /* note: everything falls through. */
    330             case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
    331             case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
    332             case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
    333             case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
    334         }
    335         target += bytesToWrite;
    336     }
    337     *sourceStart = source;
    338     *targetStart = target;
    339     return result;
    340 }
    341 
    342 /* --------------------------------------------------------------------- */
    343 
    344 /*
    345  * Utility routine to tell whether a sequence of bytes is legal UTF-8.
    346  * This must be called with the length pre-determined by the first byte.
    347  * If not calling this from ConvertUTF8to*, then the length can be set by:
    348  *  length = trailingBytesForUTF8[*source]+1;
    349  * and the sequence is illegal right away if there aren't that many bytes
    350  * available.
    351  * If presented with a length > 4, this returns false.  The Unicode
    352  * definition of UTF-8 goes up to 4-byte sequences.
    353  */
    354 
    355 static Boolean isLegalUTF8(const UTF8 *source, int length) {
    356     UTF8 a;
    357     const UTF8 *srcptr = source+length;
    358     switch (length) {
    359     default: return false;
    360         /* Everything else falls through when "true"... */
    361     case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
    362     case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
    363     case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
    364 
    365         switch (*source) {
    366             /* no fall-through in this inner switch */
    367             case 0xE0: if (a < 0xA0) return false; break;
    368             case 0xED: if (a > 0x9F) return false; break;
    369             case 0xF0: if (a < 0x90) return false; break;
    370             case 0xF4: if (a > 0x8F) return false; break;
    371             default:   if (a < 0x80) return false;
    372         }
    373 
    374     case 1: if (*source >= 0x80 && *source < 0xC2) return false;
    375     }
    376     if (*source > 0xF4) return false;
    377     return true;
    378 }
    379 
    380 /* --------------------------------------------------------------------- */
    381 
    382 /*
    383  * Exported function to return whether a UTF-8 sequence is legal or not.
    384  * This is not used here; it's just exported.
    385  */
    386 Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
    387     int length = trailingBytesForUTF8[*source]+1;
    388     if (length > sourceEnd - source) {
    389         return false;
    390     }
    391     return isLegalUTF8(source, length);
    392 }
    393 
    394 /* --------------------------------------------------------------------- */
    395 
    396 static unsigned
    397 findMaximalSubpartOfIllFormedUTF8Sequence(const UTF8 *source,
    398                                           const UTF8 *sourceEnd) {
    399   UTF8 b1, b2, b3;
    400 
    401   assert(!isLegalUTF8Sequence(source, sourceEnd));
    402 
    403   /*
    404    * Unicode 6.3.0, D93b:
    405    *
    406    *   Maximal subpart of an ill-formed subsequence: The longest code unit
    407    *   subsequence starting at an unconvertible offset that is either:
    408    *   a. the initial subsequence of a well-formed code unit sequence, or
    409    *   b. a subsequence of length one.
    410    */
    411 
    412   if (source == sourceEnd)
    413     return 0;
    414 
    415   /*
    416    * Perform case analysis.  See Unicode 6.3.0, Table 3-7. Well-Formed UTF-8
    417    * Byte Sequences.
    418    */
    419 
    420   b1 = *source;
    421   ++source;
    422   if (b1 >= 0xC2 && b1 <= 0xDF) {
    423     /*
    424      * First byte is valid, but we know that this code unit sequence is
    425      * invalid, so the maximal subpart has to end after the first byte.
    426      */
    427     return 1;
    428   }
    429 
    430   if (source == sourceEnd)
    431     return 1;
    432 
    433   b2 = *source;
    434   ++source;
    435 
    436   if (b1 == 0xE0) {
    437     return (b2 >= 0xA0 && b2 <= 0xBF) ? 2 : 1;
    438   }
    439   if (b1 >= 0xE1 && b1 <= 0xEC) {
    440     return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
    441   }
    442   if (b1 == 0xED) {
    443     return (b2 >= 0x80 && b2 <= 0x9F) ? 2 : 1;
    444   }
    445   if (b1 >= 0xEE && b1 <= 0xEF) {
    446     return (b2 >= 0x80 && b2 <= 0xBF) ? 2 : 1;
    447   }
    448   if (b1 == 0xF0) {
    449     if (b2 >= 0x90 && b2 <= 0xBF) {
    450       if (source == sourceEnd)
    451         return 2;
    452 
    453       b3 = *source;
    454       return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
    455     }
    456     return 1;
    457   }
    458   if (b1 >= 0xF1 && b1 <= 0xF3) {
    459     if (b2 >= 0x80 && b2 <= 0xBF) {
    460       if (source == sourceEnd)
    461         return 2;
    462 
    463       b3 = *source;
    464       return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
    465     }
    466     return 1;
    467   }
    468   if (b1 == 0xF4) {
    469     if (b2 >= 0x80 && b2 <= 0x8F) {
    470       if (source == sourceEnd)
    471         return 2;
    472 
    473       b3 = *source;
    474       return (b3 >= 0x80 && b3 <= 0xBF) ? 3 : 2;
    475     }
    476     return 1;
    477   }
    478 
    479   assert((b1 >= 0x80 && b1 <= 0xC1) || b1 >= 0xF5);
    480   /*
    481    * There are no valid sequences that start with these bytes.  Maximal subpart
    482    * is defined to have length 1 in these cases.
    483    */
    484   return 1;
    485 }
    486 
    487 /* --------------------------------------------------------------------- */
    488 
    489 /*
    490  * Exported function to return the total number of bytes in a codepoint
    491  * represented in UTF-8, given the value of the first byte.
    492  */
    493 unsigned getNumBytesForUTF8(UTF8 first) {
    494   return trailingBytesForUTF8[first] + 1;
    495 }
    496 
    497 /* --------------------------------------------------------------------- */
    498 
    499 /*
    500  * Exported function to return whether a UTF-8 string is legal or not.
    501  * This is not used here; it's just exported.
    502  */
    503 Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) {
    504     while (*source != sourceEnd) {
    505         int length = trailingBytesForUTF8[**source] + 1;
    506         if (length > sourceEnd - *source || !isLegalUTF8(*source, length))
    507             return false;
    508         *source += length;
    509     }
    510     return true;
    511 }
    512 
    513 /* --------------------------------------------------------------------- */
    514 
    515 ConversionResult ConvertUTF8toUTF16 (
    516         const UTF8** sourceStart, const UTF8* sourceEnd,
    517         UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
    518     ConversionResult result = conversionOK;
    519     const UTF8* source = *sourceStart;
    520     UTF16* target = *targetStart;
    521     while (source < sourceEnd) {
    522         UTF32 ch = 0;
    523         unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
    524         if (extraBytesToRead >= sourceEnd - source) {
    525             result = sourceExhausted; break;
    526         }
    527         /* Do this check whether lenient or strict */
    528         if (!isLegalUTF8(source, extraBytesToRead+1)) {
    529             result = sourceIllegal;
    530             break;
    531         }
    532         /*
    533          * The cases all fall through. See "Note A" below.
    534          */
    535         switch (extraBytesToRead) {
    536             case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
    537             case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
    538             case 3: ch += *source++; ch <<= 6;
    539             case 2: ch += *source++; ch <<= 6;
    540             case 1: ch += *source++; ch <<= 6;
    541             case 0: ch += *source++;
    542         }
    543         ch -= offsetsFromUTF8[extraBytesToRead];
    544 
    545         if (target >= targetEnd) {
    546             source -= (extraBytesToRead+1); /* Back up source pointer! */
    547             result = targetExhausted; break;
    548         }
    549         if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
    550             /* UTF-16 surrogate values are illegal in UTF-32 */
    551             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
    552                 if (flags == strictConversion) {
    553                     source -= (extraBytesToRead+1); /* return to the illegal value itself */
    554                     result = sourceIllegal;
    555                     break;
    556                 } else {
    557                     *target++ = UNI_REPLACEMENT_CHAR;
    558                 }
    559             } else {
    560                 *target++ = (UTF16)ch; /* normal case */
    561             }
    562         } else if (ch > UNI_MAX_UTF16) {
    563             if (flags == strictConversion) {
    564                 result = sourceIllegal;
    565                 source -= (extraBytesToRead+1); /* return to the start */
    566                 break; /* Bail out; shouldn't continue */
    567             } else {
    568                 *target++ = UNI_REPLACEMENT_CHAR;
    569             }
    570         } else {
    571             /* target is a character in range 0xFFFF - 0x10FFFF. */
    572             if (target + 1 >= targetEnd) {
    573                 source -= (extraBytesToRead+1); /* Back up source pointer! */
    574                 result = targetExhausted; break;
    575             }
    576             ch -= halfBase;
    577             *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
    578             *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
    579         }
    580     }
    581     *sourceStart = source;
    582     *targetStart = target;
    583     return result;
    584 }
    585 
    586 /* --------------------------------------------------------------------- */
    587 
    588 static ConversionResult ConvertUTF8toUTF32Impl(
    589         const UTF8** sourceStart, const UTF8* sourceEnd,
    590         UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags,
    591         Boolean InputIsPartial) {
    592     ConversionResult result = conversionOK;
    593     const UTF8* source = *sourceStart;
    594     UTF32* target = *targetStart;
    595     while (source < sourceEnd) {
    596         UTF32 ch = 0;
    597         unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
    598         if (extraBytesToRead >= sourceEnd - source) {
    599             if (flags == strictConversion || InputIsPartial) {
    600                 result = sourceExhausted;
    601                 break;
    602             } else {
    603                 result = sourceIllegal;
    604 
    605                 /*
    606                  * Replace the maximal subpart of ill-formed sequence with
    607                  * replacement character.
    608                  */
    609                 source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
    610                                                                     sourceEnd);
    611                 *target++ = UNI_REPLACEMENT_CHAR;
    612                 continue;
    613             }
    614         }
    615         if (target >= targetEnd) {
    616             result = targetExhausted; break;
    617         }
    618 
    619         /* Do this check whether lenient or strict */
    620         if (!isLegalUTF8(source, extraBytesToRead+1)) {
    621             result = sourceIllegal;
    622             if (flags == strictConversion) {
    623                 /* Abort conversion. */
    624                 break;
    625             } else {
    626                 /*
    627                  * Replace the maximal subpart of ill-formed sequence with
    628                  * replacement character.
    629                  */
    630                 source += findMaximalSubpartOfIllFormedUTF8Sequence(source,
    631                                                                     sourceEnd);
    632                 *target++ = UNI_REPLACEMENT_CHAR;
    633                 continue;
    634             }
    635         }
    636         /*
    637          * The cases all fall through. See "Note A" below.
    638          */
    639         switch (extraBytesToRead) {
    640             case 5: ch += *source++; ch <<= 6;
    641             case 4: ch += *source++; ch <<= 6;
    642             case 3: ch += *source++; ch <<= 6;
    643             case 2: ch += *source++; ch <<= 6;
    644             case 1: ch += *source++; ch <<= 6;
    645             case 0: ch += *source++;
    646         }
    647         ch -= offsetsFromUTF8[extraBytesToRead];
    648 
    649         if (ch <= UNI_MAX_LEGAL_UTF32) {
    650             /*
    651              * UTF-16 surrogate values are illegal in UTF-32, and anything
    652              * over Plane 17 (> 0x10FFFF) is illegal.
    653              */
    654             if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
    655                 if (flags == strictConversion) {
    656                     source -= (extraBytesToRead+1); /* return to the illegal value itself */
    657                     result = sourceIllegal;
    658                     break;
    659                 } else {
    660                     *target++ = UNI_REPLACEMENT_CHAR;
    661                 }
    662             } else {
    663                 *target++ = ch;
    664             }
    665         } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
    666             result = sourceIllegal;
    667             *target++ = UNI_REPLACEMENT_CHAR;
    668         }
    669     }
    670     *sourceStart = source;
    671     *targetStart = target;
    672     return result;
    673 }
    674 
    675 ConversionResult ConvertUTF8toUTF32Partial(const UTF8 **sourceStart,
    676                                            const UTF8 *sourceEnd,
    677                                            UTF32 **targetStart,
    678                                            UTF32 *targetEnd,
    679                                            ConversionFlags flags) {
    680   return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
    681                                 flags, /*InputIsPartial=*/true);
    682 }
    683 
    684 ConversionResult ConvertUTF8toUTF32(const UTF8 **sourceStart,
    685                                     const UTF8 *sourceEnd, UTF32 **targetStart,
    686                                     UTF32 *targetEnd, ConversionFlags flags) {
    687   return ConvertUTF8toUTF32Impl(sourceStart, sourceEnd, targetStart, targetEnd,
    688                                 flags, /*InputIsPartial=*/false);
    689 }
    690 
    691 /* ---------------------------------------------------------------------
    692 
    693     Note A.
    694     The fall-through switches in UTF-8 reading code save a
    695     temp variable, some decrements & conditionals.  The switches
    696     are equivalent to the following loop:
    697         {
    698             int tmpBytesToRead = extraBytesToRead+1;
    699             do {
    700                 ch += *source++;
    701                 --tmpBytesToRead;
    702                 if (tmpBytesToRead) ch <<= 6;
    703             } while (tmpBytesToRead > 0);
    704         }
    705     In UTF-8 writing code, the switches on "bytesToWrite" are
    706     similarly unrolled loops.
    707 
    708    --------------------------------------------------------------------- */
    709 
    710 } // namespace llvm
    711