Home | History | Annotate | Download | only in runtime
      1 /*
      2  * Copyright (C) 2011 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 #include "utf.h"
     18 
     19 #include "base/logging.h"
     20 #include "mirror/array.h"
     21 #include "mirror/object-inl.h"
     22 #include "utf-inl.h"
     23 
     24 namespace art {
     25 
     26 // This is used only from debugger and test code.
     27 size_t CountModifiedUtf8Chars(const char* utf8) {
     28   return CountModifiedUtf8Chars(utf8, strlen(utf8));
     29 }
     30 
     31 /*
     32  * This does not validate UTF8 rules (nor did older code). But it gets the right answer
     33  * for valid UTF-8 and that's fine because it's used only to size a buffer for later
     34  * conversion.
     35  *
     36  * Modified UTF-8 consists of a series of bytes up to 21 bit Unicode code points as follows:
     37  * U+0001  - U+007F   0xxxxxxx
     38  * U+0080  - U+07FF   110xxxxx 10xxxxxx
     39  * U+0800  - U+FFFF   1110xxxx 10xxxxxx 10xxxxxx
     40  * U+10000 - U+1FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
     41  *
     42  * U+0000 is encoded using the 2nd form to avoid nulls inside strings (this differs from
     43  * standard UTF-8).
     44  * The four byte encoding converts to two utf16 characters.
     45  */
     46 size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) {
     47   DCHECK_LE(byte_count, strlen(utf8));
     48   size_t len = 0;
     49   const char* end = utf8 + byte_count;
     50   for (; utf8 < end; ++utf8) {
     51     int ic = *utf8;
     52     len++;
     53     if (LIKELY((ic & 0x80) == 0)) {
     54       // One-byte encoding.
     55       continue;
     56     }
     57     // Two- or three-byte encoding.
     58     utf8++;
     59     if ((ic & 0x20) == 0) {
     60       // Two-byte encoding.
     61       continue;
     62     }
     63     utf8++;
     64     if ((ic & 0x10) == 0) {
     65       // Three-byte encoding.
     66       continue;
     67     }
     68 
     69     // Four-byte encoding: needs to be converted into a surrogate
     70     // pair.
     71     utf8++;
     72     len++;
     73   }
     74   return len;
     75 }
     76 
     77 // This is used only from debugger and test code.
     78 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, const char* utf8_data_in) {
     79   while (*utf8_data_in != '\0') {
     80     const uint32_t ch = GetUtf16FromUtf8(&utf8_data_in);
     81     const uint16_t leading = GetLeadingUtf16Char(ch);
     82     const uint16_t trailing = GetTrailingUtf16Char(ch);
     83 
     84     *utf16_data_out++ = leading;
     85     if (trailing != 0) {
     86       *utf16_data_out++ = trailing;
     87     }
     88   }
     89 }
     90 
     91 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, size_t out_chars,
     92                                 const char* utf8_data_in, size_t in_bytes) {
     93   const char *in_start = utf8_data_in;
     94   const char *in_end = utf8_data_in + in_bytes;
     95   uint16_t *out_p = utf16_data_out;
     96 
     97   if (LIKELY(out_chars == in_bytes)) {
     98     // Common case where all characters are ASCII.
     99     for (const char *p = in_start; p < in_end;) {
    100       // Safe even if char is signed because ASCII characters always have
    101       // the high bit cleared.
    102       *out_p++ = dchecked_integral_cast<uint16_t>(*p++);
    103     }
    104     return;
    105   }
    106 
    107   // String contains non-ASCII characters.
    108   for (const char *p = in_start; p < in_end;) {
    109     const uint32_t ch = GetUtf16FromUtf8(&p);
    110     const uint16_t leading = GetLeadingUtf16Char(ch);
    111     const uint16_t trailing = GetTrailingUtf16Char(ch);
    112 
    113     *out_p++ = leading;
    114     if (trailing != 0) {
    115       *out_p++ = trailing;
    116     }
    117   }
    118 }
    119 
    120 void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count,
    121                                 const uint16_t* utf16_in, size_t char_count) {
    122   if (LIKELY(byte_count == char_count)) {
    123     // Common case where all characters are ASCII.
    124     const uint16_t *utf16_end = utf16_in + char_count;
    125     for (const uint16_t *p = utf16_in; p < utf16_end;) {
    126       *utf8_out++ = dchecked_integral_cast<char>(*p++);
    127     }
    128     return;
    129   }
    130 
    131   // String contains non-ASCII characters.
    132   while (char_count--) {
    133     const uint16_t ch = *utf16_in++;
    134     if (ch > 0 && ch <= 0x7f) {
    135       *utf8_out++ = ch;
    136     } else {
    137       // Char_count == 0 here implies we've encountered an unpaired
    138       // surrogate and we have no choice but to encode it as 3-byte UTF
    139       // sequence. Note that unpaired surrogates can occur as a part of
    140       // "normal" operation.
    141       if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) {
    142         const uint16_t ch2 = *utf16_in;
    143 
    144         // Check if the other half of the pair is within the expected
    145         // range. If it isn't, we will have to emit both "halves" as
    146         // separate 3 byte sequences.
    147         if (ch2 >= 0xdc00 && ch2 <= 0xdfff) {
    148           utf16_in++;
    149           char_count--;
    150           const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00;
    151           *utf8_out++ = (code_point >> 18) | 0xf0;
    152           *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80;
    153           *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80;
    154           *utf8_out++ = (code_point & 0x3f) | 0x80;
    155           continue;
    156         }
    157       }
    158 
    159       if (ch > 0x07ff) {
    160         // Three byte encoding.
    161         *utf8_out++ = (ch >> 12) | 0xe0;
    162         *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80;
    163         *utf8_out++ = (ch & 0x3f) | 0x80;
    164       } else /*(ch > 0x7f || ch == 0)*/ {
    165         // Two byte encoding.
    166         *utf8_out++ = (ch >> 6) | 0xc0;
    167         *utf8_out++ = (ch & 0x3f) | 0x80;
    168       }
    169     }
    170   }
    171 }
    172 
    173 int32_t ComputeUtf16Hash(const uint16_t* chars, size_t char_count) {
    174   uint32_t hash = 0;
    175   while (char_count--) {
    176     hash = hash * 31 + *chars++;
    177   }
    178   return static_cast<int32_t>(hash);
    179 }
    180 
    181 int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length) {
    182   uint32_t hash = 0;
    183   while (utf16_length != 0u) {
    184     const uint32_t pair = GetUtf16FromUtf8(&utf8);
    185     const uint16_t first = GetLeadingUtf16Char(pair);
    186     hash = hash * 31 + first;
    187     --utf16_length;
    188     const uint16_t second = GetTrailingUtf16Char(pair);
    189     if (second != 0) {
    190       hash = hash * 31 + second;
    191       DCHECK_NE(utf16_length, 0u);
    192       --utf16_length;
    193     }
    194   }
    195   return static_cast<int32_t>(hash);
    196 }
    197 
    198 uint32_t ComputeModifiedUtf8Hash(const char* chars) {
    199   uint32_t hash = 0;
    200   while (*chars != '\0') {
    201     hash = hash * 31 + *chars++;
    202   }
    203   return static_cast<int32_t>(hash);
    204 }
    205 
    206 int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16,
    207                                                 size_t utf16_length) {
    208   for (;;) {
    209     if (*utf8 == '\0') {
    210       return (utf16_length == 0) ? 0 : -1;
    211     } else if (utf16_length == 0) {
    212       return 1;
    213     }
    214 
    215     const uint32_t pair = GetUtf16FromUtf8(&utf8);
    216 
    217     // First compare the leading utf16 char.
    218     const uint16_t lhs = GetLeadingUtf16Char(pair);
    219     const uint16_t rhs = *utf16++;
    220     --utf16_length;
    221     if (lhs != rhs) {
    222       return lhs > rhs ? 1 : -1;
    223     }
    224 
    225     // Then compare the trailing utf16 char. First check if there
    226     // are any characters left to consume.
    227     const uint16_t lhs2 = GetTrailingUtf16Char(pair);
    228     if (lhs2 != 0) {
    229       if (utf16_length == 0) {
    230         return 1;
    231       }
    232 
    233       const uint16_t rhs2 = *utf16++;
    234       --utf16_length;
    235       if (lhs2 != rhs2) {
    236         return lhs2 > rhs2 ? 1 : -1;
    237       }
    238     }
    239   }
    240 }
    241 
    242 size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) {
    243   size_t result = 0;
    244   const uint16_t *end = chars + char_count;
    245   while (chars < end) {
    246     const uint16_t ch = *chars++;
    247     if (LIKELY(ch != 0 && ch < 0x80)) {
    248       result++;
    249       continue;
    250     }
    251     if (ch < 0x800) {
    252       result += 2;
    253       continue;
    254     }
    255     if (ch >= 0xd800 && ch < 0xdc00) {
    256       if (chars < end) {
    257         const uint16_t ch2 = *chars;
    258         // If we find a properly paired surrogate, we emit it as a 4 byte
    259         // UTF sequence. If we find an unpaired leading or trailing surrogate,
    260         // we emit it as a 3 byte sequence like would have done earlier.
    261         if (ch2 >= 0xdc00 && ch2 < 0xe000) {
    262           chars++;
    263           result += 4;
    264           continue;
    265         }
    266       }
    267     }
    268     result += 3;
    269   }
    270   return result;
    271 }
    272 
    273 }  // namespace art
    274