1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "utf.h" 18 19 #include "base/logging.h" 20 #include "mirror/array.h" 21 #include "mirror/object-inl.h" 22 #include "utf-inl.h" 23 24 namespace art { 25 26 // This is used only from debugger and test code. 27 size_t CountModifiedUtf8Chars(const char* utf8) { 28 return CountModifiedUtf8Chars(utf8, strlen(utf8)); 29 } 30 31 /* 32 * This does not validate UTF8 rules (nor did older code). But it gets the right answer 33 * for valid UTF-8 and that's fine because it's used only to size a buffer for later 34 * conversion. 35 * 36 * Modified UTF-8 consists of a series of bytes up to 21 bit Unicode code points as follows: 37 * U+0001 - U+007F 0xxxxxxx 38 * U+0080 - U+07FF 110xxxxx 10xxxxxx 39 * U+0800 - U+FFFF 1110xxxx 10xxxxxx 10xxxxxx 40 * U+10000 - U+1FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 41 * 42 * U+0000 is encoded using the 2nd form to avoid nulls inside strings (this differs from 43 * standard UTF-8). 44 * The four byte encoding converts to two utf16 characters. 45 */ 46 size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) { 47 DCHECK_LE(byte_count, strlen(utf8)); 48 size_t len = 0; 49 const char* end = utf8 + byte_count; 50 for (; utf8 < end; ++utf8) { 51 int ic = *utf8; 52 len++; 53 if (LIKELY((ic & 0x80) == 0)) { 54 // One-byte encoding. 55 continue; 56 } 57 // Two- or three-byte encoding. 58 utf8++; 59 if ((ic & 0x20) == 0) { 60 // Two-byte encoding. 61 continue; 62 } 63 utf8++; 64 if ((ic & 0x10) == 0) { 65 // Three-byte encoding. 66 continue; 67 } 68 69 // Four-byte encoding: needs to be converted into a surrogate 70 // pair. 71 utf8++; 72 len++; 73 } 74 return len; 75 } 76 77 // This is used only from debugger and test code. 78 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, const char* utf8_data_in) { 79 while (*utf8_data_in != '\0') { 80 const uint32_t ch = GetUtf16FromUtf8(&utf8_data_in); 81 const uint16_t leading = GetLeadingUtf16Char(ch); 82 const uint16_t trailing = GetTrailingUtf16Char(ch); 83 84 *utf16_data_out++ = leading; 85 if (trailing != 0) { 86 *utf16_data_out++ = trailing; 87 } 88 } 89 } 90 91 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, size_t out_chars, 92 const char* utf8_data_in, size_t in_bytes) { 93 const char *in_start = utf8_data_in; 94 const char *in_end = utf8_data_in + in_bytes; 95 uint16_t *out_p = utf16_data_out; 96 97 if (LIKELY(out_chars == in_bytes)) { 98 // Common case where all characters are ASCII. 99 for (const char *p = in_start; p < in_end;) { 100 // Safe even if char is signed because ASCII characters always have 101 // the high bit cleared. 102 *out_p++ = dchecked_integral_cast<uint16_t>(*p++); 103 } 104 return; 105 } 106 107 // String contains non-ASCII characters. 108 for (const char *p = in_start; p < in_end;) { 109 const uint32_t ch = GetUtf16FromUtf8(&p); 110 const uint16_t leading = GetLeadingUtf16Char(ch); 111 const uint16_t trailing = GetTrailingUtf16Char(ch); 112 113 *out_p++ = leading; 114 if (trailing != 0) { 115 *out_p++ = trailing; 116 } 117 } 118 } 119 120 void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count, 121 const uint16_t* utf16_in, size_t char_count) { 122 if (LIKELY(byte_count == char_count)) { 123 // Common case where all characters are ASCII. 124 const uint16_t *utf16_end = utf16_in + char_count; 125 for (const uint16_t *p = utf16_in; p < utf16_end;) { 126 *utf8_out++ = dchecked_integral_cast<char>(*p++); 127 } 128 return; 129 } 130 131 // String contains non-ASCII characters. 132 while (char_count--) { 133 const uint16_t ch = *utf16_in++; 134 if (ch > 0 && ch <= 0x7f) { 135 *utf8_out++ = ch; 136 } else { 137 // Char_count == 0 here implies we've encountered an unpaired 138 // surrogate and we have no choice but to encode it as 3-byte UTF 139 // sequence. Note that unpaired surrogates can occur as a part of 140 // "normal" operation. 141 if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) { 142 const uint16_t ch2 = *utf16_in; 143 144 // Check if the other half of the pair is within the expected 145 // range. If it isn't, we will have to emit both "halves" as 146 // separate 3 byte sequences. 147 if (ch2 >= 0xdc00 && ch2 <= 0xdfff) { 148 utf16_in++; 149 char_count--; 150 const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00; 151 *utf8_out++ = (code_point >> 18) | 0xf0; 152 *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80; 153 *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80; 154 *utf8_out++ = (code_point & 0x3f) | 0x80; 155 continue; 156 } 157 } 158 159 if (ch > 0x07ff) { 160 // Three byte encoding. 161 *utf8_out++ = (ch >> 12) | 0xe0; 162 *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80; 163 *utf8_out++ = (ch & 0x3f) | 0x80; 164 } else /*(ch > 0x7f || ch == 0)*/ { 165 // Two byte encoding. 166 *utf8_out++ = (ch >> 6) | 0xc0; 167 *utf8_out++ = (ch & 0x3f) | 0x80; 168 } 169 } 170 } 171 } 172 173 int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length) { 174 uint32_t hash = 0; 175 while (utf16_length != 0u) { 176 const uint32_t pair = GetUtf16FromUtf8(&utf8); 177 const uint16_t first = GetLeadingUtf16Char(pair); 178 hash = hash * 31 + first; 179 --utf16_length; 180 const uint16_t second = GetTrailingUtf16Char(pair); 181 if (second != 0) { 182 hash = hash * 31 + second; 183 DCHECK_NE(utf16_length, 0u); 184 --utf16_length; 185 } 186 } 187 return static_cast<int32_t>(hash); 188 } 189 190 uint32_t ComputeModifiedUtf8Hash(const char* chars) { 191 uint32_t hash = 0; 192 while (*chars != '\0') { 193 hash = hash * 31 + *chars++; 194 } 195 return static_cast<int32_t>(hash); 196 } 197 198 int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16, 199 size_t utf16_length) { 200 for (;;) { 201 if (*utf8 == '\0') { 202 return (utf16_length == 0) ? 0 : -1; 203 } else if (utf16_length == 0) { 204 return 1; 205 } 206 207 const uint32_t pair = GetUtf16FromUtf8(&utf8); 208 209 // First compare the leading utf16 char. 210 const uint16_t lhs = GetLeadingUtf16Char(pair); 211 const uint16_t rhs = *utf16++; 212 --utf16_length; 213 if (lhs != rhs) { 214 return lhs > rhs ? 1 : -1; 215 } 216 217 // Then compare the trailing utf16 char. First check if there 218 // are any characters left to consume. 219 const uint16_t lhs2 = GetTrailingUtf16Char(pair); 220 if (lhs2 != 0) { 221 if (utf16_length == 0) { 222 return 1; 223 } 224 225 const uint16_t rhs2 = *utf16++; 226 --utf16_length; 227 if (lhs2 != rhs2) { 228 return lhs2 > rhs2 ? 1 : -1; 229 } 230 } 231 } 232 } 233 234 size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) { 235 size_t result = 0; 236 const uint16_t *end = chars + char_count; 237 while (chars < end) { 238 const uint16_t ch = *chars++; 239 if (LIKELY(ch != 0 && ch < 0x80)) { 240 result++; 241 continue; 242 } 243 if (ch < 0x800) { 244 result += 2; 245 continue; 246 } 247 if (ch >= 0xd800 && ch < 0xdc00) { 248 if (chars < end) { 249 const uint16_t ch2 = *chars; 250 // If we find a properly paired surrogate, we emit it as a 4 byte 251 // UTF sequence. If we find an unpaired leading or trailing surrogate, 252 // we emit it as a 3 byte sequence like would have done earlier. 253 if (ch2 >= 0xdc00 && ch2 < 0xe000) { 254 chars++; 255 result += 4; 256 continue; 257 } 258 } 259 } 260 result += 3; 261 } 262 return result; 263 } 264 265 } // namespace art 266