1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "utf.h" 18 19 #include <android-base/logging.h> 20 #include <android-base/stringprintf.h> 21 #include <android-base/strings.h> 22 23 #include "base/casts.h" 24 #include "utf-inl.h" 25 26 namespace art { 27 28 using android::base::StringAppendF; 29 30 // This is used only from debugger and test code. 31 size_t CountModifiedUtf8Chars(const char* utf8) { 32 return CountModifiedUtf8Chars(utf8, strlen(utf8)); 33 } 34 35 /* 36 * This does not validate UTF8 rules (nor did older code). But it gets the right answer 37 * for valid UTF-8 and that's fine because it's used only to size a buffer for later 38 * conversion. 39 * 40 * Modified UTF-8 consists of a series of bytes up to 21 bit Unicode code points as follows: 41 * U+0001 - U+007F 0xxxxxxx 42 * U+0080 - U+07FF 110xxxxx 10xxxxxx 43 * U+0800 - U+FFFF 1110xxxx 10xxxxxx 10xxxxxx 44 * U+10000 - U+1FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 45 * 46 * U+0000 is encoded using the 2nd form to avoid nulls inside strings (this differs from 47 * standard UTF-8). 48 * The four byte encoding converts to two utf16 characters. 49 */ 50 size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) { 51 DCHECK_LE(byte_count, strlen(utf8)); 52 size_t len = 0; 53 const char* end = utf8 + byte_count; 54 for (; utf8 < end; ++utf8) { 55 int ic = *utf8; 56 len++; 57 if (LIKELY((ic & 0x80) == 0)) { 58 // One-byte encoding. 59 continue; 60 } 61 // Two- or three-byte encoding. 62 utf8++; 63 if ((ic & 0x20) == 0) { 64 // Two-byte encoding. 65 continue; 66 } 67 utf8++; 68 if ((ic & 0x10) == 0) { 69 // Three-byte encoding. 70 continue; 71 } 72 73 // Four-byte encoding: needs to be converted into a surrogate 74 // pair. 75 utf8++; 76 len++; 77 } 78 return len; 79 } 80 81 // This is used only from debugger and test code. 82 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, const char* utf8_data_in) { 83 while (*utf8_data_in != '\0') { 84 const uint32_t ch = GetUtf16FromUtf8(&utf8_data_in); 85 const uint16_t leading = GetLeadingUtf16Char(ch); 86 const uint16_t trailing = GetTrailingUtf16Char(ch); 87 88 *utf16_data_out++ = leading; 89 if (trailing != 0) { 90 *utf16_data_out++ = trailing; 91 } 92 } 93 } 94 95 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, size_t out_chars, 96 const char* utf8_data_in, size_t in_bytes) { 97 const char *in_start = utf8_data_in; 98 const char *in_end = utf8_data_in + in_bytes; 99 uint16_t *out_p = utf16_data_out; 100 101 if (LIKELY(out_chars == in_bytes)) { 102 // Common case where all characters are ASCII. 103 for (const char *p = in_start; p < in_end;) { 104 // Safe even if char is signed because ASCII characters always have 105 // the high bit cleared. 106 *out_p++ = dchecked_integral_cast<uint16_t>(*p++); 107 } 108 return; 109 } 110 111 // String contains non-ASCII characters. 112 for (const char *p = in_start; p < in_end;) { 113 const uint32_t ch = GetUtf16FromUtf8(&p); 114 const uint16_t leading = GetLeadingUtf16Char(ch); 115 const uint16_t trailing = GetTrailingUtf16Char(ch); 116 117 *out_p++ = leading; 118 if (trailing != 0) { 119 *out_p++ = trailing; 120 } 121 } 122 } 123 124 void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count, 125 const uint16_t* utf16_in, size_t char_count) { 126 if (LIKELY(byte_count == char_count)) { 127 // Common case where all characters are ASCII. 128 const uint16_t *utf16_end = utf16_in + char_count; 129 for (const uint16_t *p = utf16_in; p < utf16_end;) { 130 *utf8_out++ = dchecked_integral_cast<char>(*p++); 131 } 132 return; 133 } 134 135 // String contains non-ASCII characters. 136 while (char_count--) { 137 const uint16_t ch = *utf16_in++; 138 if (ch > 0 && ch <= 0x7f) { 139 *utf8_out++ = ch; 140 } else { 141 // Char_count == 0 here implies we've encountered an unpaired 142 // surrogate and we have no choice but to encode it as 3-byte UTF 143 // sequence. Note that unpaired surrogates can occur as a part of 144 // "normal" operation. 145 if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) { 146 const uint16_t ch2 = *utf16_in; 147 148 // Check if the other half of the pair is within the expected 149 // range. If it isn't, we will have to emit both "halves" as 150 // separate 3 byte sequences. 151 if (ch2 >= 0xdc00 && ch2 <= 0xdfff) { 152 utf16_in++; 153 char_count--; 154 const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00; 155 *utf8_out++ = (code_point >> 18) | 0xf0; 156 *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80; 157 *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80; 158 *utf8_out++ = (code_point & 0x3f) | 0x80; 159 continue; 160 } 161 } 162 163 if (ch > 0x07ff) { 164 // Three byte encoding. 165 *utf8_out++ = (ch >> 12) | 0xe0; 166 *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80; 167 *utf8_out++ = (ch & 0x3f) | 0x80; 168 } else /*(ch > 0x7f || ch == 0)*/ { 169 // Two byte encoding. 170 *utf8_out++ = (ch >> 6) | 0xc0; 171 *utf8_out++ = (ch & 0x3f) | 0x80; 172 } 173 } 174 } 175 } 176 177 int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length) { 178 uint32_t hash = 0; 179 while (utf16_length != 0u) { 180 const uint32_t pair = GetUtf16FromUtf8(&utf8); 181 const uint16_t first = GetLeadingUtf16Char(pair); 182 hash = hash * 31 + first; 183 --utf16_length; 184 const uint16_t second = GetTrailingUtf16Char(pair); 185 if (second != 0) { 186 hash = hash * 31 + second; 187 DCHECK_NE(utf16_length, 0u); 188 --utf16_length; 189 } 190 } 191 return static_cast<int32_t>(hash); 192 } 193 194 uint32_t ComputeModifiedUtf8Hash(const char* chars) { 195 uint32_t hash = 0; 196 while (*chars != '\0') { 197 hash = hash * 31 + static_cast<uint8_t>(*chars); 198 ++chars; 199 } 200 return hash; 201 } 202 203 int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16, 204 size_t utf16_length) { 205 for (;;) { 206 if (*utf8 == '\0') { 207 return (utf16_length == 0) ? 0 : -1; 208 } else if (utf16_length == 0) { 209 return 1; 210 } 211 212 const uint32_t pair = GetUtf16FromUtf8(&utf8); 213 214 // First compare the leading utf16 char. 215 const uint16_t lhs = GetLeadingUtf16Char(pair); 216 const uint16_t rhs = *utf16++; 217 --utf16_length; 218 if (lhs != rhs) { 219 return lhs > rhs ? 1 : -1; 220 } 221 222 // Then compare the trailing utf16 char. First check if there 223 // are any characters left to consume. 224 const uint16_t lhs2 = GetTrailingUtf16Char(pair); 225 if (lhs2 != 0) { 226 if (utf16_length == 0) { 227 return 1; 228 } 229 230 const uint16_t rhs2 = *utf16++; 231 --utf16_length; 232 if (lhs2 != rhs2) { 233 return lhs2 > rhs2 ? 1 : -1; 234 } 235 } 236 } 237 } 238 239 size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) { 240 size_t result = 0; 241 const uint16_t *end = chars + char_count; 242 while (chars < end) { 243 const uint16_t ch = *chars++; 244 if (LIKELY(ch != 0 && ch < 0x80)) { 245 result++; 246 continue; 247 } 248 if (ch < 0x800) { 249 result += 2; 250 continue; 251 } 252 if (ch >= 0xd800 && ch < 0xdc00) { 253 if (chars < end) { 254 const uint16_t ch2 = *chars; 255 // If we find a properly paired surrogate, we emit it as a 4 byte 256 // UTF sequence. If we find an unpaired leading or trailing surrogate, 257 // we emit it as a 3 byte sequence like would have done earlier. 258 if (ch2 >= 0xdc00 && ch2 < 0xe000) { 259 chars++; 260 result += 4; 261 continue; 262 } 263 } 264 } 265 result += 3; 266 } 267 return result; 268 } 269 270 static inline constexpr bool NeedsEscaping(uint16_t ch) { 271 return (ch < ' ' || ch > '~'); 272 } 273 274 std::string PrintableChar(uint16_t ch) { 275 std::string result; 276 result += '\''; 277 if (NeedsEscaping(ch)) { 278 StringAppendF(&result, "\\u%04x", ch); 279 } else { 280 result += static_cast<std::string::value_type>(ch); 281 } 282 result += '\''; 283 return result; 284 } 285 286 std::string PrintableString(const char* utf8) { 287 std::string result; 288 result += '"'; 289 const char* p = utf8; 290 size_t char_count = CountModifiedUtf8Chars(p); 291 for (size_t i = 0; i < char_count; ++i) { 292 uint32_t ch = GetUtf16FromUtf8(&p); 293 if (ch == '\\') { 294 result += "\\\\"; 295 } else if (ch == '\n') { 296 result += "\\n"; 297 } else if (ch == '\r') { 298 result += "\\r"; 299 } else if (ch == '\t') { 300 result += "\\t"; 301 } else { 302 const uint16_t leading = GetLeadingUtf16Char(ch); 303 304 if (NeedsEscaping(leading)) { 305 StringAppendF(&result, "\\u%04x", leading); 306 } else { 307 result += static_cast<std::string::value_type>(leading); 308 } 309 310 const uint32_t trailing = GetTrailingUtf16Char(ch); 311 if (trailing != 0) { 312 // All high surrogates will need escaping. 313 StringAppendF(&result, "\\u%04x", trailing); 314 // Account for the surrogate pair. 315 ++i; 316 DCHECK_LT(i, char_count); 317 } 318 } 319 } 320 result += '"'; 321 return result; 322 } 323 324 } // namespace art 325