1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "utf.h" 18 19 #include <android-base/logging.h> 20 #include <android-base/stringprintf.h> 21 #include <android-base/strings.h> 22 23 #include "base/casts.h" 24 #include "utf-inl.h" 25 26 namespace art { 27 28 using android::base::StringAppendF; 29 using android::base::StringPrintf; 30 31 // This is used only from debugger and test code. 32 size_t CountModifiedUtf8Chars(const char* utf8) { 33 return CountModifiedUtf8Chars(utf8, strlen(utf8)); 34 } 35 36 /* 37 * This does not validate UTF8 rules (nor did older code). But it gets the right answer 38 * for valid UTF-8 and that's fine because it's used only to size a buffer for later 39 * conversion. 40 * 41 * Modified UTF-8 consists of a series of bytes up to 21 bit Unicode code points as follows: 42 * U+0001 - U+007F 0xxxxxxx 43 * U+0080 - U+07FF 110xxxxx 10xxxxxx 44 * U+0800 - U+FFFF 1110xxxx 10xxxxxx 10xxxxxx 45 * U+10000 - U+1FFFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 46 * 47 * U+0000 is encoded using the 2nd form to avoid nulls inside strings (this differs from 48 * standard UTF-8). 49 * The four byte encoding converts to two utf16 characters. 50 */ 51 size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) { 52 DCHECK_LE(byte_count, strlen(utf8)); 53 size_t len = 0; 54 const char* end = utf8 + byte_count; 55 for (; utf8 < end; ++utf8) { 56 int ic = *utf8; 57 len++; 58 if (LIKELY((ic & 0x80) == 0)) { 59 // One-byte encoding. 60 continue; 61 } 62 // Two- or three-byte encoding. 63 utf8++; 64 if ((ic & 0x20) == 0) { 65 // Two-byte encoding. 66 continue; 67 } 68 utf8++; 69 if ((ic & 0x10) == 0) { 70 // Three-byte encoding. 71 continue; 72 } 73 74 // Four-byte encoding: needs to be converted into a surrogate 75 // pair. 76 utf8++; 77 len++; 78 } 79 return len; 80 } 81 82 // This is used only from debugger and test code. 83 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, const char* utf8_data_in) { 84 while (*utf8_data_in != '\0') { 85 const uint32_t ch = GetUtf16FromUtf8(&utf8_data_in); 86 const uint16_t leading = GetLeadingUtf16Char(ch); 87 const uint16_t trailing = GetTrailingUtf16Char(ch); 88 89 *utf16_data_out++ = leading; 90 if (trailing != 0) { 91 *utf16_data_out++ = trailing; 92 } 93 } 94 } 95 96 void ConvertModifiedUtf8ToUtf16(uint16_t* utf16_data_out, size_t out_chars, 97 const char* utf8_data_in, size_t in_bytes) { 98 const char *in_start = utf8_data_in; 99 const char *in_end = utf8_data_in + in_bytes; 100 uint16_t *out_p = utf16_data_out; 101 102 if (LIKELY(out_chars == in_bytes)) { 103 // Common case where all characters are ASCII. 104 for (const char *p = in_start; p < in_end;) { 105 // Safe even if char is signed because ASCII characters always have 106 // the high bit cleared. 107 *out_p++ = dchecked_integral_cast<uint16_t>(*p++); 108 } 109 return; 110 } 111 112 // String contains non-ASCII characters. 113 for (const char *p = in_start; p < in_end;) { 114 const uint32_t ch = GetUtf16FromUtf8(&p); 115 const uint16_t leading = GetLeadingUtf16Char(ch); 116 const uint16_t trailing = GetTrailingUtf16Char(ch); 117 118 *out_p++ = leading; 119 if (trailing != 0) { 120 *out_p++ = trailing; 121 } 122 } 123 } 124 125 void ConvertUtf16ToModifiedUtf8(char* utf8_out, size_t byte_count, 126 const uint16_t* utf16_in, size_t char_count) { 127 if (LIKELY(byte_count == char_count)) { 128 // Common case where all characters are ASCII. 129 const uint16_t *utf16_end = utf16_in + char_count; 130 for (const uint16_t *p = utf16_in; p < utf16_end;) { 131 *utf8_out++ = dchecked_integral_cast<char>(*p++); 132 } 133 return; 134 } 135 136 // String contains non-ASCII characters. 137 while (char_count--) { 138 const uint16_t ch = *utf16_in++; 139 if (ch > 0 && ch <= 0x7f) { 140 *utf8_out++ = ch; 141 } else { 142 // Char_count == 0 here implies we've encountered an unpaired 143 // surrogate and we have no choice but to encode it as 3-byte UTF 144 // sequence. Note that unpaired surrogates can occur as a part of 145 // "normal" operation. 146 if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) { 147 const uint16_t ch2 = *utf16_in; 148 149 // Check if the other half of the pair is within the expected 150 // range. If it isn't, we will have to emit both "halves" as 151 // separate 3 byte sequences. 152 if (ch2 >= 0xdc00 && ch2 <= 0xdfff) { 153 utf16_in++; 154 char_count--; 155 const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00; 156 *utf8_out++ = (code_point >> 18) | 0xf0; 157 *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80; 158 *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80; 159 *utf8_out++ = (code_point & 0x3f) | 0x80; 160 continue; 161 } 162 } 163 164 if (ch > 0x07ff) { 165 // Three byte encoding. 166 *utf8_out++ = (ch >> 12) | 0xe0; 167 *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80; 168 *utf8_out++ = (ch & 0x3f) | 0x80; 169 } else /*(ch > 0x7f || ch == 0)*/ { 170 // Two byte encoding. 171 *utf8_out++ = (ch >> 6) | 0xc0; 172 *utf8_out++ = (ch & 0x3f) | 0x80; 173 } 174 } 175 } 176 } 177 178 int32_t ComputeUtf16HashFromModifiedUtf8(const char* utf8, size_t utf16_length) { 179 uint32_t hash = 0; 180 while (utf16_length != 0u) { 181 const uint32_t pair = GetUtf16FromUtf8(&utf8); 182 const uint16_t first = GetLeadingUtf16Char(pair); 183 hash = hash * 31 + first; 184 --utf16_length; 185 const uint16_t second = GetTrailingUtf16Char(pair); 186 if (second != 0) { 187 hash = hash * 31 + second; 188 DCHECK_NE(utf16_length, 0u); 189 --utf16_length; 190 } 191 } 192 return static_cast<int32_t>(hash); 193 } 194 195 uint32_t ComputeModifiedUtf8Hash(const char* chars) { 196 uint32_t hash = 0; 197 while (*chars != '\0') { 198 hash = hash * 31 + *chars++; 199 } 200 return static_cast<int32_t>(hash); 201 } 202 203 int CompareModifiedUtf8ToUtf16AsCodePointValues(const char* utf8, const uint16_t* utf16, 204 size_t utf16_length) { 205 for (;;) { 206 if (*utf8 == '\0') { 207 return (utf16_length == 0) ? 0 : -1; 208 } else if (utf16_length == 0) { 209 return 1; 210 } 211 212 const uint32_t pair = GetUtf16FromUtf8(&utf8); 213 214 // First compare the leading utf16 char. 215 const uint16_t lhs = GetLeadingUtf16Char(pair); 216 const uint16_t rhs = *utf16++; 217 --utf16_length; 218 if (lhs != rhs) { 219 return lhs > rhs ? 1 : -1; 220 } 221 222 // Then compare the trailing utf16 char. First check if there 223 // are any characters left to consume. 224 const uint16_t lhs2 = GetTrailingUtf16Char(pair); 225 if (lhs2 != 0) { 226 if (utf16_length == 0) { 227 return 1; 228 } 229 230 const uint16_t rhs2 = *utf16++; 231 --utf16_length; 232 if (lhs2 != rhs2) { 233 return lhs2 > rhs2 ? 1 : -1; 234 } 235 } 236 } 237 } 238 239 size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) { 240 size_t result = 0; 241 const uint16_t *end = chars + char_count; 242 while (chars < end) { 243 const uint16_t ch = *chars++; 244 if (LIKELY(ch != 0 && ch < 0x80)) { 245 result++; 246 continue; 247 } 248 if (ch < 0x800) { 249 result += 2; 250 continue; 251 } 252 if (ch >= 0xd800 && ch < 0xdc00) { 253 if (chars < end) { 254 const uint16_t ch2 = *chars; 255 // If we find a properly paired surrogate, we emit it as a 4 byte 256 // UTF sequence. If we find an unpaired leading or trailing surrogate, 257 // we emit it as a 3 byte sequence like would have done earlier. 258 if (ch2 >= 0xdc00 && ch2 < 0xe000) { 259 chars++; 260 result += 4; 261 continue; 262 } 263 } 264 } 265 result += 3; 266 } 267 return result; 268 } 269 270 static inline constexpr bool NeedsEscaping(uint16_t ch) { 271 return (ch < ' ' || ch > '~'); 272 } 273 274 std::string PrintableChar(uint16_t ch) { 275 std::string result; 276 result += '\''; 277 if (NeedsEscaping(ch)) { 278 StringAppendF(&result, "\\u%04x", ch); 279 } else { 280 result += static_cast<std::string::value_type>(ch); 281 } 282 result += '\''; 283 return result; 284 } 285 286 std::string PrintableString(const char* utf) { 287 std::string result; 288 result += '"'; 289 const char* p = utf; 290 size_t char_count = CountModifiedUtf8Chars(p); 291 for (size_t i = 0; i < char_count; ++i) { 292 uint32_t ch = GetUtf16FromUtf8(&p); 293 if (ch == '\\') { 294 result += "\\\\"; 295 } else if (ch == '\n') { 296 result += "\\n"; 297 } else if (ch == '\r') { 298 result += "\\r"; 299 } else if (ch == '\t') { 300 result += "\\t"; 301 } else { 302 const uint16_t leading = GetLeadingUtf16Char(ch); 303 304 if (NeedsEscaping(leading)) { 305 StringAppendF(&result, "\\u%04x", leading); 306 } else { 307 result += static_cast<std::string::value_type>(leading); 308 } 309 310 const uint32_t trailing = GetTrailingUtf16Char(ch); 311 if (trailing != 0) { 312 // All high surrogates will need escaping. 313 StringAppendF(&result, "\\u%04x", trailing); 314 } 315 } 316 } 317 result += '"'; 318 return result; 319 } 320 321 } // namespace art 322