1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ART_TEST_TI_AGENT_TI_UTF_H_ 18 #define ART_TEST_TI_AGENT_TI_UTF_H_ 19 20 #include <inttypes.h> 21 #include <string.h> 22 23 #include "android-base/logging.h" 24 25 namespace art { 26 namespace ti { 27 28 inline size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) { 29 DCHECK_LE(byte_count, strlen(utf8)); 30 size_t len = 0; 31 const char* end = utf8 + byte_count; 32 for (; utf8 < end; ++utf8) { 33 int ic = *utf8; 34 len++; 35 if (LIKELY((ic & 0x80) == 0)) { 36 // One-byte encoding. 37 continue; 38 } 39 // Two- or three-byte encoding. 40 utf8++; 41 if ((ic & 0x20) == 0) { 42 // Two-byte encoding. 43 continue; 44 } 45 utf8++; 46 if ((ic & 0x10) == 0) { 47 // Three-byte encoding. 48 continue; 49 } 50 51 // Four-byte encoding: needs to be converted into a surrogate 52 // pair. 53 utf8++; 54 len++; 55 } 56 return len; 57 } 58 59 inline uint16_t GetTrailingUtf16Char(uint32_t maybe_pair) { 60 return static_cast<uint16_t>(maybe_pair >> 16); 61 } 62 63 inline uint16_t GetLeadingUtf16Char(uint32_t maybe_pair) { 64 return static_cast<uint16_t>(maybe_pair & 0x0000FFFF); 65 } 66 67 inline uint32_t GetUtf16FromUtf8(const char** utf8_data_in) { 68 const uint8_t one = *(*utf8_data_in)++; 69 if ((one & 0x80) == 0) { 70 // one-byte encoding 71 return one; 72 } 73 74 const uint8_t two = *(*utf8_data_in)++; 75 if ((one & 0x20) == 0) { 76 // two-byte encoding 77 return ((one & 0x1f) << 6) | (two & 0x3f); 78 } 79 80 const uint8_t three = *(*utf8_data_in)++; 81 if ((one & 0x10) == 0) { 82 return ((one & 0x0f) << 12) | ((two & 0x3f) << 6) | (three & 0x3f); 83 } 84 85 // Four byte encodings need special handling. We'll have 86 // to convert them into a surrogate pair. 87 const uint8_t four = *(*utf8_data_in)++; 88 89 // Since this is a 4 byte UTF-8 sequence, it will lie between 90 // U+10000 and U+1FFFFF. 91 // 92 // TODO: What do we do about values in (U+10FFFF, U+1FFFFF) ? The 93 // spec says they're invalid but nobody appears to check for them. 94 const uint32_t code_point = ((one & 0x0f) << 18) | ((two & 0x3f) << 12) 95 | ((three & 0x3f) << 6) | (four & 0x3f); 96 97 uint32_t surrogate_pair = 0; 98 // Step two: Write out the high (leading) surrogate to the bottom 16 bits 99 // of the of the 32 bit type. 100 surrogate_pair |= ((code_point >> 10) + 0xd7c0) & 0xffff; 101 // Step three : Write out the low (trailing) surrogate to the top 16 bits. 102 surrogate_pair |= ((code_point & 0x03ff) + 0xdc00) << 16; 103 104 return surrogate_pair; 105 } 106 107 inline void ConvertUtf16ToModifiedUtf8(char* utf8_out, 108 size_t byte_count, 109 const uint16_t* utf16_in, 110 size_t char_count) { 111 if (LIKELY(byte_count == char_count)) { 112 // Common case where all characters are ASCII. 113 const uint16_t *utf16_end = utf16_in + char_count; 114 for (const uint16_t *p = utf16_in; p < utf16_end;) { 115 *utf8_out++ = static_cast<char>(*p++); 116 } 117 return; 118 } 119 120 // String contains non-ASCII characters. 121 while (char_count--) { 122 const uint16_t ch = *utf16_in++; 123 if (ch > 0 && ch <= 0x7f) { 124 *utf8_out++ = ch; 125 } else { 126 // Char_count == 0 here implies we've encountered an unpaired 127 // surrogate and we have no choice but to encode it as 3-byte UTF 128 // sequence. Note that unpaired surrogates can occur as a part of 129 // "normal" operation. 130 if ((ch >= 0xd800 && ch <= 0xdbff) && (char_count > 0)) { 131 const uint16_t ch2 = *utf16_in; 132 133 // Check if the other half of the pair is within the expected 134 // range. If it isn't, we will have to emit both "halves" as 135 // separate 3 byte sequences. 136 if (ch2 >= 0xdc00 && ch2 <= 0xdfff) { 137 utf16_in++; 138 char_count--; 139 const uint32_t code_point = (ch << 10) + ch2 - 0x035fdc00; 140 *utf8_out++ = (code_point >> 18) | 0xf0; 141 *utf8_out++ = ((code_point >> 12) & 0x3f) | 0x80; 142 *utf8_out++ = ((code_point >> 6) & 0x3f) | 0x80; 143 *utf8_out++ = (code_point & 0x3f) | 0x80; 144 continue; 145 } 146 } 147 148 if (ch > 0x07ff) { 149 // Three byte encoding. 150 *utf8_out++ = (ch >> 12) | 0xe0; 151 *utf8_out++ = ((ch >> 6) & 0x3f) | 0x80; 152 *utf8_out++ = (ch & 0x3f) | 0x80; 153 } else /*(ch > 0x7f || ch == 0)*/ { 154 // Two byte encoding. 155 *utf8_out++ = (ch >> 6) | 0xc0; 156 *utf8_out++ = (ch & 0x3f) | 0x80; 157 } 158 } 159 } 160 } 161 162 inline size_t CountUtf8Bytes(const uint16_t* chars, size_t char_count) { 163 size_t result = 0; 164 const uint16_t *end = chars + char_count; 165 while (chars < end) { 166 const uint16_t ch = *chars++; 167 if (LIKELY(ch != 0 && ch < 0x80)) { 168 result++; 169 continue; 170 } 171 if (ch < 0x800) { 172 result += 2; 173 continue; 174 } 175 if (ch >= 0xd800 && ch < 0xdc00) { 176 if (chars < end) { 177 const uint16_t ch2 = *chars; 178 // If we find a properly paired surrogate, we emit it as a 4 byte 179 // UTF sequence. If we find an unpaired leading or trailing surrogate, 180 // we emit it as a 3 byte sequence like would have done earlier. 181 if (ch2 >= 0xdc00 && ch2 < 0xe000) { 182 chars++; 183 result += 4; 184 continue; 185 } 186 } 187 } 188 result += 3; 189 } 190 return result; 191 } 192 193 } // namespace ti 194 } // namespace art 195 196 #endif // ART_TEST_TI_AGENT_TI_UTF_H_ 197