1 /* 2 * Copyright (C) 2007 Apple Inc. All rights reserved. 3 * Copyright (C) 2010 Patrick Gansterer <paroga (at) paroga.com> 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY 15 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 21 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 22 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 24 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include "config.h" 28 #include "UTF8.h" 29 30 #include "ASCIICType.h" 31 #include <wtf/StringHasher.h> 32 #include <wtf/unicode/CharacterNames.h> 33 34 namespace WTF { 35 namespace Unicode { 36 37 inline int inlineUTF8SequenceLengthNonASCII(char b0) 38 { 39 if ((b0 & 0xC0) != 0xC0) 40 return 0; 41 if ((b0 & 0xE0) == 0xC0) 42 return 2; 43 if ((b0 & 0xF0) == 0xE0) 44 return 3; 45 if ((b0 & 0xF8) == 0xF0) 46 return 4; 47 return 0; 48 } 49 50 inline int inlineUTF8SequenceLength(char b0) 51 { 52 return isASCII(b0) ? 1 : inlineUTF8SequenceLengthNonASCII(b0); 53 } 54 55 int UTF8SequenceLength(char b0) 56 { 57 return isASCII(b0) ? 1 : inlineUTF8SequenceLengthNonASCII(b0); 58 } 59 60 int decodeUTF8Sequence(const char* sequence) 61 { 62 // Handle 0-byte sequences (never valid). 63 const unsigned char b0 = sequence[0]; 64 const int length = inlineUTF8SequenceLength(b0); 65 if (length == 0) 66 return -1; 67 68 // Handle 1-byte sequences (plain ASCII). 69 const unsigned char b1 = sequence[1]; 70 if (length == 1) { 71 if (b1) 72 return -1; 73 return b0; 74 } 75 76 // Handle 2-byte sequences. 77 if ((b1 & 0xC0) != 0x80) 78 return -1; 79 const unsigned char b2 = sequence[2]; 80 if (length == 2) { 81 if (b2) 82 return -1; 83 const int c = ((b0 & 0x1F) << 6) | (b1 & 0x3F); 84 if (c < 0x80) 85 return -1; 86 return c; 87 } 88 89 // Handle 3-byte sequences. 90 if ((b2 & 0xC0) != 0x80) 91 return -1; 92 const unsigned char b3 = sequence[3]; 93 if (length == 3) { 94 if (b3) 95 return -1; 96 const int c = ((b0 & 0xF) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F); 97 if (c < 0x800) 98 return -1; 99 // UTF-16 surrogates should never appear in UTF-8 data. 100 if (c >= 0xD800 && c <= 0xDFFF) 101 return -1; 102 return c; 103 } 104 105 // Handle 4-byte sequences. 106 if ((b3 & 0xC0) != 0x80) 107 return -1; 108 const unsigned char b4 = sequence[4]; 109 if (length == 4) { 110 if (b4) 111 return -1; 112 const int c = ((b0 & 0x7) << 18) | ((b1 & 0x3F) << 12) | ((b2 & 0x3F) << 6) | (b3 & 0x3F); 113 if (c < 0x10000 || c > 0x10FFFF) 114 return -1; 115 return c; 116 } 117 118 return -1; 119 } 120 121 // Once the bits are split out into bytes of UTF-8, this is a mask OR-ed 122 // into the first byte, depending on how many bytes follow. There are 123 // as many entries in this table as there are UTF-8 sequence types. 124 // (I.e., one byte sequence, two byte... etc.). Remember that sequencs 125 // for *legal* UTF-8 will be 4 or fewer bytes total. 126 static const unsigned char firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; 127 128 ConversionResult convertUTF16ToUTF8( 129 const UChar** sourceStart, const UChar* sourceEnd, 130 char** targetStart, char* targetEnd, bool strict) 131 { 132 ConversionResult result = conversionOK; 133 const UChar* source = *sourceStart; 134 char* target = *targetStart; 135 while (source < sourceEnd) { 136 UChar32 ch; 137 unsigned short bytesToWrite = 0; 138 const UChar32 byteMask = 0xBF; 139 const UChar32 byteMark = 0x80; 140 const UChar* oldSource = source; // In case we have to back up because of target overflow. 141 ch = static_cast<unsigned short>(*source++); 142 // If we have a surrogate pair, convert to UChar32 first. 143 if (ch >= 0xD800 && ch <= 0xDBFF) { 144 // If the 16 bits following the high surrogate are in the source buffer... 145 if (source < sourceEnd) { 146 UChar32 ch2 = static_cast<unsigned short>(*source); 147 // If it's a low surrogate, convert to UChar32. 148 if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { 149 ch = ((ch - 0xD800) << 10) + (ch2 - 0xDC00) + 0x0010000; 150 ++source; 151 } else if (strict) { // it's an unpaired high surrogate 152 --source; // return to the illegal value itself 153 result = sourceIllegal; 154 break; 155 } 156 } else { // We don't have the 16 bits following the high surrogate. 157 --source; // return to the high surrogate 158 result = sourceExhausted; 159 break; 160 } 161 } else if (strict) { 162 // UTF-16 surrogate values are illegal in UTF-32 163 if (ch >= 0xDC00 && ch <= 0xDFFF) { 164 --source; // return to the illegal value itself 165 result = sourceIllegal; 166 break; 167 } 168 } 169 // Figure out how many bytes the result will require 170 if (ch < (UChar32)0x80) { 171 bytesToWrite = 1; 172 } else if (ch < (UChar32)0x800) { 173 bytesToWrite = 2; 174 } else if (ch < (UChar32)0x10000) { 175 bytesToWrite = 3; 176 } else if (ch < (UChar32)0x110000) { 177 bytesToWrite = 4; 178 } else { 179 bytesToWrite = 3; 180 ch = replacementCharacter; 181 } 182 183 target += bytesToWrite; 184 if (target > targetEnd) { 185 source = oldSource; // Back up source pointer! 186 target -= bytesToWrite; 187 result = targetExhausted; 188 break; 189 } 190 switch (bytesToWrite) { // note: everything falls through. 191 case 4: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; 192 case 3: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; 193 case 2: *--target = (char)((ch | byteMark) & byteMask); ch >>= 6; 194 case 1: *--target = (char)(ch | firstByteMark[bytesToWrite]); 195 } 196 target += bytesToWrite; 197 } 198 *sourceStart = source; 199 *targetStart = target; 200 return result; 201 } 202 203 // This must be called with the length pre-determined by the first byte. 204 // If presented with a length > 4, this returns false. The Unicode 205 // definition of UTF-8 goes up to 4-byte sequences. 206 static bool isLegalUTF8(const unsigned char* source, int length) 207 { 208 unsigned char a; 209 const unsigned char* srcptr = source + length; 210 switch (length) { 211 default: return false; 212 // Everything else falls through when "true"... 213 case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 214 case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; 215 case 2: if ((a = (*--srcptr)) > 0xBF) return false; 216 217 switch (*source) { 218 // no fall-through in this inner switch 219 case 0xE0: if (a < 0xA0) return false; break; 220 case 0xED: if (a > 0x9F) return false; break; 221 case 0xF0: if (a < 0x90) return false; break; 222 case 0xF4: if (a > 0x8F) return false; break; 223 default: if (a < 0x80) return false; 224 } 225 226 case 1: if (*source >= 0x80 && *source < 0xC2) return false; 227 } 228 if (*source > 0xF4) 229 return false; 230 return true; 231 } 232 233 // Magic values subtracted from a buffer value during UTF8 conversion. 234 // This table contains as many values as there might be trailing bytes 235 // in a UTF-8 sequence. 236 static const UChar32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, 237 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; 238 239 static inline UChar32 readUTF8Sequence(const char*& sequence, unsigned length) 240 { 241 UChar32 character = 0; 242 243 // The cases all fall through. 244 switch (length) { 245 case 6: character += static_cast<unsigned char>(*sequence++); character <<= 6; 246 case 5: character += static_cast<unsigned char>(*sequence++); character <<= 6; 247 case 4: character += static_cast<unsigned char>(*sequence++); character <<= 6; 248 case 3: character += static_cast<unsigned char>(*sequence++); character <<= 6; 249 case 2: character += static_cast<unsigned char>(*sequence++); character <<= 6; 250 case 1: character += static_cast<unsigned char>(*sequence++); 251 } 252 253 return character - offsetsFromUTF8[length - 1]; 254 } 255 256 ConversionResult convertUTF8ToUTF16( 257 const char** sourceStart, const char* sourceEnd, 258 UChar** targetStart, UChar* targetEnd, bool strict) 259 { 260 ConversionResult result = conversionOK; 261 const char* source = *sourceStart; 262 UChar* target = *targetStart; 263 while (source < sourceEnd) { 264 int utf8SequenceLength = inlineUTF8SequenceLength(*source); 265 if (sourceEnd - source < utf8SequenceLength) { 266 result = sourceExhausted; 267 break; 268 } 269 // Do this check whether lenient or strict 270 if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(source), utf8SequenceLength)) { 271 result = sourceIllegal; 272 break; 273 } 274 275 UChar32 character = readUTF8Sequence(source, utf8SequenceLength); 276 277 if (target >= targetEnd) { 278 source -= utf8SequenceLength; // Back up source pointer! 279 result = targetExhausted; 280 break; 281 } 282 283 if (U_IS_BMP(character)) { 284 // UTF-16 surrogate values are illegal in UTF-32 285 if (U_IS_SURROGATE(character)) { 286 if (strict) { 287 source -= utf8SequenceLength; // return to the illegal value itself 288 result = sourceIllegal; 289 break; 290 } else 291 *target++ = replacementCharacter; 292 } else 293 *target++ = character; // normal case 294 } else if (U_IS_SUPPLEMENTARY(character)) { 295 // target is a character in range 0xFFFF - 0x10FFFF 296 if (target + 1 >= targetEnd) { 297 source -= utf8SequenceLength; // Back up source pointer! 298 result = targetExhausted; 299 break; 300 } 301 *target++ = U16_LEAD(character); 302 *target++ = U16_TRAIL(character); 303 } else { 304 if (strict) { 305 source -= utf8SequenceLength; // return to the start 306 result = sourceIllegal; 307 break; // Bail out; shouldn't continue 308 } else 309 *target++ = replacementCharacter; 310 } 311 } 312 *sourceStart = source; 313 *targetStart = target; 314 return result; 315 } 316 317 unsigned calculateStringHashAndLengthFromUTF8(const char* data, const char* dataEnd, unsigned& dataLength, unsigned& utf16Length) 318 { 319 if (!data) 320 return 0; 321 322 StringHasher stringHasher; 323 dataLength = 0; 324 utf16Length = 0; 325 326 while (data < dataEnd || (!dataEnd && *data)) { 327 if (isASCII(*data)) { 328 stringHasher.addCharacter(*data++); 329 dataLength++; 330 utf16Length++; 331 continue; 332 } 333 334 int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*data); 335 dataLength += utf8SequenceLength; 336 337 if (!dataEnd) { 338 for (int i = 1; i < utf8SequenceLength; ++i) { 339 if (!data[i]) 340 return 0; 341 } 342 } else if (dataEnd - data < utf8SequenceLength) 343 return 0; 344 345 if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(data), utf8SequenceLength)) 346 return 0; 347 348 UChar32 character = readUTF8Sequence(data, utf8SequenceLength); 349 ASSERT(!isASCII(character)); 350 351 if (U_IS_BMP(character)) { 352 // UTF-16 surrogate values are illegal in UTF-32 353 if (U_IS_SURROGATE(character)) 354 return 0; 355 stringHasher.addCharacter(static_cast<UChar>(character)); // normal case 356 utf16Length++; 357 } else if (U_IS_SUPPLEMENTARY(character)) { 358 stringHasher.addCharacters(static_cast<UChar>(U16_LEAD(character)), 359 static_cast<UChar>(U16_TRAIL(character))); 360 utf16Length += 2; 361 } else 362 return 0; 363 } 364 365 return stringHasher.hash(); 366 } 367 368 bool equalUTF16WithUTF8(const UChar* a, const UChar* aEnd, const char* b, const char* bEnd) 369 { 370 while (b < bEnd) { 371 if (isASCII(*b)) { 372 if (*a++ != *b++) 373 return false; 374 continue; 375 } 376 377 int utf8SequenceLength = inlineUTF8SequenceLengthNonASCII(*b); 378 379 if (bEnd - b < utf8SequenceLength) 380 return false; 381 382 if (!isLegalUTF8(reinterpret_cast<const unsigned char*>(b), utf8SequenceLength)) 383 return 0; 384 385 UChar32 character = readUTF8Sequence(b, utf8SequenceLength); 386 ASSERT(!isASCII(character)); 387 388 if (U_IS_BMP(character)) { 389 // UTF-16 surrogate values are illegal in UTF-32 390 if (U_IS_SURROGATE(character)) 391 return false; 392 if (*a++ != character) 393 return false; 394 } else if (U_IS_SUPPLEMENTARY(character)) { 395 if (*a++ != U16_LEAD(character)) 396 return false; 397 if (*a++ != U16_TRAIL(character)) 398 return false; 399 } else 400 return false; 401 } 402 403 return a == aEnd; 404 } 405 406 } // namespace Unicode 407 } // namespace WTF 408
