1 /* 2 * Copyright (C) 2004, 2006, 2008 Apple Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY 14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR 17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 */ 25 26 #include "config.h" 27 #include "wtf/text/TextCodecLatin1.h" 28 29 #include "wtf/text/TextCodecASCIIFastPath.h" 30 #include "wtf/PassOwnPtr.h" 31 #include "wtf/text/CString.h" 32 #include "wtf/text/StringBuffer.h" 33 #include "wtf/text/WTFString.h" 34 35 using namespace WTF; 36 37 namespace WTF { 38 39 static const UChar table[256] = { 40 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, // 00-07 41 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, // 08-0F 42 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, // 10-17 43 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, // 18-1F 44 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, // 20-27 45 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, // 28-2F 46 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, // 30-37 47 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, // 38-3F 48 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, // 40-47 49 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, // 48-4F 50 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, // 50-57 51 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, // 58-5F 52 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, // 60-67 53 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, // 68-6F 54 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, // 70-77 55 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, // 78-7F 56 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, // 80-87 57 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, // 88-8F 58 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, // 90-97 59 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, // 98-9F 60 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, // A0-A7 61 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, // A8-AF 62 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, // B0-B7 63 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, // B8-BF 64 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, // C0-C7 65 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, // C8-CF 66 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, // D0-D7 67 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, // D8-DF 68 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, // E0-E7 69 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, // E8-EF 70 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, // F0-F7 71 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF // F8-FF 72 }; 73 74 void TextCodecLatin1::registerEncodingNames(EncodingNameRegistrar registrar) 75 { 76 registrar("windows-1252", "windows-1252"); 77 registrar("ISO-8859-1", "ISO-8859-1"); 78 registrar("US-ASCII", "US-ASCII"); 79 80 registrar("WinLatin1", "windows-1252"); 81 registrar("ibm-1252", "windows-1252"); 82 registrar("ibm-1252_P100-2000", "windows-1252"); 83 84 registrar("CP819", "ISO-8859-1"); 85 registrar("IBM819", "ISO-8859-1"); 86 registrar("csISOLatin1", "ISO-8859-1"); 87 registrar("iso-ir-100", "ISO-8859-1"); 88 registrar("iso_8859-1:1987", "ISO-8859-1"); 89 registrar("l1", "ISO-8859-1"); 90 registrar("latin1", "ISO-8859-1"); 91 92 registrar("ANSI_X3.4-1968", "US-ASCII"); 93 registrar("ANSI_X3.4-1986", "US-ASCII"); 94 registrar("ASCII", "US-ASCII"); 95 registrar("IBM367", "US-ASCII"); 96 registrar("ISO646-US", "US-ASCII"); 97 registrar("ISO_646.irv:1991", "US-ASCII"); 98 registrar("cp367", "US-ASCII"); 99 registrar("csASCII", "US-ASCII"); 100 registrar("ibm-367_P100-1995", "US-ASCII"); 101 registrar("iso-ir-6", "US-ASCII"); 102 registrar("iso-ir-6-us", "US-ASCII"); 103 registrar("us", "US-ASCII"); 104 registrar("x-ansi", "US-ASCII"); 105 } 106 107 static PassOwnPtr<TextCodec> newStreamingTextDecoderWindowsLatin1(const TextEncoding&, const void*) 108 { 109 return adoptPtr(new TextCodecLatin1); 110 } 111 112 void TextCodecLatin1::registerCodecs(TextCodecRegistrar registrar) 113 { 114 registrar("windows-1252", newStreamingTextDecoderWindowsLatin1, 0); 115 116 // ASCII and Latin-1 both decode as Windows Latin-1 although they retain unique identities. 117 registrar("ISO-8859-1", newStreamingTextDecoderWindowsLatin1, 0); 118 registrar("US-ASCII", newStreamingTextDecoderWindowsLatin1, 0); 119 } 120 121 String TextCodecLatin1::decode(const char* bytes, size_t length, bool, bool, bool&) 122 { 123 LChar* characters; 124 if (!length) 125 return emptyString(); 126 String result = String::createUninitialized(length, characters); 127 128 const uint8_t* source = reinterpret_cast<const uint8_t*>(bytes); 129 const uint8_t* end = reinterpret_cast<const uint8_t*>(bytes + length); 130 const uint8_t* alignedEnd = alignToMachineWord(end); 131 LChar* destination = characters; 132 133 while (source < end) { 134 if (isASCII(*source)) { 135 // Fast path for ASCII. Most Latin-1 text will be ASCII. 136 if (isAlignedToMachineWord(source)) { 137 while (source < alignedEnd) { 138 MachineWord chunk = *reinterpret_cast_ptr<const MachineWord*>(source); 139 140 if (!isAllASCII<LChar>(chunk)) 141 goto useLookupTable; 142 143 copyASCIIMachineWord(destination, source); 144 source += sizeof(MachineWord); 145 destination += sizeof(MachineWord); 146 } 147 148 if (source == end) 149 break; 150 } 151 *destination = *source; 152 } else { 153 useLookupTable: 154 if (table[*source] > 0xff) 155 goto upConvertTo16Bit; 156 157 *destination = table[*source]; 158 } 159 160 ++source; 161 ++destination; 162 } 163 164 return result; 165 166 upConvertTo16Bit: 167 UChar* characters16; 168 String result16 = String::createUninitialized(length, characters16); 169 170 UChar* destination16 = characters16; 171 172 // Zero extend and copy already processed 8 bit data 173 LChar* ptr8 = characters; 174 LChar* endPtr8 = destination; 175 176 while (ptr8 < endPtr8) 177 *destination16++ = *ptr8++; 178 179 // Handle the character that triggered the 16 bit path 180 *destination16 = table[*source]; 181 ++source; 182 ++destination16; 183 184 while (source < end) { 185 if (isASCII(*source)) { 186 // Fast path for ASCII. Most Latin-1 text will be ASCII. 187 if (isAlignedToMachineWord(source)) { 188 while (source < alignedEnd) { 189 MachineWord chunk = *reinterpret_cast_ptr<const MachineWord*>(source); 190 191 if (!isAllASCII<LChar>(chunk)) 192 goto useLookupTable16; 193 194 copyASCIIMachineWord(destination16, source); 195 source += sizeof(MachineWord); 196 destination16 += sizeof(MachineWord); 197 } 198 199 if (source == end) 200 break; 201 } 202 *destination16 = *source; 203 } else { 204 useLookupTable16: 205 *destination16 = table[*source]; 206 } 207 208 ++source; 209 ++destination16; 210 } 211 212 return result16; 213 } 214 215 template<typename CharType> 216 static CString encodeComplexWindowsLatin1(const CharType* characters, size_t length, UnencodableHandling handling) 217 { 218 Vector<char> result(length); 219 char* bytes = result.data(); 220 221 size_t resultLength = 0; 222 for (size_t i = 0; i < length; ) { 223 UChar32 c; 224 U16_NEXT(characters, i, length, c); 225 unsigned char b = c; 226 // Do an efficient check to detect characters other than 00-7F and A0-FF. 227 if (b != c || (c & 0xE0) == 0x80) { 228 // Look for a way to encode this with Windows Latin-1. 229 for (b = 0x80; b < 0xA0; ++b) 230 if (table[b] == c) 231 goto gotByte; 232 // No way to encode this character with Windows Latin-1. 233 UnencodableReplacementArray replacement; 234 int replacementLength = TextCodec::getUnencodableReplacement(c, handling, replacement); 235 result.grow(resultLength + replacementLength + length - i); 236 bytes = result.data(); 237 memcpy(bytes + resultLength, replacement, replacementLength); 238 resultLength += replacementLength; 239 continue; 240 } 241 gotByte: 242 bytes[resultLength++] = b; 243 } 244 245 return CString(bytes, resultLength); 246 } 247 248 template<typename CharType> 249 CString TextCodecLatin1::encodeCommon(const CharType* characters, size_t length, UnencodableHandling handling) 250 { 251 { 252 char* bytes; 253 CString string = CString::newUninitialized(length, bytes); 254 255 // Convert the string a fast way and simultaneously do an efficient check to see if it's all ASCII. 256 UChar ored = 0; 257 for (size_t i = 0; i < length; ++i) { 258 UChar c = characters[i]; 259 bytes[i] = c; 260 ored |= c; 261 } 262 263 if (!(ored & 0xFF80)) 264 return string; 265 } 266 267 // If it wasn't all ASCII, call the function that handles more-complex cases. 268 return encodeComplexWindowsLatin1(characters, length, handling); 269 } 270 271 CString TextCodecLatin1::encode(const UChar* characters, size_t length, UnencodableHandling handling) 272 { 273 return encodeCommon(characters, length, handling); 274 } 275 276 CString TextCodecLatin1::encode(const LChar* characters, size_t length, UnencodableHandling handling) 277 { 278 return encodeCommon(characters, length, handling); 279 } 280 281 } // namespace WTF 282
