1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <math.h> 6 #include <stdarg.h> 7 8 #include <limits> 9 #include <sstream> 10 11 #include "base/basictypes.h" 12 #include "base/format_macros.h" 13 #include "base/i18n/icu_string_conversions.h" 14 #include "base/logging.h" 15 #include "base/stringprintf.h" 16 #include "base/string_piece.h" 17 #include "base/utf_string_conversions.h" 18 #include "testing/gtest/include/gtest/gtest.h" 19 20 namespace base { 21 22 namespace { 23 24 // Given a null-terminated string of wchar_t with each wchar_t representing 25 // a UTF-16 code unit, returns a string16 made up of wchar_t's in the input. 26 // Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF) 27 // should be represented as a surrogate pair (two UTF-16 units) 28 // *even* where wchar_t is 32-bit (Linux and Mac). 29 // 30 // This is to help write tests for functions with string16 params until 31 // the C++ 0x UTF-16 literal is well-supported by compilers. 32 string16 BuildString16(const wchar_t* s) { 33 #if defined(WCHAR_T_IS_UTF16) 34 return string16(s); 35 #elif defined(WCHAR_T_IS_UTF32) 36 string16 u16; 37 while (*s != 0) { 38 DCHECK(static_cast<unsigned int>(*s) <= 0xFFFFu); 39 u16.push_back(*s++); 40 } 41 return u16; 42 #endif 43 } 44 45 const wchar_t* const kConvertRoundtripCases[] = { 46 L"Google Video", 47 // " " 48 L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb", 49 // " " 50 L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9" 51 L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2", 52 // " " 53 L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442" 54 L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430" 55 L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c", 56 // "" 57 L"\xc804\xccb4\xc11c\xbe44\xc2a4", 58 59 // Test characters that take more than 16 bits. This will depend on whether 60 // wchar_t is 16 or 32 bits. 61 #if defined(WCHAR_T_IS_UTF16) 62 L"\xd800\xdf00", 63 // ????? (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E) 64 L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44", 65 #elif defined(WCHAR_T_IS_UTF32) 66 L"\x10300", 67 // ????? (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E) 68 L"\x11d40\x11d41\x11d42\x11d43\x11d44", 69 #endif 70 }; 71 72 } // namespace 73 74 TEST(ICUStringConversionsTest, ConvertCodepageUTF8) { 75 // Make sure WideToCodepage works like WideToUTF8. 76 for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) { 77 SCOPED_TRACE(base::StringPrintf("Test[%" PRIuS "]: %ls", 78 i, kConvertRoundtripCases[i])); 79 80 std::string expected(WideToUTF8(kConvertRoundtripCases[i])); 81 std::string utf8; 82 EXPECT_TRUE(WideToCodepage(kConvertRoundtripCases[i], kCodepageUTF8, 83 OnStringConversionError::SKIP, &utf8)); 84 EXPECT_EQ(expected, utf8); 85 } 86 } 87 88 // kConverterCodepageCases is not comprehensive. There are a number of cases 89 // to add if we really want to have a comprehensive coverage of various 90 // codepages and their 'idiosyncrasies'. Currently, the only implementation 91 // for CodepageTo* and *ToCodepage uses ICU, which has a very extensive 92 // set of tests for the charset conversion. So, we can get away with a 93 // relatively small number of cases listed below. 94 // 95 // Note about |u16_wide| in the following struct. 96 // On Windows, the field is always identical to |wide|. On Mac and Linux, 97 // it's identical as long as there's no character outside the 98 // BMP (<= U+FFFF). When there is, it is different from |wide| and 99 // is not a real wide string (UTF-32 string) in that each wchar_t in 100 // the string is a UTF-16 code unit zero-extended to be 32-bit 101 // even when the code unit belongs to a surrogate pair. 102 // For instance, a Unicode string (U+0041 U+010000) is represented as 103 // L"\x0041\xD800\xDC00" instead of L"\x0041\x10000". 104 // To avoid the clutter, |u16_wide| will be set to NULL 105 // if it's identical to |wide| on *all* platforms. 106 107 static const struct { 108 const char* codepage_name; 109 const char* encoded; 110 OnStringConversionError::Type on_error; 111 bool success; 112 const wchar_t* wide; 113 const wchar_t* u16_wide; 114 } kConvertCodepageCases[] = { 115 // Test a case where the input cannot be decoded, using SKIP, FAIL 116 // and SUBSTITUTE error handling rules. "A7 41" is valid, but "A6" isn't. 117 {"big5", 118 "\xA7\x41\xA6", 119 OnStringConversionError::FAIL, 120 false, 121 L"", 122 NULL}, 123 {"big5", 124 "\xA7\x41\xA6", 125 OnStringConversionError::SKIP, 126 true, 127 L"\x4F60", 128 NULL}, 129 {"big5", 130 "\xA7\x41\xA6", 131 OnStringConversionError::SUBSTITUTE, 132 true, 133 L"\x4F60\xFFFD", 134 NULL}, 135 // Arabic (ISO-8859) 136 {"iso-8859-6", 137 "\xC7\xEE\xE4\xD3\xF1\xEE\xE4\xC7\xE5\xEF" " " 138 "\xD9\xEE\xE4\xEE\xEA\xF2\xE3\xEF\xE5\xF2", 139 OnStringConversionError::FAIL, 140 true, 141 L"\x0627\x064E\x0644\x0633\x0651\x064E\x0644\x0627\x0645\x064F" L" " 142 L"\x0639\x064E\x0644\x064E\x064A\x0652\x0643\x064F\x0645\x0652", 143 NULL}, 144 // Chinese Simplified (GB2312) 145 {"gb2312", 146 "\xC4\xE3\xBA\xC3", 147 OnStringConversionError::FAIL, 148 true, 149 L"\x4F60\x597D", 150 NULL}, 151 // Chinese (GB18030) : 4 byte sequences mapped to BMP characters 152 {"gb18030", 153 "\x81\x30\x84\x36\xA1\xA7", 154 OnStringConversionError::FAIL, 155 true, 156 L"\x00A5\x00A8", 157 NULL}, 158 // Chinese (GB18030) : A 4 byte sequence mapped to plane 2 (U+20000) 159 {"gb18030", 160 "\x95\x32\x82\x36\xD2\xBB", 161 OnStringConversionError::FAIL, 162 true, 163 #if defined(WCHAR_T_IS_UTF16) 164 L"\xD840\xDC00\x4E00", 165 #elif defined(WCHAR_T_IS_UTF32) 166 L"\x20000\x4E00", 167 #endif 168 L"\xD840\xDC00\x4E00"}, 169 {"big5", 170 "\xA7\x41\xA6\x6E", 171 OnStringConversionError::FAIL, 172 true, 173 L"\x4F60\x597D", 174 NULL}, 175 // Greek (ISO-8859) 176 {"iso-8859-7", 177 "\xE3\xE5\xE9\xDC" " " "\xF3\xEF\xF5", 178 OnStringConversionError::FAIL, 179 true, 180 L"\x03B3\x03B5\x03B9\x03AC" L" " L"\x03C3\x03BF\x03C5", 181 NULL}, 182 // Hebrew (Windows) 183 {"windows-1255", 184 "\xF9\xD1\xC8\xEC\xE5\xC9\xED", 185 OnStringConversionError::FAIL, 186 true, 187 L"\x05E9\x05C1\x05B8\x05DC\x05D5\x05B9\x05DD", 188 NULL}, 189 // Hindi Devanagari (ISCII) 190 {"iscii-dev", 191 "\xEF\x42" "\xC6\xCC\xD7\xE8\xB3\xDA\xCF", 192 OnStringConversionError::FAIL, 193 true, 194 L"\x0928\x092E\x0938\x094D\x0915\x093E\x0930", 195 NULL}, 196 // Korean (EUC) 197 {"euc-kr", 198 "\xBE\xC8\xB3\xE7\xC7\xCF\xBC\xBC\xBF\xE4", 199 OnStringConversionError::FAIL, 200 true, 201 L"\xC548\xB155\xD558\xC138\xC694", 202 NULL}, 203 // Japanese (EUC) 204 {"euc-jp", 205 "\xA4\xB3\xA4\xF3\xA4\xCB\xA4\xC1\xA4\xCF\xB0\xEC\x8F\xB0\xA1\x8E\xA6", 206 OnStringConversionError::FAIL, 207 true, 208 L"\x3053\x3093\x306B\x3061\x306F\x4E00\x4E02\xFF66", 209 NULL}, 210 // Japanese (ISO-2022) 211 {"iso-2022-jp", 212 "\x1B$B" "\x24\x33\x24\x73\x24\x4B\x24\x41\x24\x4F\x30\x6C" "\x1B(B" 213 "ab" "\x1B(J" "\x5C\x7E#$" "\x1B(B", 214 OnStringConversionError::FAIL, 215 true, 216 L"\x3053\x3093\x306B\x3061\x306F\x4E00" L"ab\x00A5\x203E#$", 217 NULL}, 218 // Japanese (Shift-JIS) 219 {"sjis", 220 "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x88\xEA\xA6", 221 OnStringConversionError::FAIL, 222 true, 223 L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66", 224 NULL}, 225 // Russian (KOI8) 226 {"koi8-r", 227 "\xDA\xC4\xD2\xC1\xD7\xD3\xD4\xD7\xD5\xCA\xD4\xC5", 228 OnStringConversionError::FAIL, 229 true, 230 L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432" 231 L"\x0443\x0439\x0442\x0435", 232 NULL}, 233 // Thai (windows-874) 234 {"windows-874", 235 "\xCA\xC7\xD1\xCA\xB4\xD5" "\xA4\xC3\xD1\xBA", 236 OnStringConversionError::FAIL, 237 true, 238 L"\x0E2A\x0E27\x0E31\x0E2A\x0E14\x0E35" 239 L"\x0E04\x0E23\x0e31\x0E1A", 240 NULL}, 241 }; 242 243 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndWide) { 244 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) { 245 SCOPED_TRACE(base::StringPrintf( 246 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i, 247 kConvertCodepageCases[i].encoded, 248 kConvertCodepageCases[i].codepage_name)); 249 250 std::wstring wide; 251 bool success = CodepageToWide(kConvertCodepageCases[i].encoded, 252 kConvertCodepageCases[i].codepage_name, 253 kConvertCodepageCases[i].on_error, 254 &wide); 255 EXPECT_EQ(kConvertCodepageCases[i].success, success); 256 EXPECT_EQ(kConvertCodepageCases[i].wide, wide); 257 258 // When decoding was successful and nothing was skipped, we also check the 259 // reverse conversion. Not all conversions are round-trippable, but 260 // kConverterCodepageCases does not have any one-way conversion at the 261 // moment. 262 if (success && 263 kConvertCodepageCases[i].on_error == 264 OnStringConversionError::FAIL) { 265 std::string encoded; 266 success = WideToCodepage(wide, kConvertCodepageCases[i].codepage_name, 267 kConvertCodepageCases[i].on_error, &encoded); 268 EXPECT_EQ(kConvertCodepageCases[i].success, success); 269 EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded); 270 } 271 } 272 273 // The above cases handled codepage->wide errors, but not wide->codepage. 274 // Test that here. 275 std::string encoded("Temp data"); // Make sure the string gets cleared. 276 277 // First test going to an encoding that can not represent that character. 278 EXPECT_FALSE(WideToCodepage(L"Chinese\xff27", "iso-8859-1", 279 OnStringConversionError::FAIL, &encoded)); 280 EXPECT_TRUE(encoded.empty()); 281 EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1", 282 OnStringConversionError::SKIP, &encoded)); 283 EXPECT_STREQ("Chinese", encoded.c_str()); 284 // From Unicode, SUBSTITUTE is the same as SKIP for now. 285 EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1", 286 OnStringConversionError::SUBSTITUTE, 287 &encoded)); 288 EXPECT_STREQ("Chinese", encoded.c_str()); 289 290 #if defined(WCHAR_T_IS_UTF16) 291 // When we're in UTF-16 mode, test an invalid UTF-16 character in the input. 292 EXPECT_FALSE(WideToCodepage(L"a\xd800z", "iso-8859-1", 293 OnStringConversionError::FAIL, &encoded)); 294 EXPECT_TRUE(encoded.empty()); 295 EXPECT_TRUE(WideToCodepage(L"a\xd800z", "iso-8859-1", 296 OnStringConversionError::SKIP, &encoded)); 297 EXPECT_STREQ("az", encoded.c_str()); 298 #endif // WCHAR_T_IS_UTF16 299 300 // Invalid characters should fail. 301 EXPECT_TRUE(WideToCodepage(L"a\xffffz", "iso-8859-1", 302 OnStringConversionError::SKIP, &encoded)); 303 EXPECT_STREQ("az", encoded.c_str()); 304 305 // Invalid codepages should fail. 306 EXPECT_FALSE(WideToCodepage(L"Hello, world", "awesome-8571-2", 307 OnStringConversionError::SKIP, &encoded)); 308 } 309 310 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndUTF16) { 311 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) { 312 SCOPED_TRACE(base::StringPrintf( 313 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i, 314 kConvertCodepageCases[i].encoded, 315 kConvertCodepageCases[i].codepage_name)); 316 317 string16 utf16; 318 bool success = CodepageToUTF16(kConvertCodepageCases[i].encoded, 319 kConvertCodepageCases[i].codepage_name, 320 kConvertCodepageCases[i].on_error, 321 &utf16); 322 string16 utf16_expected; 323 if (kConvertCodepageCases[i].u16_wide == NULL) 324 utf16_expected = BuildString16(kConvertCodepageCases[i].wide); 325 else 326 utf16_expected = BuildString16(kConvertCodepageCases[i].u16_wide); 327 EXPECT_EQ(kConvertCodepageCases[i].success, success); 328 EXPECT_EQ(utf16_expected, utf16); 329 330 // When decoding was successful and nothing was skipped, we also check the 331 // reverse conversion. See also the corresponding comment in 332 // ConvertBetweenCodepageAndWide. 333 if (success && 334 kConvertCodepageCases[i].on_error == OnStringConversionError::FAIL) { 335 std::string encoded; 336 success = UTF16ToCodepage(utf16, kConvertCodepageCases[i].codepage_name, 337 kConvertCodepageCases[i].on_error, &encoded); 338 EXPECT_EQ(kConvertCodepageCases[i].success, success); 339 EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded); 340 } 341 } 342 } 343 344 static const struct { 345 const char* encoded; 346 const char* codepage_name; 347 bool expected_success; 348 const char* expected_value; 349 } kConvertAndNormalizeCases[] = { 350 {"foo-\xe4.html", "iso-8859-1", true, "foo-\xc3\xa4.html"}, 351 {"foo-\xe4.html", "iso-8859-7", true, "foo-\xce\xb4.html"}, 352 {"foo-\xe4.html", "foo-bar", false, ""}, 353 {"foo-\xff.html", "ascii", false, ""}, 354 {"foo.html", "ascii", true, "foo.html"}, 355 {"foo-a\xcc\x88.html", "utf-8", true, "foo-\xc3\xa4.html"}, 356 {"\x95\x32\x82\x36\xD2\xBB", "gb18030", true, "\xF0\xA0\x80\x80\xE4\xB8\x80"}, 357 {"\xA7\x41\xA6\x6E", "big5", true, "\xE4\xBD\xA0\xE5\xA5\xBD"}, 358 // Windows-1258 does have a combining character at xD2 (which is U+0309). 359 // The sequence of (U+00E2, U+0309) is also encoded as U+1EA9. 360 {"foo\xE2\xD2", "windows-1258", true, "foo\xE1\xBA\xA9"}, 361 }; 362 TEST(ICUStringConversionsTest, ConvertToUtf8AndNormalize) { 363 std::string result; 364 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertAndNormalizeCases); ++i) { 365 SCOPED_TRACE(base::StringPrintf( 366 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i, 367 kConvertAndNormalizeCases[i].encoded, 368 kConvertAndNormalizeCases[i].codepage_name)); 369 370 bool success = ConvertToUtf8AndNormalize( 371 kConvertAndNormalizeCases[i].encoded, 372 kConvertAndNormalizeCases[i].codepage_name, &result); 373 EXPECT_EQ(kConvertAndNormalizeCases[i].expected_success, success); 374 EXPECT_EQ(kConvertAndNormalizeCases[i].expected_value, result); 375 } 376 } 377 378 } // namespace base 379
