1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <math.h> 6 #include <stdarg.h> 7 8 #include <limits> 9 #include <sstream> 10 11 #include "base/basictypes.h" 12 #include "base/format_macros.h" 13 #include "base/i18n/icu_string_conversions.h" 14 #include "base/logging.h" 15 #include "base/strings/string_piece.h" 16 #include "base/strings/stringprintf.h" 17 #include "base/strings/utf_string_conversions.h" 18 #include "testing/gtest/include/gtest/gtest.h" 19 20 namespace base { 21 22 namespace { 23 24 // Given a null-terminated string of wchar_t with each wchar_t representing 25 // a UTF-16 code unit, returns a string16 made up of wchar_t's in the input. 26 // Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF) 27 // should be represented as a surrogate pair (two UTF-16 units) 28 // *even* where wchar_t is 32-bit (Linux and Mac). 29 // 30 // This is to help write tests for functions with string16 params until 31 // the C++ 0x UTF-16 literal is well-supported by compilers. 32 string16 BuildString16(const wchar_t* s) { 33 #if defined(WCHAR_T_IS_UTF16) 34 return string16(s); 35 #elif defined(WCHAR_T_IS_UTF32) 36 string16 u16; 37 while (*s != 0) { 38 DCHECK_LE(static_cast<unsigned int>(*s), 0xFFFFu); 39 u16.push_back(*s++); 40 } 41 return u16; 42 #endif 43 } 44 45 const wchar_t* const kConvertRoundtripCases[] = { 46 L"Google Video", 47 // " " 48 L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb", 49 // " " 50 L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9" 51 L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2", 52 // " " 53 L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442" 54 L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430" 55 L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c", 56 // "" 57 L"\xc804\xccb4\xc11c\xbe44\xc2a4", 58 59 // Test characters that take more than 16 bits. This will depend on whether 60 // wchar_t is 16 or 32 bits. 61 #if defined(WCHAR_T_IS_UTF16) 62 L"\xd800\xdf00", 63 // ????? (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E) 64 L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44", 65 #elif defined(WCHAR_T_IS_UTF32) 66 L"\x10300", 67 // ????? (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E) 68 L"\x11d40\x11d41\x11d42\x11d43\x11d44", 69 #endif 70 }; 71 72 } // namespace 73 74 TEST(ICUStringConversionsTest, ConvertCodepageUTF8) { 75 // Make sure WideToCodepage works like WideToUTF8. 76 for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) { 77 SCOPED_TRACE(base::StringPrintf("Test[%" PRIuS "]: %ls", 78 i, kConvertRoundtripCases[i])); 79 80 std::string expected(WideToUTF8(kConvertRoundtripCases[i])); 81 std::string utf8; 82 EXPECT_TRUE(WideToCodepage(kConvertRoundtripCases[i], kCodepageUTF8, 83 OnStringConversionError::SKIP, &utf8)); 84 EXPECT_EQ(expected, utf8); 85 } 86 } 87 88 // kConverterCodepageCases is not comprehensive. There are a number of cases 89 // to add if we really want to have a comprehensive coverage of various 90 // codepages and their 'idiosyncrasies'. Currently, the only implementation 91 // for CodepageTo* and *ToCodepage uses ICU, which has a very extensive 92 // set of tests for the charset conversion. So, we can get away with a 93 // relatively small number of cases listed below. 94 // 95 // Note about |u16_wide| in the following struct. 96 // On Windows, the field is always identical to |wide|. On Mac and Linux, 97 // it's identical as long as there's no character outside the 98 // BMP (<= U+FFFF). When there is, it is different from |wide| and 99 // is not a real wide string (UTF-32 string) in that each wchar_t in 100 // the string is a UTF-16 code unit zero-extended to be 32-bit 101 // even when the code unit belongs to a surrogate pair. 102 // For instance, a Unicode string (U+0041 U+010000) is represented as 103 // L"\x0041\xD800\xDC00" instead of L"\x0041\x10000". 104 // To avoid the clutter, |u16_wide| will be set to NULL 105 // if it's identical to |wide| on *all* platforms. 106 107 static const struct { 108 const char* codepage_name; 109 const char* encoded; 110 OnStringConversionError::Type on_error; 111 bool success; 112 const wchar_t* wide; 113 const wchar_t* u16_wide; 114 } kConvertCodepageCases[] = { 115 // Test a case where the input cannot be decoded, using SKIP, FAIL 116 // and SUBSTITUTE error handling rules. "A7 41" is valid, but "A6" isn't. 117 {"big5", 118 "\xA7\x41\xA6", 119 OnStringConversionError::FAIL, 120 false, 121 L"", 122 NULL}, 123 {"big5", 124 "\xA7\x41\xA6", 125 OnStringConversionError::SKIP, 126 true, 127 L"\x4F60", 128 NULL}, 129 {"big5", 130 "\xA7\x41\xA6", 131 OnStringConversionError::SUBSTITUTE, 132 true, 133 L"\x4F60\xFFFD", 134 NULL}, 135 // Arabic (ISO-8859) 136 {"iso-8859-6", 137 "\xC7\xEE\xE4\xD3\xF1\xEE\xE4\xC7\xE5\xEF" " " 138 "\xD9\xEE\xE4\xEE\xEA\xF2\xE3\xEF\xE5\xF2", 139 OnStringConversionError::FAIL, 140 true, 141 L"\x0627\x064E\x0644\x0633\x0651\x064E\x0644\x0627\x0645\x064F" L" " 142 L"\x0639\x064E\x0644\x064E\x064A\x0652\x0643\x064F\x0645\x0652", 143 NULL}, 144 // Chinese Simplified (GB2312) 145 {"gb2312", 146 "\xC4\xE3\xBA\xC3", 147 OnStringConversionError::FAIL, 148 true, 149 L"\x4F60\x597D", 150 NULL}, 151 // Chinese (GB18030) : 4 byte sequences mapped to BMP characters 152 {"gb18030", 153 "\x81\x30\x84\x36\xA1\xA7", 154 OnStringConversionError::FAIL, 155 true, 156 L"\x00A5\x00A8", 157 NULL}, 158 // Chinese (GB18030) : A 4 byte sequence mapped to plane 2 (U+20000) 159 {"gb18030", 160 "\x95\x32\x82\x36\xD2\xBB", 161 OnStringConversionError::FAIL, 162 true, 163 #if defined(WCHAR_T_IS_UTF16) 164 L"\xD840\xDC00\x4E00", 165 #elif defined(WCHAR_T_IS_UTF32) 166 L"\x20000\x4E00", 167 #endif 168 L"\xD840\xDC00\x4E00"}, 169 {"big5", 170 "\xA7\x41\xA6\x6E", 171 OnStringConversionError::FAIL, 172 true, 173 L"\x4F60\x597D", 174 NULL}, 175 // Greek (ISO-8859) 176 {"iso-8859-7", 177 "\xE3\xE5\xE9\xDC" " " "\xF3\xEF\xF5", 178 OnStringConversionError::FAIL, 179 true, 180 L"\x03B3\x03B5\x03B9\x03AC" L" " L"\x03C3\x03BF\x03C5", 181 NULL}, 182 // Hebrew (Windows) 183 {"windows-1255", 184 "\xF9\xD1\xC8\xEC\xE5\xC9\xED", 185 OnStringConversionError::FAIL, 186 true, 187 L"\x05E9\x05C1\x05B8\x05DC\x05D5\x05B9\x05DD", 188 NULL}, 189 // Korean (EUC) 190 {"euc-kr", 191 "\xBE\xC8\xB3\xE7\xC7\xCF\xBC\xBC\xBF\xE4", 192 OnStringConversionError::FAIL, 193 true, 194 L"\xC548\xB155\xD558\xC138\xC694", 195 NULL}, 196 // Japanese (EUC) 197 {"euc-jp", 198 "\xA4\xB3\xA4\xF3\xA4\xCB\xA4\xC1\xA4\xCF\xB0\xEC\x8E\xA6", 199 OnStringConversionError::FAIL, 200 true, 201 L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66", 202 NULL}, 203 // Japanese (ISO-2022) 204 {"iso-2022-jp", 205 "\x1B$B" "\x24\x33\x24\x73\x24\x4B\x24\x41\x24\x4F\x30\x6C" "\x1B(B" 206 "ab" "\x1B(J" "\x5C\x7E#$" "\x1B(B", 207 OnStringConversionError::FAIL, 208 true, 209 L"\x3053\x3093\x306B\x3061\x306F\x4E00" L"ab\x00A5\x203E#$", 210 NULL}, 211 // Japanese (Shift-JIS) 212 {"sjis", 213 "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x88\xEA\xA6", 214 OnStringConversionError::FAIL, 215 true, 216 L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66", 217 NULL}, 218 // Russian (KOI8) 219 {"koi8-r", 220 "\xDA\xC4\xD2\xC1\xD7\xD3\xD4\xD7\xD5\xCA\xD4\xC5", 221 OnStringConversionError::FAIL, 222 true, 223 L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432" 224 L"\x0443\x0439\x0442\x0435", 225 NULL}, 226 // Thai (windows-874) 227 {"windows-874", 228 "\xCA\xC7\xD1\xCA\xB4\xD5" "\xA4\xC3\xD1\xBA", 229 OnStringConversionError::FAIL, 230 true, 231 L"\x0E2A\x0E27\x0E31\x0E2A\x0E14\x0E35" 232 L"\x0E04\x0E23\x0e31\x0E1A", 233 NULL}, 234 }; 235 236 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndWide) { 237 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) { 238 SCOPED_TRACE(base::StringPrintf( 239 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i, 240 kConvertCodepageCases[i].encoded, 241 kConvertCodepageCases[i].codepage_name)); 242 243 std::wstring wide; 244 bool success = CodepageToWide(kConvertCodepageCases[i].encoded, 245 kConvertCodepageCases[i].codepage_name, 246 kConvertCodepageCases[i].on_error, 247 &wide); 248 EXPECT_EQ(kConvertCodepageCases[i].success, success); 249 EXPECT_EQ(kConvertCodepageCases[i].wide, wide); 250 251 // When decoding was successful and nothing was skipped, we also check the 252 // reverse conversion. Not all conversions are round-trippable, but 253 // kConverterCodepageCases does not have any one-way conversion at the 254 // moment. 255 if (success && 256 kConvertCodepageCases[i].on_error == 257 OnStringConversionError::FAIL) { 258 std::string encoded; 259 success = WideToCodepage(wide, kConvertCodepageCases[i].codepage_name, 260 kConvertCodepageCases[i].on_error, &encoded); 261 EXPECT_EQ(kConvertCodepageCases[i].success, success); 262 EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded); 263 } 264 } 265 266 // The above cases handled codepage->wide errors, but not wide->codepage. 267 // Test that here. 268 std::string encoded("Temp data"); // Make sure the string gets cleared. 269 270 // First test going to an encoding that can not represent that character. 271 EXPECT_FALSE(WideToCodepage(L"Chinese\xff27", "iso-8859-1", 272 OnStringConversionError::FAIL, &encoded)); 273 EXPECT_TRUE(encoded.empty()); 274 EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1", 275 OnStringConversionError::SKIP, &encoded)); 276 EXPECT_STREQ("Chinese", encoded.c_str()); 277 // From Unicode, SUBSTITUTE is the same as SKIP for now. 278 EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1", 279 OnStringConversionError::SUBSTITUTE, 280 &encoded)); 281 EXPECT_STREQ("Chinese", encoded.c_str()); 282 283 #if defined(WCHAR_T_IS_UTF16) 284 // When we're in UTF-16 mode, test an invalid UTF-16 character in the input. 285 EXPECT_FALSE(WideToCodepage(L"a\xd800z", "iso-8859-1", 286 OnStringConversionError::FAIL, &encoded)); 287 EXPECT_TRUE(encoded.empty()); 288 EXPECT_TRUE(WideToCodepage(L"a\xd800z", "iso-8859-1", 289 OnStringConversionError::SKIP, &encoded)); 290 EXPECT_STREQ("az", encoded.c_str()); 291 #endif // WCHAR_T_IS_UTF16 292 293 // Invalid characters should fail. 294 EXPECT_TRUE(WideToCodepage(L"a\xffffz", "iso-8859-1", 295 OnStringConversionError::SKIP, &encoded)); 296 EXPECT_STREQ("az", encoded.c_str()); 297 298 // Invalid codepages should fail. 299 EXPECT_FALSE(WideToCodepage(L"Hello, world", "awesome-8571-2", 300 OnStringConversionError::SKIP, &encoded)); 301 } 302 303 TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndUTF16) { 304 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) { 305 SCOPED_TRACE(base::StringPrintf( 306 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i, 307 kConvertCodepageCases[i].encoded, 308 kConvertCodepageCases[i].codepage_name)); 309 310 string16 utf16; 311 bool success = CodepageToUTF16(kConvertCodepageCases[i].encoded, 312 kConvertCodepageCases[i].codepage_name, 313 kConvertCodepageCases[i].on_error, 314 &utf16); 315 string16 utf16_expected; 316 if (kConvertCodepageCases[i].u16_wide == NULL) 317 utf16_expected = BuildString16(kConvertCodepageCases[i].wide); 318 else 319 utf16_expected = BuildString16(kConvertCodepageCases[i].u16_wide); 320 EXPECT_EQ(kConvertCodepageCases[i].success, success); 321 EXPECT_EQ(utf16_expected, utf16); 322 323 // When decoding was successful and nothing was skipped, we also check the 324 // reverse conversion. See also the corresponding comment in 325 // ConvertBetweenCodepageAndWide. 326 if (success && 327 kConvertCodepageCases[i].on_error == OnStringConversionError::FAIL) { 328 std::string encoded; 329 success = UTF16ToCodepage(utf16, kConvertCodepageCases[i].codepage_name, 330 kConvertCodepageCases[i].on_error, &encoded); 331 EXPECT_EQ(kConvertCodepageCases[i].success, success); 332 EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded); 333 } 334 } 335 } 336 337 static const struct { 338 const char* encoded; 339 const char* codepage_name; 340 bool expected_success; 341 const char* expected_value; 342 } kConvertAndNormalizeCases[] = { 343 {"foo-\xe4.html", "iso-8859-1", true, "foo-\xc3\xa4.html"}, 344 {"foo-\xe4.html", "iso-8859-7", true, "foo-\xce\xb4.html"}, 345 {"foo-\xe4.html", "foo-bar", false, ""}, 346 {"foo-\xff.html", "ascii", false, ""}, 347 {"foo.html", "ascii", true, "foo.html"}, 348 {"foo-a\xcc\x88.html", "utf-8", true, "foo-\xc3\xa4.html"}, 349 {"\x95\x32\x82\x36\xD2\xBB", "gb18030", true, "\xF0\xA0\x80\x80\xE4\xB8\x80"}, 350 {"\xA7\x41\xA6\x6E", "big5", true, "\xE4\xBD\xA0\xE5\xA5\xBD"}, 351 // Windows-1258 does have a combining character at xD2 (which is U+0309). 352 // The sequence of (U+00E2, U+0309) is also encoded as U+1EA9. 353 {"foo\xE2\xD2", "windows-1258", true, "foo\xE1\xBA\xA9"}, 354 {"", "iso-8859-1", true, ""}, 355 }; 356 TEST(ICUStringConversionsTest, ConvertToUtf8AndNormalize) { 357 std::string result; 358 for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertAndNormalizeCases); ++i) { 359 SCOPED_TRACE(base::StringPrintf( 360 "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i, 361 kConvertAndNormalizeCases[i].encoded, 362 kConvertAndNormalizeCases[i].codepage_name)); 363 364 bool success = ConvertToUtf8AndNormalize( 365 kConvertAndNormalizeCases[i].encoded, 366 kConvertAndNormalizeCases[i].codepage_name, &result); 367 EXPECT_EQ(kConvertAndNormalizeCases[i].expected_success, success); 368 EXPECT_EQ(kConvertAndNormalizeCases[i].expected_value, result); 369 } 370 } 371 372 } // namespace base 373
