1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/i18n/icu_string_conversions.h" 6 7 #include <vector> 8 9 #include "base/basictypes.h" 10 #include "base/logging.h" 11 #include "base/string_util.h" 12 #include "base/utf_string_conversions.h" 13 #include "unicode/ucnv.h" 14 #include "unicode/ucnv_cb.h" 15 #include "unicode/ucnv_err.h" 16 #include "unicode/unorm.h" 17 #include "unicode/ustring.h" 18 19 namespace base { 20 21 namespace { 22 // ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUSBSTITUTE 23 // in source/common/ucnv_err.c. 24 25 // Copyright (c) 1995-2006 International Business Machines Corporation 26 // and others 27 // 28 // All rights reserved. 29 // 30 31 // Permission is hereby granted, free of charge, to any person obtaining a 32 // copy of this software and associated documentation files (the "Software"), 33 // to deal in the Software without restriction, including without limitation 34 // the rights to use, copy, modify, merge, publish, distribute, and/or 35 // sell copies of the Software, and to permit persons to whom the Software 36 // is furnished to do so, provided that the above copyright notice(s) and 37 // this permission notice appear in all copies of the Software and that 38 // both the above copyright notice(s) and this permission notice appear in 39 // supporting documentation. 40 // 41 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 42 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 43 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 44 // OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS 45 // INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT 46 // OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS 47 // OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 48 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE 49 // OR PERFORMANCE OF THIS SOFTWARE. 50 // 51 // Except as contained in this notice, the name of a copyright holder 52 // shall not be used in advertising or otherwise to promote the sale, use 53 // or other dealings in this Software without prior written authorization 54 // of the copyright holder. 55 56 // ___________________________________________________________________________ 57 // 58 // All trademarks and registered trademarks mentioned herein are the property 59 // of their respective owners. 60 61 void ToUnicodeCallbackSubstitute(const void* context, 62 UConverterToUnicodeArgs *to_args, 63 const char* code_units, 64 int32_t length, 65 UConverterCallbackReason reason, 66 UErrorCode * err) { 67 static const UChar kReplacementChar = 0xFFFD; 68 if (reason <= UCNV_IRREGULAR) { 69 if (context == NULL || 70 (*(reinterpret_cast<const char*>(context)) == 'i' && 71 reason == UCNV_UNASSIGNED)) { 72 *err = U_ZERO_ERROR; 73 ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err); 74 } 75 // else the caller must have set the error code accordingly. 76 } 77 // else ignore the reset, close and clone calls. 78 } 79 80 bool ConvertFromUTF16(UConverter* converter, const UChar* uchar_src, 81 int uchar_len, OnStringConversionError::Type on_error, 82 std::string* encoded) { 83 int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(uchar_len, 84 ucnv_getMaxCharSize(converter)); 85 encoded->resize(encoded_max_length); 86 87 UErrorCode status = U_ZERO_ERROR; 88 89 // Setup our error handler. 90 switch (on_error) { 91 case OnStringConversionError::FAIL: 92 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, 0, 93 NULL, NULL, &status); 94 break; 95 case OnStringConversionError::SKIP: 96 case OnStringConversionError::SUBSTITUTE: 97 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, 0, 98 NULL, NULL, &status); 99 break; 100 default: 101 NOTREACHED(); 102 } 103 104 // ucnv_fromUChars returns size not including terminating null 105 int actual_size = ucnv_fromUChars(converter, &(*encoded)[0], 106 encoded_max_length, uchar_src, uchar_len, &status); 107 encoded->resize(actual_size); 108 ucnv_close(converter); 109 if (U_SUCCESS(status)) 110 return true; 111 encoded->clear(); // Make sure the output is empty on error. 112 return false; 113 } 114 115 // Set up our error handler for ToUTF-16 converters 116 void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error, 117 UConverter* converter, UErrorCode* status) { 118 switch (on_error) { 119 case OnStringConversionError::FAIL: 120 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, 0, 121 NULL, NULL, status); 122 break; 123 case OnStringConversionError::SKIP: 124 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, 0, 125 NULL, NULL, status); 126 break; 127 case OnStringConversionError::SUBSTITUTE: 128 ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, 0, 129 NULL, NULL, status); 130 break; 131 default: 132 NOTREACHED(); 133 } 134 } 135 136 inline UConverterType utf32_platform_endian() { 137 #if U_IS_BIG_ENDIAN 138 return UCNV_UTF32_BigEndian; 139 #else 140 return UCNV_UTF32_LittleEndian; 141 #endif 142 } 143 144 } // namespace 145 146 const char kCodepageLatin1[] = "ISO-8859-1"; 147 const char kCodepageUTF8[] = "UTF-8"; 148 const char kCodepageUTF16BE[] = "UTF-16BE"; 149 const char kCodepageUTF16LE[] = "UTF-16LE"; 150 151 // Codepage <-> Wide/UTF-16 --------------------------------------------------- 152 153 bool UTF16ToCodepage(const string16& utf16, 154 const char* codepage_name, 155 OnStringConversionError::Type on_error, 156 std::string* encoded) { 157 encoded->clear(); 158 159 UErrorCode status = U_ZERO_ERROR; 160 UConverter* converter = ucnv_open(codepage_name, &status); 161 if (!U_SUCCESS(status)) 162 return false; 163 164 return ConvertFromUTF16(converter, utf16.c_str(), 165 static_cast<int>(utf16.length()), on_error, encoded); 166 } 167 168 bool CodepageToUTF16(const std::string& encoded, 169 const char* codepage_name, 170 OnStringConversionError::Type on_error, 171 string16* utf16) { 172 utf16->clear(); 173 174 UErrorCode status = U_ZERO_ERROR; 175 UConverter* converter = ucnv_open(codepage_name, &status); 176 if (!U_SUCCESS(status)) 177 return false; 178 179 // Even in the worst case, the maximum length in 2-byte units of UTF-16 180 // output would be at most the same as the number of bytes in input. There 181 // is no single-byte encoding in which a character is mapped to a 182 // non-BMP character requiring two 2-byte units. 183 // 184 // Moreover, non-BMP characters in legacy multibyte encodings 185 // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are 186 // BOCU and SCSU, but we don't care about them. 187 size_t uchar_max_length = encoded.length() + 1; 188 189 SetUpErrorHandlerForToUChars(on_error, converter, &status); 190 int actual_size = ucnv_toUChars(converter, WriteInto(utf16, uchar_max_length), 191 static_cast<int>(uchar_max_length), encoded.data(), 192 static_cast<int>(encoded.length()), &status); 193 ucnv_close(converter); 194 if (!U_SUCCESS(status)) { 195 utf16->clear(); // Make sure the output is empty on error. 196 return false; 197 } 198 199 utf16->resize(actual_size); 200 return true; 201 } 202 203 bool WideToCodepage(const std::wstring& wide, 204 const char* codepage_name, 205 OnStringConversionError::Type on_error, 206 std::string* encoded) { 207 #if defined(WCHAR_T_IS_UTF16) 208 return UTF16ToCodepage(wide, codepage_name, on_error, encoded); 209 #elif defined(WCHAR_T_IS_UTF32) 210 encoded->clear(); 211 212 UErrorCode status = U_ZERO_ERROR; 213 UConverter* converter = ucnv_open(codepage_name, &status); 214 if (!U_SUCCESS(status)) 215 return false; 216 217 int utf16_len; 218 // When wchar_t is wider than UChar (16 bits), transform |wide| into a 219 // UChar* string. Size the UChar* buffer to be large enough to hold twice 220 // as many UTF-16 code units (UChar's) as there are Unicode code points, 221 // in case each code points translates to a UTF-16 surrogate pair, 222 // and leave room for a NUL terminator. 223 std::vector<UChar> utf16(wide.length() * 2 + 1); 224 u_strFromUTF32(&utf16[0], utf16.size(), &utf16_len, 225 reinterpret_cast<const UChar32*>(wide.c_str()), 226 wide.length(), &status); 227 DCHECK(U_SUCCESS(status)) << "failed to convert wstring to UChar*"; 228 229 return ConvertFromUTF16(converter, &utf16[0], utf16_len, on_error, encoded); 230 #endif // defined(WCHAR_T_IS_UTF32) 231 } 232 233 bool CodepageToWide(const std::string& encoded, 234 const char* codepage_name, 235 OnStringConversionError::Type on_error, 236 std::wstring* wide) { 237 #if defined(WCHAR_T_IS_UTF16) 238 return CodepageToUTF16(encoded, codepage_name, on_error, wide); 239 #elif defined(WCHAR_T_IS_UTF32) 240 wide->clear(); 241 242 UErrorCode status = U_ZERO_ERROR; 243 UConverter* converter = ucnv_open(codepage_name, &status); 244 if (!U_SUCCESS(status)) 245 return false; 246 247 // The maximum length in 4 byte unit of UTF-32 output would be 248 // at most the same as the number of bytes in input. In the worst 249 // case of GB18030 (excluding escaped-based encodings like ISO-2022-JP), 250 // this can be 4 times larger than actually needed. 251 size_t wchar_max_length = encoded.length() + 1; 252 253 SetUpErrorHandlerForToUChars(on_error, converter, &status); 254 int actual_size = ucnv_toAlgorithmic(utf32_platform_endian(), converter, 255 reinterpret_cast<char*>(WriteInto(wide, wchar_max_length)), 256 static_cast<int>(wchar_max_length) * sizeof(wchar_t), encoded.data(), 257 static_cast<int>(encoded.length()), &status); 258 ucnv_close(converter); 259 if (!U_SUCCESS(status)) { 260 wide->clear(); // Make sure the output is empty on error. 261 return false; 262 } 263 264 // actual_size is # of bytes. 265 wide->resize(actual_size / sizeof(wchar_t)); 266 return true; 267 #endif // defined(WCHAR_T_IS_UTF32) 268 } 269 270 bool ConvertToUtf8AndNormalize(const std::string& text, 271 const std::string& charset, 272 std::string* result) { 273 result->clear(); 274 string16 utf16; 275 if (!CodepageToUTF16( 276 text, charset.c_str(), OnStringConversionError::FAIL, &utf16)) 277 return false; 278 279 UErrorCode status = U_ZERO_ERROR; 280 size_t max_length = utf16.length() + 1; 281 string16 normalized_utf16; 282 int actual_length = unorm_normalize( 283 utf16.c_str(), utf16.length(), UNORM_NFC, 0, 284 WriteInto(&normalized_utf16, max_length), 285 static_cast<int>(max_length), &status); 286 if (!U_SUCCESS(status)) 287 return false; 288 normalized_utf16.resize(actual_length); 289 290 return UTF16ToUTF8(normalized_utf16.data(), 291 normalized_utf16.length(), result); 292 } 293 294 } // namespace base 295
