1 // Copyright (c) 2009 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/i18n/icu_string_conversions.h" 6 7 #include <vector> 8 9 #include "base/basictypes.h" 10 #include "base/logging.h" 11 #include "base/string_util.h" 12 #include "unicode/ucnv.h" 13 #include "unicode/ucnv_cb.h" 14 #include "unicode/ucnv_err.h" 15 #include "unicode/ustring.h" 16 17 namespace base { 18 19 namespace { 20 21 inline bool IsValidCodepoint(uint32 code_point) { 22 // Excludes the surrogate code points ([0xD800, 0xDFFF]) and 23 // codepoints larger than 0x10FFFF (the highest codepoint allowed). 24 // Non-characters and unassigned codepoints are allowed. 25 return code_point < 0xD800u || 26 (code_point >= 0xE000u && code_point <= 0x10FFFFu); 27 } 28 29 // ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUSBSTITUTE 30 // in source/common/ucnv_err.c. 31 32 // Copyright (c) 1995-2006 International Business Machines Corporation 33 // and others 34 // 35 // All rights reserved. 36 // 37 38 // Permission is hereby granted, free of charge, to any person obtaining a 39 // copy of this software and associated documentation files (the "Software"), 40 // to deal in the Software without restriction, including without limitation 41 // the rights to use, copy, modify, merge, publish, distribute, and/or 42 // sell copies of the Software, and to permit persons to whom the Software 43 // is furnished to do so, provided that the above copyright notice(s) and 44 // this permission notice appear in all copies of the Software and that 45 // both the above copyright notice(s) and this permission notice appear in 46 // supporting documentation. 47 // 48 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 49 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 50 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT 51 // OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS 52 // INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT 53 // OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS 54 // OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE 55 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE 56 // OR PERFORMANCE OF THIS SOFTWARE. 57 // 58 // Except as contained in this notice, the name of a copyright holder 59 // shall not be used in advertising or otherwise to promote the sale, use 60 // or other dealings in this Software without prior written authorization 61 // of the copyright holder. 62 63 // ___________________________________________________________________________ 64 // 65 // All trademarks and registered trademarks mentioned herein are the property 66 // of their respective owners. 67 68 void ToUnicodeCallbackSubstitute(const void* context, 69 UConverterToUnicodeArgs *to_args, 70 const char* code_units, 71 int32_t length, 72 UConverterCallbackReason reason, 73 UErrorCode * err) { 74 static const UChar kReplacementChar = 0xFFFD; 75 if (reason <= UCNV_IRREGULAR) { 76 if (context == NULL || 77 (*(reinterpret_cast<const char*>(context)) == 'i' && 78 reason == UCNV_UNASSIGNED)) { 79 *err = U_ZERO_ERROR; 80 ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err); 81 } 82 // else the caller must have set the error code accordingly. 83 } 84 // else ignore the reset, close and clone calls. 85 } 86 87 bool ConvertFromUTF16(UConverter* converter, const UChar* uchar_src, 88 int uchar_len, OnStringConversionError::Type on_error, 89 std::string* encoded) { 90 int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(uchar_len, 91 ucnv_getMaxCharSize(converter)); 92 encoded->resize(encoded_max_length); 93 94 UErrorCode status = U_ZERO_ERROR; 95 96 // Setup our error handler. 97 switch (on_error) { 98 case OnStringConversionError::FAIL: 99 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, 0, 100 NULL, NULL, &status); 101 break; 102 case OnStringConversionError::SKIP: 103 case OnStringConversionError::SUBSTITUTE: 104 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, 0, 105 NULL, NULL, &status); 106 break; 107 default: 108 NOTREACHED(); 109 } 110 111 // ucnv_fromUChars returns size not including terminating null 112 int actual_size = ucnv_fromUChars(converter, &(*encoded)[0], 113 encoded_max_length, uchar_src, uchar_len, &status); 114 encoded->resize(actual_size); 115 ucnv_close(converter); 116 if (U_SUCCESS(status)) 117 return true; 118 encoded->clear(); // Make sure the output is empty on error. 119 return false; 120 } 121 122 // Set up our error handler for ToUTF-16 converters 123 void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error, 124 UConverter* converter, UErrorCode* status) { 125 switch (on_error) { 126 case OnStringConversionError::FAIL: 127 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, 0, 128 NULL, NULL, status); 129 break; 130 case OnStringConversionError::SKIP: 131 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, 0, 132 NULL, NULL, status); 133 break; 134 case OnStringConversionError::SUBSTITUTE: 135 ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, 0, 136 NULL, NULL, status); 137 break; 138 default: 139 NOTREACHED(); 140 } 141 } 142 143 inline UConverterType utf32_platform_endian() { 144 #if U_IS_BIG_ENDIAN 145 return UCNV_UTF32_BigEndian; 146 #else 147 return UCNV_UTF32_LittleEndian; 148 #endif 149 } 150 151 } // namespace 152 153 const char kCodepageLatin1[] = "ISO-8859-1"; 154 const char kCodepageUTF8[] = "UTF-8"; 155 const char kCodepageUTF16BE[] = "UTF-16BE"; 156 const char kCodepageUTF16LE[] = "UTF-16LE"; 157 158 // Codepage <-> Wide/UTF-16 --------------------------------------------------- 159 160 bool UTF16ToCodepage(const string16& utf16, 161 const char* codepage_name, 162 OnStringConversionError::Type on_error, 163 std::string* encoded) { 164 encoded->clear(); 165 166 UErrorCode status = U_ZERO_ERROR; 167 UConverter* converter = ucnv_open(codepage_name, &status); 168 if (!U_SUCCESS(status)) 169 return false; 170 171 return ConvertFromUTF16(converter, utf16.c_str(), 172 static_cast<int>(utf16.length()), on_error, encoded); 173 } 174 175 bool CodepageToUTF16(const std::string& encoded, 176 const char* codepage_name, 177 OnStringConversionError::Type on_error, 178 string16* utf16) { 179 utf16->clear(); 180 181 UErrorCode status = U_ZERO_ERROR; 182 UConverter* converter = ucnv_open(codepage_name, &status); 183 if (!U_SUCCESS(status)) 184 return false; 185 186 // Even in the worst case, the maximum length in 2-byte units of UTF-16 187 // output would be at most the same as the number of bytes in input. There 188 // is no single-byte encoding in which a character is mapped to a 189 // non-BMP character requiring two 2-byte units. 190 // 191 // Moreover, non-BMP characters in legacy multibyte encodings 192 // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are 193 // BOCU and SCSU, but we don't care about them. 194 size_t uchar_max_length = encoded.length() + 1; 195 196 SetUpErrorHandlerForToUChars(on_error, converter, &status); 197 int actual_size = ucnv_toUChars(converter, WriteInto(utf16, uchar_max_length), 198 static_cast<int>(uchar_max_length), encoded.data(), 199 static_cast<int>(encoded.length()), &status); 200 ucnv_close(converter); 201 if (!U_SUCCESS(status)) { 202 utf16->clear(); // Make sure the output is empty on error. 203 return false; 204 } 205 206 utf16->resize(actual_size); 207 return true; 208 } 209 210 bool WideToCodepage(const std::wstring& wide, 211 const char* codepage_name, 212 OnStringConversionError::Type on_error, 213 std::string* encoded) { 214 #if defined(WCHAR_T_IS_UTF16) 215 return UTF16ToCodepage(wide, codepage_name, on_error, encoded); 216 #elif defined(WCHAR_T_IS_UTF32) 217 encoded->clear(); 218 219 UErrorCode status = U_ZERO_ERROR; 220 UConverter* converter = ucnv_open(codepage_name, &status); 221 if (!U_SUCCESS(status)) 222 return false; 223 224 int utf16_len; 225 // When wchar_t is wider than UChar (16 bits), transform |wide| into a 226 // UChar* string. Size the UChar* buffer to be large enough to hold twice 227 // as many UTF-16 code units (UChar's) as there are Unicode code points, 228 // in case each code points translates to a UTF-16 surrogate pair, 229 // and leave room for a NUL terminator. 230 std::vector<UChar> utf16(wide.length() * 2 + 1); 231 u_strFromWCS(&utf16[0], utf16.size(), &utf16_len, 232 wide.c_str(), wide.length(), &status); 233 DCHECK(U_SUCCESS(status)) << "failed to convert wstring to UChar*"; 234 235 return ConvertFromUTF16(converter, &utf16[0], utf16_len, on_error, encoded); 236 #endif // defined(WCHAR_T_IS_UTF32) 237 } 238 239 bool CodepageToWide(const std::string& encoded, 240 const char* codepage_name, 241 OnStringConversionError::Type on_error, 242 std::wstring* wide) { 243 #if defined(WCHAR_T_IS_UTF16) 244 return CodepageToUTF16(encoded, codepage_name, on_error, wide); 245 #elif defined(WCHAR_T_IS_UTF32) 246 wide->clear(); 247 248 UErrorCode status = U_ZERO_ERROR; 249 UConverter* converter = ucnv_open(codepage_name, &status); 250 if (!U_SUCCESS(status)) 251 return false; 252 253 // The maximum length in 4 byte unit of UTF-32 output would be 254 // at most the same as the number of bytes in input. In the worst 255 // case of GB18030 (excluding escaped-based encodings like ISO-2022-JP), 256 // this can be 4 times larger than actually needed. 257 size_t wchar_max_length = encoded.length() + 1; 258 259 SetUpErrorHandlerForToUChars(on_error, converter, &status); 260 int actual_size = ucnv_toAlgorithmic(utf32_platform_endian(), converter, 261 reinterpret_cast<char*>(WriteInto(wide, wchar_max_length)), 262 static_cast<int>(wchar_max_length) * sizeof(wchar_t), encoded.data(), 263 static_cast<int>(encoded.length()), &status); 264 ucnv_close(converter); 265 if (!U_SUCCESS(status)) { 266 wide->clear(); // Make sure the output is empty on error. 267 return false; 268 } 269 270 // actual_size is # of bytes. 271 wide->resize(actual_size / sizeof(wchar_t)); 272 return true; 273 #endif // defined(WCHAR_T_IS_UTF32) 274 } 275 276 } // namespace base 277
