1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/i18n/icu_encoding_detection.h" 6 7 #include <set> 8 9 #include "base/string_util.h" 10 #include "unicode/ucsdet.h" 11 12 namespace base { 13 14 bool DetectEncoding(const std::string& text, std::string* encoding) { 15 if (IsStringASCII(text)) { 16 *encoding = std::string(); 17 return true; 18 } 19 20 UErrorCode status = U_ZERO_ERROR; 21 UCharsetDetector* detector = ucsdet_open(&status); 22 ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()), 23 &status); 24 const UCharsetMatch* match = ucsdet_detect(detector, &status); 25 if (match == NULL) 26 return false; 27 const char* detected_encoding = ucsdet_getName(match, &status); 28 ucsdet_close(detector); 29 30 if (U_FAILURE(status)) 31 return false; 32 33 *encoding = detected_encoding; 34 return true; 35 } 36 37 bool DetectAllEncodings(const std::string& text, 38 std::vector<std::string>* encodings) { 39 UErrorCode status = U_ZERO_ERROR; 40 UCharsetDetector* detector = ucsdet_open(&status); 41 ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()), 42 &status); 43 int matches_count = 0; 44 const UCharsetMatch** matches = ucsdet_detectAll(detector, 45 &matches_count, 46 &status); 47 if (U_FAILURE(status)) { 48 ucsdet_close(detector); 49 return false; 50 } 51 52 // ICU has some heuristics for encoding detection, such that the more likely 53 // encodings should be returned first. However, it doesn't always return 54 // all encodings that properly decode |text|, so we'll append more encodings 55 // later. To make that efficient, keep track of encodings sniffed in this 56 // first phase. 57 std::set<std::string> sniffed_encodings; 58 59 encodings->clear(); 60 for (int i = 0; i < matches_count; i++) { 61 UErrorCode get_name_status = U_ZERO_ERROR; 62 const char* encoding_name = ucsdet_getName(matches[i], &get_name_status); 63 64 // If we failed to get the encoding's name, ignore the error. 65 if (U_FAILURE(get_name_status)) 66 continue; 67 68 int32_t confidence = ucsdet_getConfidence(matches[i], &get_name_status); 69 70 // We also treat this error as non-fatal. 71 if (U_FAILURE(get_name_status)) 72 continue; 73 74 // A confidence level >= 10 means that the encoding is expected to properly 75 // decode the text. Drop all encodings with lower confidence level. 76 if (confidence < 10) 77 continue; 78 79 encodings->push_back(encoding_name); 80 sniffed_encodings.insert(encoding_name); 81 } 82 83 // Append all encodings not included earlier, in arbitrary order. 84 // TODO(jshin): This shouldn't be necessary, possible ICU bug. 85 // See also http://crbug.com/65917. 86 UEnumeration* detectable_encodings = ucsdet_getAllDetectableCharsets(detector, 87 &status); 88 int detectable_count = uenum_count(detectable_encodings, &status); 89 for (int i = 0; i < detectable_count; i++) { 90 int name_length; 91 const char* name_raw = uenum_next(detectable_encodings, 92 &name_length, 93 &status); 94 std::string name(name_raw, name_length); 95 if (sniffed_encodings.find(name) == sniffed_encodings.end()) 96 encodings->push_back(name); 97 } 98 uenum_close(detectable_encodings); 99 100 ucsdet_close(detector); 101 return !encodings->empty(); 102 } 103 104 } // namespace base 105
