1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/translate/translate_language_list.h" 6 7 #include <set> 8 9 #include "base/bind.h" 10 #include "base/json/json_reader.h" 11 #include "base/lazy_instance.h" 12 #include "base/logging.h" 13 #include "base/strings/string_util.h" 14 #include "base/strings/stringprintf.h" 15 #include "base/values.h" 16 #include "chrome/browser/browser_process.h" 17 #include "chrome/browser/translate/translate_browser_metrics.h" 18 #include "chrome/browser/translate/translate_event_details.h" 19 #include "chrome/browser/translate/translate_manager.h" 20 #include "chrome/browser/translate/translate_url_fetcher.h" 21 #include "chrome/browser/translate/translate_url_util.h" 22 #include "net/base/url_util.h" 23 #include "ui/base/l10n/l10n_util.h" 24 #include "url/gurl.h" 25 26 namespace { 27 28 // The default list of languages the Google translation server supports. 29 // We use this list until we receive the list that the server exposes. 30 // For information, here is the list of languages that Chrome can be run in 31 // but that the translation server does not support: 32 // am Amharic 33 // bn Bengali 34 // gu Gujarati 35 // kn Kannada 36 // ml Malayalam 37 // mr Marathi 38 // ta Tamil 39 // te Telugu 40 const char* const kDefaultSupportedLanguages[] = { 41 "af", // Afrikaans 42 "sq", // Albanian 43 "ar", // Arabic 44 "be", // Belarusian 45 "bg", // Bulgarian 46 "ca", // Catalan 47 "zh-CN", // Chinese (Simplified) 48 "zh-TW", // Chinese (Traditional) 49 "hr", // Croatian 50 "cs", // Czech 51 "da", // Danish 52 "nl", // Dutch 53 "en", // English 54 "eo", // Esperanto 55 "et", // Estonian 56 "tl", // Filipino 57 "fi", // Finnish 58 "fr", // French 59 "gl", // Galician 60 "de", // German 61 "el", // Greek 62 "ht", // Haitian Creole 63 "iw", // Hebrew 64 "hi", // Hindi 65 "hu", // Hungarian 66 "is", // Icelandic 67 "id", // Indonesian 68 "ga", // Irish 69 "it", // Italian 70 "ja", // Japanese 71 "ko", // Korean 72 "lv", // Latvian 73 "lt", // Lithuanian 74 "mk", // Macedonian 75 "ms", // Malay 76 "mt", // Maltese 77 "no", // Norwegian 78 "fa", // Persian 79 "pl", // Polish 80 "pt", // Portuguese 81 "ro", // Romanian 82 "ru", // Russian 83 "sr", // Serbian 84 "sk", // Slovak 85 "sl", // Slovenian 86 "es", // Spanish 87 "sw", // Swahili 88 "sv", // Swedish 89 "th", // Thai 90 "tr", // Turkish 91 "uk", // Ukrainian 92 "vi", // Vietnamese 93 "cy", // Welsh 94 "yi", // Yiddish 95 }; 96 97 // Constant URL string to fetch server supporting language list. 98 const char kLanguageListFetchURL[] = 99 "https://translate.googleapis.com/translate_a/l?client=chrome&cb=sl"; 100 101 // Used in kTranslateScriptURL to request supporting languages list including 102 // "alpha languages". 103 const char kAlphaLanguageQueryName[] = "alpha"; 104 const char kAlphaLanguageQueryValue[] = "1"; 105 106 // Represent if the language list updater is disabled. 107 bool update_is_disabled = false; 108 109 // Retry parameter for fetching. 110 const int kMaxRetryOn5xx = 5; 111 112 // Show a message in chrome:://translate-internals Event Logs. 113 void NotifyEvent(int line, const std::string& message) { 114 TranslateManager* manager = TranslateManager::GetInstance(); 115 DCHECK(manager); 116 117 TranslateEventDetails details(__FILE__, line, message); 118 manager->NotifyTranslateEvent(details); 119 } 120 121 // Parses |language_list| containing the list of languages that the translate 122 // server can translate to and from, and fills |set| with them. 123 void SetSupportedLanguages(const std::string& language_list, 124 std::set<std::string>* target_language_set, 125 std::set<std::string>* alpha_language_set) { 126 DCHECK(target_language_set); 127 DCHECK(alpha_language_set); 128 129 // The format is: 130 // sl({ 131 // "sl": {"XX": "LanguageName", ...}, 132 // "tl": {"XX": "LanguageName", ...}, 133 // "al": {"XX": 1, ...} 134 // }) 135 // Where "sl(" is set in kLanguageListCallbackName, "tl" is 136 // kTargetLanguagesKey and "al" kAlphaLanguagesKey. 137 if (!StartsWithASCII(language_list, 138 TranslateLanguageList::kLanguageListCallbackName, 139 false) || 140 !EndsWith(language_list, ")", false)) { 141 // We don't have a NOTREACHED here since this can happen in ui_tests, even 142 // though the the BrowserMain function won't call us with parameters.ui_task 143 // is NULL some tests don't set it, so we must bail here. 144 return; 145 } 146 static const size_t kLanguageListCallbackNameLength = 147 strlen(TranslateLanguageList::kLanguageListCallbackName); 148 std::string languages_json = language_list.substr( 149 kLanguageListCallbackNameLength, 150 language_list.size() - kLanguageListCallbackNameLength - 1); 151 scoped_ptr<Value> json_value( 152 base::JSONReader::Read(languages_json, base::JSON_ALLOW_TRAILING_COMMAS)); 153 if (json_value == NULL || !json_value->IsType(Value::TYPE_DICTIONARY)) { 154 NOTREACHED(); 155 return; 156 } 157 // The first level dictionary contains three sub-dict, first for source 158 // languages and second for target languages, we want to use the target 159 // languages. The last is for alpha languages. 160 DictionaryValue* language_dict = 161 static_cast<DictionaryValue*>(json_value.get()); 162 DictionaryValue* target_languages = NULL; 163 if (!language_dict->GetDictionary(TranslateLanguageList::kTargetLanguagesKey, 164 &target_languages) || 165 target_languages == NULL) { 166 NOTREACHED(); 167 return; 168 } 169 170 const std::string& locale = g_browser_process->GetApplicationLocale(); 171 172 // Now we can clear language list. 173 target_language_set->clear(); 174 std::string message; 175 // ... and replace it with the values we just fetched from the server. 176 for (DictionaryValue::Iterator iter(*target_languages); 177 !iter.IsAtEnd(); 178 iter.Advance()) { 179 const std::string& lang = iter.key(); 180 if (!l10n_util::IsLocaleNameTranslated(lang.c_str(), locale)) { 181 TranslateBrowserMetrics::ReportUndisplayableLanguage(lang); 182 continue; 183 } 184 target_language_set->insert(lang); 185 if (message.empty()) 186 message += lang; 187 else 188 message += ", " + lang; 189 } 190 NotifyEvent(__LINE__, message); 191 192 // Get the alpha languages. The "al" parameter could be abandoned. 193 DictionaryValue* alpha_languages = NULL; 194 if (!language_dict->GetDictionary(TranslateLanguageList::kAlphaLanguagesKey, 195 &alpha_languages) || 196 alpha_languages == NULL) { 197 return; 198 } 199 200 // We assume that the alpha languages are included in the above target 201 // languages, and don't use UMA or NotifyEvent. 202 alpha_language_set->clear(); 203 for (DictionaryValue::Iterator iter(*alpha_languages); 204 !iter.IsAtEnd(); iter.Advance()) { 205 const std::string& lang = iter.key(); 206 if (!l10n_util::IsLocaleNameTranslated(lang.c_str(), locale)) 207 continue; 208 alpha_language_set->insert(lang); 209 } 210 } 211 212 } // namespace 213 214 // This must be kept in sync with the &cb= value in the kLanguageListFetchURL. 215 const char TranslateLanguageList::kLanguageListCallbackName[] = "sl("; 216 const char TranslateLanguageList::kTargetLanguagesKey[] = "tl"; 217 const char TranslateLanguageList::kAlphaLanguagesKey[] = "al"; 218 219 TranslateLanguageList::TranslateLanguageList() { 220 // We default to our hard coded list of languages in 221 // |kDefaultSupportedLanguages|. This list will be overriden by a server 222 // providing supported langauges list. 223 for (size_t i = 0; i < arraysize(kDefaultSupportedLanguages); ++i) 224 all_supported_languages_.insert(kDefaultSupportedLanguages[i]); 225 226 if (update_is_disabled) 227 return; 228 229 language_list_fetcher_.reset(new TranslateURLFetcher(kFetcherId)); 230 language_list_fetcher_->set_max_retry_on_5xx(kMaxRetryOn5xx); 231 } 232 233 TranslateLanguageList::~TranslateLanguageList() { 234 } 235 236 void TranslateLanguageList::GetSupportedLanguages( 237 std::vector<std::string>* languages) { 238 DCHECK(languages && languages->empty()); 239 std::set<std::string>::const_iterator iter = all_supported_languages_.begin(); 240 for (; iter != all_supported_languages_.end(); ++iter) 241 languages->push_back(*iter); 242 243 // Update language lists if they are not updated after Chrome was launched 244 // for later requests. 245 if (language_list_fetcher_.get()) 246 RequestLanguageList(); 247 } 248 249 std::string TranslateLanguageList::GetLanguageCode( 250 const std::string& chrome_locale) { 251 // Only remove the country code for country specific languages we don't 252 // support specifically yet. 253 if (IsSupportedLanguage(chrome_locale)) 254 return chrome_locale; 255 256 size_t hypen_index = chrome_locale.find('-'); 257 if (hypen_index == std::string::npos) 258 return chrome_locale; 259 return chrome_locale.substr(0, hypen_index); 260 } 261 262 bool TranslateLanguageList::IsSupportedLanguage(const std::string& language) { 263 return all_supported_languages_.count(language) != 0; 264 } 265 266 bool TranslateLanguageList::IsAlphaLanguage(const std::string& language) { 267 return alpha_languages_.count(language) != 0; 268 } 269 270 void TranslateLanguageList::RequestLanguageList() { 271 // If resource requests are not allowed, we'll get a callback when they are. 272 if (resource_request_allowed_notifier_.ResourceRequestsAllowed()) 273 OnResourceRequestsAllowed(); 274 } 275 276 void TranslateLanguageList::OnResourceRequestsAllowed() { 277 if (language_list_fetcher_.get() && 278 (language_list_fetcher_->state() == TranslateURLFetcher::IDLE || 279 language_list_fetcher_->state() == TranslateURLFetcher::FAILED)) { 280 GURL url = GURL(kLanguageListFetchURL); 281 url = TranslateURLUtil::AddHostLocaleToUrl(url); 282 url = TranslateURLUtil::AddApiKeyToUrl(url); 283 url = net::AppendQueryParameter(url, 284 kAlphaLanguageQueryName, 285 kAlphaLanguageQueryValue); 286 287 std::string message = base::StringPrintf( 288 "Language list includeing alpha languages fetch starts (URL: %s)", 289 url.spec().c_str()); 290 NotifyEvent(__LINE__, message); 291 292 bool result = language_list_fetcher_->Request( 293 url, 294 base::Bind(&TranslateLanguageList::OnLanguageListFetchComplete, 295 base::Unretained(this))); 296 if (!result) 297 NotifyEvent(__LINE__, "Request is omitted due to retry limit"); 298 } 299 } 300 301 // static 302 void TranslateLanguageList::DisableUpdate() { 303 update_is_disabled = true; 304 } 305 306 void TranslateLanguageList::OnLanguageListFetchComplete( 307 int id, 308 bool success, 309 const std::string& data) { 310 if (!success) { 311 // Since it fails just now, omit to schedule resource requests if 312 // ResourceRequestAllowedNotifier think it's ready. Otherwise, a callback 313 // will be invoked later to request resources again. 314 // The TranslateURLFetcher has a limit for retried requests and aborts 315 // re-try not to invoke OnLanguageListFetchComplete anymore if it's asked to 316 // re-try too many times. 317 NotifyEvent(__LINE__, "Failed to fetch languages"); 318 return; 319 } 320 321 NotifyEvent(__LINE__, "Language list is updated"); 322 323 DCHECK_EQ(kFetcherId, id); 324 325 SetSupportedLanguages(data, &all_supported_languages_, &alpha_languages_); 326 language_list_fetcher_.reset(); 327 328 last_updated_ = base::Time::Now(); 329 } 330
