1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "components/translate/core/browser/translate_language_list.h" 6 7 #include <set> 8 9 #include "base/bind.h" 10 #include "base/json/json_reader.h" 11 #include "base/lazy_instance.h" 12 #include "base/logging.h" 13 #include "base/strings/string_util.h" 14 #include "base/strings/stringprintf.h" 15 #include "base/values.h" 16 #include "components/translate/core/browser/translate_browser_metrics.h" 17 #include "components/translate/core/browser/translate_download_manager.h" 18 #include "components/translate/core/browser/translate_event_details.h" 19 #include "components/translate/core/browser/translate_url_fetcher.h" 20 #include "components/translate/core/browser/translate_url_util.h" 21 #include "components/translate/core/common/translate_util.h" 22 #include "net/base/url_util.h" 23 #include "ui/base/l10n/l10n_util.h" 24 #include "url/gurl.h" 25 26 namespace { 27 28 // The default list of languages the Google translation server supports. 29 // We use this list until we receive the list that the server exposes. 30 // For information, here is the list of languages that Chrome can be run in 31 // but that the translation server does not support: 32 // am Amharic 33 // bn Bengali 34 // gu Gujarati 35 // kn Kannada 36 // ml Malayalam 37 // mr Marathi 38 // ta Tamil 39 // te Telugu 40 const char* const kDefaultSupportedLanguages[] = { 41 "af", // Afrikaans 42 "sq", // Albanian 43 "ar", // Arabic 44 "be", // Belarusian 45 "bg", // Bulgarian 46 "ca", // Catalan 47 "zh-CN", // Chinese (Simplified) 48 "zh-TW", // Chinese (Traditional) 49 "hr", // Croatian 50 "cs", // Czech 51 "da", // Danish 52 "nl", // Dutch 53 "en", // English 54 "eo", // Esperanto 55 "et", // Estonian 56 "tl", // Filipino 57 "fi", // Finnish 58 "fr", // French 59 "gl", // Galician 60 "de", // German 61 "el", // Greek 62 "ht", // Haitian Creole 63 "iw", // Hebrew 64 "hi", // Hindi 65 "hu", // Hungarian 66 "is", // Icelandic 67 "id", // Indonesian 68 "ga", // Irish 69 "it", // Italian 70 "ja", // Japanese 71 "ko", // Korean 72 "lv", // Latvian 73 "lt", // Lithuanian 74 "mk", // Macedonian 75 "ms", // Malay 76 "mt", // Maltese 77 "no", // Norwegian 78 "fa", // Persian 79 "pl", // Polish 80 "pt", // Portuguese 81 "ro", // Romanian 82 "ru", // Russian 83 "sr", // Serbian 84 "sk", // Slovak 85 "sl", // Slovenian 86 "es", // Spanish 87 "sw", // Swahili 88 "sv", // Swedish 89 "th", // Thai 90 "tr", // Turkish 91 "uk", // Ukrainian 92 "vi", // Vietnamese 93 "cy", // Welsh 94 "yi", // Yiddish 95 }; 96 97 // Constant URL string to fetch server supporting language list. 98 const char kLanguageListFetchPath[] = "translate_a/l?client=chrome&cb=sl"; 99 100 // Used in kTranslateScriptURL to request supporting languages list including 101 // "alpha languages". 102 const char kAlphaLanguageQueryName[] = "alpha"; 103 const char kAlphaLanguageQueryValue[] = "1"; 104 105 // Represent if the language list updater is disabled. 106 bool update_is_disabled = false; 107 108 // Retry parameter for fetching. 109 const int kMaxRetryOn5xx = 5; 110 111 } // namespace 112 113 // This must be kept in sync with the &cb= value in the kLanguageListFetchURL. 114 const char TranslateLanguageList::kLanguageListCallbackName[] = "sl("; 115 const char TranslateLanguageList::kTargetLanguagesKey[] = "tl"; 116 const char TranslateLanguageList::kAlphaLanguagesKey[] = "al"; 117 118 TranslateLanguageList::TranslateLanguageList() 119 : resource_requests_allowed_(false), request_pending_(false) { 120 // We default to our hard coded list of languages in 121 // |kDefaultSupportedLanguages|. This list will be overriden by a server 122 // providing supported langauges list. 123 for (size_t i = 0; i < arraysize(kDefaultSupportedLanguages); ++i) 124 all_supported_languages_.insert(kDefaultSupportedLanguages[i]); 125 126 if (update_is_disabled) 127 return; 128 129 language_list_fetcher_.reset(new TranslateURLFetcher(kFetcherId)); 130 language_list_fetcher_->set_max_retry_on_5xx(kMaxRetryOn5xx); 131 } 132 133 TranslateLanguageList::~TranslateLanguageList() {} 134 135 void TranslateLanguageList::GetSupportedLanguages( 136 std::vector<std::string>* languages) { 137 DCHECK(languages && languages->empty()); 138 std::set<std::string>::const_iterator iter = all_supported_languages_.begin(); 139 for (; iter != all_supported_languages_.end(); ++iter) 140 languages->push_back(*iter); 141 142 // Update language lists if they are not updated after Chrome was launched 143 // for later requests. 144 if (!update_is_disabled && language_list_fetcher_.get()) 145 RequestLanguageList(); 146 } 147 148 std::string TranslateLanguageList::GetLanguageCode( 149 const std::string& language) { 150 // Only remove the country code for country specific languages we don't 151 // support specifically yet. 152 if (IsSupportedLanguage(language)) 153 return language; 154 155 size_t hypen_index = language.find('-'); 156 if (hypen_index == std::string::npos) 157 return language; 158 return language.substr(0, hypen_index); 159 } 160 161 bool TranslateLanguageList::IsSupportedLanguage(const std::string& language) { 162 return all_supported_languages_.count(language) != 0; 163 } 164 165 bool TranslateLanguageList::IsAlphaLanguage(const std::string& language) { 166 return alpha_languages_.count(language) != 0; 167 } 168 169 GURL TranslateLanguageList::TranslateLanguageUrl() { 170 std::string url = translate::GetTranslateSecurityOrigin().spec() + 171 kLanguageListFetchPath; 172 return GURL(url); 173 } 174 175 void TranslateLanguageList::RequestLanguageList() { 176 // If resource requests are not allowed, we'll get a callback when they are. 177 if (!resource_requests_allowed_) { 178 request_pending_ = true; 179 return; 180 } 181 182 request_pending_ = false; 183 184 if (language_list_fetcher_.get() && 185 (language_list_fetcher_->state() == TranslateURLFetcher::IDLE || 186 language_list_fetcher_->state() == TranslateURLFetcher::FAILED)) { 187 GURL url = TranslateLanguageUrl(); 188 url = TranslateURLUtil::AddHostLocaleToUrl(url); 189 url = TranslateURLUtil::AddApiKeyToUrl(url); 190 url = net::AppendQueryParameter( 191 url, kAlphaLanguageQueryName, kAlphaLanguageQueryValue); 192 193 std::string message = base::StringPrintf( 194 "Language list including alpha languages fetch starts (URL: %s)", 195 url.spec().c_str()); 196 NotifyEvent(__LINE__, message); 197 198 bool result = language_list_fetcher_->Request( 199 url, 200 base::Bind(&TranslateLanguageList::OnLanguageListFetchComplete, 201 base::Unretained(this))); 202 if (!result) 203 NotifyEvent(__LINE__, "Request is omitted due to retry limit"); 204 } 205 } 206 207 void TranslateLanguageList::SetResourceRequestsAllowed(bool allowed) { 208 resource_requests_allowed_ = allowed; 209 if (resource_requests_allowed_ && request_pending_) { 210 RequestLanguageList(); 211 DCHECK(!request_pending_); 212 } 213 } 214 215 scoped_ptr<TranslateLanguageList::EventCallbackList::Subscription> 216 TranslateLanguageList::RegisterEventCallback(const EventCallback& callback) { 217 return callback_list_.Add(callback); 218 } 219 220 // static 221 void TranslateLanguageList::DisableUpdate() { 222 update_is_disabled = true; 223 } 224 225 void TranslateLanguageList::OnLanguageListFetchComplete( 226 int id, 227 bool success, 228 const std::string& data) { 229 if (!success) { 230 // Since it fails just now, omit to schedule resource requests if 231 // ResourceRequestAllowedNotifier think it's ready. Otherwise, a callback 232 // will be invoked later to request resources again. 233 // The TranslateURLFetcher has a limit for retried requests and aborts 234 // re-try not to invoke OnLanguageListFetchComplete anymore if it's asked to 235 // re-try too many times. 236 NotifyEvent(__LINE__, "Failed to fetch languages"); 237 return; 238 } 239 240 NotifyEvent(__LINE__, "Language list is updated"); 241 242 DCHECK_EQ(kFetcherId, id); 243 244 SetSupportedLanguages(data); 245 language_list_fetcher_.reset(); 246 247 last_updated_ = base::Time::Now(); 248 } 249 250 void TranslateLanguageList::NotifyEvent(int line, const std::string& message) { 251 TranslateEventDetails details(__FILE__, line, message); 252 callback_list_.Notify(details); 253 } 254 255 void TranslateLanguageList::SetSupportedLanguages( 256 const std::string& language_list) { 257 // The format is: 258 // sl({ 259 // "sl": {"XX": "LanguageName", ...}, 260 // "tl": {"XX": "LanguageName", ...}, 261 // "al": {"XX": 1, ...} 262 // }) 263 // Where "sl(" is set in kLanguageListCallbackName, "tl" is 264 // kTargetLanguagesKey and "al" kAlphaLanguagesKey. 265 if (!StartsWithASCII(language_list, 266 TranslateLanguageList::kLanguageListCallbackName, 267 false) || 268 !EndsWith(language_list, ")", false)) { 269 // We don't have a NOTREACHED here since this can happen in ui_tests, even 270 // though the the BrowserMain function won't call us with parameters.ui_task 271 // is NULL some tests don't set it, so we must bail here. 272 return; 273 } 274 static const size_t kLanguageListCallbackNameLength = 275 strlen(TranslateLanguageList::kLanguageListCallbackName); 276 std::string languages_json = language_list.substr( 277 kLanguageListCallbackNameLength, 278 language_list.size() - kLanguageListCallbackNameLength - 1); 279 scoped_ptr<base::Value> json_value( 280 base::JSONReader::Read(languages_json, base::JSON_ALLOW_TRAILING_COMMAS)); 281 if (json_value == NULL || !json_value->IsType(base::Value::TYPE_DICTIONARY)) { 282 NOTREACHED(); 283 return; 284 } 285 // The first level dictionary contains three sub-dict, first for source 286 // languages and second for target languages, we want to use the target 287 // languages. The last is for alpha languages. 288 base::DictionaryValue* language_dict = 289 static_cast<base::DictionaryValue*>(json_value.get()); 290 base::DictionaryValue* target_languages = NULL; 291 if (!language_dict->GetDictionary(TranslateLanguageList::kTargetLanguagesKey, 292 &target_languages) || 293 target_languages == NULL) { 294 NOTREACHED(); 295 return; 296 } 297 298 const std::string& locale = 299 TranslateDownloadManager::GetInstance()->application_locale(); 300 301 // Now we can clear language list. 302 all_supported_languages_.clear(); 303 std::string message; 304 // ... and replace it with the values we just fetched from the server. 305 for (base::DictionaryValue::Iterator iter(*target_languages); 306 !iter.IsAtEnd(); 307 iter.Advance()) { 308 const std::string& lang = iter.key(); 309 if (!l10n_util::IsLocaleNameTranslated(lang.c_str(), locale)) { 310 TranslateBrowserMetrics::ReportUndisplayableLanguage(lang); 311 continue; 312 } 313 all_supported_languages_.insert(lang); 314 if (message.empty()) 315 message += lang; 316 else 317 message += ", " + lang; 318 } 319 NotifyEvent(__LINE__, message); 320 321 // Get the alpha languages. The "al" parameter could be abandoned. 322 base::DictionaryValue* alpha_languages = NULL; 323 if (!language_dict->GetDictionary(TranslateLanguageList::kAlphaLanguagesKey, 324 &alpha_languages) || 325 alpha_languages == NULL) { 326 return; 327 } 328 329 // We assume that the alpha languages are included in the above target 330 // languages, and don't use UMA or NotifyEvent. 331 alpha_languages_.clear(); 332 for (base::DictionaryValue::Iterator iter(*alpha_languages); 333 !iter.IsAtEnd(); iter.Advance()) { 334 const std::string& lang = iter.key(); 335 if (!l10n_util::IsLocaleNameTranslated(lang.c_str(), locale)) 336 continue; 337 alpha_languages_.insert(lang); 338 } 339 } 340
