1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/autocomplete/zero_suggest_provider.h" 6 7 #include "base/callback.h" 8 #include "base/i18n/case_conversion.h" 9 #include "base/json/json_string_value_serializer.h" 10 #include "base/metrics/histogram.h" 11 #include "base/prefs/pref_service.h" 12 #include "base/strings/string16.h" 13 #include "base/strings/string_util.h" 14 #include "base/strings/utf_string_conversions.h" 15 #include "base/time/time.h" 16 #include "chrome/browser/autocomplete/autocomplete_classifier.h" 17 #include "chrome/browser/autocomplete/autocomplete_classifier_factory.h" 18 #include "chrome/browser/autocomplete/autocomplete_input.h" 19 #include "chrome/browser/autocomplete/autocomplete_match.h" 20 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h" 21 #include "chrome/browser/autocomplete/history_url_provider.h" 22 #include "chrome/browser/autocomplete/search_provider.h" 23 #include "chrome/browser/autocomplete/url_prefix.h" 24 #include "chrome/browser/google/google_util.h" 25 #include "chrome/browser/metrics/variations/variations_http_header_provider.h" 26 #include "chrome/browser/omnibox/omnibox_field_trial.h" 27 #include "chrome/browser/profiles/profile.h" 28 #include "chrome/browser/search/search.h" 29 #include "chrome/browser/search_engines/template_url_service.h" 30 #include "chrome/browser/search_engines/template_url_service_factory.h" 31 #include "chrome/browser/sync/profile_sync_service.h" 32 #include "chrome/browser/sync/profile_sync_service_factory.h" 33 #include "chrome/common/net/url_fixer_upper.h" 34 #include "chrome/common/pref_names.h" 35 #include "chrome/common/url_constants.h" 36 #include "net/base/escape.h" 37 #include "net/base/load_flags.h" 38 #include "net/base/net_util.h" 39 #include "net/http/http_request_headers.h" 40 #include "net/http/http_response_headers.h" 41 #include "net/url_request/url_fetcher.h" 42 #include "net/url_request/url_request_status.h" 43 #include "url/gurl.h" 44 45 namespace { 46 47 // TODO(hfung): The histogram code was copied and modified from 48 // search_provider.cc. Refactor and consolidate the code. 49 // We keep track in a histogram how many suggest requests we send, how 50 // many suggest requests we invalidate (e.g., due to a user typing 51 // another character), and how many replies we receive. 52 // *** ADD NEW ENUMS AFTER ALL PREVIOUSLY DEFINED ONES! *** 53 // (excluding the end-of-list enum value) 54 // We do not want values of existing enums to change or else it screws 55 // up the statistics. 56 enum ZeroSuggestRequestsHistogramValue { 57 ZERO_SUGGEST_REQUEST_SENT = 1, 58 ZERO_SUGGEST_REQUEST_INVALIDATED, 59 ZERO_SUGGEST_REPLY_RECEIVED, 60 ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE 61 }; 62 63 void LogOmniboxZeroSuggestRequest( 64 ZeroSuggestRequestsHistogramValue request_value) { 65 UMA_HISTOGRAM_ENUMERATION("Omnibox.ZeroSuggestRequests", request_value, 66 ZERO_SUGGEST_MAX_REQUEST_HISTOGRAM_VALUE); 67 } 68 69 // The maximum relevance of the top match from this provider. 70 const int kDefaultVerbatimZeroSuggestRelevance = 1300; 71 72 // Relevance value to use if it was not set explicitly by the server. 73 const int kDefaultZeroSuggestRelevance = 100; 74 75 } // namespace 76 77 // static 78 ZeroSuggestProvider* ZeroSuggestProvider::Create( 79 AutocompleteProviderListener* listener, 80 Profile* profile) { 81 return new ZeroSuggestProvider(listener, profile); 82 } 83 84 void ZeroSuggestProvider::Start(const AutocompleteInput& input, 85 bool /*minimal_changes*/) { 86 } 87 88 void ZeroSuggestProvider::Stop(bool clear_cached_results) { 89 if (have_pending_request_) 90 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_INVALIDATED); 91 have_pending_request_ = false; 92 fetcher_.reset(); 93 done_ = true; 94 if (clear_cached_results) { 95 query_matches_map_.clear(); 96 navigation_results_.clear(); 97 current_query_.clear(); 98 matches_.clear(); 99 } 100 } 101 102 void ZeroSuggestProvider::AddProviderInfo(ProvidersInfo* provider_info) const { 103 provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo()); 104 metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back(); 105 new_entry.set_provider(AsOmniboxEventProviderType()); 106 new_entry.set_provider_done(done_); 107 std::vector<uint32> field_trial_hashes; 108 OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes); 109 for (size_t i = 0; i < field_trial_hashes.size(); ++i) { 110 if (field_trial_triggered_) 111 new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]); 112 if (field_trial_triggered_in_session_) { 113 new_entry.mutable_field_trial_triggered_in_session()->Add( 114 field_trial_hashes[i]); 115 } 116 } 117 } 118 119 void ZeroSuggestProvider::ResetSession() { 120 // The user has started editing in the omnibox, so leave 121 // |field_trial_triggered_in_session_| unchanged and set 122 // |field_trial_triggered_| to false since zero suggest is inactive now. 123 field_trial_triggered_ = false; 124 Stop(true); 125 } 126 127 void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) { 128 have_pending_request_ = false; 129 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REPLY_RECEIVED); 130 131 std::string json_data; 132 source->GetResponseAsString(&json_data); 133 const bool request_succeeded = 134 source->GetStatus().is_success() && source->GetResponseCode() == 200; 135 136 bool have_results = false; 137 if (request_succeeded) { 138 JSONStringValueSerializer deserializer(json_data); 139 deserializer.set_allow_trailing_comma(true); 140 scoped_ptr<Value> data(deserializer.Deserialize(NULL, NULL)); 141 if (data.get()) { 142 ParseSuggestResults(*data.get()); 143 have_results = !query_matches_map_.empty() || 144 !navigation_results_.empty(); 145 } 146 } 147 done_ = true; 148 149 if (have_results) { 150 ConvertResultsToAutocompleteMatches(); 151 listener_->OnProviderUpdate(true); 152 } 153 } 154 155 void ZeroSuggestProvider::StartZeroSuggest( 156 const GURL& url, 157 AutocompleteInput::PageClassification page_classification, 158 const string16& permanent_text) { 159 Stop(true); 160 field_trial_triggered_ = false; 161 field_trial_triggered_in_session_ = false; 162 if (!ShouldRunZeroSuggest(url)) 163 return; 164 verbatim_relevance_ = kDefaultVerbatimZeroSuggestRelevance; 165 done_ = false; 166 permanent_text_ = permanent_text; 167 current_query_ = url.spec(); 168 current_page_classification_ = page_classification; 169 current_url_match_ = MatchForCurrentURL(); 170 // TODO(jered): Consider adding locally-sourced zero-suggestions here too. 171 // These may be useful on the NTP or more relevant to the user than server 172 // suggestions, if based on local browsing history. 173 Run(); 174 } 175 176 ZeroSuggestProvider::ZeroSuggestProvider( 177 AutocompleteProviderListener* listener, 178 Profile* profile) 179 : AutocompleteProvider(listener, profile, 180 AutocompleteProvider::TYPE_ZERO_SUGGEST), 181 template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)), 182 have_pending_request_(false), 183 verbatim_relevance_(kDefaultVerbatimZeroSuggestRelevance), 184 field_trial_triggered_(false), 185 field_trial_triggered_in_session_(false) { 186 } 187 188 ZeroSuggestProvider::~ZeroSuggestProvider() { 189 } 190 191 bool ZeroSuggestProvider::ShouldRunZeroSuggest(const GURL& url) const { 192 if (!ShouldSendURL(url)) 193 return false; 194 195 // Don't run if there's no profile or in incognito mode. 196 if (profile_ == NULL || profile_->IsOffTheRecord()) 197 return false; 198 199 // Don't run if we can't get preferences or search suggest is not enabled. 200 PrefService* prefs = profile_->GetPrefs(); 201 if (prefs == NULL || !prefs->GetBoolean(prefs::kSearchSuggestEnabled)) 202 return false; 203 204 ProfileSyncService* service = 205 ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile_); 206 browser_sync::SyncPrefs sync_prefs(prefs); 207 // The user has needs to have Chrome Sync enabled (for permissions to 208 // transmit their current URL) and be in the field trial. 209 if (!OmniboxFieldTrial::InZeroSuggestFieldTrial() || 210 service == NULL || 211 !service->IsSyncEnabledAndLoggedIn() || 212 !sync_prefs.HasKeepEverythingSynced()) { 213 return false; 214 } 215 return true; 216 } 217 218 bool ZeroSuggestProvider::ShouldSendURL(const GURL& url) const { 219 if (!url.is_valid()) 220 return false; 221 222 // Only allow HTTP URLs or Google HTTPS URLs (including Google search 223 // result pages). For the latter case, Google was already sent the HTTPS 224 // URLs when requesting the page, so the information is just re-sent. 225 return (url.scheme() == chrome::kHttpScheme) || 226 google_util::IsGoogleDomainUrl(url, google_util::ALLOW_SUBDOMAIN, 227 google_util::ALLOW_NON_STANDARD_PORTS); 228 } 229 230 void ZeroSuggestProvider::FillResults( 231 const Value& root_val, 232 int* verbatim_relevance, 233 SearchProvider::SuggestResults* suggest_results, 234 SearchProvider::NavigationResults* navigation_results) { 235 string16 query; 236 const ListValue* root_list = NULL; 237 const ListValue* results = NULL; 238 const ListValue* relevances = NULL; 239 // The response includes the query, which should be empty for ZeroSuggest 240 // responses. 241 if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) || 242 (!query.empty()) || !root_list->GetList(1, &results)) 243 return; 244 245 // 3rd element: Description list. 246 const ListValue* descriptions = NULL; 247 root_list->GetList(2, &descriptions); 248 249 // 4th element: Disregard the query URL list for now. 250 251 // Reset suggested relevance information from the provider. 252 *verbatim_relevance = kDefaultVerbatimZeroSuggestRelevance; 253 254 // 5th element: Optional key-value pairs from the Suggest server. 255 const ListValue* types = NULL; 256 const DictionaryValue* extras = NULL; 257 if (root_list->GetDictionary(4, &extras)) { 258 extras->GetList("google:suggesttype", &types); 259 260 // Discard this list if its size does not match that of the suggestions. 261 if (extras->GetList("google:suggestrelevance", &relevances) && 262 relevances->GetSize() != results->GetSize()) 263 relevances = NULL; 264 extras->GetInteger("google:verbatimrelevance", verbatim_relevance); 265 266 // Check if the active suggest field trial (if any) has triggered. 267 bool triggered = false; 268 extras->GetBoolean("google:fieldtrialtriggered", &triggered); 269 field_trial_triggered_ |= triggered; 270 field_trial_triggered_in_session_ |= triggered; 271 } 272 273 // Clear the previous results now that new results are available. 274 suggest_results->clear(); 275 navigation_results->clear(); 276 277 string16 result, title; 278 std::string type; 279 for (size_t index = 0; results->GetString(index, &result); ++index) { 280 // Google search may return empty suggestions for weird input characters, 281 // they make no sense at all and can cause problems in our code. 282 if (result.empty()) 283 continue; 284 285 int relevance = kDefaultZeroSuggestRelevance; 286 287 // Apply valid suggested relevance scores; discard invalid lists. 288 if (relevances != NULL && !relevances->GetInteger(index, &relevance)) 289 relevances = NULL; 290 if (types && types->GetString(index, &type) && (type == "NAVIGATION")) { 291 // Do not blindly trust the URL coming from the server to be valid. 292 GURL url(URLFixerUpper::FixupURL(UTF16ToUTF8(result), std::string())); 293 if (url.is_valid()) { 294 if (descriptions != NULL) 295 descriptions->GetString(index, &title); 296 navigation_results->push_back(SearchProvider::NavigationResult( 297 *this, url, title, false, relevance, relevances != NULL)); 298 } 299 } else { 300 suggest_results->push_back(SearchProvider::SuggestResult( 301 result, false, relevance, relevances != NULL)); 302 } 303 } 304 } 305 306 void ZeroSuggestProvider::AddSuggestResultsToMap( 307 const SearchProvider::SuggestResults& results, 308 const TemplateURL* template_url, 309 SearchProvider::MatchMap* map) { 310 for (size_t i = 0; i < results.size(); ++i) { 311 AddMatchToMap(results[i].relevance(), AutocompleteMatchType::SEARCH_SUGGEST, 312 template_url, results[i].suggestion(), i, map); 313 } 314 } 315 316 void ZeroSuggestProvider::AddMatchToMap(int relevance, 317 AutocompleteMatch::Type type, 318 const TemplateURL* template_url, 319 const string16& query_string, 320 int accepted_suggestion, 321 SearchProvider::MatchMap* map) { 322 // Pass in query_string as the input_text since we don't want any bolding. 323 // TODO(samarth|melevin): use the actual omnibox margin here as well instead 324 // of passing in -1. 325 AutocompleteMatch match = SearchProvider::CreateSearchSuggestion( 326 this, relevance, type, template_url, query_string, query_string, 327 AutocompleteInput(), false, accepted_suggestion, -1, true); 328 if (!match.destination_url.is_valid()) 329 return; 330 331 // Try to add |match| to |map|. If a match for |query_string| is already in 332 // |map|, replace it if |match| is more relevant. 333 // NOTE: Keep this ToLower() call in sync with url_database.cc. 334 const std::pair<SearchProvider::MatchMap::iterator, bool> i(map->insert( 335 std::make_pair(base::i18n::ToLower(query_string), match))); 336 // NOTE: We purposefully do a direct relevance comparison here instead of 337 // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items added 338 // first" rather than "items alphabetically first" when the scores are equal. 339 // The only case this matters is when a user has results with the same score 340 // that differ only by capitalization; because the history system returns 341 // results sorted by recency, this means we'll pick the most recent such 342 // result even if the precision of our relevance score is too low to 343 // distinguish the two. 344 if (!i.second && (match.relevance > i.first->second.relevance)) 345 i.first->second = match; 346 } 347 348 AutocompleteMatch ZeroSuggestProvider::NavigationToMatch( 349 const SearchProvider::NavigationResult& navigation) { 350 AutocompleteMatch match(this, navigation.relevance(), false, 351 AutocompleteMatchType::NAVSUGGEST); 352 match.destination_url = navigation.url(); 353 354 const std::string languages( 355 profile_->GetPrefs()->GetString(prefs::kAcceptLanguages)); 356 match.contents = net::FormatUrl(navigation.url(), languages, 357 net::kFormatUrlOmitAll, net::UnescapeRule::SPACES, NULL, NULL, NULL); 358 match.fill_into_edit += 359 AutocompleteInput::FormattedStringWithEquivalentMeaning(navigation.url(), 360 match.contents); 361 362 AutocompleteMatch::ClassifyLocationInString(string16::npos, 0, 363 match.contents.length(), ACMatchClassification::URL, 364 &match.contents_class); 365 366 match.description = 367 AutocompleteMatch::SanitizeString(navigation.description()); 368 AutocompleteMatch::ClassifyLocationInString(string16::npos, 0, 369 match.description.length(), ACMatchClassification::NONE, 370 &match.description_class); 371 return match; 372 } 373 374 void ZeroSuggestProvider::Run() { 375 have_pending_request_ = false; 376 const int kFetcherID = 1; 377 378 const TemplateURL* default_provider = 379 template_url_service_->GetDefaultSearchProvider(); 380 // TODO(hfung): Generalize if the default provider supports zero suggest. 381 // Only make the request if we know that the provider supports zero suggest 382 // (currently only the prepopulated Google provider). 383 if (default_provider == NULL || !default_provider->SupportsReplacement() || 384 default_provider->prepopulate_id() != 1) { 385 Stop(true); 386 return; 387 } 388 string16 prefix; 389 TemplateURLRef::SearchTermsArgs search_term_args(prefix); 390 search_term_args.zero_prefix_url = current_query_; 391 std::string req_url = default_provider->suggestions_url_ref(). 392 ReplaceSearchTerms(search_term_args); 393 GURL suggest_url(req_url); 394 // Make sure we are sending the suggest request through HTTPS. 395 if (!suggest_url.SchemeIs(chrome::kHttpsScheme)) { 396 Stop(true); 397 return; 398 } 399 400 fetcher_.reset( 401 net::URLFetcher::Create(kFetcherID, 402 suggest_url, 403 net::URLFetcher::GET, this)); 404 fetcher_->SetRequestContext(profile_->GetRequestContext()); 405 fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES); 406 // Add Chrome experiment state to the request headers. 407 net::HttpRequestHeaders headers; 408 chrome_variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders( 409 fetcher_->GetOriginalURL(), profile_->IsOffTheRecord(), false, &headers); 410 fetcher_->SetExtraRequestHeaders(headers.ToString()); 411 412 fetcher_->Start(); 413 have_pending_request_ = true; 414 LogOmniboxZeroSuggestRequest(ZERO_SUGGEST_REQUEST_SENT); 415 } 416 417 void ZeroSuggestProvider::ParseSuggestResults(const Value& root_val) { 418 SearchProvider::SuggestResults suggest_results; 419 FillResults(root_val, &verbatim_relevance_, 420 &suggest_results, &navigation_results_); 421 422 query_matches_map_.clear(); 423 AddSuggestResultsToMap(suggest_results, 424 template_url_service_->GetDefaultSearchProvider(), 425 &query_matches_map_); 426 } 427 428 void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() { 429 matches_.clear(); 430 431 const TemplateURL* default_provider = 432 template_url_service_->GetDefaultSearchProvider(); 433 // Fail if we can't set the clickthrough URL for query suggestions. 434 if (default_provider == NULL || !default_provider->SupportsReplacement()) 435 return; 436 437 const int num_query_results = query_matches_map_.size(); 438 const int num_nav_results = navigation_results_.size(); 439 const int num_results = num_query_results + num_nav_results; 440 UMA_HISTOGRAM_COUNTS("ZeroSuggest.QueryResults", num_query_results); 441 UMA_HISTOGRAM_COUNTS("ZeroSuggest.URLResults", num_nav_results); 442 UMA_HISTOGRAM_COUNTS("ZeroSuggest.AllResults", num_results); 443 444 if (num_results == 0) 445 return; 446 447 // TODO(jered): Rip this out once the first match is decoupled from the 448 // current typing in the omnibox. 449 matches_.push_back(current_url_match_); 450 451 for (SearchProvider::MatchMap::const_iterator it(query_matches_map_.begin()); 452 it != query_matches_map_.end(); ++it) 453 matches_.push_back(it->second); 454 455 for (SearchProvider::NavigationResults::const_iterator it( 456 navigation_results_.begin()); it != navigation_results_.end(); ++it) 457 matches_.push_back(NavigationToMatch(*it)); 458 } 459 460 AutocompleteMatch ZeroSuggestProvider::MatchForCurrentURL() { 461 AutocompleteInput input(permanent_text_, string16::npos, string16(), 462 GURL(current_query_), current_page_classification_, 463 false, false, true, AutocompleteInput::ALL_MATCHES); 464 465 AutocompleteMatch match; 466 AutocompleteClassifierFactory::GetForProfile(profile_)->Classify( 467 permanent_text_, false, true, &match, NULL); 468 match.is_history_what_you_typed_match = false; 469 match.allowed_to_be_default_match = true; 470 471 // The placeholder suggestion for the current URL has high relevance so 472 // that it is in the first suggestion slot and inline autocompleted. It 473 // gets dropped as soon as the user types something. 474 match.relevance = verbatim_relevance_; 475 476 return match; 477 } 478
