1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/autocomplete/base_search_provider.h" 6 7 #include "base/i18n/case_conversion.h" 8 #include "base/i18n/icu_string_conversions.h" 9 #include "base/json/json_string_value_serializer.h" 10 #include "base/json/json_writer.h" 11 #include "base/prefs/pref_registry_simple.h" 12 #include "base/prefs/pref_service.h" 13 #include "base/strings/string_util.h" 14 #include "base/strings/utf_string_conversions.h" 15 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h" 16 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service.h" 17 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service_factory.h" 18 #include "chrome/browser/history/history_service.h" 19 #include "chrome/browser/history/history_service_factory.h" 20 #include "chrome/browser/omnibox/omnibox_field_trial.h" 21 #include "chrome/browser/profiles/profile.h" 22 #include "chrome/browser/search/instant_service.h" 23 #include "chrome/browser/search/instant_service_factory.h" 24 #include "chrome/browser/search/search.h" 25 #include "chrome/browser/search_engines/template_url.h" 26 #include "chrome/browser/search_engines/template_url_prepopulate_data.h" 27 #include "chrome/browser/search_engines/template_url_service.h" 28 #include "chrome/browser/search_engines/template_url_service_factory.h" 29 #include "chrome/browser/search_engines/ui_thread_search_terms_data.h" 30 #include "chrome/browser/sync/profile_sync_service.h" 31 #include "chrome/browser/sync/profile_sync_service_factory.h" 32 #include "chrome/common/pref_names.h" 33 #include "components/autocomplete/url_prefix.h" 34 #include "components/metrics/proto/omnibox_event.pb.h" 35 #include "components/metrics/proto/omnibox_input_type.pb.h" 36 #include "components/sync_driver/sync_prefs.h" 37 #include "components/url_fixer/url_fixer.h" 38 #include "content/public/common/url_constants.h" 39 #include "net/base/escape.h" 40 #include "net/base/net_util.h" 41 #include "net/base/registry_controlled_domains/registry_controlled_domain.h" 42 #include "net/http/http_response_headers.h" 43 #include "net/url_request/url_fetcher.h" 44 #include "net/url_request/url_fetcher_delegate.h" 45 #include "url/gurl.h" 46 47 using metrics::OmniboxEventProto; 48 49 namespace { 50 51 AutocompleteMatchType::Type GetAutocompleteMatchType(const std::string& type) { 52 if (type == "ENTITY") 53 return AutocompleteMatchType::SEARCH_SUGGEST_ENTITY; 54 if (type == "INFINITE") 55 return AutocompleteMatchType::SEARCH_SUGGEST_INFINITE; 56 if (type == "PERSONALIZED_QUERY") 57 return AutocompleteMatchType::SEARCH_SUGGEST_PERSONALIZED; 58 if (type == "PROFILE") 59 return AutocompleteMatchType::SEARCH_SUGGEST_PROFILE; 60 if (type == "NAVIGATION") 61 return AutocompleteMatchType::NAVSUGGEST; 62 if (type == "PERSONALIZED_NAVIGATION") 63 return AutocompleteMatchType::NAVSUGGEST_PERSONALIZED; 64 return AutocompleteMatchType::SEARCH_SUGGEST; 65 } 66 67 } // namespace 68 69 // SuggestionDeletionHandler ------------------------------------------------- 70 71 // This class handles making requests to the server in order to delete 72 // personalized suggestions. 73 class SuggestionDeletionHandler : public net::URLFetcherDelegate { 74 public: 75 typedef base::Callback<void(bool, SuggestionDeletionHandler*)> 76 DeletionCompletedCallback; 77 78 SuggestionDeletionHandler( 79 const std::string& deletion_url, 80 Profile* profile, 81 const DeletionCompletedCallback& callback); 82 83 virtual ~SuggestionDeletionHandler(); 84 85 private: 86 // net::URLFetcherDelegate: 87 virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE; 88 89 scoped_ptr<net::URLFetcher> deletion_fetcher_; 90 DeletionCompletedCallback callback_; 91 92 DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler); 93 }; 94 95 SuggestionDeletionHandler::SuggestionDeletionHandler( 96 const std::string& deletion_url, 97 Profile* profile, 98 const DeletionCompletedCallback& callback) : callback_(callback) { 99 GURL url(deletion_url); 100 DCHECK(url.is_valid()); 101 102 deletion_fetcher_.reset(net::URLFetcher::Create( 103 BaseSearchProvider::kDeletionURLFetcherID, 104 url, 105 net::URLFetcher::GET, 106 this)); 107 deletion_fetcher_->SetRequestContext(profile->GetRequestContext()); 108 deletion_fetcher_->Start(); 109 } 110 111 SuggestionDeletionHandler::~SuggestionDeletionHandler() { 112 } 113 114 void SuggestionDeletionHandler::OnURLFetchComplete( 115 const net::URLFetcher* source) { 116 DCHECK(source == deletion_fetcher_.get()); 117 callback_.Run( 118 source->GetStatus().is_success() && (source->GetResponseCode() == 200), 119 this); 120 } 121 122 // BaseSearchProvider --------------------------------------------------------- 123 124 // static 125 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1; 126 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2; 127 const int BaseSearchProvider::kDeletionURLFetcherID = 3; 128 129 BaseSearchProvider::BaseSearchProvider(AutocompleteProviderListener* listener, 130 Profile* profile, 131 AutocompleteProvider::Type type) 132 : AutocompleteProvider(listener, profile, type), 133 field_trial_triggered_(false), 134 field_trial_triggered_in_session_(false), 135 suggest_results_pending_(0), 136 in_app_list_(false) { 137 } 138 139 // static 140 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) { 141 return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue; 142 } 143 144 // static 145 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion( 146 const base::string16& suggestion, 147 AutocompleteMatchType::Type type, 148 bool from_keyword_provider, 149 const TemplateURL* template_url, 150 const SearchTermsData& search_terms_data) { 151 return CreateSearchSuggestion( 152 NULL, AutocompleteInput(), BaseSearchProvider::SuggestResult( 153 suggestion, type, suggestion, base::string16(), base::string16(), 154 base::string16(), base::string16(), std::string(), std::string(), 155 from_keyword_provider, 0, false, false, base::string16()), 156 template_url, search_terms_data, 0, 0, false, false); 157 } 158 159 void BaseSearchProvider::Stop(bool clear_cached_results) { 160 StopSuggest(); 161 done_ = true; 162 163 if (clear_cached_results) 164 ClearAllResults(); 165 } 166 167 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) { 168 DCHECK(match.deletable); 169 if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) { 170 deletion_handlers_.push_back(new SuggestionDeletionHandler( 171 match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey), 172 profile_, 173 base::Bind(&BaseSearchProvider::OnDeletionComplete, 174 base::Unretained(this)))); 175 } 176 177 HistoryService* const history_service = 178 HistoryServiceFactory::GetForProfile(profile_, Profile::EXPLICIT_ACCESS); 179 TemplateURL* template_url = match.GetTemplateURL(profile_, false); 180 // This may be NULL if the template corresponding to the keyword has been 181 // deleted or there is no keyword set. 182 if (template_url != NULL) { 183 history_service->DeleteMatchingURLsForKeyword(template_url->id(), 184 match.contents); 185 } 186 187 // Immediately update the list of matches to show the match was deleted, 188 // regardless of whether the server request actually succeeds. 189 DeleteMatchFromMatches(match); 190 } 191 192 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const { 193 provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo()); 194 metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back(); 195 new_entry.set_provider(AsOmniboxEventProviderType()); 196 new_entry.set_provider_done(done_); 197 std::vector<uint32> field_trial_hashes; 198 OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes); 199 for (size_t i = 0; i < field_trial_hashes.size(); ++i) { 200 if (field_trial_triggered_) 201 new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]); 202 if (field_trial_triggered_in_session_) { 203 new_entry.mutable_field_trial_triggered_in_session()->Add( 204 field_trial_hashes[i]); 205 } 206 } 207 ModifyProviderInfo(&new_entry); 208 } 209 210 // static 211 const char BaseSearchProvider::kRelevanceFromServerKey[] = 212 "relevance_from_server"; 213 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch"; 214 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata"; 215 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url"; 216 const char BaseSearchProvider::kTrue[] = "true"; 217 const char BaseSearchProvider::kFalse[] = "false"; 218 219 BaseSearchProvider::~BaseSearchProvider() {} 220 221 // BaseSearchProvider::Result -------------------------------------------------- 222 223 BaseSearchProvider::Result::Result(bool from_keyword_provider, 224 int relevance, 225 bool relevance_from_server, 226 AutocompleteMatchType::Type type, 227 const std::string& deletion_url) 228 : from_keyword_provider_(from_keyword_provider), 229 type_(type), 230 relevance_(relevance), 231 relevance_from_server_(relevance_from_server), 232 deletion_url_(deletion_url) {} 233 234 BaseSearchProvider::Result::~Result() {} 235 236 // BaseSearchProvider::SuggestResult ------------------------------------------- 237 238 BaseSearchProvider::SuggestResult::SuggestResult( 239 const base::string16& suggestion, 240 AutocompleteMatchType::Type type, 241 const base::string16& match_contents, 242 const base::string16& match_contents_prefix, 243 const base::string16& annotation, 244 const base::string16& answer_contents, 245 const base::string16& answer_type, 246 const std::string& suggest_query_params, 247 const std::string& deletion_url, 248 bool from_keyword_provider, 249 int relevance, 250 bool relevance_from_server, 251 bool should_prefetch, 252 const base::string16& input_text) 253 : Result(from_keyword_provider, 254 relevance, 255 relevance_from_server, 256 type, 257 deletion_url), 258 suggestion_(suggestion), 259 match_contents_prefix_(match_contents_prefix), 260 annotation_(annotation), 261 suggest_query_params_(suggest_query_params), 262 answer_contents_(answer_contents), 263 answer_type_(answer_type), 264 should_prefetch_(should_prefetch) { 265 match_contents_ = match_contents; 266 DCHECK(!match_contents_.empty()); 267 ClassifyMatchContents(true, input_text); 268 } 269 270 BaseSearchProvider::SuggestResult::~SuggestResult() {} 271 272 void BaseSearchProvider::SuggestResult::ClassifyMatchContents( 273 const bool allow_bolding_all, 274 const base::string16& input_text) { 275 if (input_text.empty()) { 276 // In case of zero-suggest results, do not highlight matches. 277 match_contents_class_.push_back( 278 ACMatchClassification(0, ACMatchClassification::NONE)); 279 return; 280 } 281 282 base::string16 lookup_text = input_text; 283 if (type_ == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) { 284 const size_t contents_index = 285 suggestion_.length() - match_contents_.length(); 286 // Ensure the query starts with the input text, and ends with the match 287 // contents, and the input text has an overlap with contents. 288 if (StartsWith(suggestion_, input_text, true) && 289 EndsWith(suggestion_, match_contents_, true) && 290 (input_text.length() > contents_index)) { 291 lookup_text = input_text.substr(contents_index); 292 } 293 } 294 size_t lookup_position = match_contents_.find(lookup_text); 295 if (!allow_bolding_all && (lookup_position == base::string16::npos)) { 296 // Bail if the code below to update the bolding would bold the whole 297 // string. Note that the string may already be entirely bolded; if 298 // so, leave it as is. 299 return; 300 } 301 match_contents_class_.clear(); 302 // We do intra-string highlighting for suggestions - the suggested segment 303 // will be highlighted, e.g. for input_text = "you" the suggestion may be 304 // "youtube", so we'll bold the "tube" section: you*tube*. 305 if (input_text != match_contents_) { 306 if (lookup_position == base::string16::npos) { 307 // The input text is not a substring of the query string, e.g. input 308 // text is "slasdot" and the query string is "slashdot", so we bold the 309 // whole thing. 310 match_contents_class_.push_back( 311 ACMatchClassification(0, ACMatchClassification::MATCH)); 312 } else { 313 // We don't iterate over the string here annotating all matches because 314 // it looks odd to have every occurrence of a substring that may be as 315 // short as a single character highlighted in a query suggestion result, 316 // e.g. for input text "s" and query string "southwest airlines", it 317 // looks odd if both the first and last s are highlighted. 318 if (lookup_position != 0) { 319 match_contents_class_.push_back( 320 ACMatchClassification(0, ACMatchClassification::MATCH)); 321 } 322 match_contents_class_.push_back( 323 ACMatchClassification(lookup_position, ACMatchClassification::NONE)); 324 size_t next_fragment_position = lookup_position + lookup_text.length(); 325 if (next_fragment_position < match_contents_.length()) { 326 match_contents_class_.push_back(ACMatchClassification( 327 next_fragment_position, ACMatchClassification::MATCH)); 328 } 329 } 330 } else { 331 // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either 332 // for the default provider or a keyword search provider. 333 match_contents_class_.push_back( 334 ACMatchClassification(0, ACMatchClassification::NONE)); 335 } 336 } 337 338 bool BaseSearchProvider::SuggestResult::IsInlineable( 339 const base::string16& input) const { 340 return StartsWith(suggestion_, input, false); 341 } 342 343 int BaseSearchProvider::SuggestResult::CalculateRelevance( 344 const AutocompleteInput& input, 345 bool keyword_provider_requested) const { 346 if (!from_keyword_provider_ && keyword_provider_requested) 347 return 100; 348 return ((input.type() == metrics::OmniboxInputType::URL) ? 300 : 600); 349 } 350 351 // BaseSearchProvider::NavigationResult ---------------------------------------- 352 353 BaseSearchProvider::NavigationResult::NavigationResult( 354 const AutocompleteProvider& provider, 355 const GURL& url, 356 AutocompleteMatchType::Type type, 357 const base::string16& description, 358 const std::string& deletion_url, 359 bool from_keyword_provider, 360 int relevance, 361 bool relevance_from_server, 362 const base::string16& input_text, 363 const std::string& languages) 364 : Result(from_keyword_provider, 365 relevance, 366 relevance_from_server, 367 type, 368 deletion_url), 369 url_(url), 370 formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning( 371 url, 372 provider.StringForURLDisplay(url, true, false))), 373 description_(description) { 374 DCHECK(url_.is_valid()); 375 CalculateAndClassifyMatchContents(true, input_text, languages); 376 } 377 378 BaseSearchProvider::NavigationResult::~NavigationResult() {} 379 380 void BaseSearchProvider::NavigationResult::CalculateAndClassifyMatchContents( 381 const bool allow_bolding_nothing, 382 const base::string16& input_text, 383 const std::string& languages) { 384 if (input_text.empty()) { 385 // In case of zero-suggest results, do not highlight matches. 386 match_contents_class_.push_back( 387 ACMatchClassification(0, ACMatchClassification::NONE)); 388 return; 389 } 390 391 // First look for the user's input inside the formatted url as it would be 392 // without trimming the scheme, so we can find matches at the beginning of the 393 // scheme. 394 const URLPrefix* prefix = 395 URLPrefix::BestURLPrefix(formatted_url_, input_text); 396 size_t match_start = (prefix == NULL) ? 397 formatted_url_.find(input_text) : prefix->prefix.length(); 398 bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) && 399 (!prefix || (match_start != 0)); 400 const net::FormatUrlTypes format_types = 401 net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP); 402 403 base::string16 match_contents = net::FormatUrl(url_, languages, format_types, 404 net::UnescapeRule::SPACES, NULL, NULL, &match_start); 405 // If the first match in the untrimmed string was inside a scheme that we 406 // trimmed, look for a subsequent match. 407 if (match_start == base::string16::npos) 408 match_start = match_contents.find(input_text); 409 // Update |match_contents_| and |match_contents_class_| if it's allowed. 410 if (allow_bolding_nothing || (match_start != base::string16::npos)) { 411 match_contents_ = match_contents; 412 // Safe if |match_start| is npos; also safe if the input is longer than the 413 // remaining contents after |match_start|. 414 AutocompleteMatch::ClassifyLocationInString(match_start, 415 input_text.length(), match_contents_.length(), 416 ACMatchClassification::URL, &match_contents_class_); 417 } 418 } 419 420 bool BaseSearchProvider::NavigationResult::IsInlineable( 421 const base::string16& input) const { 422 return 423 URLPrefix::BestURLPrefix(base::UTF8ToUTF16(url_.spec()), input) != NULL; 424 } 425 426 int BaseSearchProvider::NavigationResult::CalculateRelevance( 427 const AutocompleteInput& input, 428 bool keyword_provider_requested) const { 429 return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150; 430 } 431 432 // BaseSearchProvider::Results ------------------------------------------------- 433 434 BaseSearchProvider::Results::Results() : verbatim_relevance(-1) {} 435 436 BaseSearchProvider::Results::~Results() {} 437 438 void BaseSearchProvider::Results::Clear() { 439 suggest_results.clear(); 440 navigation_results.clear(); 441 verbatim_relevance = -1; 442 metadata.clear(); 443 } 444 445 bool BaseSearchProvider::Results::HasServerProvidedScores() const { 446 if (verbatim_relevance >= 0) 447 return true; 448 449 // Right now either all results of one type will be server-scored or they will 450 // all be locally scored, but in case we change this later, we'll just check 451 // them all. 452 for (SuggestResults::const_iterator i(suggest_results.begin()); 453 i != suggest_results.end(); ++i) { 454 if (i->relevance_from_server()) 455 return true; 456 } 457 for (NavigationResults::const_iterator i(navigation_results.begin()); 458 i != navigation_results.end(); ++i) { 459 if (i->relevance_from_server()) 460 return true; 461 } 462 463 return false; 464 } 465 466 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url, 467 AutocompleteMatch* match) { 468 if (deletion_url.empty()) 469 return; 470 TemplateURLService* template_service = 471 TemplateURLServiceFactory::GetForProfile(profile_); 472 if (!template_service) 473 return; 474 GURL url = template_service->GetDefaultSearchProvider()->GenerateSearchURL( 475 template_service->search_terms_data()); 476 url = url.GetOrigin().Resolve(deletion_url); 477 if (url.is_valid()) { 478 match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey, 479 url.spec()); 480 match->deletable = true; 481 } 482 } 483 484 // BaseSearchProvider --------------------------------------------------------- 485 486 // static 487 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion( 488 AutocompleteProvider* autocomplete_provider, 489 const AutocompleteInput& input, 490 const SuggestResult& suggestion, 491 const TemplateURL* template_url, 492 const SearchTermsData& search_terms_data, 493 int accepted_suggestion, 494 int omnibox_start_margin, 495 bool append_extra_query_params, 496 bool from_app_list) { 497 AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false, 498 suggestion.type()); 499 500 if (!template_url) 501 return match; 502 match.keyword = template_url->keyword(); 503 match.contents = suggestion.match_contents(); 504 match.contents_class = suggestion.match_contents_class(); 505 match.answer_contents = suggestion.answer_contents(); 506 match.answer_type = suggestion.answer_type(); 507 if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) { 508 match.RecordAdditionalInfo( 509 kACMatchPropertyInputText, base::UTF16ToUTF8(input.text())); 510 match.RecordAdditionalInfo( 511 kACMatchPropertyContentsPrefix, 512 base::UTF16ToUTF8(suggestion.match_contents_prefix())); 513 match.RecordAdditionalInfo( 514 kACMatchPropertyContentsStartIndex, 515 static_cast<int>( 516 suggestion.suggestion().length() - match.contents.length())); 517 } 518 519 if (!suggestion.annotation().empty()) 520 match.description = suggestion.annotation(); 521 522 // suggestion.match_contents() should have already been collapsed. 523 match.allowed_to_be_default_match = 524 (base::CollapseWhitespace(input.text(), false) == 525 suggestion.match_contents()); 526 527 // When the user forced a query, we need to make sure all the fill_into_edit 528 // values preserve that property. Otherwise, if the user starts editing a 529 // suggestion, non-Search results will suddenly appear. 530 if (input.type() == metrics::OmniboxInputType::FORCED_QUERY) 531 match.fill_into_edit.assign(base::ASCIIToUTF16("?")); 532 if (suggestion.from_keyword_provider()) 533 match.fill_into_edit.append(match.keyword + base::char16(' ')); 534 if (!input.prevent_inline_autocomplete() && 535 StartsWith(suggestion.suggestion(), input.text(), false)) { 536 match.inline_autocompletion = 537 suggestion.suggestion().substr(input.text().length()); 538 match.allowed_to_be_default_match = true; 539 } 540 match.fill_into_edit.append(suggestion.suggestion()); 541 542 const TemplateURLRef& search_url = template_url->url_ref(); 543 DCHECK(search_url.SupportsReplacement(search_terms_data)); 544 match.search_terms_args.reset( 545 new TemplateURLRef::SearchTermsArgs(suggestion.suggestion())); 546 match.search_terms_args->original_query = input.text(); 547 match.search_terms_args->accepted_suggestion = accepted_suggestion; 548 match.search_terms_args->omnibox_start_margin = omnibox_start_margin; 549 match.search_terms_args->suggest_query_params = 550 suggestion.suggest_query_params(); 551 match.search_terms_args->append_extra_query_params = 552 append_extra_query_params; 553 match.search_terms_args->from_app_list = from_app_list; 554 // This is the destination URL sans assisted query stats. This must be set 555 // so the AutocompleteController can properly de-dupe; the controller will 556 // eventually overwrite it before it reaches the user. 557 match.destination_url = 558 GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(), 559 search_terms_data)); 560 561 // Search results don't look like URLs. 562 match.transition = suggestion.from_keyword_provider() ? 563 content::PAGE_TRANSITION_KEYWORD : content::PAGE_TRANSITION_GENERATED; 564 565 return match; 566 } 567 568 // static 569 scoped_ptr<base::Value> BaseSearchProvider::DeserializeJsonData( 570 std::string json_data) { 571 // The JSON response should be an array. 572 for (size_t response_start_index = json_data.find("["), i = 0; 573 response_start_index != std::string::npos && i < 5; 574 response_start_index = json_data.find("[", 1), i++) { 575 // Remove any XSSI guards to allow for JSON parsing. 576 if (response_start_index > 0) 577 json_data.erase(0, response_start_index); 578 579 JSONStringValueSerializer deserializer(json_data); 580 deserializer.set_allow_trailing_comma(true); 581 int error_code = 0; 582 scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL)); 583 if (error_code == 0) 584 return data.Pass(); 585 } 586 return scoped_ptr<base::Value>(); 587 } 588 589 // static 590 bool BaseSearchProvider::ZeroSuggestEnabled( 591 const GURL& suggest_url, 592 const TemplateURL* template_url, 593 OmniboxEventProto::PageClassification page_classification, 594 Profile* profile) { 595 if (!OmniboxFieldTrial::InZeroSuggestFieldTrial()) 596 return false; 597 598 // Make sure we are sending the suggest request through HTTPS to prevent 599 // exposing the current page URL or personalized results without encryption. 600 if (!suggest_url.SchemeIs(url::kHttpsScheme)) 601 return false; 602 603 // Don't show zero suggest on the NTP. 604 // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP 605 // under the conditions described in crbug.com/305366. 606 if ((page_classification == 607 OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) || 608 (page_classification == 609 OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS)) 610 return false; 611 612 // Don't run if there's no profile or in incognito mode. 613 if (profile == NULL || profile->IsOffTheRecord()) 614 return false; 615 616 // Don't run if we can't get preferences or search suggest is not enabled. 617 PrefService* prefs = profile->GetPrefs(); 618 if (!prefs->GetBoolean(prefs::kSearchSuggestEnabled)) 619 return false; 620 621 // Only make the request if we know that the provider supports zero suggest 622 // (currently only the prepopulated Google provider). 623 UIThreadSearchTermsData search_terms_data(profile); 624 if (template_url == NULL || 625 !template_url->SupportsReplacement(search_terms_data) || 626 TemplateURLPrepopulateData::GetEngineType( 627 *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE) 628 return false; 629 630 return true; 631 } 632 633 // static 634 bool BaseSearchProvider::CanSendURL( 635 const GURL& current_page_url, 636 const GURL& suggest_url, 637 const TemplateURL* template_url, 638 OmniboxEventProto::PageClassification page_classification, 639 Profile* profile) { 640 if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification, 641 profile)) 642 return false; 643 644 if (!current_page_url.is_valid()) 645 return false; 646 647 // Only allow HTTP URLs or HTTPS URLs for the same domain as the search 648 // provider. 649 if ((current_page_url.scheme() != url::kHttpScheme) && 650 ((current_page_url.scheme() != url::kHttpsScheme) || 651 !net::registry_controlled_domains::SameDomainOrHost( 652 current_page_url, suggest_url, 653 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES))) 654 return false; 655 656 // Check field trials and settings allow sending the URL on suggest requests. 657 ProfileSyncService* service = 658 ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile); 659 sync_driver::SyncPrefs sync_prefs(profile->GetPrefs()); 660 if (service == NULL || 661 !service->IsSyncEnabledAndLoggedIn() || 662 !sync_prefs.GetPreferredDataTypes(syncer::UserTypes()).Has( 663 syncer::PROXY_TABS) || 664 service->GetEncryptedDataTypes().Has(syncer::SESSIONS)) 665 return false; 666 667 return true; 668 } 669 670 void BaseSearchProvider::OnURLFetchComplete(const net::URLFetcher* source) { 671 DCHECK(!done_); 672 suggest_results_pending_--; 673 DCHECK_GE(suggest_results_pending_, 0); // Should never go negative. 674 675 const bool is_keyword = IsKeywordFetcher(source); 676 677 // Ensure the request succeeded and that the provider used is still available. 678 // A verbatim match cannot be generated without this provider, causing errors. 679 const bool request_succeeded = 680 source->GetStatus().is_success() && (source->GetResponseCode() == 200) && 681 GetTemplateURL(is_keyword); 682 683 LogFetchComplete(request_succeeded, is_keyword); 684 685 bool results_updated = false; 686 if (request_succeeded) { 687 const net::HttpResponseHeaders* const response_headers = 688 source->GetResponseHeaders(); 689 std::string json_data; 690 source->GetResponseAsString(&json_data); 691 692 // JSON is supposed to be UTF-8, but some suggest service providers send 693 // JSON files in non-UTF-8 encodings. The actual encoding is usually 694 // specified in the Content-Type header field. 695 if (response_headers) { 696 std::string charset; 697 if (response_headers->GetCharset(&charset)) { 698 base::string16 data_16; 699 // TODO(jungshik): Switch to CodePageToUTF8 after it's added. 700 if (base::CodepageToUTF16(json_data, charset.c_str(), 701 base::OnStringConversionError::FAIL, 702 &data_16)) 703 json_data = base::UTF16ToUTF8(data_16); 704 } 705 } 706 707 scoped_ptr<base::Value> data(DeserializeJsonData(json_data)); 708 if (data && StoreSuggestionResponse(json_data, *data.get())) 709 return; 710 711 results_updated = data.get() && ParseSuggestResults( 712 *data.get(), is_keyword, GetResultsToFill(is_keyword)); 713 } 714 715 UpdateMatches(); 716 if (done_ || results_updated) 717 listener_->OnProviderUpdate(results_updated); 718 } 719 720 void BaseSearchProvider::AddMatchToMap(const SuggestResult& result, 721 const std::string& metadata, 722 int accepted_suggestion, 723 bool mark_as_deletable, 724 MatchMap* map) { 725 InstantService* instant_service = 726 InstantServiceFactory::GetForProfile(profile_); 727 // Android and iOS have no InstantService. 728 const int omnibox_start_margin = instant_service ? 729 instant_service->omnibox_start_margin() : chrome::kDisableStartMargin; 730 731 AutocompleteMatch match = CreateSearchSuggestion( 732 this, GetInput(result.from_keyword_provider()), result, 733 GetTemplateURL(result.from_keyword_provider()), 734 UIThreadSearchTermsData(profile_), accepted_suggestion, 735 omnibox_start_margin, ShouldAppendExtraParams(result), 736 in_app_list_); 737 if (!match.destination_url.is_valid()) 738 return; 739 match.search_terms_args->bookmark_bar_pinned = 740 profile_->GetPrefs()->GetBoolean(prefs::kShowBookmarkBar); 741 match.RecordAdditionalInfo(kRelevanceFromServerKey, 742 result.relevance_from_server() ? kTrue : kFalse); 743 match.RecordAdditionalInfo(kShouldPrefetchKey, 744 result.should_prefetch() ? kTrue : kFalse); 745 SetDeletionURL(result.deletion_url(), &match); 746 if (mark_as_deletable) 747 match.deletable = true; 748 // Metadata is needed only for prefetching queries. 749 if (result.should_prefetch()) 750 match.RecordAdditionalInfo(kSuggestMetadataKey, metadata); 751 752 // Try to add |match| to |map|. If a match for this suggestion is 753 // already in |map|, replace it if |match| is more relevant. 754 // NOTE: Keep this ToLower() call in sync with url_database.cc. 755 MatchKey match_key( 756 std::make_pair(base::i18n::ToLower(result.suggestion()), 757 match.search_terms_args->suggest_query_params)); 758 const std::pair<MatchMap::iterator, bool> i( 759 map->insert(std::make_pair(match_key, match))); 760 761 bool should_prefetch = result.should_prefetch(); 762 if (!i.second) { 763 // NOTE: We purposefully do a direct relevance comparison here instead of 764 // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items 765 // added first" rather than "items alphabetically first" when the scores 766 // are equal. The only case this matters is when a user has results with 767 // the same score that differ only by capitalization; because the history 768 // system returns results sorted by recency, this means we'll pick the most 769 // recent such result even if the precision of our relevance score is too 770 // low to distinguish the two. 771 if (match.relevance > i.first->second.relevance) { 772 match.duplicate_matches.insert(match.duplicate_matches.end(), 773 i.first->second.duplicate_matches.begin(), 774 i.first->second.duplicate_matches.end()); 775 i.first->second.duplicate_matches.clear(); 776 match.duplicate_matches.push_back(i.first->second); 777 i.first->second = match; 778 } else { 779 i.first->second.duplicate_matches.push_back(match); 780 if (match.keyword == i.first->second.keyword) { 781 // Old and new matches are from the same search provider. It is okay to 782 // record one match's prefetch data onto a different match (for the same 783 // query string) for the following reasons: 784 // 1. Because the suggest server only sends down a query string from 785 // which we construct a URL, rather than sending a full URL, and because 786 // we construct URLs from query strings in the same way every time, the 787 // URLs for the two matches will be the same. Therefore, we won't end up 788 // prefetching something the server didn't intend. 789 // 2. Presumably the server sets the prefetch bit on a match it things 790 // is sufficiently relevant that the user is likely to choose it. 791 // Surely setting the prefetch bit on a match of even higher relevance 792 // won't violate this assumption. 793 should_prefetch |= ShouldPrefetch(i.first->second); 794 i.first->second.RecordAdditionalInfo(kShouldPrefetchKey, 795 should_prefetch ? kTrue : kFalse); 796 if (should_prefetch) 797 i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata); 798 } 799 } 800 } 801 } 802 803 bool BaseSearchProvider::ParseSuggestResults(const base::Value& root_val, 804 bool is_keyword_result, 805 Results* results) { 806 base::string16 query; 807 const base::ListValue* root_list = NULL; 808 const base::ListValue* results_list = NULL; 809 const AutocompleteInput& input = GetInput(is_keyword_result); 810 811 if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) || 812 query != input.text() || !root_list->GetList(1, &results_list)) 813 return false; 814 815 // 3rd element: Description list. 816 const base::ListValue* descriptions = NULL; 817 root_list->GetList(2, &descriptions); 818 819 // 4th element: Disregard the query URL list for now. 820 821 // Reset suggested relevance information. 822 results->verbatim_relevance = -1; 823 824 // 5th element: Optional key-value pairs from the Suggest server. 825 const base::ListValue* types = NULL; 826 const base::ListValue* relevances = NULL; 827 const base::ListValue* suggestion_details = NULL; 828 const base::DictionaryValue* extras = NULL; 829 int prefetch_index = -1; 830 if (root_list->GetDictionary(4, &extras)) { 831 extras->GetList("google:suggesttype", &types); 832 833 // Discard this list if its size does not match that of the suggestions. 834 if (extras->GetList("google:suggestrelevance", &relevances) && 835 (relevances->GetSize() != results_list->GetSize())) 836 relevances = NULL; 837 extras->GetInteger("google:verbatimrelevance", 838 &results->verbatim_relevance); 839 840 // Check if the active suggest field trial (if any) has triggered either 841 // for the default provider or keyword provider. 842 bool triggered = false; 843 extras->GetBoolean("google:fieldtrialtriggered", &triggered); 844 field_trial_triggered_ |= triggered; 845 field_trial_triggered_in_session_ |= triggered; 846 847 const base::DictionaryValue* client_data = NULL; 848 if (extras->GetDictionary("google:clientdata", &client_data) && client_data) 849 client_data->GetInteger("phi", &prefetch_index); 850 851 if (extras->GetList("google:suggestdetail", &suggestion_details) && 852 suggestion_details->GetSize() != results_list->GetSize()) 853 suggestion_details = NULL; 854 855 // Store the metadata that came with the response in case we need to pass it 856 // along with the prefetch query to Instant. 857 JSONStringValueSerializer json_serializer(&results->metadata); 858 json_serializer.Serialize(*extras); 859 } 860 861 // Clear the previous results now that new results are available. 862 results->suggest_results.clear(); 863 results->navigation_results.clear(); 864 865 base::string16 suggestion; 866 std::string type; 867 int relevance = GetDefaultResultRelevance(); 868 // Prohibit navsuggest in FORCED_QUERY mode. Users wants queries, not URLs. 869 const bool allow_navsuggest = 870 input.type() != metrics::OmniboxInputType::FORCED_QUERY; 871 const std::string languages( 872 profile_->GetPrefs()->GetString(prefs::kAcceptLanguages)); 873 const base::string16& trimmed_input = 874 base::CollapseWhitespace(input.text(), false); 875 for (size_t index = 0; results_list->GetString(index, &suggestion); ++index) { 876 // Google search may return empty suggestions for weird input characters, 877 // they make no sense at all and can cause problems in our code. 878 if (suggestion.empty()) 879 continue; 880 881 // Apply valid suggested relevance scores; discard invalid lists. 882 if (relevances != NULL && !relevances->GetInteger(index, &relevance)) 883 relevances = NULL; 884 AutocompleteMatchType::Type match_type = 885 AutocompleteMatchType::SEARCH_SUGGEST; 886 if (types && types->GetString(index, &type)) 887 match_type = GetAutocompleteMatchType(type); 888 const base::DictionaryValue* suggestion_detail = NULL; 889 std::string deletion_url; 890 891 if (suggestion_details && 892 suggestion_details->GetDictionary(index, &suggestion_detail)) 893 suggestion_detail->GetString("du", &deletion_url); 894 895 if ((match_type == AutocompleteMatchType::NAVSUGGEST) || 896 (match_type == AutocompleteMatchType::NAVSUGGEST_PERSONALIZED)) { 897 // Do not blindly trust the URL coming from the server to be valid. 898 GURL url( 899 url_fixer::FixupURL(base::UTF16ToUTF8(suggestion), std::string())); 900 if (url.is_valid() && allow_navsuggest) { 901 base::string16 title; 902 if (descriptions != NULL) 903 descriptions->GetString(index, &title); 904 results->navigation_results.push_back(NavigationResult( 905 *this, url, match_type, title, deletion_url, is_keyword_result, 906 relevance, relevances != NULL, input.text(), languages)); 907 } 908 } else { 909 base::string16 match_contents = suggestion; 910 base::string16 match_contents_prefix; 911 base::string16 annotation; 912 base::string16 answer_contents; 913 base::string16 answer_type; 914 std::string suggest_query_params; 915 916 if (suggestion_details) { 917 suggestion_details->GetDictionary(index, &suggestion_detail); 918 if (suggestion_detail) { 919 suggestion_detail->GetString("t", &match_contents); 920 suggestion_detail->GetString("mp", &match_contents_prefix); 921 // Error correction for bad data from server. 922 if (match_contents.empty()) 923 match_contents = suggestion; 924 suggestion_detail->GetString("a", &annotation); 925 suggestion_detail->GetString("q", &suggest_query_params); 926 927 // Extract Answers, if provided. 928 const base::DictionaryValue* answer_json = NULL; 929 if (suggestion_detail->GetDictionary("ansa", &answer_json)) { 930 match_type = AutocompleteMatchType::SEARCH_SUGGEST_ANSWER; 931 PrefetchAnswersImages(answer_json); 932 std::string contents; 933 base::JSONWriter::Write(answer_json, &contents); 934 answer_contents = base::UTF8ToUTF16(contents); 935 suggestion_detail->GetString("ansb", &answer_type); 936 } 937 } 938 } 939 940 bool should_prefetch = static_cast<int>(index) == prefetch_index; 941 // TODO(kochi): Improve calculator suggestion presentation. 942 results->suggest_results.push_back(SuggestResult( 943 base::CollapseWhitespace(suggestion, false), match_type, 944 base::CollapseWhitespace(match_contents, false), 945 match_contents_prefix, annotation, answer_contents, answer_type, 946 suggest_query_params, deletion_url, is_keyword_result, relevance, 947 relevances != NULL, should_prefetch, trimmed_input)); 948 } 949 } 950 SortResults(is_keyword_result, relevances, results); 951 return true; 952 } 953 954 void BaseSearchProvider::PrefetchAnswersImages( 955 const base::DictionaryValue* answer_json) { 956 DCHECK(answer_json); 957 const base::ListValue* lines = NULL; 958 answer_json->GetList("l", &lines); 959 if (!lines || lines->GetSize() == 0) 960 return; 961 962 BitmapFetcherService* image_service = 963 BitmapFetcherServiceFactory::GetForBrowserContext(profile_); 964 DCHECK(image_service); 965 966 for (size_t line = 0; line < lines->GetSize(); ++line) { 967 const base::DictionaryValue* imageLine = NULL; 968 lines->GetDictionary(line, &imageLine); 969 if (!imageLine) 970 continue; 971 const base::DictionaryValue* imageData = NULL; 972 imageLine->GetDictionary("i", &imageData); 973 if (!imageData) 974 continue; 975 std::string imageUrl; 976 imageData->GetString("d", &imageUrl); 977 image_service->Prefetch(GURL(imageUrl)); 978 } 979 } 980 981 void BaseSearchProvider::SortResults(bool is_keyword, 982 const base::ListValue* relevances, 983 Results* results) { 984 } 985 986 bool BaseSearchProvider::StoreSuggestionResponse( 987 const std::string& json_data, 988 const base::Value& parsed_data) { 989 return false; 990 } 991 992 void BaseSearchProvider::ModifyProviderInfo( 993 metrics::OmniboxEventProto_ProviderInfo* provider_info) const { 994 } 995 996 void BaseSearchProvider::DeleteMatchFromMatches( 997 const AutocompleteMatch& match) { 998 for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) { 999 // Find the desired match to delete by checking the type and contents. 1000 // We can't check the destination URL, because the autocomplete controller 1001 // may have reformulated that. Not that while checking for matching 1002 // contents works for personalized suggestions, if more match types gain 1003 // deletion support, this algorithm may need to be re-examined. 1004 if (i->contents == match.contents && i->type == match.type) { 1005 matches_.erase(i); 1006 break; 1007 } 1008 } 1009 } 1010 1011 void BaseSearchProvider::OnDeletionComplete( 1012 bool success, SuggestionDeletionHandler* handler) { 1013 RecordDeletionResult(success); 1014 SuggestionDeletionHandlers::iterator it = std::find( 1015 deletion_handlers_.begin(), deletion_handlers_.end(), handler); 1016 DCHECK(it != deletion_handlers_.end()); 1017 deletion_handlers_.erase(it); 1018 } 1019
