1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "components/omnibox/base_search_provider.h" 6 7 #include "base/i18n/case_conversion.h" 8 #include "base/strings/string_util.h" 9 #include "base/strings/utf_string_conversions.h" 10 #include "components/metrics/proto/omnibox_event.pb.h" 11 #include "components/metrics/proto/omnibox_input_type.pb.h" 12 #include "components/omnibox/autocomplete_provider_client.h" 13 #include "components/omnibox/autocomplete_provider_listener.h" 14 #include "components/omnibox/omnibox_field_trial.h" 15 #include "components/search_engines/template_url.h" 16 #include "components/search_engines/template_url_prepopulate_data.h" 17 #include "components/search_engines/template_url_service.h" 18 #include "net/base/registry_controlled_domains/registry_controlled_domain.h" 19 #include "net/url_request/url_fetcher.h" 20 #include "net/url_request/url_fetcher_delegate.h" 21 #include "url/gurl.h" 22 23 using metrics::OmniboxEventProto; 24 25 // SuggestionDeletionHandler ------------------------------------------------- 26 27 // This class handles making requests to the server in order to delete 28 // personalized suggestions. 29 class SuggestionDeletionHandler : public net::URLFetcherDelegate { 30 public: 31 typedef base::Callback<void(bool, SuggestionDeletionHandler*)> 32 DeletionCompletedCallback; 33 34 SuggestionDeletionHandler( 35 const std::string& deletion_url, 36 net::URLRequestContextGetter* request_context, 37 const DeletionCompletedCallback& callback); 38 39 virtual ~SuggestionDeletionHandler(); 40 41 private: 42 // net::URLFetcherDelegate: 43 virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE; 44 45 scoped_ptr<net::URLFetcher> deletion_fetcher_; 46 DeletionCompletedCallback callback_; 47 48 DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler); 49 }; 50 51 SuggestionDeletionHandler::SuggestionDeletionHandler( 52 const std::string& deletion_url, 53 net::URLRequestContextGetter* request_context, 54 const DeletionCompletedCallback& callback) : callback_(callback) { 55 GURL url(deletion_url); 56 DCHECK(url.is_valid()); 57 58 deletion_fetcher_.reset(net::URLFetcher::Create( 59 BaseSearchProvider::kDeletionURLFetcherID, 60 url, 61 net::URLFetcher::GET, 62 this)); 63 deletion_fetcher_->SetRequestContext(request_context); 64 deletion_fetcher_->Start(); 65 } 66 67 SuggestionDeletionHandler::~SuggestionDeletionHandler() { 68 } 69 70 void SuggestionDeletionHandler::OnURLFetchComplete( 71 const net::URLFetcher* source) { 72 DCHECK(source == deletion_fetcher_.get()); 73 callback_.Run( 74 source->GetStatus().is_success() && (source->GetResponseCode() == 200), 75 this); 76 } 77 78 // BaseSearchProvider --------------------------------------------------------- 79 80 // static 81 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1; 82 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2; 83 const int BaseSearchProvider::kDeletionURLFetcherID = 3; 84 85 BaseSearchProvider::BaseSearchProvider( 86 TemplateURLService* template_url_service, 87 scoped_ptr<AutocompleteProviderClient> client, 88 AutocompleteProvider::Type type) 89 : AutocompleteProvider(type), 90 template_url_service_(template_url_service), 91 client_(client.Pass()), 92 field_trial_triggered_(false), 93 field_trial_triggered_in_session_(false) { 94 } 95 96 // static 97 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) { 98 return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue; 99 } 100 101 // static 102 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion( 103 const base::string16& suggestion, 104 AutocompleteMatchType::Type type, 105 bool from_keyword_provider, 106 const TemplateURL* template_url, 107 const SearchTermsData& search_terms_data) { 108 // These calls use a number of default values. For instance, they assume 109 // that if this match is from a keyword provider, then the user is in keyword 110 // mode. They also assume the caller knows what it's doing and we set 111 // this match to look as if it was received/created synchronously. 112 SearchSuggestionParser::SuggestResult suggest_result( 113 suggestion, type, suggestion, base::string16(), base::string16(), 114 base::string16(), base::string16(), std::string(), std::string(), 115 from_keyword_provider, 0, false, false, base::string16()); 116 suggest_result.set_received_after_last_keystroke(false); 117 return CreateSearchSuggestion( 118 NULL, AutocompleteInput(), from_keyword_provider, suggest_result, 119 template_url, search_terms_data, 0, false); 120 } 121 122 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) { 123 DCHECK(match.deletable); 124 if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) { 125 deletion_handlers_.push_back(new SuggestionDeletionHandler( 126 match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey), 127 client_->RequestContext(), 128 base::Bind(&BaseSearchProvider::OnDeletionComplete, 129 base::Unretained(this)))); 130 } 131 132 TemplateURL* template_url = 133 match.GetTemplateURL(template_url_service_, false); 134 // This may be NULL if the template corresponding to the keyword has been 135 // deleted or there is no keyword set. 136 if (template_url != NULL) { 137 client_->DeleteMatchingURLsForKeywordFromHistory(template_url->id(), 138 match.contents); 139 } 140 141 // Immediately update the list of matches to show the match was deleted, 142 // regardless of whether the server request actually succeeds. 143 DeleteMatchFromMatches(match); 144 } 145 146 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const { 147 provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo()); 148 metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back(); 149 new_entry.set_provider(AsOmniboxEventProviderType()); 150 new_entry.set_provider_done(done_); 151 std::vector<uint32> field_trial_hashes; 152 OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes); 153 for (size_t i = 0; i < field_trial_hashes.size(); ++i) { 154 if (field_trial_triggered_) 155 new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]); 156 if (field_trial_triggered_in_session_) { 157 new_entry.mutable_field_trial_triggered_in_session()->Add( 158 field_trial_hashes[i]); 159 } 160 } 161 } 162 163 // static 164 const char BaseSearchProvider::kRelevanceFromServerKey[] = 165 "relevance_from_server"; 166 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch"; 167 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata"; 168 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url"; 169 const char BaseSearchProvider::kTrue[] = "true"; 170 const char BaseSearchProvider::kFalse[] = "false"; 171 172 BaseSearchProvider::~BaseSearchProvider() {} 173 174 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url, 175 AutocompleteMatch* match) { 176 if (deletion_url.empty()) 177 return; 178 if (!template_url_service_) 179 return; 180 GURL url = 181 template_url_service_->GetDefaultSearchProvider()->GenerateSearchURL( 182 template_url_service_->search_terms_data()); 183 url = url.GetOrigin().Resolve(deletion_url); 184 if (url.is_valid()) { 185 match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey, 186 url.spec()); 187 match->deletable = true; 188 } 189 } 190 191 // static 192 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion( 193 AutocompleteProvider* autocomplete_provider, 194 const AutocompleteInput& input, 195 const bool in_keyword_mode, 196 const SearchSuggestionParser::SuggestResult& suggestion, 197 const TemplateURL* template_url, 198 const SearchTermsData& search_terms_data, 199 int accepted_suggestion, 200 bool append_extra_query_params) { 201 AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false, 202 suggestion.type()); 203 204 if (!template_url) 205 return match; 206 match.keyword = template_url->keyword(); 207 match.contents = suggestion.match_contents(); 208 match.contents_class = suggestion.match_contents_class(); 209 match.answer_contents = suggestion.answer_contents(); 210 match.answer_type = suggestion.answer_type(); 211 if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) { 212 match.RecordAdditionalInfo( 213 kACMatchPropertyInputText, base::UTF16ToUTF8(input.text())); 214 match.RecordAdditionalInfo( 215 kACMatchPropertyContentsPrefix, 216 base::UTF16ToUTF8(suggestion.match_contents_prefix())); 217 match.RecordAdditionalInfo( 218 kACMatchPropertyContentsStartIndex, 219 static_cast<int>( 220 suggestion.suggestion().length() - match.contents.length())); 221 } 222 223 if (!suggestion.annotation().empty()) 224 match.description = suggestion.annotation(); 225 226 // suggestion.match_contents() should have already been collapsed. 227 match.allowed_to_be_default_match = 228 (!in_keyword_mode || suggestion.from_keyword_provider()) && 229 (base::CollapseWhitespace(input.text(), false) == 230 suggestion.match_contents()); 231 232 // When the user forced a query, we need to make sure all the fill_into_edit 233 // values preserve that property. Otherwise, if the user starts editing a 234 // suggestion, non-Search results will suddenly appear. 235 if (input.type() == metrics::OmniboxInputType::FORCED_QUERY) 236 match.fill_into_edit.assign(base::ASCIIToUTF16("?")); 237 if (suggestion.from_keyword_provider()) 238 match.fill_into_edit.append(match.keyword + base::char16(' ')); 239 // We only allow inlinable navsuggestions that were received before the 240 // last keystroke because we don't want asynchronous inline autocompletions. 241 if (!input.prevent_inline_autocomplete() && 242 !suggestion.received_after_last_keystroke() && 243 (!in_keyword_mode || suggestion.from_keyword_provider()) && 244 StartsWith(suggestion.suggestion(), input.text(), false)) { 245 match.inline_autocompletion = 246 suggestion.suggestion().substr(input.text().length()); 247 match.allowed_to_be_default_match = true; 248 } 249 match.fill_into_edit.append(suggestion.suggestion()); 250 251 const TemplateURLRef& search_url = template_url->url_ref(); 252 DCHECK(search_url.SupportsReplacement(search_terms_data)); 253 match.search_terms_args.reset( 254 new TemplateURLRef::SearchTermsArgs(suggestion.suggestion())); 255 match.search_terms_args->original_query = input.text(); 256 match.search_terms_args->accepted_suggestion = accepted_suggestion; 257 match.search_terms_args->enable_omnibox_start_margin = true; 258 match.search_terms_args->suggest_query_params = 259 suggestion.suggest_query_params(); 260 match.search_terms_args->append_extra_query_params = 261 append_extra_query_params; 262 // This is the destination URL sans assisted query stats. This must be set 263 // so the AutocompleteController can properly de-dupe; the controller will 264 // eventually overwrite it before it reaches the user. 265 match.destination_url = 266 GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(), 267 search_terms_data)); 268 269 // Search results don't look like URLs. 270 match.transition = suggestion.from_keyword_provider() ? 271 ui::PAGE_TRANSITION_KEYWORD : ui::PAGE_TRANSITION_GENERATED; 272 273 return match; 274 } 275 276 // static 277 bool BaseSearchProvider::ZeroSuggestEnabled( 278 const GURL& suggest_url, 279 const TemplateURL* template_url, 280 OmniboxEventProto::PageClassification page_classification, 281 const SearchTermsData& search_terms_data, 282 AutocompleteProviderClient* client) { 283 if (!OmniboxFieldTrial::InZeroSuggestFieldTrial()) 284 return false; 285 286 // Make sure we are sending the suggest request through HTTPS to prevent 287 // exposing the current page URL or personalized results without encryption. 288 if (!suggest_url.SchemeIs(url::kHttpsScheme)) 289 return false; 290 291 // Don't show zero suggest on the NTP. 292 // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP 293 // under the conditions described in crbug.com/305366. 294 if ((page_classification == 295 OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) || 296 (page_classification == 297 OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS)) 298 return false; 299 300 // Don't run if in incognito mode. 301 if (client->IsOffTheRecord()) 302 return false; 303 304 // Don't run if we can't get preferences or search suggest is not enabled. 305 if (!client->SearchSuggestEnabled()) 306 return false; 307 308 // Only make the request if we know that the provider supports zero suggest 309 // (currently only the prepopulated Google provider). 310 if (template_url == NULL || 311 !template_url->SupportsReplacement(search_terms_data) || 312 TemplateURLPrepopulateData::GetEngineType( 313 *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE) 314 return false; 315 316 return true; 317 } 318 319 // static 320 bool BaseSearchProvider::CanSendURL( 321 const GURL& current_page_url, 322 const GURL& suggest_url, 323 const TemplateURL* template_url, 324 OmniboxEventProto::PageClassification page_classification, 325 const SearchTermsData& search_terms_data, 326 AutocompleteProviderClient* client) { 327 if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification, 328 search_terms_data, client)) 329 return false; 330 331 if (!current_page_url.is_valid()) 332 return false; 333 334 // Only allow HTTP URLs or HTTPS URLs for the same domain as the search 335 // provider. 336 if ((current_page_url.scheme() != url::kHttpScheme) && 337 ((current_page_url.scheme() != url::kHttpsScheme) || 338 !net::registry_controlled_domains::SameDomainOrHost( 339 current_page_url, suggest_url, 340 net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES))) 341 return false; 342 343 if (!client->TabSyncEnabledAndUnencrypted()) 344 return false; 345 346 return true; 347 } 348 349 void BaseSearchProvider::AddMatchToMap( 350 const SearchSuggestionParser::SuggestResult& result, 351 const std::string& metadata, 352 int accepted_suggestion, 353 bool mark_as_deletable, 354 bool in_keyword_mode, 355 MatchMap* map) { 356 AutocompleteMatch match = CreateSearchSuggestion( 357 this, GetInput(result.from_keyword_provider()), in_keyword_mode, result, 358 GetTemplateURL(result.from_keyword_provider()), 359 template_url_service_->search_terms_data(), accepted_suggestion, 360 ShouldAppendExtraParams(result)); 361 if (!match.destination_url.is_valid()) 362 return; 363 match.search_terms_args->bookmark_bar_pinned = client_->ShowBookmarkBar(); 364 match.RecordAdditionalInfo(kRelevanceFromServerKey, 365 result.relevance_from_server() ? kTrue : kFalse); 366 match.RecordAdditionalInfo(kShouldPrefetchKey, 367 result.should_prefetch() ? kTrue : kFalse); 368 SetDeletionURL(result.deletion_url(), &match); 369 if (mark_as_deletable) 370 match.deletable = true; 371 // Metadata is needed only for prefetching queries. 372 if (result.should_prefetch()) 373 match.RecordAdditionalInfo(kSuggestMetadataKey, metadata); 374 375 // Try to add |match| to |map|. If a match for this suggestion is 376 // already in |map|, replace it if |match| is more relevant. 377 // NOTE: Keep this ToLower() call in sync with url_database.cc. 378 MatchKey match_key( 379 std::make_pair(base::i18n::ToLower(result.suggestion()), 380 match.search_terms_args->suggest_query_params)); 381 const std::pair<MatchMap::iterator, bool> i( 382 map->insert(std::make_pair(match_key, match))); 383 384 bool should_prefetch = result.should_prefetch(); 385 if (!i.second) { 386 // NOTE: We purposefully do a direct relevance comparison here instead of 387 // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items 388 // added first" rather than "items alphabetically first" when the scores 389 // are equal. The only case this matters is when a user has results with 390 // the same score that differ only by capitalization; because the history 391 // system returns results sorted by recency, this means we'll pick the most 392 // recent such result even if the precision of our relevance score is too 393 // low to distinguish the two. 394 if (match.relevance > i.first->second.relevance) { 395 match.duplicate_matches.insert(match.duplicate_matches.end(), 396 i.first->second.duplicate_matches.begin(), 397 i.first->second.duplicate_matches.end()); 398 i.first->second.duplicate_matches.clear(); 399 match.duplicate_matches.push_back(i.first->second); 400 i.first->second = match; 401 } else { 402 i.first->second.duplicate_matches.push_back(match); 403 if (match.keyword == i.first->second.keyword) { 404 // Old and new matches are from the same search provider. It is okay to 405 // record one match's prefetch data onto a different match (for the same 406 // query string) for the following reasons: 407 // 1. Because the suggest server only sends down a query string from 408 // which we construct a URL, rather than sending a full URL, and because 409 // we construct URLs from query strings in the same way every time, the 410 // URLs for the two matches will be the same. Therefore, we won't end up 411 // prefetching something the server didn't intend. 412 // 2. Presumably the server sets the prefetch bit on a match it things 413 // is sufficiently relevant that the user is likely to choose it. 414 // Surely setting the prefetch bit on a match of even higher relevance 415 // won't violate this assumption. 416 should_prefetch |= ShouldPrefetch(i.first->second); 417 i.first->second.RecordAdditionalInfo(kShouldPrefetchKey, 418 should_prefetch ? kTrue : kFalse); 419 if (should_prefetch) 420 i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata); 421 } 422 } 423 // Copy over answer data from lower-ranking item, if necessary. 424 // This depends on the lower-ranking item always being added last - see 425 // use of push_back above. 426 AutocompleteMatch& more_relevant_match = i.first->second; 427 const AutocompleteMatch& less_relevant_match = 428 more_relevant_match.duplicate_matches.back(); 429 if (!less_relevant_match.answer_type.empty() && 430 more_relevant_match.answer_type.empty()) { 431 more_relevant_match.answer_type = less_relevant_match.answer_type; 432 more_relevant_match.answer_contents = less_relevant_match.answer_contents; 433 } 434 } 435 } 436 437 bool BaseSearchProvider::ParseSuggestResults( 438 const base::Value& root_val, 439 int default_result_relevance, 440 bool is_keyword_result, 441 SearchSuggestionParser::Results* results) { 442 if (!SearchSuggestionParser::ParseSuggestResults( 443 root_val, GetInput(is_keyword_result), 444 client_->SchemeClassifier(), default_result_relevance, 445 client_->AcceptLanguages(), is_keyword_result, results)) 446 return false; 447 448 for (std::vector<GURL>::const_iterator it = 449 results->answers_image_urls.begin(); 450 it != results->answers_image_urls.end(); ++it) 451 client_->PrefetchImage(*it); 452 453 field_trial_triggered_ |= results->field_trial_triggered; 454 field_trial_triggered_in_session_ |= results->field_trial_triggered; 455 return true; 456 } 457 458 void BaseSearchProvider::DeleteMatchFromMatches( 459 const AutocompleteMatch& match) { 460 for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) { 461 // Find the desired match to delete by checking the type and contents. 462 // We can't check the destination URL, because the autocomplete controller 463 // may have reformulated that. Not that while checking for matching 464 // contents works for personalized suggestions, if more match types gain 465 // deletion support, this algorithm may need to be re-examined. 466 if (i->contents == match.contents && i->type == match.type) { 467 matches_.erase(i); 468 break; 469 } 470 } 471 } 472 473 void BaseSearchProvider::OnDeletionComplete( 474 bool success, SuggestionDeletionHandler* handler) { 475 RecordDeletionResult(success); 476 SuggestionDeletionHandlers::iterator it = std::find( 477 deletion_handlers_.begin(), deletion_handlers_.end(), handler); 478 DCHECK(it != deletion_handlers_.end()); 479 deletion_handlers_.erase(it); 480 } 481
