Home | History | Annotate | Download | only in autocomplete 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/autocomplete/base_search_provider.h"
      6 
      7 #include "base/i18n/case_conversion.h"
      8 #include "base/i18n/icu_string_conversions.h"
      9 #include "base/json/json_string_value_serializer.h"
     10 #include "base/json/json_writer.h"
     11 #include "base/prefs/pref_registry_simple.h"
     12 #include "base/prefs/pref_service.h"
     13 #include "base/strings/string_util.h"
     14 #include "base/strings/utf_string_conversions.h"
     15 #include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
     16 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service.h"
     17 #include "chrome/browser/bitmap_fetcher/bitmap_fetcher_service_factory.h"
     18 #include "chrome/browser/history/history_service.h"
     19 #include "chrome/browser/history/history_service_factory.h"
     20 #include "chrome/browser/omnibox/omnibox_field_trial.h"
     21 #include "chrome/browser/profiles/profile.h"
     22 #include "chrome/browser/search/instant_service.h"
     23 #include "chrome/browser/search/instant_service_factory.h"
     24 #include "chrome/browser/search/search.h"
     25 #include "chrome/browser/search_engines/template_url.h"
     26 #include "chrome/browser/search_engines/template_url_prepopulate_data.h"
     27 #include "chrome/browser/search_engines/template_url_service.h"
     28 #include "chrome/browser/search_engines/template_url_service_factory.h"
     29 #include "chrome/browser/search_engines/ui_thread_search_terms_data.h"
     30 #include "chrome/browser/sync/profile_sync_service.h"
     31 #include "chrome/browser/sync/profile_sync_service_factory.h"
     32 #include "chrome/common/pref_names.h"
     33 #include "components/autocomplete/url_prefix.h"
     34 #include "components/metrics/proto/omnibox_event.pb.h"
     35 #include "components/metrics/proto/omnibox_input_type.pb.h"
     36 #include "components/sync_driver/sync_prefs.h"
     37 #include "components/url_fixer/url_fixer.h"
     38 #include "content/public/common/url_constants.h"
     39 #include "net/base/escape.h"
     40 #include "net/base/net_util.h"
     41 #include "net/base/registry_controlled_domains/registry_controlled_domain.h"
     42 #include "net/http/http_response_headers.h"
     43 #include "net/url_request/url_fetcher.h"
     44 #include "net/url_request/url_fetcher_delegate.h"
     45 #include "url/gurl.h"
     46 
     47 using metrics::OmniboxEventProto;
     48 
     49 namespace {
     50 
     51 AutocompleteMatchType::Type GetAutocompleteMatchType(const std::string& type) {
     52   if (type == "ENTITY")
     53     return AutocompleteMatchType::SEARCH_SUGGEST_ENTITY;
     54   if (type == "INFINITE")
     55     return AutocompleteMatchType::SEARCH_SUGGEST_INFINITE;
     56   if (type == "PERSONALIZED_QUERY")
     57     return AutocompleteMatchType::SEARCH_SUGGEST_PERSONALIZED;
     58   if (type == "PROFILE")
     59     return AutocompleteMatchType::SEARCH_SUGGEST_PROFILE;
     60   if (type == "NAVIGATION")
     61     return AutocompleteMatchType::NAVSUGGEST;
     62   if (type == "PERSONALIZED_NAVIGATION")
     63     return AutocompleteMatchType::NAVSUGGEST_PERSONALIZED;
     64   return AutocompleteMatchType::SEARCH_SUGGEST;
     65 }
     66 
     67 } // namespace
     68 
     69 // SuggestionDeletionHandler -------------------------------------------------
     70 
     71 // This class handles making requests to the server in order to delete
     72 // personalized suggestions.
     73 class SuggestionDeletionHandler : public net::URLFetcherDelegate {
     74  public:
     75   typedef base::Callback<void(bool, SuggestionDeletionHandler*)>
     76       DeletionCompletedCallback;
     77 
     78   SuggestionDeletionHandler(
     79       const std::string& deletion_url,
     80       Profile* profile,
     81       const DeletionCompletedCallback& callback);
     82 
     83   virtual ~SuggestionDeletionHandler();
     84 
     85  private:
     86   // net::URLFetcherDelegate:
     87   virtual void OnURLFetchComplete(const net::URLFetcher* source) OVERRIDE;
     88 
     89   scoped_ptr<net::URLFetcher> deletion_fetcher_;
     90   DeletionCompletedCallback callback_;
     91 
     92   DISALLOW_COPY_AND_ASSIGN(SuggestionDeletionHandler);
     93 };
     94 
     95 SuggestionDeletionHandler::SuggestionDeletionHandler(
     96     const std::string& deletion_url,
     97     Profile* profile,
     98     const DeletionCompletedCallback& callback) : callback_(callback) {
     99   GURL url(deletion_url);
    100   DCHECK(url.is_valid());
    101 
    102   deletion_fetcher_.reset(net::URLFetcher::Create(
    103       BaseSearchProvider::kDeletionURLFetcherID,
    104       url,
    105       net::URLFetcher::GET,
    106       this));
    107   deletion_fetcher_->SetRequestContext(profile->GetRequestContext());
    108   deletion_fetcher_->Start();
    109 }
    110 
    111 SuggestionDeletionHandler::~SuggestionDeletionHandler() {
    112 }
    113 
    114 void SuggestionDeletionHandler::OnURLFetchComplete(
    115     const net::URLFetcher* source) {
    116   DCHECK(source == deletion_fetcher_.get());
    117   callback_.Run(
    118       source->GetStatus().is_success() && (source->GetResponseCode() == 200),
    119       this);
    120 }
    121 
    122 // BaseSearchProvider ---------------------------------------------------------
    123 
    124 // static
    125 const int BaseSearchProvider::kDefaultProviderURLFetcherID = 1;
    126 const int BaseSearchProvider::kKeywordProviderURLFetcherID = 2;
    127 const int BaseSearchProvider::kDeletionURLFetcherID = 3;
    128 
    129 BaseSearchProvider::BaseSearchProvider(AutocompleteProviderListener* listener,
    130                                        Profile* profile,
    131                                        AutocompleteProvider::Type type)
    132     : AutocompleteProvider(listener, profile, type),
    133       field_trial_triggered_(false),
    134       field_trial_triggered_in_session_(false),
    135       suggest_results_pending_(0),
    136       in_app_list_(false) {
    137 }
    138 
    139 // static
    140 bool BaseSearchProvider::ShouldPrefetch(const AutocompleteMatch& match) {
    141   return match.GetAdditionalInfo(kShouldPrefetchKey) == kTrue;
    142 }
    143 
    144 // static
    145 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
    146     const base::string16& suggestion,
    147     AutocompleteMatchType::Type type,
    148     bool from_keyword_provider,
    149     const TemplateURL* template_url,
    150     const SearchTermsData& search_terms_data) {
    151   return CreateSearchSuggestion(
    152       NULL, AutocompleteInput(), BaseSearchProvider::SuggestResult(
    153           suggestion, type, suggestion, base::string16(), base::string16(),
    154           base::string16(), base::string16(), std::string(), std::string(),
    155           from_keyword_provider, 0, false, false, base::string16()),
    156       template_url, search_terms_data, 0, 0, false, false);
    157 }
    158 
    159 void BaseSearchProvider::Stop(bool clear_cached_results) {
    160   StopSuggest();
    161   done_ = true;
    162 
    163   if (clear_cached_results)
    164     ClearAllResults();
    165 }
    166 
    167 void BaseSearchProvider::DeleteMatch(const AutocompleteMatch& match) {
    168   DCHECK(match.deletable);
    169   if (!match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey).empty()) {
    170     deletion_handlers_.push_back(new SuggestionDeletionHandler(
    171         match.GetAdditionalInfo(BaseSearchProvider::kDeletionUrlKey),
    172         profile_,
    173         base::Bind(&BaseSearchProvider::OnDeletionComplete,
    174                    base::Unretained(this))));
    175   }
    176 
    177   HistoryService* const history_service =
    178       HistoryServiceFactory::GetForProfile(profile_, Profile::EXPLICIT_ACCESS);
    179   TemplateURL* template_url = match.GetTemplateURL(profile_, false);
    180   // This may be NULL if the template corresponding to the keyword has been
    181   // deleted or there is no keyword set.
    182   if (template_url != NULL) {
    183     history_service->DeleteMatchingURLsForKeyword(template_url->id(),
    184                                                   match.contents);
    185   }
    186 
    187   // Immediately update the list of matches to show the match was deleted,
    188   // regardless of whether the server request actually succeeds.
    189   DeleteMatchFromMatches(match);
    190 }
    191 
    192 void BaseSearchProvider::AddProviderInfo(ProvidersInfo* provider_info) const {
    193   provider_info->push_back(metrics::OmniboxEventProto_ProviderInfo());
    194   metrics::OmniboxEventProto_ProviderInfo& new_entry = provider_info->back();
    195   new_entry.set_provider(AsOmniboxEventProviderType());
    196   new_entry.set_provider_done(done_);
    197   std::vector<uint32> field_trial_hashes;
    198   OmniboxFieldTrial::GetActiveSuggestFieldTrialHashes(&field_trial_hashes);
    199   for (size_t i = 0; i < field_trial_hashes.size(); ++i) {
    200     if (field_trial_triggered_)
    201       new_entry.mutable_field_trial_triggered()->Add(field_trial_hashes[i]);
    202     if (field_trial_triggered_in_session_) {
    203       new_entry.mutable_field_trial_triggered_in_session()->Add(
    204           field_trial_hashes[i]);
    205     }
    206   }
    207   ModifyProviderInfo(&new_entry);
    208 }
    209 
    210 // static
    211 const char BaseSearchProvider::kRelevanceFromServerKey[] =
    212     "relevance_from_server";
    213 const char BaseSearchProvider::kShouldPrefetchKey[] = "should_prefetch";
    214 const char BaseSearchProvider::kSuggestMetadataKey[] = "suggest_metadata";
    215 const char BaseSearchProvider::kDeletionUrlKey[] = "deletion_url";
    216 const char BaseSearchProvider::kTrue[] = "true";
    217 const char BaseSearchProvider::kFalse[] = "false";
    218 
    219 BaseSearchProvider::~BaseSearchProvider() {}
    220 
    221 // BaseSearchProvider::Result --------------------------------------------------
    222 
    223 BaseSearchProvider::Result::Result(bool from_keyword_provider,
    224                                    int relevance,
    225                                    bool relevance_from_server,
    226                                    AutocompleteMatchType::Type type,
    227                                    const std::string& deletion_url)
    228     : from_keyword_provider_(from_keyword_provider),
    229        type_(type),
    230        relevance_(relevance),
    231        relevance_from_server_(relevance_from_server),
    232        deletion_url_(deletion_url) {}
    233 
    234 BaseSearchProvider::Result::~Result() {}
    235 
    236 // BaseSearchProvider::SuggestResult -------------------------------------------
    237 
    238 BaseSearchProvider::SuggestResult::SuggestResult(
    239     const base::string16& suggestion,
    240     AutocompleteMatchType::Type type,
    241     const base::string16& match_contents,
    242     const base::string16& match_contents_prefix,
    243     const base::string16& annotation,
    244     const base::string16& answer_contents,
    245     const base::string16& answer_type,
    246     const std::string& suggest_query_params,
    247     const std::string& deletion_url,
    248     bool from_keyword_provider,
    249     int relevance,
    250     bool relevance_from_server,
    251     bool should_prefetch,
    252     const base::string16& input_text)
    253     : Result(from_keyword_provider,
    254              relevance,
    255              relevance_from_server,
    256              type,
    257              deletion_url),
    258       suggestion_(suggestion),
    259       match_contents_prefix_(match_contents_prefix),
    260       annotation_(annotation),
    261       suggest_query_params_(suggest_query_params),
    262       answer_contents_(answer_contents),
    263       answer_type_(answer_type),
    264       should_prefetch_(should_prefetch) {
    265   match_contents_ = match_contents;
    266   DCHECK(!match_contents_.empty());
    267   ClassifyMatchContents(true, input_text);
    268 }
    269 
    270 BaseSearchProvider::SuggestResult::~SuggestResult() {}
    271 
    272 void BaseSearchProvider::SuggestResult::ClassifyMatchContents(
    273     const bool allow_bolding_all,
    274     const base::string16& input_text) {
    275   if (input_text.empty()) {
    276     // In case of zero-suggest results, do not highlight matches.
    277     match_contents_class_.push_back(
    278         ACMatchClassification(0, ACMatchClassification::NONE));
    279     return;
    280   }
    281 
    282   base::string16 lookup_text = input_text;
    283   if (type_ == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
    284     const size_t contents_index =
    285         suggestion_.length() - match_contents_.length();
    286     // Ensure the query starts with the input text, and ends with the match
    287     // contents, and the input text has an overlap with contents.
    288     if (StartsWith(suggestion_, input_text, true) &&
    289         EndsWith(suggestion_, match_contents_, true) &&
    290         (input_text.length() > contents_index)) {
    291       lookup_text = input_text.substr(contents_index);
    292     }
    293   }
    294   size_t lookup_position = match_contents_.find(lookup_text);
    295   if (!allow_bolding_all && (lookup_position == base::string16::npos)) {
    296     // Bail if the code below to update the bolding would bold the whole
    297     // string.  Note that the string may already be entirely bolded; if
    298     // so, leave it as is.
    299     return;
    300   }
    301   match_contents_class_.clear();
    302   // We do intra-string highlighting for suggestions - the suggested segment
    303   // will be highlighted, e.g. for input_text = "you" the suggestion may be
    304   // "youtube", so we'll bold the "tube" section: you*tube*.
    305   if (input_text != match_contents_) {
    306     if (lookup_position == base::string16::npos) {
    307       // The input text is not a substring of the query string, e.g. input
    308       // text is "slasdot" and the query string is "slashdot", so we bold the
    309       // whole thing.
    310       match_contents_class_.push_back(
    311           ACMatchClassification(0, ACMatchClassification::MATCH));
    312     } else {
    313       // We don't iterate over the string here annotating all matches because
    314       // it looks odd to have every occurrence of a substring that may be as
    315       // short as a single character highlighted in a query suggestion result,
    316       // e.g. for input text "s" and query string "southwest airlines", it
    317       // looks odd if both the first and last s are highlighted.
    318       if (lookup_position != 0) {
    319         match_contents_class_.push_back(
    320             ACMatchClassification(0, ACMatchClassification::MATCH));
    321       }
    322       match_contents_class_.push_back(
    323           ACMatchClassification(lookup_position, ACMatchClassification::NONE));
    324       size_t next_fragment_position = lookup_position + lookup_text.length();
    325       if (next_fragment_position < match_contents_.length()) {
    326         match_contents_class_.push_back(ACMatchClassification(
    327             next_fragment_position, ACMatchClassification::MATCH));
    328       }
    329     }
    330   } else {
    331     // Otherwise, match_contents_ is a verbatim (what-you-typed) match, either
    332     // for the default provider or a keyword search provider.
    333     match_contents_class_.push_back(
    334         ACMatchClassification(0, ACMatchClassification::NONE));
    335   }
    336 }
    337 
    338 bool BaseSearchProvider::SuggestResult::IsInlineable(
    339     const base::string16& input) const {
    340   return StartsWith(suggestion_, input, false);
    341 }
    342 
    343 int BaseSearchProvider::SuggestResult::CalculateRelevance(
    344     const AutocompleteInput& input,
    345     bool keyword_provider_requested) const {
    346   if (!from_keyword_provider_ && keyword_provider_requested)
    347     return 100;
    348   return ((input.type() == metrics::OmniboxInputType::URL) ? 300 : 600);
    349 }
    350 
    351 // BaseSearchProvider::NavigationResult ----------------------------------------
    352 
    353 BaseSearchProvider::NavigationResult::NavigationResult(
    354     const AutocompleteProvider& provider,
    355     const GURL& url,
    356     AutocompleteMatchType::Type type,
    357     const base::string16& description,
    358     const std::string& deletion_url,
    359     bool from_keyword_provider,
    360     int relevance,
    361     bool relevance_from_server,
    362     const base::string16& input_text,
    363     const std::string& languages)
    364     : Result(from_keyword_provider,
    365              relevance,
    366              relevance_from_server,
    367              type,
    368              deletion_url),
    369       url_(url),
    370       formatted_url_(AutocompleteInput::FormattedStringWithEquivalentMeaning(
    371           url,
    372           provider.StringForURLDisplay(url, true, false))),
    373       description_(description) {
    374   DCHECK(url_.is_valid());
    375   CalculateAndClassifyMatchContents(true, input_text, languages);
    376 }
    377 
    378 BaseSearchProvider::NavigationResult::~NavigationResult() {}
    379 
    380 void BaseSearchProvider::NavigationResult::CalculateAndClassifyMatchContents(
    381     const bool allow_bolding_nothing,
    382     const base::string16& input_text,
    383     const std::string& languages) {
    384   if (input_text.empty()) {
    385     // In case of zero-suggest results, do not highlight matches.
    386     match_contents_class_.push_back(
    387         ACMatchClassification(0, ACMatchClassification::NONE));
    388     return;
    389   }
    390 
    391   // First look for the user's input inside the formatted url as it would be
    392   // without trimming the scheme, so we can find matches at the beginning of the
    393   // scheme.
    394   const URLPrefix* prefix =
    395       URLPrefix::BestURLPrefix(formatted_url_, input_text);
    396   size_t match_start = (prefix == NULL) ?
    397       formatted_url_.find(input_text) : prefix->prefix.length();
    398   bool trim_http = !AutocompleteInput::HasHTTPScheme(input_text) &&
    399                    (!prefix || (match_start != 0));
    400   const net::FormatUrlTypes format_types =
    401       net::kFormatUrlOmitAll & ~(trim_http ? 0 : net::kFormatUrlOmitHTTP);
    402 
    403   base::string16 match_contents = net::FormatUrl(url_, languages, format_types,
    404       net::UnescapeRule::SPACES, NULL, NULL, &match_start);
    405   // If the first match in the untrimmed string was inside a scheme that we
    406   // trimmed, look for a subsequent match.
    407   if (match_start == base::string16::npos)
    408     match_start = match_contents.find(input_text);
    409   // Update |match_contents_| and |match_contents_class_| if it's allowed.
    410   if (allow_bolding_nothing || (match_start != base::string16::npos)) {
    411     match_contents_ = match_contents;
    412     // Safe if |match_start| is npos; also safe if the input is longer than the
    413     // remaining contents after |match_start|.
    414     AutocompleteMatch::ClassifyLocationInString(match_start,
    415         input_text.length(), match_contents_.length(),
    416         ACMatchClassification::URL, &match_contents_class_);
    417   }
    418 }
    419 
    420 bool BaseSearchProvider::NavigationResult::IsInlineable(
    421     const base::string16& input) const {
    422   return
    423       URLPrefix::BestURLPrefix(base::UTF8ToUTF16(url_.spec()), input) != NULL;
    424 }
    425 
    426 int BaseSearchProvider::NavigationResult::CalculateRelevance(
    427     const AutocompleteInput& input,
    428     bool keyword_provider_requested) const {
    429   return (from_keyword_provider_ || !keyword_provider_requested) ? 800 : 150;
    430 }
    431 
    432 // BaseSearchProvider::Results -------------------------------------------------
    433 
    434 BaseSearchProvider::Results::Results() : verbatim_relevance(-1) {}
    435 
    436 BaseSearchProvider::Results::~Results() {}
    437 
    438 void BaseSearchProvider::Results::Clear() {
    439   suggest_results.clear();
    440   navigation_results.clear();
    441   verbatim_relevance = -1;
    442   metadata.clear();
    443 }
    444 
    445 bool BaseSearchProvider::Results::HasServerProvidedScores() const {
    446   if (verbatim_relevance >= 0)
    447     return true;
    448 
    449   // Right now either all results of one type will be server-scored or they will
    450   // all be locally scored, but in case we change this later, we'll just check
    451   // them all.
    452   for (SuggestResults::const_iterator i(suggest_results.begin());
    453        i != suggest_results.end(); ++i) {
    454     if (i->relevance_from_server())
    455       return true;
    456   }
    457   for (NavigationResults::const_iterator i(navigation_results.begin());
    458        i != navigation_results.end(); ++i) {
    459     if (i->relevance_from_server())
    460       return true;
    461   }
    462 
    463   return false;
    464 }
    465 
    466 void BaseSearchProvider::SetDeletionURL(const std::string& deletion_url,
    467                                         AutocompleteMatch* match) {
    468   if (deletion_url.empty())
    469     return;
    470   TemplateURLService* template_service =
    471       TemplateURLServiceFactory::GetForProfile(profile_);
    472   if (!template_service)
    473     return;
    474   GURL url = template_service->GetDefaultSearchProvider()->GenerateSearchURL(
    475       template_service->search_terms_data());
    476   url = url.GetOrigin().Resolve(deletion_url);
    477   if (url.is_valid()) {
    478     match->RecordAdditionalInfo(BaseSearchProvider::kDeletionUrlKey,
    479         url.spec());
    480     match->deletable = true;
    481   }
    482 }
    483 
    484 // BaseSearchProvider ---------------------------------------------------------
    485 
    486 // static
    487 AutocompleteMatch BaseSearchProvider::CreateSearchSuggestion(
    488     AutocompleteProvider* autocomplete_provider,
    489     const AutocompleteInput& input,
    490     const SuggestResult& suggestion,
    491     const TemplateURL* template_url,
    492     const SearchTermsData& search_terms_data,
    493     int accepted_suggestion,
    494     int omnibox_start_margin,
    495     bool append_extra_query_params,
    496     bool from_app_list) {
    497   AutocompleteMatch match(autocomplete_provider, suggestion.relevance(), false,
    498                           suggestion.type());
    499 
    500   if (!template_url)
    501     return match;
    502   match.keyword = template_url->keyword();
    503   match.contents = suggestion.match_contents();
    504   match.contents_class = suggestion.match_contents_class();
    505   match.answer_contents = suggestion.answer_contents();
    506   match.answer_type = suggestion.answer_type();
    507   if (suggestion.type() == AutocompleteMatchType::SEARCH_SUGGEST_INFINITE) {
    508     match.RecordAdditionalInfo(
    509         kACMatchPropertyInputText, base::UTF16ToUTF8(input.text()));
    510     match.RecordAdditionalInfo(
    511         kACMatchPropertyContentsPrefix,
    512         base::UTF16ToUTF8(suggestion.match_contents_prefix()));
    513     match.RecordAdditionalInfo(
    514         kACMatchPropertyContentsStartIndex,
    515         static_cast<int>(
    516             suggestion.suggestion().length() - match.contents.length()));
    517   }
    518 
    519   if (!suggestion.annotation().empty())
    520     match.description = suggestion.annotation();
    521 
    522   // suggestion.match_contents() should have already been collapsed.
    523   match.allowed_to_be_default_match =
    524       (base::CollapseWhitespace(input.text(), false) ==
    525        suggestion.match_contents());
    526 
    527   // When the user forced a query, we need to make sure all the fill_into_edit
    528   // values preserve that property.  Otherwise, if the user starts editing a
    529   // suggestion, non-Search results will suddenly appear.
    530   if (input.type() == metrics::OmniboxInputType::FORCED_QUERY)
    531     match.fill_into_edit.assign(base::ASCIIToUTF16("?"));
    532   if (suggestion.from_keyword_provider())
    533     match.fill_into_edit.append(match.keyword + base::char16(' '));
    534   if (!input.prevent_inline_autocomplete() &&
    535       StartsWith(suggestion.suggestion(), input.text(), false)) {
    536     match.inline_autocompletion =
    537         suggestion.suggestion().substr(input.text().length());
    538     match.allowed_to_be_default_match = true;
    539   }
    540   match.fill_into_edit.append(suggestion.suggestion());
    541 
    542   const TemplateURLRef& search_url = template_url->url_ref();
    543   DCHECK(search_url.SupportsReplacement(search_terms_data));
    544   match.search_terms_args.reset(
    545       new TemplateURLRef::SearchTermsArgs(suggestion.suggestion()));
    546   match.search_terms_args->original_query = input.text();
    547   match.search_terms_args->accepted_suggestion = accepted_suggestion;
    548   match.search_terms_args->omnibox_start_margin = omnibox_start_margin;
    549   match.search_terms_args->suggest_query_params =
    550       suggestion.suggest_query_params();
    551   match.search_terms_args->append_extra_query_params =
    552       append_extra_query_params;
    553   match.search_terms_args->from_app_list = from_app_list;
    554   // This is the destination URL sans assisted query stats.  This must be set
    555   // so the AutocompleteController can properly de-dupe; the controller will
    556   // eventually overwrite it before it reaches the user.
    557   match.destination_url =
    558       GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get(),
    559                                          search_terms_data));
    560 
    561   // Search results don't look like URLs.
    562   match.transition = suggestion.from_keyword_provider() ?
    563       content::PAGE_TRANSITION_KEYWORD : content::PAGE_TRANSITION_GENERATED;
    564 
    565   return match;
    566 }
    567 
    568 // static
    569 scoped_ptr<base::Value> BaseSearchProvider::DeserializeJsonData(
    570     std::string json_data) {
    571   // The JSON response should be an array.
    572   for (size_t response_start_index = json_data.find("["), i = 0;
    573        response_start_index != std::string::npos && i < 5;
    574        response_start_index = json_data.find("[", 1), i++) {
    575     // Remove any XSSI guards to allow for JSON parsing.
    576     if (response_start_index > 0)
    577       json_data.erase(0, response_start_index);
    578 
    579     JSONStringValueSerializer deserializer(json_data);
    580     deserializer.set_allow_trailing_comma(true);
    581     int error_code = 0;
    582     scoped_ptr<base::Value> data(deserializer.Deserialize(&error_code, NULL));
    583     if (error_code == 0)
    584       return data.Pass();
    585   }
    586   return scoped_ptr<base::Value>();
    587 }
    588 
    589 // static
    590 bool BaseSearchProvider::ZeroSuggestEnabled(
    591     const GURL& suggest_url,
    592     const TemplateURL* template_url,
    593     OmniboxEventProto::PageClassification page_classification,
    594     Profile* profile) {
    595   if (!OmniboxFieldTrial::InZeroSuggestFieldTrial())
    596     return false;
    597 
    598   // Make sure we are sending the suggest request through HTTPS to prevent
    599   // exposing the current page URL or personalized results without encryption.
    600   if (!suggest_url.SchemeIs(url::kHttpsScheme))
    601     return false;
    602 
    603   // Don't show zero suggest on the NTP.
    604   // TODO(hfung): Experiment with showing MostVisited zero suggest on NTP
    605   // under the conditions described in crbug.com/305366.
    606   if ((page_classification ==
    607        OmniboxEventProto::INSTANT_NTP_WITH_FAKEBOX_AS_STARTING_FOCUS) ||
    608       (page_classification ==
    609        OmniboxEventProto::INSTANT_NTP_WITH_OMNIBOX_AS_STARTING_FOCUS))
    610     return false;
    611 
    612   // Don't run if there's no profile or in incognito mode.
    613   if (profile == NULL || profile->IsOffTheRecord())
    614     return false;
    615 
    616   // Don't run if we can't get preferences or search suggest is not enabled.
    617   PrefService* prefs = profile->GetPrefs();
    618   if (!prefs->GetBoolean(prefs::kSearchSuggestEnabled))
    619     return false;
    620 
    621   // Only make the request if we know that the provider supports zero suggest
    622   // (currently only the prepopulated Google provider).
    623   UIThreadSearchTermsData search_terms_data(profile);
    624   if (template_url == NULL ||
    625       !template_url->SupportsReplacement(search_terms_data) ||
    626       TemplateURLPrepopulateData::GetEngineType(
    627           *template_url, search_terms_data) != SEARCH_ENGINE_GOOGLE)
    628     return false;
    629 
    630   return true;
    631 }
    632 
    633 // static
    634 bool BaseSearchProvider::CanSendURL(
    635     const GURL& current_page_url,
    636     const GURL& suggest_url,
    637     const TemplateURL* template_url,
    638     OmniboxEventProto::PageClassification page_classification,
    639     Profile* profile) {
    640   if (!ZeroSuggestEnabled(suggest_url, template_url, page_classification,
    641                           profile))
    642     return false;
    643 
    644   if (!current_page_url.is_valid())
    645     return false;
    646 
    647   // Only allow HTTP URLs or HTTPS URLs for the same domain as the search
    648   // provider.
    649   if ((current_page_url.scheme() != url::kHttpScheme) &&
    650       ((current_page_url.scheme() != url::kHttpsScheme) ||
    651        !net::registry_controlled_domains::SameDomainOrHost(
    652            current_page_url, suggest_url,
    653            net::registry_controlled_domains::EXCLUDE_PRIVATE_REGISTRIES)))
    654     return false;
    655 
    656   // Check field trials and settings allow sending the URL on suggest requests.
    657   ProfileSyncService* service =
    658       ProfileSyncServiceFactory::GetInstance()->GetForProfile(profile);
    659   sync_driver::SyncPrefs sync_prefs(profile->GetPrefs());
    660   if (service == NULL ||
    661       !service->IsSyncEnabledAndLoggedIn() ||
    662       !sync_prefs.GetPreferredDataTypes(syncer::UserTypes()).Has(
    663           syncer::PROXY_TABS) ||
    664       service->GetEncryptedDataTypes().Has(syncer::SESSIONS))
    665     return false;
    666 
    667   return true;
    668 }
    669 
    670 void BaseSearchProvider::OnURLFetchComplete(const net::URLFetcher* source) {
    671   DCHECK(!done_);
    672   suggest_results_pending_--;
    673   DCHECK_GE(suggest_results_pending_, 0);  // Should never go negative.
    674 
    675   const bool is_keyword = IsKeywordFetcher(source);
    676 
    677   // Ensure the request succeeded and that the provider used is still available.
    678   // A verbatim match cannot be generated without this provider, causing errors.
    679   const bool request_succeeded =
    680       source->GetStatus().is_success() && (source->GetResponseCode() == 200) &&
    681       GetTemplateURL(is_keyword);
    682 
    683   LogFetchComplete(request_succeeded, is_keyword);
    684 
    685   bool results_updated = false;
    686   if (request_succeeded) {
    687     const net::HttpResponseHeaders* const response_headers =
    688         source->GetResponseHeaders();
    689     std::string json_data;
    690     source->GetResponseAsString(&json_data);
    691 
    692     // JSON is supposed to be UTF-8, but some suggest service providers send
    693     // JSON files in non-UTF-8 encodings.  The actual encoding is usually
    694     // specified in the Content-Type header field.
    695     if (response_headers) {
    696       std::string charset;
    697       if (response_headers->GetCharset(&charset)) {
    698         base::string16 data_16;
    699         // TODO(jungshik): Switch to CodePageToUTF8 after it's added.
    700         if (base::CodepageToUTF16(json_data, charset.c_str(),
    701                                   base::OnStringConversionError::FAIL,
    702                                   &data_16))
    703           json_data = base::UTF16ToUTF8(data_16);
    704       }
    705     }
    706 
    707     scoped_ptr<base::Value> data(DeserializeJsonData(json_data));
    708     if (data && StoreSuggestionResponse(json_data, *data.get()))
    709       return;
    710 
    711     results_updated = data.get() && ParseSuggestResults(
    712         *data.get(), is_keyword, GetResultsToFill(is_keyword));
    713   }
    714 
    715   UpdateMatches();
    716   if (done_ || results_updated)
    717     listener_->OnProviderUpdate(results_updated);
    718 }
    719 
    720 void BaseSearchProvider::AddMatchToMap(const SuggestResult& result,
    721                                        const std::string& metadata,
    722                                        int accepted_suggestion,
    723                                        bool mark_as_deletable,
    724                                        MatchMap* map) {
    725   InstantService* instant_service =
    726       InstantServiceFactory::GetForProfile(profile_);
    727   // Android and iOS have no InstantService.
    728   const int omnibox_start_margin = instant_service ?
    729       instant_service->omnibox_start_margin() : chrome::kDisableStartMargin;
    730 
    731   AutocompleteMatch match = CreateSearchSuggestion(
    732       this, GetInput(result.from_keyword_provider()), result,
    733       GetTemplateURL(result.from_keyword_provider()),
    734       UIThreadSearchTermsData(profile_), accepted_suggestion,
    735       omnibox_start_margin, ShouldAppendExtraParams(result),
    736       in_app_list_);
    737   if (!match.destination_url.is_valid())
    738     return;
    739   match.search_terms_args->bookmark_bar_pinned =
    740       profile_->GetPrefs()->GetBoolean(prefs::kShowBookmarkBar);
    741   match.RecordAdditionalInfo(kRelevanceFromServerKey,
    742                              result.relevance_from_server() ? kTrue : kFalse);
    743   match.RecordAdditionalInfo(kShouldPrefetchKey,
    744                              result.should_prefetch() ? kTrue : kFalse);
    745   SetDeletionURL(result.deletion_url(), &match);
    746   if (mark_as_deletable)
    747     match.deletable = true;
    748   // Metadata is needed only for prefetching queries.
    749   if (result.should_prefetch())
    750     match.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
    751 
    752   // Try to add |match| to |map|.  If a match for this suggestion is
    753   // already in |map|, replace it if |match| is more relevant.
    754   // NOTE: Keep this ToLower() call in sync with url_database.cc.
    755   MatchKey match_key(
    756       std::make_pair(base::i18n::ToLower(result.suggestion()),
    757                      match.search_terms_args->suggest_query_params));
    758   const std::pair<MatchMap::iterator, bool> i(
    759        map->insert(std::make_pair(match_key, match)));
    760 
    761   bool should_prefetch = result.should_prefetch();
    762   if (!i.second) {
    763     // NOTE: We purposefully do a direct relevance comparison here instead of
    764     // using AutocompleteMatch::MoreRelevant(), so that we'll prefer "items
    765     // added first" rather than "items alphabetically first" when the scores
    766     // are equal. The only case this matters is when a user has results with
    767     // the same score that differ only by capitalization; because the history
    768     // system returns results sorted by recency, this means we'll pick the most
    769     // recent such result even if the precision of our relevance score is too
    770     // low to distinguish the two.
    771     if (match.relevance > i.first->second.relevance) {
    772       match.duplicate_matches.insert(match.duplicate_matches.end(),
    773                                      i.first->second.duplicate_matches.begin(),
    774                                      i.first->second.duplicate_matches.end());
    775       i.first->second.duplicate_matches.clear();
    776       match.duplicate_matches.push_back(i.first->second);
    777       i.first->second = match;
    778     } else {
    779       i.first->second.duplicate_matches.push_back(match);
    780       if (match.keyword == i.first->second.keyword) {
    781         // Old and new matches are from the same search provider. It is okay to
    782         // record one match's prefetch data onto a different match (for the same
    783         // query string) for the following reasons:
    784         // 1. Because the suggest server only sends down a query string from
    785         // which we construct a URL, rather than sending a full URL, and because
    786         // we construct URLs from query strings in the same way every time, the
    787         // URLs for the two matches will be the same. Therefore, we won't end up
    788         // prefetching something the server didn't intend.
    789         // 2. Presumably the server sets the prefetch bit on a match it things
    790         // is sufficiently relevant that the user is likely to choose it.
    791         // Surely setting the prefetch bit on a match of even higher relevance
    792         // won't violate this assumption.
    793         should_prefetch |= ShouldPrefetch(i.first->second);
    794         i.first->second.RecordAdditionalInfo(kShouldPrefetchKey,
    795                                              should_prefetch ? kTrue : kFalse);
    796         if (should_prefetch)
    797           i.first->second.RecordAdditionalInfo(kSuggestMetadataKey, metadata);
    798       }
    799     }
    800   }
    801 }
    802 
    803 bool BaseSearchProvider::ParseSuggestResults(const base::Value& root_val,
    804                                              bool is_keyword_result,
    805                                              Results* results) {
    806   base::string16 query;
    807   const base::ListValue* root_list = NULL;
    808   const base::ListValue* results_list = NULL;
    809   const AutocompleteInput& input = GetInput(is_keyword_result);
    810 
    811   if (!root_val.GetAsList(&root_list) || !root_list->GetString(0, &query) ||
    812       query != input.text() || !root_list->GetList(1, &results_list))
    813     return false;
    814 
    815   // 3rd element: Description list.
    816   const base::ListValue* descriptions = NULL;
    817   root_list->GetList(2, &descriptions);
    818 
    819   // 4th element: Disregard the query URL list for now.
    820 
    821   // Reset suggested relevance information.
    822   results->verbatim_relevance = -1;
    823 
    824   // 5th element: Optional key-value pairs from the Suggest server.
    825   const base::ListValue* types = NULL;
    826   const base::ListValue* relevances = NULL;
    827   const base::ListValue* suggestion_details = NULL;
    828   const base::DictionaryValue* extras = NULL;
    829   int prefetch_index = -1;
    830   if (root_list->GetDictionary(4, &extras)) {
    831     extras->GetList("google:suggesttype", &types);
    832 
    833     // Discard this list if its size does not match that of the suggestions.
    834     if (extras->GetList("google:suggestrelevance", &relevances) &&
    835         (relevances->GetSize() != results_list->GetSize()))
    836       relevances = NULL;
    837     extras->GetInteger("google:verbatimrelevance",
    838                        &results->verbatim_relevance);
    839 
    840     // Check if the active suggest field trial (if any) has triggered either
    841     // for the default provider or keyword provider.
    842     bool triggered = false;
    843     extras->GetBoolean("google:fieldtrialtriggered", &triggered);
    844     field_trial_triggered_ |= triggered;
    845     field_trial_triggered_in_session_ |= triggered;
    846 
    847     const base::DictionaryValue* client_data = NULL;
    848     if (extras->GetDictionary("google:clientdata", &client_data) && client_data)
    849       client_data->GetInteger("phi", &prefetch_index);
    850 
    851     if (extras->GetList("google:suggestdetail", &suggestion_details) &&
    852         suggestion_details->GetSize() != results_list->GetSize())
    853       suggestion_details = NULL;
    854 
    855     // Store the metadata that came with the response in case we need to pass it
    856     // along with the prefetch query to Instant.
    857     JSONStringValueSerializer json_serializer(&results->metadata);
    858     json_serializer.Serialize(*extras);
    859   }
    860 
    861   // Clear the previous results now that new results are available.
    862   results->suggest_results.clear();
    863   results->navigation_results.clear();
    864 
    865   base::string16 suggestion;
    866   std::string type;
    867   int relevance = GetDefaultResultRelevance();
    868   // Prohibit navsuggest in FORCED_QUERY mode.  Users wants queries, not URLs.
    869   const bool allow_navsuggest =
    870       input.type() != metrics::OmniboxInputType::FORCED_QUERY;
    871   const std::string languages(
    872       profile_->GetPrefs()->GetString(prefs::kAcceptLanguages));
    873   const base::string16& trimmed_input =
    874       base::CollapseWhitespace(input.text(), false);
    875   for (size_t index = 0; results_list->GetString(index, &suggestion); ++index) {
    876     // Google search may return empty suggestions for weird input characters,
    877     // they make no sense at all and can cause problems in our code.
    878     if (suggestion.empty())
    879       continue;
    880 
    881     // Apply valid suggested relevance scores; discard invalid lists.
    882     if (relevances != NULL && !relevances->GetInteger(index, &relevance))
    883       relevances = NULL;
    884     AutocompleteMatchType::Type match_type =
    885         AutocompleteMatchType::SEARCH_SUGGEST;
    886     if (types && types->GetString(index, &type))
    887       match_type = GetAutocompleteMatchType(type);
    888     const base::DictionaryValue* suggestion_detail = NULL;
    889     std::string deletion_url;
    890 
    891     if (suggestion_details &&
    892         suggestion_details->GetDictionary(index, &suggestion_detail))
    893       suggestion_detail->GetString("du", &deletion_url);
    894 
    895     if ((match_type == AutocompleteMatchType::NAVSUGGEST) ||
    896         (match_type == AutocompleteMatchType::NAVSUGGEST_PERSONALIZED)) {
    897       // Do not blindly trust the URL coming from the server to be valid.
    898       GURL url(
    899           url_fixer::FixupURL(base::UTF16ToUTF8(suggestion), std::string()));
    900       if (url.is_valid() && allow_navsuggest) {
    901         base::string16 title;
    902         if (descriptions != NULL)
    903           descriptions->GetString(index, &title);
    904         results->navigation_results.push_back(NavigationResult(
    905             *this, url, match_type, title, deletion_url, is_keyword_result,
    906             relevance, relevances != NULL, input.text(), languages));
    907       }
    908     } else {
    909       base::string16 match_contents = suggestion;
    910       base::string16 match_contents_prefix;
    911       base::string16 annotation;
    912       base::string16 answer_contents;
    913       base::string16 answer_type;
    914       std::string suggest_query_params;
    915 
    916       if (suggestion_details) {
    917         suggestion_details->GetDictionary(index, &suggestion_detail);
    918         if (suggestion_detail) {
    919           suggestion_detail->GetString("t", &match_contents);
    920           suggestion_detail->GetString("mp", &match_contents_prefix);
    921           // Error correction for bad data from server.
    922           if (match_contents.empty())
    923             match_contents = suggestion;
    924           suggestion_detail->GetString("a", &annotation);
    925           suggestion_detail->GetString("q", &suggest_query_params);
    926 
    927           // Extract Answers, if provided.
    928           const base::DictionaryValue* answer_json = NULL;
    929           if (suggestion_detail->GetDictionary("ansa", &answer_json)) {
    930             match_type = AutocompleteMatchType::SEARCH_SUGGEST_ANSWER;
    931             PrefetchAnswersImages(answer_json);
    932             std::string contents;
    933             base::JSONWriter::Write(answer_json, &contents);
    934             answer_contents = base::UTF8ToUTF16(contents);
    935             suggestion_detail->GetString("ansb", &answer_type);
    936           }
    937         }
    938       }
    939 
    940       bool should_prefetch = static_cast<int>(index) == prefetch_index;
    941       // TODO(kochi): Improve calculator suggestion presentation.
    942       results->suggest_results.push_back(SuggestResult(
    943           base::CollapseWhitespace(suggestion, false), match_type,
    944           base::CollapseWhitespace(match_contents, false),
    945           match_contents_prefix, annotation, answer_contents, answer_type,
    946           suggest_query_params, deletion_url, is_keyword_result, relevance,
    947           relevances != NULL, should_prefetch, trimmed_input));
    948     }
    949   }
    950   SortResults(is_keyword_result, relevances, results);
    951   return true;
    952 }
    953 
    954 void BaseSearchProvider::PrefetchAnswersImages(
    955     const base::DictionaryValue* answer_json) {
    956   DCHECK(answer_json);
    957   const base::ListValue* lines = NULL;
    958   answer_json->GetList("l", &lines);
    959   if (!lines || lines->GetSize() == 0)
    960     return;
    961 
    962   BitmapFetcherService* image_service =
    963       BitmapFetcherServiceFactory::GetForBrowserContext(profile_);
    964   DCHECK(image_service);
    965 
    966   for (size_t line = 0; line < lines->GetSize(); ++line) {
    967     const base::DictionaryValue* imageLine = NULL;
    968     lines->GetDictionary(line, &imageLine);
    969     if (!imageLine)
    970       continue;
    971     const base::DictionaryValue* imageData = NULL;
    972     imageLine->GetDictionary("i", &imageData);
    973     if (!imageData)
    974       continue;
    975     std::string imageUrl;
    976     imageData->GetString("d", &imageUrl);
    977     image_service->Prefetch(GURL(imageUrl));
    978   }
    979 }
    980 
    981 void BaseSearchProvider::SortResults(bool is_keyword,
    982                                      const base::ListValue* relevances,
    983                                      Results* results) {
    984 }
    985 
    986 bool BaseSearchProvider::StoreSuggestionResponse(
    987     const std::string& json_data,
    988     const base::Value& parsed_data) {
    989   return false;
    990 }
    991 
    992 void BaseSearchProvider::ModifyProviderInfo(
    993     metrics::OmniboxEventProto_ProviderInfo* provider_info) const {
    994 }
    995 
    996 void BaseSearchProvider::DeleteMatchFromMatches(
    997     const AutocompleteMatch& match) {
    998   for (ACMatches::iterator i(matches_.begin()); i != matches_.end(); ++i) {
    999     // Find the desired match to delete by checking the type and contents.
   1000     // We can't check the destination URL, because the autocomplete controller
   1001     // may have reformulated that. Not that while checking for matching
   1002     // contents works for personalized suggestions, if more match types gain
   1003     // deletion support, this algorithm may need to be re-examined.
   1004     if (i->contents == match.contents && i->type == match.type) {
   1005       matches_.erase(i);
   1006       break;
   1007     }
   1008   }
   1009 }
   1010 
   1011 void BaseSearchProvider::OnDeletionComplete(
   1012     bool success, SuggestionDeletionHandler* handler) {
   1013   RecordDeletionResult(success);
   1014   SuggestionDeletionHandlers::iterator it = std::find(
   1015       deletion_handlers_.begin(), deletion_handlers_.end(), handler);
   1016   DCHECK(it != deletion_handlers_.end());
   1017   deletion_handlers_.erase(it);
   1018 }
   1019