Home | History | Annotate | Download | only in suggestions 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "components/suggestions/suggestions_service.h"
      6 
      7 #include <sstream>
      8 #include <string>
      9 
     10 #include "base/memory/scoped_ptr.h"
     11 #include "base/message_loop/message_loop_proxy.h"
     12 #include "base/metrics/histogram.h"
     13 #include "base/metrics/sparse_histogram.h"
     14 #include "base/strings/string_number_conversions.h"
     15 #include "base/strings/string_util.h"
     16 #include "base/time/time.h"
     17 #include "components/pref_registry/pref_registry_syncable.h"
     18 #include "components/suggestions/blacklist_store.h"
     19 #include "components/suggestions/suggestions_store.h"
     20 #include "components/variations/variations_associated_data.h"
     21 #include "components/variations/variations_http_header_provider.h"
     22 #include "net/base/escape.h"
     23 #include "net/base/load_flags.h"
     24 #include "net/base/net_errors.h"
     25 #include "net/base/url_util.h"
     26 #include "net/http/http_response_headers.h"
     27 #include "net/http/http_status_code.h"
     28 #include "net/http/http_util.h"
     29 #include "net/url_request/url_fetcher.h"
     30 #include "net/url_request/url_request_status.h"
     31 #include "url/gurl.h"
     32 
     33 using base::CancelableClosure;
     34 
     35 namespace suggestions {
     36 
     37 namespace {
     38 
     39 // Used to UMA log the state of the last response from the server.
     40 enum SuggestionsResponseState {
     41   RESPONSE_EMPTY,
     42   RESPONSE_INVALID,
     43   RESPONSE_VALID,
     44   RESPONSE_STATE_SIZE
     45 };
     46 
     47 // Will log the supplied response |state|.
     48 void LogResponseState(SuggestionsResponseState state) {
     49   UMA_HISTOGRAM_ENUMERATION("Suggestions.ResponseState", state,
     50                             RESPONSE_STATE_SIZE);
     51 }
     52 
     53 // Obtains the experiment parameter under the supplied |key|, or empty string
     54 // if the parameter does not exist.
     55 std::string GetExperimentParam(const std::string& key) {
     56   return variations::GetVariationParamValue(kSuggestionsFieldTrialName, key);
     57 }
     58 
     59 GURL BuildBlacklistRequestURL(const std::string& blacklist_url_prefix,
     60                               const GURL& candidate_url) {
     61   return GURL(blacklist_url_prefix +
     62               net::EscapeQueryParamValue(candidate_url.spec(), true));
     63 }
     64 
     65 // Runs each callback in |requestors| on |suggestions|, then deallocates
     66 // |requestors|.
     67 void DispatchRequestsAndClear(
     68     const SuggestionsProfile& suggestions,
     69     std::vector<SuggestionsService::ResponseCallback>* requestors) {
     70   std::vector<SuggestionsService::ResponseCallback>::iterator it;
     71   for (it = requestors->begin(); it != requestors->end(); ++it) {
     72     if (!it->is_null()) it->Run(suggestions);
     73   }
     74   std::vector<SuggestionsService::ResponseCallback>().swap(*requestors);
     75 }
     76 
     77 const int kDefaultRequestTimeoutMs = 200;
     78 
     79 // Default delay used when scheduling a blacklist request.
     80 const int kBlacklistDefaultDelaySec = 1;
     81 
     82 // Multiplier on the delay used when scheduling a blacklist request, in case the
     83 // last observed request was unsuccessful.
     84 const int kBlacklistBackoffMultiplier = 2;
     85 
     86 // Maximum valid delay for scheduling a request. Candidate delays larger than
     87 // this are rejected. This means the maximum backoff is at least 300 / 2, i.e.
     88 // 2.5 minutes.
     89 const int kBlacklistMaxDelaySec = 300;  // 5 minutes
     90 
     91 }  // namespace
     92 
     93 const char kSuggestionsFieldTrialName[] = "ChromeSuggestions";
     94 const char kSuggestionsFieldTrialURLParam[] = "url";
     95 const char kSuggestionsFieldTrialCommonParamsParam[] = "common_params";
     96 const char kSuggestionsFieldTrialBlacklistPathParam[] = "blacklist_path";
     97 const char kSuggestionsFieldTrialBlacklistUrlParam[] = "blacklist_url_param";
     98 const char kSuggestionsFieldTrialStateParam[] = "state";
     99 const char kSuggestionsFieldTrialControlParam[] = "control";
    100 const char kSuggestionsFieldTrialStateEnabled[] = "enabled";
    101 const char kSuggestionsFieldTrialTimeoutMs[] = "timeout_ms";
    102 
    103 // The default expiry timeout is 72 hours.
    104 const int64 kDefaultExpiryUsec = 72 * base::Time::kMicrosecondsPerHour;
    105 
    106 namespace {
    107 
    108 std::string GetBlacklistUrlPrefix() {
    109   std::stringstream blacklist_url_prefix_stream;
    110   blacklist_url_prefix_stream
    111       << GetExperimentParam(kSuggestionsFieldTrialURLParam)
    112       << GetExperimentParam(kSuggestionsFieldTrialBlacklistPathParam) << "?"
    113       << GetExperimentParam(kSuggestionsFieldTrialCommonParamsParam) << "&"
    114       << GetExperimentParam(kSuggestionsFieldTrialBlacklistUrlParam) << "=";
    115   return blacklist_url_prefix_stream.str();
    116 }
    117 
    118 }  // namespace
    119 
    120 SuggestionsService::SuggestionsService(
    121     net::URLRequestContextGetter* url_request_context,
    122     scoped_ptr<SuggestionsStore> suggestions_store,
    123     scoped_ptr<ImageManager> thumbnail_manager,
    124     scoped_ptr<BlacklistStore> blacklist_store)
    125     : suggestions_store_(suggestions_store.Pass()),
    126       blacklist_store_(blacklist_store.Pass()),
    127       thumbnail_manager_(thumbnail_manager.Pass()),
    128       url_request_context_(url_request_context),
    129       blacklist_delay_sec_(kBlacklistDefaultDelaySec),
    130       request_timeout_ms_(kDefaultRequestTimeoutMs),
    131       weak_ptr_factory_(this) {
    132   // Obtain various parameters from Variations.
    133   suggestions_url_ =
    134       GURL(GetExperimentParam(kSuggestionsFieldTrialURLParam) + "?" +
    135            GetExperimentParam(kSuggestionsFieldTrialCommonParamsParam));
    136   blacklist_url_prefix_ = GetBlacklistUrlPrefix();
    137   std::string timeout = GetExperimentParam(kSuggestionsFieldTrialTimeoutMs);
    138   int temp_timeout;
    139   if (!timeout.empty() && base::StringToInt(timeout, &temp_timeout)) {
    140     request_timeout_ms_ = temp_timeout;
    141   }
    142 }
    143 
    144 SuggestionsService::~SuggestionsService() {}
    145 
    146 // static
    147 bool SuggestionsService::IsEnabled() {
    148   return GetExperimentParam(kSuggestionsFieldTrialStateParam) ==
    149          kSuggestionsFieldTrialStateEnabled;
    150 }
    151 
    152 // static
    153 bool SuggestionsService::IsControlGroup() {
    154   return GetExperimentParam(kSuggestionsFieldTrialControlParam) ==
    155          kSuggestionsFieldTrialStateEnabled;
    156 }
    157 
    158 void SuggestionsService::FetchSuggestionsData(
    159     SyncState sync_state,
    160     SuggestionsService::ResponseCallback callback) {
    161   DCHECK(thread_checker_.CalledOnValidThread());
    162   if (sync_state == NOT_INITIALIZED_ENABLED) {
    163     // Sync is not initialized yet, but enabled. Serve previously cached
    164     // suggestions if available.
    165     waiting_requestors_.push_back(callback);
    166     ServeFromCache();
    167     return;
    168   } else if (sync_state == SYNC_OR_HISTORY_SYNC_DISABLED) {
    169     // Cancel any ongoing request (and the timeout closure). We must no longer
    170     // interact with the server.
    171     pending_request_.reset(NULL);
    172     pending_timeout_closure_.reset(NULL);
    173     suggestions_store_->ClearSuggestions();
    174     callback.Run(SuggestionsProfile());
    175     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
    176     return;
    177   }
    178 
    179   FetchSuggestionsDataNoTimeout(callback);
    180 
    181   // Post a task to serve the cached suggestions if the request hasn't completed
    182   // after some time. Cancels the previous such task, if one existed.
    183   pending_timeout_closure_.reset(new CancelableClosure(base::Bind(
    184       &SuggestionsService::OnRequestTimeout, weak_ptr_factory_.GetWeakPtr())));
    185   base::MessageLoopProxy::current()->PostDelayedTask(
    186       FROM_HERE, pending_timeout_closure_->callback(),
    187       base::TimeDelta::FromMilliseconds(request_timeout_ms_));
    188 }
    189 
    190 void SuggestionsService::GetPageThumbnail(
    191     const GURL& url,
    192     base::Callback<void(const GURL&, const SkBitmap*)> callback) {
    193   thumbnail_manager_->GetImageForURL(url, callback);
    194 }
    195 
    196 void SuggestionsService::BlacklistURL(
    197     const GURL& candidate_url,
    198     const SuggestionsService::ResponseCallback& callback) {
    199   DCHECK(thread_checker_.CalledOnValidThread());
    200   waiting_requestors_.push_back(callback);
    201 
    202   // Blacklist locally, for immediate effect.
    203   if (!blacklist_store_->BlacklistUrl(candidate_url)) {
    204     DVLOG(1) << "Failed blacklisting attempt.";
    205     return;
    206   }
    207 
    208   // If there's an ongoing request, let it complete.
    209   if (pending_request_.get()) return;
    210   IssueRequest(BuildBlacklistRequestURL(blacklist_url_prefix_, candidate_url));
    211 }
    212 
    213 // static
    214 bool SuggestionsService::GetBlacklistedUrl(const net::URLFetcher& request,
    215                                            GURL* url) {
    216   bool is_blacklist_request = StartsWithASCII(request.GetOriginalURL().spec(),
    217                                               GetBlacklistUrlPrefix(), true);
    218   if (!is_blacklist_request) return false;
    219 
    220   // Extract the blacklisted URL from the blacklist request.
    221   std::string blacklisted;
    222   if (!net::GetValueForKeyInQuery(
    223           request.GetOriginalURL(),
    224           GetExperimentParam(kSuggestionsFieldTrialBlacklistUrlParam),
    225           &blacklisted))
    226     return false;
    227 
    228   GURL blacklisted_url(blacklisted);
    229   blacklisted_url.Swap(url);
    230   return true;
    231 }
    232 
    233 // static
    234 void SuggestionsService::RegisterProfilePrefs(
    235     user_prefs::PrefRegistrySyncable* registry) {
    236   SuggestionsStore::RegisterProfilePrefs(registry);
    237   BlacklistStore::RegisterProfilePrefs(registry);
    238 }
    239 
    240 void SuggestionsService::SetDefaultExpiryTimestamp(
    241     SuggestionsProfile* suggestions, int64 default_timestamp_usec) {
    242   for (int i = 0; i < suggestions->suggestions_size(); ++i) {
    243     ChromeSuggestion* suggestion = suggestions->mutable_suggestions(i);
    244     // Do not set expiry if the server has already provided a more specific
    245     // expiry time for this suggestion.
    246     if (!suggestion->has_expiry_ts()) {
    247       suggestion->set_expiry_ts(default_timestamp_usec);
    248     }
    249   }
    250 }
    251 
    252 void SuggestionsService::FetchSuggestionsDataNoTimeout(
    253     SuggestionsService::ResponseCallback callback) {
    254   DCHECK(thread_checker_.CalledOnValidThread());
    255   if (pending_request_.get()) {
    256     // Request already exists, so just add requestor to queue.
    257     waiting_requestors_.push_back(callback);
    258     return;
    259   }
    260 
    261   // Form new request.
    262   DCHECK(waiting_requestors_.empty());
    263   waiting_requestors_.push_back(callback);
    264   IssueRequest(suggestions_url_);
    265 }
    266 
    267 void SuggestionsService::IssueRequest(const GURL& url) {
    268   pending_request_.reset(CreateSuggestionsRequest(url));
    269   pending_request_->Start();
    270   last_request_started_time_ = base::TimeTicks::Now();
    271 }
    272 
    273 net::URLFetcher* SuggestionsService::CreateSuggestionsRequest(const GURL& url) {
    274   net::URLFetcher* request =
    275       net::URLFetcher::Create(0, url, net::URLFetcher::GET, this);
    276   request->SetLoadFlags(net::LOAD_DISABLE_CACHE);
    277   request->SetRequestContext(url_request_context_);
    278   // Add Chrome experiment state to the request headers.
    279   net::HttpRequestHeaders headers;
    280   variations::VariationsHttpHeaderProvider::GetInstance()->AppendHeaders(
    281       request->GetOriginalURL(), false, false, &headers);
    282   request->SetExtraRequestHeaders(headers.ToString());
    283   return request;
    284 }
    285 
    286 void SuggestionsService::OnRequestTimeout() {
    287   DCHECK(thread_checker_.CalledOnValidThread());
    288   ServeFromCache();
    289 }
    290 
    291 void SuggestionsService::OnURLFetchComplete(const net::URLFetcher* source) {
    292   DCHECK(thread_checker_.CalledOnValidThread());
    293   DCHECK_EQ(pending_request_.get(), source);
    294   // We no longer need the timeout closure. Delete it whether or not it has run.
    295   // If it hasn't, this cancels it.
    296   pending_timeout_closure_.reset();
    297 
    298   // The fetcher will be deleted when the request is handled.
    299   scoped_ptr<const net::URLFetcher> request(pending_request_.release());
    300   const net::URLRequestStatus& request_status = request->GetStatus();
    301   if (request_status.status() != net::URLRequestStatus::SUCCESS) {
    302     UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FailedRequestErrorCode",
    303                                 -request_status.error());
    304     DVLOG(1) << "Suggestions server request failed with error: "
    305              << request_status.error() << ": "
    306              << net::ErrorToString(request_status.error());
    307     // Dispatch the cached profile on error.
    308     ServeFromCache();
    309     ScheduleBlacklistUpload(false);
    310     return;
    311   }
    312 
    313   // Log the response code.
    314   const int response_code = request->GetResponseCode();
    315   UMA_HISTOGRAM_SPARSE_SLOWLY("Suggestions.FetchResponseCode", response_code);
    316   if (response_code != net::HTTP_OK) {
    317     // Aggressively clear the store.
    318     suggestions_store_->ClearSuggestions();
    319     DispatchRequestsAndClear(SuggestionsProfile(), &waiting_requestors_);
    320     ScheduleBlacklistUpload(false);
    321     return;
    322   }
    323 
    324   const base::TimeDelta latency =
    325       base::TimeTicks::Now() - last_request_started_time_;
    326   UMA_HISTOGRAM_MEDIUM_TIMES("Suggestions.FetchSuccessLatency", latency);
    327 
    328   // Handle a successful blacklisting.
    329   GURL blacklisted_url;
    330   if (GetBlacklistedUrl(*source, &blacklisted_url)) {
    331     blacklist_store_->RemoveUrl(blacklisted_url);
    332   }
    333 
    334   std::string suggestions_data;
    335   bool success = request->GetResponseAsString(&suggestions_data);
    336   DCHECK(success);
    337 
    338   // Compute suggestions, and dispatch them to requestors. On error still
    339   // dispatch empty suggestions.
    340   SuggestionsProfile suggestions;
    341   if (suggestions_data.empty()) {
    342     LogResponseState(RESPONSE_EMPTY);
    343     suggestions_store_->ClearSuggestions();
    344   } else if (suggestions.ParseFromString(suggestions_data)) {
    345     LogResponseState(RESPONSE_VALID);
    346     thumbnail_manager_->Initialize(suggestions);
    347 
    348     int64 now_usec = (base::Time::NowFromSystemTime() - base::Time::UnixEpoch())
    349         .ToInternalValue();
    350     SetDefaultExpiryTimestamp(&suggestions, now_usec + kDefaultExpiryUsec);
    351     suggestions_store_->StoreSuggestions(suggestions);
    352   } else {
    353     LogResponseState(RESPONSE_INVALID);
    354     suggestions_store_->LoadSuggestions(&suggestions);
    355     thumbnail_manager_->Initialize(suggestions);
    356   }
    357 
    358   FilterAndServe(&suggestions);
    359   ScheduleBlacklistUpload(true);
    360 }
    361 
    362 void SuggestionsService::Shutdown() {
    363   // Cancel pending request and timeout closure, then serve existing requestors
    364   // from cache.
    365   pending_request_.reset(NULL);
    366   pending_timeout_closure_.reset(NULL);
    367   ServeFromCache();
    368 }
    369 
    370 void SuggestionsService::ServeFromCache() {
    371   SuggestionsProfile suggestions;
    372   suggestions_store_->LoadSuggestions(&suggestions);
    373   thumbnail_manager_->Initialize(suggestions);
    374   FilterAndServe(&suggestions);
    375 }
    376 
    377 void SuggestionsService::FilterAndServe(SuggestionsProfile* suggestions) {
    378   blacklist_store_->FilterSuggestions(suggestions);
    379   DispatchRequestsAndClear(*suggestions, &waiting_requestors_);
    380 }
    381 
    382 void SuggestionsService::ScheduleBlacklistUpload(bool last_request_successful) {
    383   DCHECK(thread_checker_.CalledOnValidThread());
    384 
    385   UpdateBlacklistDelay(last_request_successful);
    386 
    387   // Schedule a blacklist upload task.
    388   GURL blacklist_url;
    389   if (blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url)) {
    390     base::Closure blacklist_cb =
    391         base::Bind(&SuggestionsService::UploadOneFromBlacklist,
    392                    weak_ptr_factory_.GetWeakPtr());
    393     base::MessageLoopProxy::current()->PostDelayedTask(
    394         FROM_HERE, blacklist_cb,
    395         base::TimeDelta::FromSeconds(blacklist_delay_sec_));
    396   }
    397 }
    398 
    399 void SuggestionsService::UploadOneFromBlacklist() {
    400   DCHECK(thread_checker_.CalledOnValidThread());
    401 
    402   // If there's an ongoing request, let it complete.
    403   if (pending_request_.get()) return;
    404 
    405   GURL blacklist_url;
    406   if (!blacklist_store_->GetFirstUrlFromBlacklist(&blacklist_url))
    407     return;  // Local blacklist is empty.
    408 
    409   // Send blacklisting request.
    410   IssueRequest(BuildBlacklistRequestURL(blacklist_url_prefix_, blacklist_url));
    411 }
    412 
    413 void SuggestionsService::UpdateBlacklistDelay(bool last_request_successful) {
    414   DCHECK(thread_checker_.CalledOnValidThread());
    415 
    416   if (last_request_successful) {
    417     blacklist_delay_sec_ = kBlacklistDefaultDelaySec;
    418   } else {
    419     int candidate_delay = blacklist_delay_sec_ * kBlacklistBackoffMultiplier;
    420     if (candidate_delay < kBlacklistMaxDelaySec)
    421       blacklist_delay_sec_ = candidate_delay;
    422   }
    423 }
    424 
    425 }  // namespace suggestions
    426