Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/url_request/url_request_throttler_entry.h"
      6 
      7 #include <cmath>
      8 
      9 #include "base/logging.h"
     10 #include "base/rand_util.h"
     11 #include "base/string_number_conversions.h"
     12 #include "net/url_request/url_request_throttler_header_interface.h"
     13 #include "net/url_request/url_request_throttler_manager.h"
     14 
     15 namespace net {
     16 
     17 const int URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs = 2000;
     18 const int URLRequestThrottlerEntry::kDefaultMaxSendThreshold = 20;
     19 
     20 // This set of back-off parameters will (at maximum values, i.e. without
     21 // the reduction caused by jitter) add 0-41% (distributed uniformly
     22 // in that range) to the "perceived downtime" of the remote server, once
     23 // exponential back-off kicks in and is throttling requests for more than
     24 // about a second at a time.  Once the maximum back-off is reached, the added
     25 // perceived downtime decreases rapidly, percentage-wise.
     26 //
     27 // Another way to put it is that the maximum additional perceived downtime
     28 // with these numbers is a couple of seconds shy of 15 minutes, and such
     29 // a delay would not occur until the remote server has been actually
     30 // unavailable at the end of each back-off period for a total of about
     31 // 48 minutes.
     32 //
     33 // Ignoring the first 4 errors helps avoid back-off from kicking in on
     34 // flaky connections.
     35 const int URLRequestThrottlerEntry::kDefaultNumErrorsToIgnore = 4;
     36 const int URLRequestThrottlerEntry::kDefaultInitialBackoffMs = 700;
     37 const double URLRequestThrottlerEntry::kDefaultMultiplyFactor = 1.4;
     38 const double URLRequestThrottlerEntry::kDefaultJitterFactor = 0.4;
     39 const int URLRequestThrottlerEntry::kDefaultMaximumBackoffMs = 15 * 60 * 1000;
     40 const int URLRequestThrottlerEntry::kDefaultEntryLifetimeMs = 2 * 60 * 1000;
     41 const char URLRequestThrottlerEntry::kRetryHeaderName[] = "X-Retry-After";
     42 const char URLRequestThrottlerEntry::kExponentialThrottlingHeader[] =
     43     "X-Chrome-Exponential-Throttling";
     44 const char URLRequestThrottlerEntry::kExponentialThrottlingDisableValue[] =
     45     "disable";
     46 
     47 URLRequestThrottlerEntry::URLRequestThrottlerEntry(
     48     URLRequestThrottlerManager* manager)
     49     : sliding_window_period_(
     50           base::TimeDelta::FromMilliseconds(kDefaultSlidingWindowPeriodMs)),
     51       max_send_threshold_(kDefaultMaxSendThreshold),
     52       is_backoff_disabled_(false),
     53       backoff_entry_(&backoff_policy_),
     54       manager_(manager) {
     55   DCHECK(manager_);
     56   Initialize();
     57 }
     58 
     59 URLRequestThrottlerEntry::URLRequestThrottlerEntry(
     60     URLRequestThrottlerManager* manager,
     61     int sliding_window_period_ms,
     62     int max_send_threshold,
     63     int initial_backoff_ms,
     64     double multiply_factor,
     65     double jitter_factor,
     66     int maximum_backoff_ms)
     67     : sliding_window_period_(
     68           base::TimeDelta::FromMilliseconds(sliding_window_period_ms)),
     69       max_send_threshold_(max_send_threshold),
     70       is_backoff_disabled_(false),
     71       backoff_entry_(&backoff_policy_),
     72       manager_(manager) {
     73   DCHECK_GT(sliding_window_period_ms, 0);
     74   DCHECK_GT(max_send_threshold_, 0);
     75   DCHECK_GE(initial_backoff_ms, 0);
     76   DCHECK_GT(multiply_factor, 0);
     77   DCHECK_GE(jitter_factor, 0.0);
     78   DCHECK_LT(jitter_factor, 1.0);
     79   DCHECK_GE(maximum_backoff_ms, 0);
     80   DCHECK(manager_);
     81 
     82   Initialize();
     83   backoff_policy_.initial_backoff_ms = initial_backoff_ms;
     84   backoff_policy_.multiply_factor = multiply_factor;
     85   backoff_policy_.jitter_factor = jitter_factor;
     86   backoff_policy_.maximum_backoff_ms = maximum_backoff_ms;
     87   backoff_policy_.entry_lifetime_ms = -1;
     88   backoff_policy_.num_errors_to_ignore = 0;
     89 }
     90 
     91 bool URLRequestThrottlerEntry::IsEntryOutdated() const {
     92   // This function is called by the URLRequestThrottlerManager to determine
     93   // whether entries should be discarded from its url_entries_ map.  We
     94   // want to ensure that it does not remove entries from the map while there
     95   // are clients (objects other than the manager) holding references to
     96   // the entry, otherwise separate clients could end up holding separate
     97   // entries for a request to the same URL, which is undesirable.  Therefore,
     98   // if an entry has more than one reference (the map will always hold one),
     99   // it should not be considered outdated.
    100   //
    101   // TODO(joi): Once the manager is not a Singleton, revisit whether
    102   // refcounting is needed at all.
    103   if (!HasOneRef())
    104     return false;
    105 
    106   // If there are send events in the sliding window period, we still need this
    107   // entry.
    108   if (!send_log_.empty() &&
    109       send_log_.back() + sliding_window_period_ > GetTimeNow()) {
    110     return false;
    111   }
    112 
    113   return GetBackoffEntry()->CanDiscard();
    114 }
    115 
    116 void URLRequestThrottlerEntry::DisableBackoffThrottling() {
    117   is_backoff_disabled_ = true;
    118 }
    119 
    120 void URLRequestThrottlerEntry::DetachManager() {
    121   manager_ = NULL;
    122 }
    123 
    124 bool URLRequestThrottlerEntry::IsDuringExponentialBackoff() const {
    125   if (is_backoff_disabled_)
    126     return false;
    127 
    128   return GetBackoffEntry()->ShouldRejectRequest();
    129 }
    130 
    131 int64 URLRequestThrottlerEntry::ReserveSendingTimeForNextRequest(
    132     const base::TimeTicks& earliest_time) {
    133   base::TimeTicks now = GetTimeNow();
    134 
    135   // If a lot of requests were successfully made recently,
    136   // sliding_window_release_time_ may be greater than
    137   // exponential_backoff_release_time_.
    138   base::TimeTicks recommended_sending_time =
    139       std::max(std::max(now, earliest_time),
    140                std::max(GetBackoffEntry()->GetReleaseTime(),
    141                         sliding_window_release_time_));
    142 
    143   DCHECK(send_log_.empty() ||
    144          recommended_sending_time >= send_log_.back());
    145   // Log the new send event.
    146   send_log_.push(recommended_sending_time);
    147 
    148   sliding_window_release_time_ = recommended_sending_time;
    149 
    150   // Drop the out-of-date events in the event list.
    151   // We don't need to worry that the queue may become empty during this
    152   // operation, since the last element is sliding_window_release_time_.
    153   while ((send_log_.front() + sliding_window_period_ <=
    154           sliding_window_release_time_) ||
    155          send_log_.size() > static_cast<unsigned>(max_send_threshold_)) {
    156     send_log_.pop();
    157   }
    158 
    159   // Check if there are too many send events in recent time.
    160   if (send_log_.size() == static_cast<unsigned>(max_send_threshold_))
    161     sliding_window_release_time_ = send_log_.front() + sliding_window_period_;
    162 
    163   return (recommended_sending_time - now).InMillisecondsRoundedUp();
    164 }
    165 
    166 base::TimeTicks
    167     URLRequestThrottlerEntry::GetExponentialBackoffReleaseTime() const {
    168   // If a site opts out, it's likely because they have problems that trigger
    169   // the back-off mechanism when it shouldn't be triggered, in which case
    170   // returning the calculated back-off release time would probably be the
    171   // wrong thing to do (i.e. it would likely be too long).  Therefore, we
    172   // return "now" so that retries are not delayed.
    173   if (is_backoff_disabled_)
    174     return GetTimeNow();
    175 
    176   return GetBackoffEntry()->GetReleaseTime();
    177 }
    178 
    179 void URLRequestThrottlerEntry::UpdateWithResponse(
    180     const std::string& host,
    181     const URLRequestThrottlerHeaderInterface* response) {
    182   if (response->GetResponseCode() >= 500) {
    183     GetBackoffEntry()->InformOfRequest(false);
    184   } else {
    185     GetBackoffEntry()->InformOfRequest(true);
    186 
    187     std::string retry_header = response->GetNormalizedValue(kRetryHeaderName);
    188     if (!retry_header.empty())
    189       HandleCustomRetryAfter(retry_header);
    190 
    191     std::string throttling_header = response->GetNormalizedValue(
    192         kExponentialThrottlingHeader);
    193     if (!throttling_header.empty())
    194       HandleThrottlingHeader(throttling_header, host);
    195   }
    196 }
    197 
    198 void URLRequestThrottlerEntry::ReceivedContentWasMalformed() {
    199   // A malformed body can only occur when the request to fetch a resource
    200   // was successful.  Therefore, in such a situation, we will receive one
    201   // call to ReceivedContentWasMalformed() and one call to UpdateWithResponse()
    202   // with a response categorized as "good".  To end up counting one failure,
    203   // we need to count two failures here against the one success in
    204   // UpdateWithResponse().
    205   GetBackoffEntry()->InformOfRequest(false);
    206   GetBackoffEntry()->InformOfRequest(false);
    207 }
    208 
    209 URLRequestThrottlerEntry::~URLRequestThrottlerEntry() {
    210 }
    211 
    212 void URLRequestThrottlerEntry::Initialize() {
    213   sliding_window_release_time_ = base::TimeTicks::Now();
    214   backoff_policy_.num_errors_to_ignore = kDefaultNumErrorsToIgnore;
    215   backoff_policy_.initial_backoff_ms = kDefaultInitialBackoffMs;
    216   backoff_policy_.multiply_factor = kDefaultMultiplyFactor;
    217   backoff_policy_.jitter_factor = kDefaultJitterFactor;
    218   backoff_policy_.maximum_backoff_ms = kDefaultMaximumBackoffMs;
    219   backoff_policy_.entry_lifetime_ms = kDefaultEntryLifetimeMs;
    220 }
    221 
    222 base::TimeTicks URLRequestThrottlerEntry::GetTimeNow() const {
    223   return base::TimeTicks::Now();
    224 }
    225 
    226 void URLRequestThrottlerEntry::HandleCustomRetryAfter(
    227     const std::string& header_value) {
    228   // Input parameter is the number of seconds to wait in a floating point value.
    229   double time_in_sec = 0;
    230   bool conversion_is_ok = base::StringToDouble(header_value, &time_in_sec);
    231 
    232   // Conversion of custom retry-after header value failed.
    233   if (!conversion_is_ok)
    234     return;
    235 
    236   // We must use an int value later so we transform this in milliseconds.
    237   int64 value_ms = static_cast<int64>(0.5 + time_in_sec * 1000);
    238 
    239   // We do not check for an upper bound; the server can set any Retry-After it
    240   // desires. Recovery from error would involve restarting the browser.
    241   if (value_ms < 0)
    242     return;
    243 
    244   GetBackoffEntry()->SetCustomReleaseTime(
    245       GetTimeNow() + base::TimeDelta::FromMilliseconds(value_ms));
    246 }
    247 
    248 void URLRequestThrottlerEntry::HandleThrottlingHeader(
    249     const std::string& header_value,
    250     const std::string& host) {
    251   if (header_value == kExponentialThrottlingDisableValue) {
    252     DisableBackoffThrottling();
    253     if (manager_)
    254       manager_->AddToOptOutList(host);
    255   } else {
    256     // TODO(joi): Log this.
    257   }
    258 }
    259 
    260 const BackoffEntry* URLRequestThrottlerEntry::GetBackoffEntry() const {
    261   return &backoff_entry_;
    262 }
    263 
    264 BackoffEntry* URLRequestThrottlerEntry::GetBackoffEntry() {
    265   return &backoff_entry_;
    266 }
    267 
    268 }  // namespace net
    269