Home | History | Annotate | Download | only in url_request
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/url_request/url_request_throttler_manager.h"
      6 
      7 #include "base/logging.h"
      8 #include "base/metrics/field_trial.h"
      9 #include "base/metrics/histogram.h"
     10 #include "base/strings/string_util.h"
     11 #include "net/base/net_log.h"
     12 #include "net/base/net_util.h"
     13 
     14 namespace net {
     15 
     16 const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500;
     17 const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200;
     18 
     19 URLRequestThrottlerManager::URLRequestThrottlerManager()
     20     : requests_since_last_gc_(0),
     21       enable_thread_checks_(false),
     22       logged_for_localhost_disabled_(false),
     23       registered_from_thread_(base::kInvalidThreadId) {
     24   url_id_replacements_.ClearPassword();
     25   url_id_replacements_.ClearUsername();
     26   url_id_replacements_.ClearQuery();
     27   url_id_replacements_.ClearRef();
     28 
     29   NetworkChangeNotifier::AddIPAddressObserver(this);
     30   NetworkChangeNotifier::AddConnectionTypeObserver(this);
     31 }
     32 
     33 URLRequestThrottlerManager::~URLRequestThrottlerManager() {
     34   NetworkChangeNotifier::RemoveIPAddressObserver(this);
     35   NetworkChangeNotifier::RemoveConnectionTypeObserver(this);
     36 
     37   // Since the manager object might conceivably go away before the
     38   // entries, detach the entries' back-pointer to the manager.
     39   UrlEntryMap::iterator i = url_entries_.begin();
     40   while (i != url_entries_.end()) {
     41     if (i->second.get() != NULL) {
     42       i->second->DetachManager();
     43     }
     44     ++i;
     45   }
     46 
     47   // Delete all entries.
     48   url_entries_.clear();
     49 }
     50 
     51 scoped_refptr<URLRequestThrottlerEntryInterface>
     52     URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
     53   DCHECK(!enable_thread_checks_ || CalledOnValidThread());
     54 
     55   // Normalize the url.
     56   std::string url_id = GetIdFromUrl(url);
     57 
     58   // Periodically garbage collect old entries.
     59   GarbageCollectEntriesIfNecessary();
     60 
     61   // Find the entry in the map or create a new NULL entry.
     62   scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id];
     63 
     64   // If the entry exists but could be garbage collected at this point, we
     65   // start with a fresh entry so that we possibly back off a bit less
     66   // aggressively (i.e. this resets the error count when the entry's URL
     67   // hasn't been requested in long enough).
     68   if (entry.get() && entry->IsEntryOutdated()) {
     69     entry = NULL;
     70   }
     71 
     72   // Create the entry if needed.
     73   if (entry.get() == NULL) {
     74     entry = new URLRequestThrottlerEntry(this, url_id);
     75 
     76     // We only disable back-off throttling on an entry that we have
     77     // just constructed.  This is to allow unit tests to explicitly override
     78     // the entry for localhost URLs.  Given that we do not attempt to
     79     // disable throttling for entries already handed out (see comment
     80     // in AddToOptOutList), this is not a problem.
     81     std::string host = url.host();
     82     if (opt_out_hosts_.find(host) != opt_out_hosts_.end() ||
     83         IsLocalhost(host)) {
     84       if (!logged_for_localhost_disabled_ && IsLocalhost(host)) {
     85         logged_for_localhost_disabled_ = true;
     86         net_log_.AddEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
     87                           NetLog::StringCallback("host", &host));
     88       }
     89 
     90       // TODO(joi): Once sliding window is separate from back-off throttling,
     91       // we can simply return a dummy implementation of
     92       // URLRequestThrottlerEntryInterface here that never blocks anything (and
     93       // not keep entries in url_entries_ for opted-out sites).
     94       entry->DisableBackoffThrottling();
     95     }
     96   }
     97 
     98   return entry;
     99 }
    100 
    101 void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) {
    102   // There is an edge case here that we are not handling, to keep things
    103   // simple.  If a host starts adding the opt-out header to its responses
    104   // after there are already one or more entries in url_entries_ for that
    105   // host, the pre-existing entries may still perform back-off throttling.
    106   // In practice, this would almost never occur.
    107   if (opt_out_hosts_.find(host) == opt_out_hosts_.end()) {
    108     UMA_HISTOGRAM_COUNTS("Throttling.SiteOptedOut", 1);
    109 
    110     net_log_.EndEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST,
    111                       NetLog::StringCallback("host", &host));
    112     opt_out_hosts_.insert(host);
    113   }
    114 }
    115 
    116 void URLRequestThrottlerManager::OverrideEntryForTests(
    117     const GURL& url,
    118     URLRequestThrottlerEntry* entry) {
    119   // Normalize the url.
    120   std::string url_id = GetIdFromUrl(url);
    121 
    122   // Periodically garbage collect old entries.
    123   GarbageCollectEntriesIfNecessary();
    124 
    125   url_entries_[url_id] = entry;
    126 }
    127 
    128 void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) {
    129   // Normalize the url.
    130   std::string url_id = GetIdFromUrl(url);
    131   url_entries_.erase(url_id);
    132 }
    133 
    134 void URLRequestThrottlerManager::set_enable_thread_checks(bool enable) {
    135   enable_thread_checks_ = enable;
    136 }
    137 
    138 bool URLRequestThrottlerManager::enable_thread_checks() const {
    139   return enable_thread_checks_;
    140 }
    141 
    142 void URLRequestThrottlerManager::set_net_log(NetLog* net_log) {
    143   DCHECK(net_log);
    144   net_log_ = BoundNetLog::Make(net_log,
    145                                NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING);
    146 }
    147 
    148 NetLog* URLRequestThrottlerManager::net_log() const {
    149   return net_log_.net_log();
    150 }
    151 
    152 void URLRequestThrottlerManager::OnIPAddressChanged() {
    153   OnNetworkChange();
    154 }
    155 
    156 void URLRequestThrottlerManager::OnConnectionTypeChanged(
    157     NetworkChangeNotifier::ConnectionType type) {
    158   OnNetworkChange();
    159 }
    160 
    161 std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const {
    162   if (!url.is_valid())
    163     return url.possibly_invalid_spec();
    164 
    165   GURL id = url.ReplaceComponents(url_id_replacements_);
    166   return StringToLowerASCII(id.spec()).c_str();
    167 }
    168 
    169 void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
    170   requests_since_last_gc_++;
    171   if (requests_since_last_gc_ < kRequestsBetweenCollecting)
    172     return;
    173   requests_since_last_gc_ = 0;
    174 
    175   GarbageCollectEntries();
    176 }
    177 
    178 void URLRequestThrottlerManager::GarbageCollectEntries() {
    179   UrlEntryMap::iterator i = url_entries_.begin();
    180   while (i != url_entries_.end()) {
    181     if ((i->second)->IsEntryOutdated()) {
    182       url_entries_.erase(i++);
    183     } else {
    184       ++i;
    185     }
    186   }
    187 
    188   // In case something broke we want to make sure not to grow indefinitely.
    189   while (url_entries_.size() > kMaximumNumberOfEntries) {
    190     url_entries_.erase(url_entries_.begin());
    191   }
    192 }
    193 
    194 void URLRequestThrottlerManager::OnNetworkChange() {
    195   // Remove all entries.  Any entries that in-flight requests have a reference
    196   // to will live until those requests end, and these entries may be
    197   // inconsistent with new entries for the same URLs, but since what we
    198   // want is a clean slate for the new connection type, this is OK.
    199   url_entries_.clear();
    200   requests_since_last_gc_ = 0;
    201 }
    202 
    203 }  // namespace net
    204