Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/url_request/url_request_throttler_manager.h"
      6 
      7 #include "base/logging.h"
      8 #include "base/string_util.h"
      9 #include "net/base/net_util.h"
     10 
     11 namespace net {
     12 
     13 const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500;
     14 const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200;
     15 
     16 URLRequestThrottlerManager* URLRequestThrottlerManager::GetInstance() {
     17   return Singleton<URLRequestThrottlerManager>::get();
     18 }
     19 
     20 scoped_refptr<URLRequestThrottlerEntryInterface>
     21     URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) {
     22   DCHECK(!enable_thread_checks_ || CalledOnValidThread());
     23 
     24   // Normalize the url.
     25   std::string url_id = GetIdFromUrl(url);
     26 
     27   // Periodically garbage collect old entries.
     28   GarbageCollectEntriesIfNecessary();
     29 
     30   // Find the entry in the map or create it.
     31   scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id];
     32   if (entry.get() == NULL) {
     33     entry = new URLRequestThrottlerEntry(this);
     34 
     35     // We only disable back-off throttling on an entry that we have
     36     // just constructed.  This is to allow unit tests to explicitly override
     37     // the entry for localhost URLs.  Given that we do not attempt to
     38     // disable throttling for entries already handed out (see comment
     39     // in AddToOptOutList), this is not a problem.
     40     std::string host = url.host();
     41     if (opt_out_hosts_.find(host) != opt_out_hosts_.end() ||
     42         IsLocalhost(host)) {
     43       // TODO(joi): Once sliding window is separate from back-off throttling,
     44       // we can simply return a dummy implementation of
     45       // URLRequestThrottlerEntryInterface here that never blocks anything (and
     46       // not keep entries in url_entries_ for opted-out sites).
     47       entry->DisableBackoffThrottling();
     48     }
     49   }
     50 
     51   return entry;
     52 }
     53 
     54 void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) {
     55   // There is an edge case here that we are not handling, to keep things
     56   // simple.  If a host starts adding the opt-out header to its responses
     57   // after there are already one or more entries in url_entries_ for that
     58   // host, the pre-existing entries may still perform back-off throttling.
     59   // In practice, this would almost never occur.
     60   opt_out_hosts_.insert(host);
     61 }
     62 
     63 void URLRequestThrottlerManager::OverrideEntryForTests(
     64     const GURL& url,
     65     URLRequestThrottlerEntry* entry) {
     66   // Normalize the url.
     67   std::string url_id = GetIdFromUrl(url);
     68 
     69   // Periodically garbage collect old entries.
     70   GarbageCollectEntriesIfNecessary();
     71 
     72   url_entries_[url_id] = entry;
     73 }
     74 
     75 void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) {
     76   // Normalize the url.
     77   std::string url_id = GetIdFromUrl(url);
     78   url_entries_.erase(url_id);
     79 }
     80 
     81 void URLRequestThrottlerManager::set_enable_thread_checks(bool enable) {
     82   enable_thread_checks_ = enable;
     83 }
     84 
     85 bool URLRequestThrottlerManager::enable_thread_checks() const {
     86   return enable_thread_checks_;
     87 }
     88 
     89 void URLRequestThrottlerManager::set_enforce_throttling(bool enforce) {
     90   enforce_throttling_ = enforce;
     91 }
     92 
     93 bool URLRequestThrottlerManager::enforce_throttling() {
     94   return enforce_throttling_;
     95 }
     96 
     97 // TODO(joi): Turn throttling on by default when appropriate.
     98 URLRequestThrottlerManager::URLRequestThrottlerManager()
     99     : requests_since_last_gc_(0),
    100       enforce_throttling_(false),
    101       enable_thread_checks_(false) {
    102   // Construction/destruction is on main thread (because BrowserMain
    103   // retrieves an instance to call InitializeOptions), but is from then on
    104   // used on I/O thread.
    105   DetachFromThread();
    106 
    107   url_id_replacements_.ClearPassword();
    108   url_id_replacements_.ClearUsername();
    109   url_id_replacements_.ClearQuery();
    110   url_id_replacements_.ClearRef();
    111 }
    112 
    113 URLRequestThrottlerManager::~URLRequestThrottlerManager() {
    114   // Destruction is on main thread (AtExit), but real use is on I/O thread.
    115   DetachFromThread();
    116 
    117   // Since, for now, the manager object might conceivably go away before
    118   // the entries, detach the entries' back-pointer to the manager.
    119   //
    120   // TODO(joi): Revisit whether to make entries non-refcounted.
    121   UrlEntryMap::iterator i = url_entries_.begin();
    122   while (i != url_entries_.end()) {
    123     if (i->second != NULL) {
    124       i->second->DetachManager();
    125     }
    126     ++i;
    127   }
    128 
    129   // Delete all entries.
    130   url_entries_.clear();
    131 }
    132 
    133 std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const {
    134   if (!url.is_valid())
    135     return url.possibly_invalid_spec();
    136 
    137   GURL id = url.ReplaceComponents(url_id_replacements_);
    138   return StringToLowerASCII(id.spec()).c_str();
    139 }
    140 
    141 void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() {
    142   requests_since_last_gc_++;
    143   if (requests_since_last_gc_ < kRequestsBetweenCollecting)
    144     return;
    145   requests_since_last_gc_ = 0;
    146 
    147   GarbageCollectEntries();
    148 }
    149 
    150 void URLRequestThrottlerManager::GarbageCollectEntries() {
    151   UrlEntryMap::iterator i = url_entries_.begin();
    152   while (i != url_entries_.end()) {
    153     if ((i->second)->IsEntryOutdated()) {
    154       url_entries_.erase(i++);
    155     } else {
    156       ++i;
    157     }
    158   }
    159 
    160   // In case something broke we want to make sure not to grow indefinitely.
    161   while (url_entries_.size() > kMaximumNumberOfEntries) {
    162     url_entries_.erase(url_entries_.begin());
    163   }
    164 }
    165 
    166 }  // namespace net
    167