1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "net/url_request/url_request_throttler_manager.h" 6 7 #include "base/logging.h" 8 #include "base/metrics/field_trial.h" 9 #include "base/metrics/histogram.h" 10 #include "base/strings/string_util.h" 11 #include "net/base/net_log.h" 12 #include "net/base/net_util.h" 13 14 namespace net { 15 16 const unsigned int URLRequestThrottlerManager::kMaximumNumberOfEntries = 1500; 17 const unsigned int URLRequestThrottlerManager::kRequestsBetweenCollecting = 200; 18 19 URLRequestThrottlerManager::URLRequestThrottlerManager() 20 : requests_since_last_gc_(0), 21 enable_thread_checks_(false), 22 logged_for_localhost_disabled_(false), 23 registered_from_thread_(base::kInvalidThreadId) { 24 url_id_replacements_.ClearPassword(); 25 url_id_replacements_.ClearUsername(); 26 url_id_replacements_.ClearQuery(); 27 url_id_replacements_.ClearRef(); 28 29 NetworkChangeNotifier::AddIPAddressObserver(this); 30 NetworkChangeNotifier::AddConnectionTypeObserver(this); 31 } 32 33 URLRequestThrottlerManager::~URLRequestThrottlerManager() { 34 NetworkChangeNotifier::RemoveIPAddressObserver(this); 35 NetworkChangeNotifier::RemoveConnectionTypeObserver(this); 36 37 // Since the manager object might conceivably go away before the 38 // entries, detach the entries' back-pointer to the manager. 39 UrlEntryMap::iterator i = url_entries_.begin(); 40 while (i != url_entries_.end()) { 41 if (i->second.get() != NULL) { 42 i->second->DetachManager(); 43 } 44 ++i; 45 } 46 47 // Delete all entries. 48 url_entries_.clear(); 49 } 50 51 scoped_refptr<URLRequestThrottlerEntryInterface> 52 URLRequestThrottlerManager::RegisterRequestUrl(const GURL &url) { 53 DCHECK(!enable_thread_checks_ || CalledOnValidThread()); 54 55 // Normalize the url. 56 std::string url_id = GetIdFromUrl(url); 57 58 // Periodically garbage collect old entries. 59 GarbageCollectEntriesIfNecessary(); 60 61 // Find the entry in the map or create a new NULL entry. 62 scoped_refptr<URLRequestThrottlerEntry>& entry = url_entries_[url_id]; 63 64 // If the entry exists but could be garbage collected at this point, we 65 // start with a fresh entry so that we possibly back off a bit less 66 // aggressively (i.e. this resets the error count when the entry's URL 67 // hasn't been requested in long enough). 68 if (entry.get() && entry->IsEntryOutdated()) { 69 entry = NULL; 70 } 71 72 // Create the entry if needed. 73 if (entry.get() == NULL) { 74 entry = new URLRequestThrottlerEntry(this, url_id); 75 76 // We only disable back-off throttling on an entry that we have 77 // just constructed. This is to allow unit tests to explicitly override 78 // the entry for localhost URLs. Given that we do not attempt to 79 // disable throttling for entries already handed out (see comment 80 // in AddToOptOutList), this is not a problem. 81 std::string host = url.host(); 82 if (opt_out_hosts_.find(host) != opt_out_hosts_.end() || 83 IsLocalhost(host)) { 84 if (!logged_for_localhost_disabled_ && IsLocalhost(host)) { 85 logged_for_localhost_disabled_ = true; 86 net_log_.AddEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST, 87 NetLog::StringCallback("host", &host)); 88 } 89 90 // TODO(joi): Once sliding window is separate from back-off throttling, 91 // we can simply return a dummy implementation of 92 // URLRequestThrottlerEntryInterface here that never blocks anything (and 93 // not keep entries in url_entries_ for opted-out sites). 94 entry->DisableBackoffThrottling(); 95 } 96 } 97 98 return entry; 99 } 100 101 void URLRequestThrottlerManager::AddToOptOutList(const std::string& host) { 102 // There is an edge case here that we are not handling, to keep things 103 // simple. If a host starts adding the opt-out header to its responses 104 // after there are already one or more entries in url_entries_ for that 105 // host, the pre-existing entries may still perform back-off throttling. 106 // In practice, this would almost never occur. 107 if (opt_out_hosts_.find(host) == opt_out_hosts_.end()) { 108 UMA_HISTOGRAM_COUNTS("Throttling.SiteOptedOut", 1); 109 110 net_log_.EndEvent(NetLog::TYPE_THROTTLING_DISABLED_FOR_HOST, 111 NetLog::StringCallback("host", &host)); 112 opt_out_hosts_.insert(host); 113 } 114 } 115 116 void URLRequestThrottlerManager::OverrideEntryForTests( 117 const GURL& url, 118 URLRequestThrottlerEntry* entry) { 119 // Normalize the url. 120 std::string url_id = GetIdFromUrl(url); 121 122 // Periodically garbage collect old entries. 123 GarbageCollectEntriesIfNecessary(); 124 125 url_entries_[url_id] = entry; 126 } 127 128 void URLRequestThrottlerManager::EraseEntryForTests(const GURL& url) { 129 // Normalize the url. 130 std::string url_id = GetIdFromUrl(url); 131 url_entries_.erase(url_id); 132 } 133 134 void URLRequestThrottlerManager::set_enable_thread_checks(bool enable) { 135 enable_thread_checks_ = enable; 136 } 137 138 bool URLRequestThrottlerManager::enable_thread_checks() const { 139 return enable_thread_checks_; 140 } 141 142 void URLRequestThrottlerManager::set_net_log(NetLog* net_log) { 143 DCHECK(net_log); 144 net_log_ = BoundNetLog::Make(net_log, 145 NetLog::SOURCE_EXPONENTIAL_BACKOFF_THROTTLING); 146 } 147 148 NetLog* URLRequestThrottlerManager::net_log() const { 149 return net_log_.net_log(); 150 } 151 152 void URLRequestThrottlerManager::OnIPAddressChanged() { 153 OnNetworkChange(); 154 } 155 156 void URLRequestThrottlerManager::OnConnectionTypeChanged( 157 NetworkChangeNotifier::ConnectionType type) { 158 OnNetworkChange(); 159 } 160 161 std::string URLRequestThrottlerManager::GetIdFromUrl(const GURL& url) const { 162 if (!url.is_valid()) 163 return url.possibly_invalid_spec(); 164 165 GURL id = url.ReplaceComponents(url_id_replacements_); 166 return base::StringToLowerASCII(id.spec()).c_str(); 167 } 168 169 void URLRequestThrottlerManager::GarbageCollectEntriesIfNecessary() { 170 requests_since_last_gc_++; 171 if (requests_since_last_gc_ < kRequestsBetweenCollecting) 172 return; 173 requests_since_last_gc_ = 0; 174 175 GarbageCollectEntries(); 176 } 177 178 void URLRequestThrottlerManager::GarbageCollectEntries() { 179 UrlEntryMap::iterator i = url_entries_.begin(); 180 while (i != url_entries_.end()) { 181 if ((i->second)->IsEntryOutdated()) { 182 url_entries_.erase(i++); 183 } else { 184 ++i; 185 } 186 } 187 188 // In case something broke we want to make sure not to grow indefinitely. 189 while (url_entries_.size() > kMaximumNumberOfEntries) { 190 url_entries_.erase(url_entries_.begin()); 191 } 192 } 193 194 void URLRequestThrottlerManager::OnNetworkChange() { 195 // Remove all entries. Any entries that in-flight requests have a reference 196 // to will live until those requests end, and these entries may be 197 // inconsistent with new entries for the same URLs, but since what we 198 // want is a clean slate for the new connection type, this is OK. 199 url_entries_.clear(); 200 requests_since_last_gc_ = 0; 201 } 202 203 } // namespace net 204