1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "net/url_request/url_request_throttler_manager.h" 6 7 #include "base/memory/scoped_ptr.h" 8 #include "base/metrics/histogram_samples.h" 9 #include "base/pickle.h" 10 #include "base/stl_util.h" 11 #include "base/strings/string_number_conversions.h" 12 #include "base/strings/stringprintf.h" 13 #include "base/test/statistics_delta_reader.h" 14 #include "base/time/time.h" 15 #include "net/base/load_flags.h" 16 #include "net/base/request_priority.h" 17 #include "net/base/test_completion_callback.h" 18 #include "net/url_request/url_request_context.h" 19 #include "net/url_request/url_request_test_util.h" 20 #include "net/url_request/url_request_throttler_header_interface.h" 21 #include "net/url_request/url_request_throttler_test_support.h" 22 #include "testing/gtest/include/gtest/gtest.h" 23 24 using base::TimeDelta; 25 using base::TimeTicks; 26 27 namespace net { 28 29 namespace { 30 31 const char kRequestThrottledHistogramName[] = "Throttling.RequestThrottled"; 32 33 class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry { 34 public: 35 explicit MockURLRequestThrottlerEntry( 36 net::URLRequestThrottlerManager* manager) 37 : net::URLRequestThrottlerEntry(manager, std::string()), 38 mock_backoff_entry_(&backoff_policy_) { 39 InitPolicy(); 40 } 41 MockURLRequestThrottlerEntry( 42 net::URLRequestThrottlerManager* manager, 43 const TimeTicks& exponential_backoff_release_time, 44 const TimeTicks& sliding_window_release_time, 45 const TimeTicks& fake_now) 46 : net::URLRequestThrottlerEntry(manager, std::string()), 47 fake_time_now_(fake_now), 48 mock_backoff_entry_(&backoff_policy_) { 49 InitPolicy(); 50 51 mock_backoff_entry_.set_fake_now(fake_now); 52 set_exponential_backoff_release_time(exponential_backoff_release_time); 53 set_sliding_window_release_time(sliding_window_release_time); 54 } 55 56 void InitPolicy() { 57 // Some tests become flaky if we have jitter. 58 backoff_policy_.jitter_factor = 0.0; 59 60 // This lets us avoid having to make multiple failures initially (this 61 // logic is already tested in the BackoffEntry unit tests). 62 backoff_policy_.num_errors_to_ignore = 0; 63 } 64 65 virtual const BackoffEntry* GetBackoffEntry() const OVERRIDE { 66 return &mock_backoff_entry_; 67 } 68 69 virtual BackoffEntry* GetBackoffEntry() OVERRIDE { 70 return &mock_backoff_entry_; 71 } 72 73 static bool ExplicitUserRequest(int load_flags) { 74 return URLRequestThrottlerEntry::ExplicitUserRequest(load_flags); 75 } 76 77 void ResetToBlank(const TimeTicks& time_now) { 78 fake_time_now_ = time_now; 79 mock_backoff_entry_.set_fake_now(time_now); 80 81 GetBackoffEntry()->Reset(); 82 GetBackoffEntry()->SetCustomReleaseTime(time_now); 83 set_sliding_window_release_time(time_now); 84 } 85 86 // Overridden for tests. 87 virtual TimeTicks ImplGetTimeNow() const OVERRIDE { return fake_time_now_; } 88 89 void set_exponential_backoff_release_time( 90 const base::TimeTicks& release_time) { 91 GetBackoffEntry()->SetCustomReleaseTime(release_time); 92 } 93 94 base::TimeTicks sliding_window_release_time() const { 95 return URLRequestThrottlerEntry::sliding_window_release_time(); 96 } 97 98 void set_sliding_window_release_time( 99 const base::TimeTicks& release_time) { 100 URLRequestThrottlerEntry::set_sliding_window_release_time( 101 release_time); 102 } 103 104 TimeTicks fake_time_now_; 105 MockBackoffEntry mock_backoff_entry_; 106 107 protected: 108 virtual ~MockURLRequestThrottlerEntry() {} 109 }; 110 111 class MockURLRequestThrottlerManager : public URLRequestThrottlerManager { 112 public: 113 MockURLRequestThrottlerManager() : create_entry_index_(0) {} 114 115 // Method to process the URL using URLRequestThrottlerManager protected 116 // method. 117 std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); } 118 119 // Method to use the garbage collecting method of URLRequestThrottlerManager. 120 void DoGarbageCollectEntries() { GarbageCollectEntries(); } 121 122 // Returns the number of entries in the map. 123 int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); } 124 125 void CreateEntry(bool is_outdated) { 126 TimeTicks time = TimeTicks::Now(); 127 if (is_outdated) { 128 time -= TimeDelta::FromMilliseconds( 129 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000); 130 } 131 std::string fake_url_string("http://www.fakeurl.com/"); 132 fake_url_string.append(base::IntToString(create_entry_index_++)); 133 GURL fake_url(fake_url_string); 134 OverrideEntryForTests( 135 fake_url, 136 new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(), 137 TimeTicks::Now())); 138 } 139 140 private: 141 int create_entry_index_; 142 }; 143 144 struct TimeAndBool { 145 TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) { 146 time = time_value; 147 result = expected; 148 line = line_num; 149 } 150 TimeTicks time; 151 bool result; 152 int line; 153 }; 154 155 struct GurlAndString { 156 GurlAndString(const GURL& url_value, 157 const std::string& expected, 158 int line_num) { 159 url = url_value; 160 result = expected; 161 line = line_num; 162 } 163 GURL url; 164 std::string result; 165 int line; 166 }; 167 168 } // namespace 169 170 class URLRequestThrottlerEntryTest : public testing::Test { 171 protected: 172 URLRequestThrottlerEntryTest() 173 : request_(GURL(), DEFAULT_PRIORITY, NULL, &context_) {} 174 175 virtual void SetUp(); 176 177 TimeTicks now_; 178 MockURLRequestThrottlerManager manager_; // Dummy object, not used. 179 scoped_refptr<MockURLRequestThrottlerEntry> entry_; 180 181 TestURLRequestContext context_; 182 TestURLRequest request_; 183 }; 184 185 void URLRequestThrottlerEntryTest::SetUp() { 186 request_.SetLoadFlags(0); 187 188 now_ = TimeTicks::Now(); 189 entry_ = new MockURLRequestThrottlerEntry(&manager_); 190 entry_->ResetToBlank(now_); 191 } 192 193 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) { 194 return out << time.ToInternalValue(); 195 } 196 197 TEST_F(URLRequestThrottlerEntryTest, CanThrottleRequest) { 198 TestNetworkDelegate d; 199 context_.set_network_delegate(&d); 200 entry_->set_exponential_backoff_release_time( 201 entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1)); 202 203 d.set_can_throttle_requests(false); 204 EXPECT_FALSE(entry_->ShouldRejectRequest(request_)); 205 d.set_can_throttle_requests(true); 206 EXPECT_TRUE(entry_->ShouldRejectRequest(request_)); 207 } 208 209 TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) { 210 base::StatisticsDeltaReader statistics_delta_reader; 211 entry_->set_exponential_backoff_release_time( 212 entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1)); 213 EXPECT_TRUE(entry_->ShouldRejectRequest(request_)); 214 215 // Also end-to-end test the load flags exceptions. 216 request_.SetLoadFlags(LOAD_MAYBE_USER_GESTURE); 217 EXPECT_FALSE(entry_->ShouldRejectRequest(request_)); 218 219 scoped_ptr<base::HistogramSamples> samples( 220 statistics_delta_reader.GetHistogramSamplesSinceCreation( 221 kRequestThrottledHistogramName)); 222 ASSERT_EQ(1, samples->GetCount(0)); 223 ASSERT_EQ(1, samples->GetCount(1)); 224 } 225 226 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) { 227 base::StatisticsDeltaReader statistics_delta_reader; 228 entry_->set_exponential_backoff_release_time(entry_->fake_time_now_); 229 EXPECT_FALSE(entry_->ShouldRejectRequest(request_)); 230 entry_->set_exponential_backoff_release_time( 231 entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1)); 232 EXPECT_FALSE(entry_->ShouldRejectRequest(request_)); 233 234 scoped_ptr<base::HistogramSamples> samples( 235 statistics_delta_reader.GetHistogramSamplesSinceCreation( 236 kRequestThrottledHistogramName)); 237 ASSERT_EQ(2, samples->GetCount(0)); 238 ASSERT_EQ(0, samples->GetCount(1)); 239 } 240 241 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) { 242 MockURLRequestThrottlerHeaderAdapter failure_response(503); 243 entry_->UpdateWithResponse(std::string(), &failure_response); 244 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_) 245 << "A failure should increase the release_time"; 246 } 247 248 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) { 249 MockURLRequestThrottlerHeaderAdapter success_response(200); 250 entry_->UpdateWithResponse(std::string(), &success_response); 251 EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_) 252 << "A success should not add any delay"; 253 } 254 255 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) { 256 MockURLRequestThrottlerHeaderAdapter failure_response(503); 257 MockURLRequestThrottlerHeaderAdapter success_response(200); 258 entry_->UpdateWithResponse(std::string(), &success_response); 259 entry_->UpdateWithResponse(std::string(), &failure_response); 260 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_) 261 << "This scenario should add delay"; 262 entry_->UpdateWithResponse(std::string(), &success_response); 263 } 264 265 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) { 266 TimeDelta lifetime = TimeDelta::FromMilliseconds( 267 MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs); 268 const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5); 269 270 TimeAndBool test_values[] = { 271 TimeAndBool(now_, false, __LINE__), 272 TimeAndBool(now_ - kFiveMs, false, __LINE__), 273 TimeAndBool(now_ + kFiveMs, false, __LINE__), 274 TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__), 275 TimeAndBool(now_ - lifetime, true, __LINE__), 276 TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)}; 277 278 for (unsigned int i = 0; i < arraysize(test_values); ++i) { 279 entry_->set_exponential_backoff_release_time(test_values[i].time); 280 EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) << 281 "Test case #" << i << " line " << test_values[i].line << " failed"; 282 } 283 } 284 285 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) { 286 for (int i = 0; i < 30; ++i) { 287 MockURLRequestThrottlerHeaderAdapter response_adapter(503); 288 entry_->UpdateWithResponse(std::string(), &response_adapter); 289 } 290 291 TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_; 292 EXPECT_EQ(delay.InMilliseconds(), 293 MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs); 294 } 295 296 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) { 297 MockURLRequestThrottlerHeaderAdapter response_adapter(503); 298 for (int i = 0; i < 5; ++i) 299 entry_->UpdateWithResponse(std::string(), &response_adapter); 300 301 TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime(); 302 303 // Inform the entry that a response body was malformed, which is supposed to 304 // increase the back-off time. Note that we also submit a successful 305 // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that 306 // is what happens in practice (if a body is received, then a non-500 307 // response must also have been received). 308 entry_->ReceivedContentWasMalformed(200); 309 MockURLRequestThrottlerHeaderAdapter success_adapter(200); 310 entry_->UpdateWithResponse(std::string(), &success_adapter); 311 EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures); 312 } 313 314 TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) { 315 int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold; 316 int sliding_window = 317 URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs; 318 319 TimeTicks time_1 = entry_->fake_time_now_ + 320 TimeDelta::FromMilliseconds(sliding_window / 3); 321 TimeTicks time_2 = entry_->fake_time_now_ + 322 TimeDelta::FromMilliseconds(2 * sliding_window / 3); 323 TimeTicks time_3 = entry_->fake_time_now_ + 324 TimeDelta::FromMilliseconds(sliding_window); 325 TimeTicks time_4 = entry_->fake_time_now_ + 326 TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3); 327 328 entry_->set_exponential_backoff_release_time(time_1); 329 330 for (int i = 0; i < max_send / 2; ++i) { 331 EXPECT_EQ(2 * sliding_window / 3, 332 entry_->ReserveSendingTimeForNextRequest(time_2)); 333 } 334 EXPECT_EQ(time_2, entry_->sliding_window_release_time()); 335 336 entry_->fake_time_now_ = time_3; 337 338 for (int i = 0; i < (max_send + 1) / 2; ++i) 339 EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks())); 340 341 EXPECT_EQ(time_4, entry_->sliding_window_release_time()); 342 } 343 344 TEST_F(URLRequestThrottlerEntryTest, ExplicitUserRequest) { 345 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest(0)); 346 ASSERT_TRUE(MockURLRequestThrottlerEntry::ExplicitUserRequest( 347 LOAD_MAYBE_USER_GESTURE)); 348 ASSERT_FALSE(MockURLRequestThrottlerEntry::ExplicitUserRequest( 349 ~LOAD_MAYBE_USER_GESTURE)); 350 } 351 352 class URLRequestThrottlerManagerTest : public testing::Test { 353 protected: 354 URLRequestThrottlerManagerTest() 355 : request_(GURL(), DEFAULT_PRIORITY, NULL, &context_) {} 356 357 virtual void SetUp() { 358 request_.SetLoadFlags(0); 359 } 360 361 // context_ must be declared before request_. 362 TestURLRequestContext context_; 363 TestURLRequest request_; 364 }; 365 366 TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) { 367 MockURLRequestThrottlerManager manager; 368 GurlAndString test_values[] = { 369 GurlAndString(GURL("http://www.example.com"), 370 std::string("http://www.example.com/"), 371 __LINE__), 372 GurlAndString(GURL("http://www.Example.com"), 373 std::string("http://www.example.com/"), 374 __LINE__), 375 GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"), 376 std::string("http://www.ex4mple.com/pr4c71c41"), 377 __LINE__), 378 GurlAndString(GURL("http://www.example.com/0/token/false"), 379 std::string("http://www.example.com/0/token/false"), 380 __LINE__), 381 GurlAndString(GURL("http://www.example.com/index.php?code=javascript"), 382 std::string("http://www.example.com/index.php"), 383 __LINE__), 384 GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"), 385 std::string("http://www.example.com/index.php"), 386 __LINE__), 387 GurlAndString(GURL("http://www.example.com/index.php#superEntry"), 388 std::string("http://www.example.com/index.php"), 389 __LINE__), 390 GurlAndString(GURL("http://www.example.com:1234/"), 391 std::string("http://www.example.com:1234/"), 392 __LINE__)}; 393 394 for (unsigned int i = 0; i < arraysize(test_values); ++i) { 395 std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url); 396 EXPECT_EQ(temp, test_values[i].result) << 397 "Test case #" << i << " line " << test_values[i].line << " failed"; 398 } 399 } 400 401 TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) { 402 MockURLRequestThrottlerManager manager; 403 404 manager.CreateEntry(true); // true = Entry is outdated. 405 manager.CreateEntry(true); 406 manager.CreateEntry(true); 407 manager.DoGarbageCollectEntries(); 408 EXPECT_EQ(0, manager.GetNumberOfEntries()); 409 410 manager.CreateEntry(false); 411 manager.CreateEntry(false); 412 manager.CreateEntry(false); 413 manager.CreateEntry(true); 414 manager.DoGarbageCollectEntries(); 415 EXPECT_EQ(3, manager.GetNumberOfEntries()); 416 } 417 418 TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) { 419 MockURLRequestThrottlerManager manager; 420 421 manager.RegisterRequestUrl(GURL("http://www.example.com/")); 422 manager.RegisterRequestUrl(GURL("http://www.google.com/")); 423 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0")); 424 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1")); 425 manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure")); 426 427 EXPECT_EQ(3, manager.GetNumberOfEntries()); 428 } 429 430 void ExpectEntryAllowsAllOnErrorIfOptedOut( 431 net::URLRequestThrottlerEntryInterface* entry, 432 bool opted_out, 433 const URLRequest& request) { 434 EXPECT_FALSE(entry->ShouldRejectRequest(request)); 435 MockURLRequestThrottlerHeaderAdapter failure_adapter(503); 436 for (int i = 0; i < 10; ++i) { 437 // Host doesn't really matter in this scenario so we skip it. 438 entry->UpdateWithResponse(std::string(), &failure_adapter); 439 } 440 EXPECT_NE(opted_out, entry->ShouldRejectRequest(request)); 441 442 if (opted_out) { 443 // We're not mocking out GetTimeNow() in this scenario 444 // so add a 100 ms buffer to avoid flakiness (that should always 445 // give enough time to get from the TimeTicks::Now() call here 446 // to the TimeTicks::Now() call in the entry class). 447 EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100), 448 entry->GetExponentialBackoffReleaseTime()); 449 } else { 450 // As above, add 100 ms. 451 EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100), 452 entry->GetExponentialBackoffReleaseTime()); 453 } 454 } 455 456 TEST_F(URLRequestThrottlerManagerTest, OptOutHeader) { 457 MockURLRequestThrottlerManager manager; 458 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry = 459 manager.RegisterRequestUrl(GURL("http://www.google.com/yodude")); 460 461 // Fake a response with the opt-out header. 462 MockURLRequestThrottlerHeaderAdapter response_adapter( 463 std::string(), 464 MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue, 465 200); 466 entry->UpdateWithResponse("www.google.com", &response_adapter); 467 468 // Ensure that the same entry on error always allows everything. 469 ExpectEntryAllowsAllOnErrorIfOptedOut(entry.get(), true, request_); 470 471 // Ensure that a freshly created entry (for a different URL on an 472 // already opted-out host) also gets "always allow" behavior. 473 scoped_refptr<net::URLRequestThrottlerEntryInterface> other_entry = 474 manager.RegisterRequestUrl(GURL("http://www.google.com/bingobob")); 475 ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry.get(), true, request_); 476 477 // Fake a response with the opt-out header incorrectly specified. 478 scoped_refptr<net::URLRequestThrottlerEntryInterface> no_opt_out_entry = 479 manager.RegisterRequestUrl(GURL("http://www.nike.com/justdoit")); 480 MockURLRequestThrottlerHeaderAdapter wrong_adapter( 481 std::string(), "yesplease", 200); 482 no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter); 483 ExpectEntryAllowsAllOnErrorIfOptedOut( 484 no_opt_out_entry.get(), false, request_); 485 486 // A localhost entry should always be opted out. 487 scoped_refptr<net::URLRequestThrottlerEntryInterface> localhost_entry = 488 manager.RegisterRequestUrl(GURL("http://localhost/hello")); 489 ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry.get(), true, request_); 490 } 491 492 TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) { 493 for (int i = 0; i < 3; ++i) { 494 MockURLRequestThrottlerManager manager; 495 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_before = 496 manager.RegisterRequestUrl(GURL("http://www.example.com/")); 497 MockURLRequestThrottlerHeaderAdapter failure_adapter(503); 498 for (int j = 0; j < 10; ++j) { 499 // Host doesn't really matter in this scenario so we skip it. 500 entry_before->UpdateWithResponse(std::string(), &failure_adapter); 501 } 502 EXPECT_TRUE(entry_before->ShouldRejectRequest(request_)); 503 504 switch (i) { 505 case 0: 506 manager.OnIPAddressChanged(); 507 break; 508 case 1: 509 manager.OnConnectionTypeChanged( 510 net::NetworkChangeNotifier::CONNECTION_UNKNOWN); 511 break; 512 case 2: 513 manager.OnConnectionTypeChanged( 514 net::NetworkChangeNotifier::CONNECTION_NONE); 515 break; 516 default: 517 FAIL(); 518 } 519 520 scoped_refptr<net::URLRequestThrottlerEntryInterface> entry_after = 521 manager.RegisterRequestUrl(GURL("http://www.example.com/")); 522 EXPECT_FALSE(entry_after->ShouldRejectRequest(request_)); 523 } 524 } 525 526 } // namespace net 527