Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/memory/scoped_ptr.h"
      6 #include "base/pickle.h"
      7 #include "base/stringprintf.h"
      8 #include "base/string_number_conversions.h"
      9 #include "base/time.h"
     10 #include "net/base/test_completion_callback.h"
     11 #include "net/url_request/url_request_context.h"
     12 #include "net/url_request/url_request_throttler_header_interface.h"
     13 #include "net/url_request/url_request_throttler_manager.h"
     14 #include "testing/gtest/include/gtest/gtest.h"
     15 
     16 using base::TimeDelta;
     17 using base::TimeTicks;
     18 
     19 namespace net {
     20 
     21 namespace {
     22 class MockURLRequestThrottlerManager;
     23 
     24 class MockBackoffEntry : public BackoffEntry {
     25  public:
     26   explicit MockBackoffEntry(const BackoffEntry::Policy* const policy)
     27       : BackoffEntry(policy), fake_now_(TimeTicks()) {
     28   }
     29 
     30   virtual ~MockBackoffEntry() {}
     31 
     32   TimeTicks GetTimeNow() const {
     33     return fake_now_;
     34   }
     35 
     36   void SetFakeNow(const TimeTicks& now) {
     37     fake_now_ = now;
     38   }
     39 
     40  private:
     41   TimeTicks fake_now_;
     42 };
     43 
     44 class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
     45  public :
     46   explicit MockURLRequestThrottlerEntry(
     47       net::URLRequestThrottlerManager* manager)
     48       : net::URLRequestThrottlerEntry(manager),
     49         mock_backoff_entry_(&backoff_policy_) {
     50     InitPolicy();
     51   }
     52   MockURLRequestThrottlerEntry(
     53       net::URLRequestThrottlerManager* manager,
     54       const TimeTicks& exponential_backoff_release_time,
     55       const TimeTicks& sliding_window_release_time,
     56       const TimeTicks& fake_now)
     57       : net::URLRequestThrottlerEntry(manager),
     58         fake_time_now_(fake_now),
     59         mock_backoff_entry_(&backoff_policy_) {
     60     InitPolicy();
     61 
     62     mock_backoff_entry_.SetFakeNow(fake_now);
     63     set_exponential_backoff_release_time(exponential_backoff_release_time);
     64     set_sliding_window_release_time(sliding_window_release_time);
     65   }
     66   virtual ~MockURLRequestThrottlerEntry() {}
     67 
     68   void InitPolicy() {
     69     // Some tests become flaky if we have jitter.
     70     backoff_policy_.jitter_factor = 0.0;
     71 
     72     // This lets us avoid having to make multiple failures initially (this
     73     // logic is already tested in the BackoffEntry unit tests).
     74     backoff_policy_.num_errors_to_ignore = 0;
     75   }
     76 
     77   const BackoffEntry* GetBackoffEntry() const {
     78     return &mock_backoff_entry_;
     79   }
     80 
     81   BackoffEntry* GetBackoffEntry() {
     82     return &mock_backoff_entry_;
     83   }
     84 
     85   void ResetToBlank(const TimeTicks& time_now) {
     86     fake_time_now_ = time_now;
     87     mock_backoff_entry_.SetFakeNow(time_now);
     88 
     89     GetBackoffEntry()->InformOfRequest(true);  // Sets failure count to 0.
     90     GetBackoffEntry()->SetCustomReleaseTime(time_now);
     91     set_sliding_window_release_time(time_now);
     92   }
     93 
     94   // Overridden for tests.
     95   virtual TimeTicks GetTimeNow() const { return fake_time_now_; }
     96 
     97   void set_exponential_backoff_release_time(
     98       const base::TimeTicks& release_time) {
     99     GetBackoffEntry()->SetCustomReleaseTime(release_time);
    100   }
    101 
    102   base::TimeTicks sliding_window_release_time() const {
    103     return URLRequestThrottlerEntry::sliding_window_release_time();
    104   }
    105 
    106   void set_sliding_window_release_time(
    107       const base::TimeTicks& release_time) {
    108     URLRequestThrottlerEntry::set_sliding_window_release_time(
    109         release_time);
    110   }
    111 
    112   TimeTicks fake_time_now_;
    113   MockBackoffEntry mock_backoff_entry_;
    114 };
    115 
    116 class MockURLRequestThrottlerHeaderAdapter
    117     : public URLRequestThrottlerHeaderInterface {
    118  public:
    119   MockURLRequestThrottlerHeaderAdapter()
    120       : fake_retry_value_(""),
    121         fake_opt_out_value_(""),
    122         fake_response_code_(0) {
    123   }
    124 
    125   explicit MockURLRequestThrottlerHeaderAdapter(int response_code)
    126       : fake_retry_value_(""),
    127         fake_opt_out_value_(""),
    128         fake_response_code_(response_code) {
    129   }
    130 
    131   MockURLRequestThrottlerHeaderAdapter(const std::string& retry_value,
    132                                        const std::string& opt_out_value,
    133                                        int response_code)
    134       : fake_retry_value_(retry_value),
    135         fake_opt_out_value_(opt_out_value),
    136         fake_response_code_(response_code) {
    137   }
    138 
    139   virtual ~MockURLRequestThrottlerHeaderAdapter() {}
    140 
    141   virtual std::string GetNormalizedValue(const std::string& key) const {
    142     if (key == MockURLRequestThrottlerEntry::kRetryHeaderName &&
    143         !fake_retry_value_.empty()) {
    144       return fake_retry_value_;
    145     } else if (key ==
    146         MockURLRequestThrottlerEntry::kExponentialThrottlingHeader &&
    147         !fake_opt_out_value_.empty()) {
    148       return fake_opt_out_value_;
    149     }
    150     return "";
    151   }
    152 
    153   virtual int GetResponseCode() const { return fake_response_code_; }
    154 
    155   std::string fake_retry_value_;
    156   std::string fake_opt_out_value_;
    157   int fake_response_code_;
    158 };
    159 
    160 class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
    161  public:
    162   MockURLRequestThrottlerManager() : create_entry_index_(0) {}
    163 
    164   // Method to process the URL using URLRequestThrottlerManager protected
    165   // method.
    166   std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
    167 
    168   // Method to use the garbage collecting method of URLRequestThrottlerManager.
    169   void DoGarbageCollectEntries() { GarbageCollectEntries(); }
    170 
    171   // Returns the number of entries in the map.
    172   int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
    173 
    174   void CreateEntry(bool is_outdated) {
    175     TimeTicks time = TimeTicks::Now();
    176     if (is_outdated) {
    177       time -= TimeDelta::FromMilliseconds(
    178           MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
    179     }
    180     std::string fake_url_string("http://www.fakeurl.com/");
    181     fake_url_string.append(base::IntToString(create_entry_index_++));
    182     GURL fake_url(fake_url_string);
    183     OverrideEntryForTests(
    184         fake_url,
    185         new MockURLRequestThrottlerEntry(this, time, TimeTicks::Now(),
    186                                          TimeTicks::Now()));
    187   }
    188 
    189  private:
    190   int create_entry_index_;
    191 };
    192 
    193 struct TimeAndBool {
    194   TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
    195     time = time_value;
    196     result = expected;
    197     line = line_num;
    198   }
    199   TimeTicks time;
    200   bool result;
    201   int line;
    202 };
    203 
    204 struct GurlAndString {
    205   GurlAndString(const GURL& url_value,
    206                 const std::string& expected,
    207                 int line_num) {
    208     url = url_value;
    209     result = expected;
    210     line = line_num;
    211   }
    212   GURL url;
    213   std::string result;
    214   int line;
    215 };
    216 
    217 }  // namespace
    218 
    219 class URLRequestThrottlerEntryTest : public testing::Test {
    220  protected:
    221   virtual void SetUp();
    222   TimeTicks now_;
    223   MockURLRequestThrottlerManager manager_;  // Dummy object, not used.
    224   scoped_refptr<MockURLRequestThrottlerEntry> entry_;
    225 };
    226 
    227 void URLRequestThrottlerEntryTest::SetUp() {
    228   now_ = TimeTicks::Now();
    229   entry_ = new MockURLRequestThrottlerEntry(&manager_);
    230   entry_->ResetToBlank(now_);
    231 }
    232 
    233 std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
    234   return out << time.ToInternalValue();
    235 }
    236 
    237 TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
    238   entry_->set_exponential_backoff_release_time(
    239       entry_->fake_time_now_ + TimeDelta::FromMilliseconds(1));
    240   EXPECT_TRUE(entry_->IsDuringExponentialBackoff());
    241 }
    242 
    243 TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
    244   entry_->set_exponential_backoff_release_time(entry_->fake_time_now_);
    245   EXPECT_FALSE(entry_->IsDuringExponentialBackoff());
    246   entry_->set_exponential_backoff_release_time(
    247       entry_->fake_time_now_ - TimeDelta::FromMilliseconds(1));
    248   EXPECT_FALSE(entry_->IsDuringExponentialBackoff());
    249 }
    250 
    251 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateRetryAfter) {
    252   // If the response we received has a retry-after field,
    253   // the request should be delayed.
    254   MockURLRequestThrottlerHeaderAdapter header_w_delay_header("5.5", "", 200);
    255   entry_->UpdateWithResponse("", &header_w_delay_header);
    256   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
    257       << "When the server put a positive value in retry-after we should "
    258          "increase release_time";
    259 
    260   entry_->ResetToBlank(now_);
    261   header_w_delay_header.fake_retry_value_ = "-5.5";
    262   EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
    263       << "When given a negative value, it should not change the release_time";
    264 }
    265 
    266 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
    267   MockURLRequestThrottlerHeaderAdapter failure_response(505);
    268   entry_->UpdateWithResponse("", &failure_response);
    269   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
    270       << "A failure should increase the release_time";
    271 }
    272 
    273 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
    274   MockURLRequestThrottlerHeaderAdapter success_response(200);
    275   entry_->UpdateWithResponse("", &success_response);
    276   EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
    277       << "A success should not add any delay";
    278 }
    279 
    280 TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
    281   MockURLRequestThrottlerHeaderAdapter failure_response(500);
    282   MockURLRequestThrottlerHeaderAdapter success_response(200);
    283   entry_->UpdateWithResponse("", &success_response);
    284   entry_->UpdateWithResponse("", &failure_response);
    285   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), entry_->fake_time_now_)
    286       << "This scenario should add delay";
    287 }
    288 
    289 TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
    290   TimeDelta lifetime = TimeDelta::FromMilliseconds(
    291       MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
    292   const TimeDelta kFiveMs = TimeDelta::FromMilliseconds(5);
    293 
    294   TimeAndBool test_values[] = {
    295       TimeAndBool(now_, false, __LINE__),
    296       TimeAndBool(now_ - kFiveMs, false, __LINE__),
    297       TimeAndBool(now_ + kFiveMs, false, __LINE__),
    298       TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
    299       TimeAndBool(now_ - lifetime, true, __LINE__),
    300       TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
    301 
    302   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
    303     entry_->set_exponential_backoff_release_time(test_values[i].time);
    304     EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
    305         "Test case #" << i << " line " << test_values[i].line << " failed";
    306   }
    307 }
    308 
    309 TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
    310   for (int i = 0; i < 30; ++i) {
    311     MockURLRequestThrottlerHeaderAdapter response_adapter(505);
    312     entry_->UpdateWithResponse("", &response_adapter);
    313   }
    314 
    315   TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
    316   EXPECT_EQ(delay.InMilliseconds(),
    317             MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
    318 }
    319 
    320 TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
    321   MockURLRequestThrottlerHeaderAdapter response_adapter(505);
    322   for (int i = 0; i < 5; ++i)
    323     entry_->UpdateWithResponse("", &response_adapter);
    324 
    325   TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
    326 
    327   // Inform the entry that a response body was malformed, which is supposed to
    328   // increase the back-off time.  Note that we also submit a successful
    329   // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
    330   // is what happens in practice (if a body is received, then a non-500
    331   // response must also have been received).
    332   entry_->ReceivedContentWasMalformed();
    333   MockURLRequestThrottlerHeaderAdapter success_adapter(200);
    334   entry_->UpdateWithResponse("", &success_adapter);
    335   EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
    336 }
    337 
    338 TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
    339   int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
    340   int sliding_window =
    341       URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
    342 
    343   TimeTicks time_1 = entry_->fake_time_now_ +
    344       TimeDelta::FromMilliseconds(sliding_window / 3);
    345   TimeTicks time_2 = entry_->fake_time_now_ +
    346       TimeDelta::FromMilliseconds(2 * sliding_window / 3);
    347   TimeTicks time_3 = entry_->fake_time_now_ +
    348       TimeDelta::FromMilliseconds(sliding_window);
    349   TimeTicks time_4 = entry_->fake_time_now_ +
    350       TimeDelta::FromMilliseconds(sliding_window + 2 * sliding_window / 3);
    351 
    352   entry_->set_exponential_backoff_release_time(time_1);
    353 
    354   for (int i = 0; i < max_send / 2; ++i) {
    355     EXPECT_EQ(2 * sliding_window / 3,
    356               entry_->ReserveSendingTimeForNextRequest(time_2));
    357   }
    358   EXPECT_EQ(time_2, entry_->sliding_window_release_time());
    359 
    360   entry_->fake_time_now_ = time_3;
    361 
    362   for (int i = 0; i < (max_send + 1) / 2; ++i)
    363     EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
    364 
    365   EXPECT_EQ(time_4, entry_->sliding_window_release_time());
    366 }
    367 
    368 TEST(URLRequestThrottlerManager, IsUrlStandardised) {
    369   MockURLRequestThrottlerManager manager;
    370   GurlAndString test_values[] = {
    371       GurlAndString(GURL("http://www.example.com"),
    372                     std::string("http://www.example.com/"),
    373                     __LINE__),
    374       GurlAndString(GURL("http://www.Example.com"),
    375                     std::string("http://www.example.com/"),
    376                     __LINE__),
    377       GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
    378                     std::string("http://www.ex4mple.com/pr4c71c41"),
    379                     __LINE__),
    380       GurlAndString(GURL("http://www.example.com/0/token/false"),
    381                     std::string("http://www.example.com/0/token/false"),
    382                     __LINE__),
    383       GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
    384                     std::string("http://www.example.com/index.php"),
    385                     __LINE__),
    386       GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
    387                     std::string("http://www.example.com/index.php"),
    388                     __LINE__),
    389       GurlAndString(GURL("http://www.example.com:1234/"),
    390                     std::string("http://www.example.com:1234/"),
    391                     __LINE__)};
    392 
    393   for (unsigned int i = 0; i < arraysize(test_values); ++i) {
    394     std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
    395     EXPECT_EQ(temp, test_values[i].result) <<
    396         "Test case #" << i << " line " << test_values[i].line << " failed";
    397   }
    398 }
    399 
    400 TEST(URLRequestThrottlerManager, AreEntriesBeingCollected) {
    401   MockURLRequestThrottlerManager manager;
    402 
    403   manager.CreateEntry(true);  // true = Entry is outdated.
    404   manager.CreateEntry(true);
    405   manager.CreateEntry(true);
    406   manager.DoGarbageCollectEntries();
    407   EXPECT_EQ(0, manager.GetNumberOfEntries());
    408 
    409   manager.CreateEntry(false);
    410   manager.CreateEntry(false);
    411   manager.CreateEntry(false);
    412   manager.CreateEntry(true);
    413   manager.DoGarbageCollectEntries();
    414   EXPECT_EQ(3, manager.GetNumberOfEntries());
    415 }
    416 
    417 TEST(URLRequestThrottlerManager, IsHostBeingRegistered) {
    418   MockURLRequestThrottlerManager manager;
    419 
    420   manager.RegisterRequestUrl(GURL("http://www.example.com/"));
    421   manager.RegisterRequestUrl(GURL("http://www.google.com/"));
    422   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
    423   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
    424   manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
    425 
    426   EXPECT_EQ(3, manager.GetNumberOfEntries());
    427 }
    428 
    429 void ExpectEntryAllowsAllOnErrorIfOptedOut(
    430     net::URLRequestThrottlerEntryInterface* entry,
    431     bool opted_out) {
    432   EXPECT_FALSE(entry->IsDuringExponentialBackoff());
    433   MockURLRequestThrottlerHeaderAdapter failure_adapter(503);
    434   for (int i = 0; i < 10; ++i) {
    435     // Host doesn't really matter in this scenario so we skip it.
    436     entry->UpdateWithResponse("", &failure_adapter);
    437   }
    438   EXPECT_NE(opted_out, entry->IsDuringExponentialBackoff());
    439 
    440   if (opted_out) {
    441     // We're not mocking out GetTimeNow() in this scenario
    442     // so add a 100 ms buffer to avoid flakiness (that should always
    443     // give enough time to get from the TimeTicks::Now() call here
    444     // to the TimeTicks::Now() call in the entry class).
    445     EXPECT_GT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
    446               entry->GetExponentialBackoffReleaseTime());
    447   } else {
    448     // As above, add 100 ms.
    449     EXPECT_LT(TimeTicks::Now() + TimeDelta::FromMilliseconds(100),
    450               entry->GetExponentialBackoffReleaseTime());
    451   }
    452 }
    453 
    454 TEST(URLRequestThrottlerManager, OptOutHeader) {
    455   MockURLRequestThrottlerManager manager;
    456   scoped_refptr<net::URLRequestThrottlerEntryInterface> entry =
    457       manager.RegisterRequestUrl(GURL("http://www.google.com/yodude"));
    458 
    459   // Fake a response with the opt-out header.
    460   MockURLRequestThrottlerHeaderAdapter response_adapter(
    461       "",
    462       MockURLRequestThrottlerEntry::kExponentialThrottlingDisableValue,
    463       200);
    464   entry->UpdateWithResponse("www.google.com", &response_adapter);
    465 
    466   // Ensure that the same entry on error always allows everything.
    467   ExpectEntryAllowsAllOnErrorIfOptedOut(entry, true);
    468 
    469   // Ensure that a freshly created entry (for a different URL on an
    470   // already opted-out host) also gets "always allow" behavior.
    471   scoped_refptr<net::URLRequestThrottlerEntryInterface> other_entry =
    472       manager.RegisterRequestUrl(GURL("http://www.google.com/bingobob"));
    473   ExpectEntryAllowsAllOnErrorIfOptedOut(other_entry, true);
    474 
    475   // Fake a response with the opt-out header incorrectly specified.
    476   scoped_refptr<net::URLRequestThrottlerEntryInterface> no_opt_out_entry =
    477       manager.RegisterRequestUrl(GURL("http://www.nike.com/justdoit"));
    478   MockURLRequestThrottlerHeaderAdapter wrong_adapter("", "yesplease", 200);
    479   no_opt_out_entry->UpdateWithResponse("www.nike.com", &wrong_adapter);
    480   ExpectEntryAllowsAllOnErrorIfOptedOut(no_opt_out_entry, false);
    481 
    482   // A localhost entry should always be opted out.
    483   scoped_refptr<net::URLRequestThrottlerEntryInterface> localhost_entry =
    484       manager.RegisterRequestUrl(GURL("http://localhost/hello"));
    485   ExpectEntryAllowsAllOnErrorIfOptedOut(localhost_entry, true);
    486 }
    487 
    488 }  // namespace net
    489