Home | History | Annotate | Download | only in net 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include <time.h>
      6 
      7 #include <algorithm>
      8 #include <sstream>
      9 #include <string>
     10 
     11 #include "base/memory/scoped_ptr.h"
     12 #include "base/message_loop/message_loop.h"
     13 #include "base/strings/string_number_conversions.h"
     14 #include "base/timer/timer.h"
     15 #include "base/values.h"
     16 #include "chrome/browser/net/predictor.h"
     17 #include "chrome/browser/net/spdyproxy/proxy_advisor.h"
     18 #include "chrome/browser/net/url_info.h"
     19 #include "chrome/common/net/predictor_common.h"
     20 #include "content/public/test/test_browser_thread.h"
     21 #include "net/base/address_list.h"
     22 #include "net/base/winsock_init.h"
     23 #include "net/dns/mock_host_resolver.h"
     24 #include "net/http/transport_security_state.h"
     25 #include "testing/gmock/include/gmock/gmock.h"
     26 #include "testing/gtest/include/gtest/gtest.h"
     27 
     28 using base::Time;
     29 using base::TimeDelta;
     30 using content::BrowserThread;
     31 
     32 namespace chrome_browser_net {
     33 
     34 class WaitForResolutionHelper;
     35 
     36 typedef base::RepeatingTimer<WaitForResolutionHelper> HelperTimer;
     37 
     38 class WaitForResolutionHelper {
     39  public:
     40   WaitForResolutionHelper(Predictor* predictor, const UrlList& hosts,
     41                           HelperTimer* timer, int checks_until_quit)
     42       : predictor_(predictor),
     43         hosts_(hosts),
     44         timer_(timer),
     45         checks_until_quit_(checks_until_quit) {
     46   }
     47 
     48   void CheckIfResolutionsDone() {
     49     if (--checks_until_quit_ > 0) {
     50       for (UrlList::const_iterator i = hosts_.begin(); i != hosts_.end(); ++i)
     51         if (predictor_->GetResolutionDuration(*i) ==
     52             UrlInfo::NullDuration())
     53           return;  // We don't have resolution for that host.
     54     }
     55 
     56     // When all hostnames have been resolved, or we've hit the limit,
     57     // exit the loop.
     58     timer_->Stop();
     59     base::MessageLoop::current()->Quit();
     60     delete timer_;
     61     delete this;
     62   }
     63 
     64  private:
     65   Predictor* predictor_;
     66   const UrlList hosts_;
     67   HelperTimer* timer_;
     68   int checks_until_quit_;
     69 };
     70 
     71 class PredictorTest : public testing::Test {
     72  public:
     73   PredictorTest()
     74       : ui_thread_(BrowserThread::UI, &loop_),
     75         io_thread_(BrowserThread::IO, &loop_),
     76         host_resolver_(new net::MockCachingHostResolver()) {
     77   }
     78 
     79  protected:
     80   virtual void SetUp() {
     81 #if defined(OS_WIN)
     82     net::EnsureWinsockInit();
     83 #endif
     84     Predictor::set_max_parallel_resolves(
     85         Predictor::kMaxSpeculativeParallelResolves);
     86     Predictor::set_max_queueing_delay(
     87         Predictor::kMaxSpeculativeResolveQueueDelayMs);
     88     // Since we are using a caching HostResolver, the following latencies will
     89     // only be incurred by the first request, after which the result will be
     90     // cached internally by |host_resolver_|.
     91     net::RuleBasedHostResolverProc* rules = host_resolver_->rules();
     92     rules->AddRuleWithLatency("www.google.com", "127.0.0.1", 50);
     93     rules->AddRuleWithLatency("gmail.google.com.com", "127.0.0.1", 70);
     94     rules->AddRuleWithLatency("mail.google.com", "127.0.0.1", 44);
     95     rules->AddRuleWithLatency("gmail.com", "127.0.0.1", 63);
     96   }
     97 
     98   void WaitForResolution(Predictor* predictor, const UrlList& hosts) {
     99     HelperTimer* timer = new HelperTimer();
    100     // By default allow the loop to run for a minute -- 600 iterations.
    101     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
    102                  new WaitForResolutionHelper(predictor, hosts, timer, 600),
    103                  &WaitForResolutionHelper::CheckIfResolutionsDone);
    104     base::MessageLoop::current()->Run();
    105   }
    106 
    107   void WaitForResolutionWithLimit(
    108       Predictor* predictor, const UrlList& hosts, int limit) {
    109     HelperTimer* timer = new HelperTimer();
    110     timer->Start(FROM_HERE, TimeDelta::FromMilliseconds(100),
    111                  new WaitForResolutionHelper(predictor, hosts, timer, limit),
    112                  &WaitForResolutionHelper::CheckIfResolutionsDone);
    113     base::MessageLoop::current()->Run();
    114   }
    115 
    116  private:
    117   // IMPORTANT: do not move this below |host_resolver_|; the host resolver
    118   // must not outlive the message loop, otherwise bad things can happen
    119   // (like posting to a deleted message loop).
    120   base::MessageLoopForUI loop_;
    121   content::TestBrowserThread ui_thread_;
    122   content::TestBrowserThread io_thread_;
    123 
    124  protected:
    125   scoped_ptr<net::MockCachingHostResolver> host_resolver_;
    126 };
    127 
    128 //------------------------------------------------------------------------------
    129 
    130 TEST_F(PredictorTest, StartupShutdownTest) {
    131   Predictor testing_master(true);
    132   testing_master.Shutdown();
    133 }
    134 
    135 
    136 TEST_F(PredictorTest, ShutdownWhenResolutionIsPendingTest) {
    137   scoped_ptr<net::HostResolver> host_resolver(new net::HangingHostResolver());
    138 
    139   Predictor testing_master(true);
    140   testing_master.SetHostResolver(host_resolver.get());
    141 
    142   GURL localhost("http://localhost:80");
    143   UrlList names;
    144   names.push_back(localhost);
    145 
    146   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
    147 
    148   base::MessageLoop::current()->PostDelayedTask(
    149       FROM_HERE,
    150       base::MessageLoop::QuitClosure(),
    151       base::TimeDelta::FromMilliseconds(500));
    152   base::MessageLoop::current()->Run();
    153 
    154   EXPECT_FALSE(testing_master.WasFound(localhost));
    155 
    156   testing_master.Shutdown();
    157 
    158   // Clean up after ourselves.
    159   base::MessageLoop::current()->RunUntilIdle();
    160 }
    161 
    162 TEST_F(PredictorTest, SingleLookupTest) {
    163   Predictor testing_master(true);
    164   testing_master.SetHostResolver(host_resolver_.get());
    165 
    166   GURL goog("http://www.google.com:80");
    167 
    168   UrlList names;
    169   names.push_back(goog);
    170 
    171   // Try to flood the predictor with many concurrent requests.
    172   for (int i = 0; i < 10; i++)
    173     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
    174 
    175   WaitForResolution(&testing_master, names);
    176 
    177   EXPECT_TRUE(testing_master.WasFound(goog));
    178 
    179   base::MessageLoop::current()->RunUntilIdle();
    180 
    181   EXPECT_GT(testing_master.peak_pending_lookups(), names.size() / 2);
    182   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
    183   EXPECT_LE(testing_master.peak_pending_lookups(),
    184             testing_master.max_concurrent_dns_lookups());
    185 
    186   testing_master.Shutdown();
    187 }
    188 
    189 TEST_F(PredictorTest, ConcurrentLookupTest) {
    190   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
    191 
    192   Predictor testing_master(true);
    193   testing_master.SetHostResolver(host_resolver_.get());
    194 
    195   GURL goog("http://www.google.com:80"),
    196       goog2("http://gmail.google.com.com:80"),
    197       goog3("http://mail.google.com:80"),
    198       goog4("http://gmail.com:80");
    199   GURL bad1("http://bad1.notfound:80"),
    200       bad2("http://bad2.notfound:80");
    201 
    202   UrlList names;
    203   names.push_back(goog);
    204   names.push_back(goog3);
    205   names.push_back(bad1);
    206   names.push_back(goog2);
    207   names.push_back(bad2);
    208   names.push_back(goog4);
    209   names.push_back(goog);
    210 
    211   // Try to flood the predictor with many concurrent requests.
    212   for (int i = 0; i < 10; i++)
    213     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
    214 
    215   WaitForResolution(&testing_master, names);
    216 
    217   EXPECT_TRUE(testing_master.WasFound(goog));
    218   EXPECT_TRUE(testing_master.WasFound(goog3));
    219   EXPECT_TRUE(testing_master.WasFound(goog2));
    220   EXPECT_TRUE(testing_master.WasFound(goog4));
    221   EXPECT_FALSE(testing_master.WasFound(bad1));
    222   EXPECT_FALSE(testing_master.WasFound(bad2));
    223 
    224   base::MessageLoop::current()->RunUntilIdle();
    225 
    226   EXPECT_FALSE(testing_master.WasFound(bad1));
    227   EXPECT_FALSE(testing_master.WasFound(bad2));
    228 
    229   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
    230   EXPECT_LE(testing_master.peak_pending_lookups(),
    231             testing_master.max_concurrent_dns_lookups());
    232 
    233   testing_master.Shutdown();
    234 }
    235 
    236 TEST_F(PredictorTest, MassiveConcurrentLookupTest) {
    237   host_resolver_->rules()->AddSimulatedFailure("*.notfound");
    238 
    239   Predictor testing_master(true);
    240   testing_master.SetHostResolver(host_resolver_.get());
    241 
    242   UrlList names;
    243   for (int i = 0; i < 100; i++)
    244     names.push_back(GURL(
    245         "http://host" + base::IntToString(i) + ".notfound:80"));
    246 
    247   // Try to flood the predictor with many concurrent requests.
    248   for (int i = 0; i < 10; i++)
    249     testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
    250 
    251   WaitForResolution(&testing_master, names);
    252 
    253   base::MessageLoop::current()->RunUntilIdle();
    254 
    255   EXPECT_LE(testing_master.peak_pending_lookups(), names.size());
    256   EXPECT_LE(testing_master.peak_pending_lookups(),
    257             testing_master.max_concurrent_dns_lookups());
    258 
    259   testing_master.Shutdown();
    260 }
    261 
    262 //------------------------------------------------------------------------------
    263 // Functions to help synthesize and test serializations of subresource referrer
    264 // lists.
    265 
    266 // Return a motivation_list if we can find one for the given motivating_host (or
    267 // NULL if a match is not found).
    268 static const base::ListValue* FindSerializationMotivation(
    269     const GURL& motivation,
    270     const base::ListValue* referral_list) {
    271   CHECK_LT(0u, referral_list->GetSize());  // Room for version.
    272   int format_version = -1;
    273   CHECK(referral_list->GetInteger(0, &format_version));
    274   CHECK_EQ(Predictor::kPredictorReferrerVersion, format_version);
    275   const base::ListValue* motivation_list(NULL);
    276   for (size_t i = 1; i < referral_list->GetSize(); ++i) {
    277     referral_list->GetList(i, &motivation_list);
    278     std::string existing_spec;
    279     EXPECT_TRUE(motivation_list->GetString(0, &existing_spec));
    280     if (motivation == GURL(existing_spec))
    281       return motivation_list;
    282   }
    283   return NULL;
    284 }
    285 
    286 static base::ListValue* FindSerializationMotivation(
    287     const GURL& motivation,
    288     base::ListValue* referral_list) {
    289   return const_cast<base::ListValue*>(FindSerializationMotivation(
    290       motivation, static_cast<const base::ListValue*>(referral_list)));
    291 }
    292 
    293 // Create a new empty serialization list.
    294 static base::ListValue* NewEmptySerializationList() {
    295   base::ListValue* list = new base::ListValue;
    296   list->Append(
    297       new base::FundamentalValue(Predictor::kPredictorReferrerVersion));
    298   return list;
    299 }
    300 
    301 // Add a motivating_url and a subresource_url to a serialized list, using
    302 // this given latency. This is a helper function for quickly building these
    303 // lists.
    304 static void AddToSerializedList(const GURL& motivation,
    305                                 const GURL& subresource,
    306                                 double use_rate,
    307                                 base::ListValue* referral_list) {
    308   // Find the motivation if it is already used.
    309   base::ListValue* motivation_list = FindSerializationMotivation(motivation,
    310                                                            referral_list);
    311   if (!motivation_list) {
    312     // This is the first mention of this motivation, so build a list.
    313     motivation_list = new base::ListValue;
    314     motivation_list->Append(new base::StringValue(motivation.spec()));
    315     // Provide empty subresource list.
    316     motivation_list->Append(new base::ListValue());
    317 
    318     // ...and make it part of the serialized referral_list.
    319     referral_list->Append(motivation_list);
    320   }
    321 
    322   base::ListValue* subresource_list(NULL);
    323   // 0 == url; 1 == subresource_list.
    324   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
    325 
    326   // We won't bother to check for the subresource being there already.  Worst
    327   // case, during deserialization, the latency value we supply plus the
    328   // existing value(s) will be added to the referrer.
    329 
    330   subresource_list->Append(new base::StringValue(subresource.spec()));
    331   subresource_list->Append(new base::FundamentalValue(use_rate));
    332 }
    333 
    334 // For a given motivation, and subresource, find what latency is currently
    335 // listed.  This assume a well formed serialization, which has at most one such
    336 // entry for any pair of names.  If no such pair is found, then return false.
    337 // Data is written into use_rate arguments.
    338 static bool GetDataFromSerialization(const GURL& motivation,
    339                                      const GURL& subresource,
    340                                      const base::ListValue& referral_list,
    341                                      double* use_rate) {
    342   const base::ListValue* motivation_list =
    343       FindSerializationMotivation(motivation, &referral_list);
    344   if (!motivation_list)
    345     return false;
    346   const base::ListValue* subresource_list;
    347   EXPECT_TRUE(motivation_list->GetList(1, &subresource_list));
    348   for (size_t i = 0; i < subresource_list->GetSize();) {
    349     std::string url_spec;
    350     EXPECT_TRUE(subresource_list->GetString(i++, &url_spec));
    351     EXPECT_TRUE(subresource_list->GetDouble(i++, use_rate));
    352     if (subresource == GURL(url_spec)) {
    353       return true;
    354     }
    355   }
    356   return false;
    357 }
    358 
    359 //------------------------------------------------------------------------------
    360 
    361 // Make sure nil referral lists really have no entries, and no latency listed.
    362 TEST_F(PredictorTest, ReferrerSerializationNilTest) {
    363   Predictor predictor(true);
    364   predictor.SetHostResolver(host_resolver_.get());
    365 
    366   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
    367   predictor.SerializeReferrers(referral_list.get());
    368   EXPECT_EQ(1U, referral_list->GetSize());
    369   EXPECT_FALSE(GetDataFromSerialization(
    370     GURL("http://a.com:79"), GURL("http://b.com:78"),
    371       *referral_list.get(), NULL));
    372 
    373   predictor.Shutdown();
    374 }
    375 
    376 // Make sure that when a serialization list includes a value, that it can be
    377 // deserialized into the database, and can be extracted back out via
    378 // serialization without being changed.
    379 TEST_F(PredictorTest, ReferrerSerializationSingleReferrerTest) {
    380   Predictor predictor(true);
    381   predictor.SetHostResolver(host_resolver_.get());
    382   const GURL motivation_url("http://www.google.com:91");
    383   const GURL subresource_url("http://icons.google.com:90");
    384   const double kUseRate = 23.4;
    385   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
    386 
    387   AddToSerializedList(motivation_url, subresource_url,
    388       kUseRate, referral_list.get());
    389 
    390   predictor.DeserializeReferrers(*referral_list.get());
    391 
    392   base::ListValue recovered_referral_list;
    393   predictor.SerializeReferrers(&recovered_referral_list);
    394   EXPECT_EQ(2U, recovered_referral_list.GetSize());
    395   double rate;
    396   EXPECT_TRUE(GetDataFromSerialization(
    397       motivation_url, subresource_url, recovered_referral_list, &rate));
    398   EXPECT_EQ(rate, kUseRate);
    399 
    400   predictor.Shutdown();
    401 }
    402 
    403 // Check that GetHtmlReferrerLists() doesn't crash when given duplicated
    404 // domains for referring URL, and that it sorts the results in the
    405 // correct order.
    406 TEST_F(PredictorTest, GetHtmlReferrerLists) {
    407   Predictor predictor(true);
    408   predictor.SetHostResolver(host_resolver_.get());
    409   const double kUseRate = 23.4;
    410   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
    411 
    412   AddToSerializedList(
    413       GURL("http://d.google.com/x1"),
    414       GURL("http://foo.com/"),
    415       kUseRate, referral_list.get());
    416 
    417   // Duplicated hostname (d.google.com). This should not cause any crashes
    418   // (i.e. crbug.com/116345)
    419   AddToSerializedList(
    420       GURL("http://d.google.com/x2"),
    421       GURL("http://foo.com/"),
    422       kUseRate, referral_list.get());
    423 
    424   AddToSerializedList(
    425       GURL("http://a.yahoo.com/y"),
    426       GURL("http://foo1.com/"),
    427       kUseRate, referral_list.get());
    428 
    429   AddToSerializedList(
    430       GURL("http://b.google.com/x3"),
    431       GURL("http://foo2.com/"),
    432       kUseRate, referral_list.get());
    433 
    434   AddToSerializedList(
    435       GURL("http://d.yahoo.com/x5"),
    436       GURL("http://i.like.turtles/"),
    437       kUseRate, referral_list.get());
    438 
    439   AddToSerializedList(
    440       GURL("http://c.yahoo.com/x4"),
    441       GURL("http://foo3.com/"),
    442       kUseRate, referral_list.get());
    443 
    444   predictor.DeserializeReferrers(*referral_list.get());
    445 
    446   std::string html;
    447   predictor.GetHtmlReferrerLists(&html);
    448 
    449   // The lexicographic sorting of hostnames would be:
    450   //   a.yahoo.com
    451   //   b.google.com
    452   //   c.yahoo.com
    453   //   d.google.com
    454   //   d.yahoo.com
    455   //
    456   // However we expect to sort them by domain in the output:
    457   //   b.google.com
    458   //   d.google.com
    459   //   a.yahoo.com
    460   //   c.yahoo.com
    461   //   d.yahoo.com
    462 
    463   size_t pos[] = {
    464       html.find("<td rowspan=1>http://b.google.com/x3"),
    465       html.find("<td rowspan=1>http://d.google.com/x1"),
    466       html.find("<td rowspan=1>http://d.google.com/x2"),
    467       html.find("<td rowspan=1>http://a.yahoo.com/y"),
    468       html.find("<td rowspan=1>http://c.yahoo.com/x4"),
    469       html.find("<td rowspan=1>http://d.yahoo.com/x5"),
    470   };
    471 
    472   // Make sure things appeared in the expected order.
    473   for (size_t i = 1; i < arraysize(pos); ++i) {
    474     EXPECT_LT(pos[i - 1], pos[i]) << "Mismatch for pos[" << i << "]";
    475   }
    476 
    477   predictor.Shutdown();
    478 }
    479 
    480 // Verify that two floats are within 1% of each other in value.
    481 #define EXPECT_SIMILAR(a, b) do { \
    482     double espilon_ratio = 1.01;  \
    483     if ((a) < 0.)  \
    484       espilon_ratio = 1 / espilon_ratio;  \
    485     EXPECT_LT(a, espilon_ratio * (b));   \
    486     EXPECT_GT((a) * espilon_ratio, b);   \
    487     } while (0)
    488 
    489 
    490 // Make sure the Trim() functionality works as expected.
    491 TEST_F(PredictorTest, ReferrerSerializationTrimTest) {
    492   Predictor predictor(true);
    493   predictor.SetHostResolver(host_resolver_.get());
    494   GURL motivation_url("http://www.google.com:110");
    495 
    496   GURL icon_subresource_url("http://icons.google.com:111");
    497   const double kRateIcon = 16.0 * Predictor::kDiscardableExpectedValue;
    498   GURL img_subresource_url("http://img.google.com:118");
    499   const double kRateImg = 8.0 * Predictor::kDiscardableExpectedValue;
    500 
    501   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
    502   AddToSerializedList(
    503       motivation_url, icon_subresource_url, kRateIcon, referral_list.get());
    504   AddToSerializedList(
    505       motivation_url, img_subresource_url, kRateImg, referral_list.get());
    506 
    507   predictor.DeserializeReferrers(*referral_list.get());
    508 
    509   base::ListValue recovered_referral_list;
    510   predictor.SerializeReferrers(&recovered_referral_list);
    511   EXPECT_EQ(2U, recovered_referral_list.GetSize());
    512   double rate;
    513   EXPECT_TRUE(GetDataFromSerialization(
    514       motivation_url, icon_subresource_url, recovered_referral_list,
    515       &rate));
    516   EXPECT_SIMILAR(rate, kRateIcon);
    517 
    518   EXPECT_TRUE(GetDataFromSerialization(
    519       motivation_url, img_subresource_url, recovered_referral_list, &rate));
    520   EXPECT_SIMILAR(rate, kRateImg);
    521 
    522   // Each time we Trim 24 times, the user_rate figures should reduce by a factor
    523   // of two,  until they are small, and then a trim will delete the whole entry.
    524   for (int i = 0; i < 24; ++i)
    525     predictor.TrimReferrersNow();
    526   predictor.SerializeReferrers(&recovered_referral_list);
    527   EXPECT_EQ(2U, recovered_referral_list.GetSize());
    528   EXPECT_TRUE(GetDataFromSerialization(
    529       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
    530   EXPECT_SIMILAR(rate, kRateIcon / 2);
    531 
    532   EXPECT_TRUE(GetDataFromSerialization(
    533       motivation_url, img_subresource_url, recovered_referral_list, &rate));
    534   EXPECT_SIMILAR(rate, kRateImg / 2);
    535 
    536   for (int i = 0; i < 24; ++i)
    537     predictor.TrimReferrersNow();
    538   predictor.SerializeReferrers(&recovered_referral_list);
    539   EXPECT_EQ(2U, recovered_referral_list.GetSize());
    540   EXPECT_TRUE(GetDataFromSerialization(
    541       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
    542   EXPECT_SIMILAR(rate, kRateIcon / 4);
    543   EXPECT_TRUE(GetDataFromSerialization(
    544       motivation_url, img_subresource_url, recovered_referral_list, &rate));
    545   EXPECT_SIMILAR(rate, kRateImg / 4);
    546 
    547   for (int i = 0; i < 24; ++i)
    548     predictor.TrimReferrersNow();
    549   predictor.SerializeReferrers(&recovered_referral_list);
    550   EXPECT_EQ(2U, recovered_referral_list.GetSize());
    551   EXPECT_TRUE(GetDataFromSerialization(
    552       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
    553   EXPECT_SIMILAR(rate, kRateIcon / 8);
    554 
    555   // Img is below threshold, and so it gets deleted.
    556   EXPECT_FALSE(GetDataFromSerialization(
    557       motivation_url, img_subresource_url, recovered_referral_list, &rate));
    558 
    559   for (int i = 0; i < 24; ++i)
    560     predictor.TrimReferrersNow();
    561   predictor.SerializeReferrers(&recovered_referral_list);
    562   // Icon is also trimmed away, so entire set gets discarded.
    563   EXPECT_EQ(1U, recovered_referral_list.GetSize());
    564   EXPECT_FALSE(GetDataFromSerialization(
    565       motivation_url, icon_subresource_url, recovered_referral_list, &rate));
    566   EXPECT_FALSE(GetDataFromSerialization(
    567       motivation_url, img_subresource_url, recovered_referral_list, &rate));
    568 
    569   predictor.Shutdown();
    570 }
    571 
    572 
    573 TEST_F(PredictorTest, PriorityQueuePushPopTest) {
    574   Predictor::HostNameQueue queue;
    575 
    576   GURL first("http://first:80"), second("http://second:90");
    577 
    578   // First check high priority queue FIFO functionality.
    579   EXPECT_TRUE(queue.IsEmpty());
    580   queue.Push(first, UrlInfo::LEARNED_REFERAL_MOTIVATED);
    581   EXPECT_FALSE(queue.IsEmpty());
    582   queue.Push(second, UrlInfo::MOUSE_OVER_MOTIVATED);
    583   EXPECT_FALSE(queue.IsEmpty());
    584   EXPECT_EQ(queue.Pop(), first);
    585   EXPECT_FALSE(queue.IsEmpty());
    586   EXPECT_EQ(queue.Pop(), second);
    587   EXPECT_TRUE(queue.IsEmpty());
    588 
    589   // Then check low priority queue FIFO functionality.
    590   queue.Push(first, UrlInfo::PAGE_SCAN_MOTIVATED);
    591   EXPECT_FALSE(queue.IsEmpty());
    592   queue.Push(second, UrlInfo::OMNIBOX_MOTIVATED);
    593   EXPECT_FALSE(queue.IsEmpty());
    594   EXPECT_EQ(queue.Pop(), first);
    595   EXPECT_FALSE(queue.IsEmpty());
    596   EXPECT_EQ(queue.Pop(), second);
    597   EXPECT_TRUE(queue.IsEmpty());
    598 }
    599 
    600 TEST_F(PredictorTest, PriorityQueueReorderTest) {
    601   Predictor::HostNameQueue queue;
    602 
    603   // Push all the low priority items.
    604   GURL low1("http://low1:80"),
    605       low2("http://low2:80"),
    606       low3("http://low3:443"),
    607       low4("http://low4:80"),
    608       low5("http://low5:80"),
    609       hi1("http://hi1:80"),
    610       hi2("http://hi2:80"),
    611       hi3("http://hi3:80");
    612 
    613   EXPECT_TRUE(queue.IsEmpty());
    614   queue.Push(low1, UrlInfo::PAGE_SCAN_MOTIVATED);
    615   queue.Push(low2, UrlInfo::UNIT_TEST_MOTIVATED);
    616   queue.Push(low3, UrlInfo::LINKED_MAX_MOTIVATED);
    617   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
    618   queue.Push(low5, UrlInfo::STARTUP_LIST_MOTIVATED);
    619   queue.Push(low4, UrlInfo::OMNIBOX_MOTIVATED);
    620 
    621   // Push all the high prority items
    622   queue.Push(hi1, UrlInfo::LEARNED_REFERAL_MOTIVATED);
    623   queue.Push(hi2, UrlInfo::STATIC_REFERAL_MOTIVATED);
    624   queue.Push(hi3, UrlInfo::MOUSE_OVER_MOTIVATED);
    625 
    626   // Check that high priority stuff comes out first, and in FIFO order.
    627   EXPECT_EQ(queue.Pop(), hi1);
    628   EXPECT_EQ(queue.Pop(), hi2);
    629   EXPECT_EQ(queue.Pop(), hi3);
    630 
    631   // ...and then low priority strings.
    632   EXPECT_EQ(queue.Pop(), low1);
    633   EXPECT_EQ(queue.Pop(), low2);
    634   EXPECT_EQ(queue.Pop(), low3);
    635   EXPECT_EQ(queue.Pop(), low4);
    636   EXPECT_EQ(queue.Pop(), low5);
    637   EXPECT_EQ(queue.Pop(), low4);
    638 
    639   EXPECT_TRUE(queue.IsEmpty());
    640 }
    641 
    642 TEST_F(PredictorTest, CanonicalizeUrl) {
    643   // Base case, only handles HTTP and HTTPS.
    644   EXPECT_EQ(GURL(), Predictor::CanonicalizeUrl(GURL("ftp://anything")));
    645 
    646   // Remove path testing.
    647   GURL long_url("http://host:999/path?query=value");
    648   EXPECT_EQ(Predictor::CanonicalizeUrl(long_url), long_url.GetWithEmptyPath());
    649 
    650   // Default port cannoncalization.
    651   GURL implied_port("http://test");
    652   GURL explicit_port("http://test:80");
    653   EXPECT_EQ(Predictor::CanonicalizeUrl(implied_port),
    654             Predictor::CanonicalizeUrl(explicit_port));
    655 
    656   // Port is still maintained.
    657   GURL port_80("http://test:80");
    658   GURL port_90("http://test:90");
    659   EXPECT_NE(Predictor::CanonicalizeUrl(port_80),
    660             Predictor::CanonicalizeUrl(port_90));
    661 
    662   // Host is still maintained.
    663   GURL host_1("http://test_1");
    664   GURL host_2("http://test_2");
    665   EXPECT_NE(Predictor::CanonicalizeUrl(host_1),
    666             Predictor::CanonicalizeUrl(host_2));
    667 
    668   // Scheme is maintained (mismatch identified).
    669   GURL http("http://test");
    670   GURL https("https://test");
    671   EXPECT_NE(Predictor::CanonicalizeUrl(http),
    672             Predictor::CanonicalizeUrl(https));
    673 
    674   // Https works fine.
    675   GURL long_https("https://host:999/path?query=value");
    676   EXPECT_EQ(Predictor::CanonicalizeUrl(long_https),
    677             long_https.GetWithEmptyPath());
    678 }
    679 
    680 TEST_F(PredictorTest, DiscardPredictorResults) {
    681   Predictor predictor(true);
    682   predictor.SetHostResolver(host_resolver_.get());
    683   base::ListValue referral_list;
    684   predictor.SerializeReferrers(&referral_list);
    685   EXPECT_EQ(1U, referral_list.GetSize());
    686 
    687   GURL host_1("http://test_1");
    688   GURL host_2("http://test_2");
    689   predictor.LearnFromNavigation(host_1, host_2);
    690 
    691   predictor.SerializeReferrers(&referral_list);
    692   EXPECT_EQ(2U, referral_list.GetSize());
    693 
    694   predictor.DiscardAllResults();
    695   predictor.SerializeReferrers(&referral_list);
    696   EXPECT_EQ(1U, referral_list.GetSize());
    697 
    698   predictor.Shutdown();
    699 }
    700 
    701 class TestPredictorObserver : public PredictorObserver {
    702  public:
    703   // PredictorObserver implementation:
    704   virtual void OnPreconnectUrl(const GURL& url,
    705                                const GURL& first_party_for_cookies,
    706                                UrlInfo::ResolutionMotivation motivation,
    707                                int count) OVERRIDE {
    708     preconnected_urls_.push_back(url);
    709   }
    710 
    711   std::vector<GURL> preconnected_urls_;
    712 };
    713 
    714 // Tests that preconnects apply the HSTS list.
    715 TEST_F(PredictorTest, HSTSRedirect) {
    716   const GURL kHttpUrl("http://example.com");
    717   const GURL kHttpsUrl("https://example.com");
    718 
    719   const base::Time expiry =
    720       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
    721   net::TransportSecurityState state;
    722   state.AddHSTS(kHttpUrl.host(), expiry, false);
    723 
    724   Predictor predictor(true);
    725   TestPredictorObserver observer;
    726   predictor.SetObserver(&observer);
    727   predictor.SetTransportSecurityState(&state);
    728 
    729   predictor.PreconnectUrl(kHttpUrl, GURL(), UrlInfo::OMNIBOX_MOTIVATED, 2);
    730   ASSERT_EQ(1u, observer.preconnected_urls_.size());
    731   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
    732 
    733   predictor.Shutdown();
    734 }
    735 
    736 // Tests that preconnecting a URL on the HSTS list preconnects the subresources
    737 // for the SSL version.
    738 TEST_F(PredictorTest, HSTSRedirectSubresources) {
    739   const GURL kHttpUrl("http://example.com");
    740   const GURL kHttpsUrl("https://example.com");
    741   const GURL kSubresourceUrl("https://images.example.com");
    742   const double kUseRate = 23.4;
    743 
    744   const base::Time expiry =
    745       base::Time::Now() + base::TimeDelta::FromSeconds(1000);
    746   net::TransportSecurityState state;
    747   state.AddHSTS(kHttpUrl.host(), expiry, false);
    748 
    749   Predictor predictor(true);
    750   TestPredictorObserver observer;
    751   predictor.SetObserver(&observer);
    752   predictor.SetTransportSecurityState(&state);
    753 
    754   scoped_ptr<base::ListValue> referral_list(NewEmptySerializationList());
    755   AddToSerializedList(
    756       kHttpsUrl, kSubresourceUrl, kUseRate, referral_list.get());
    757   predictor.DeserializeReferrers(*referral_list.get());
    758 
    759   predictor.PreconnectUrlAndSubresources(kHttpUrl, GURL());
    760   ASSERT_EQ(2u, observer.preconnected_urls_.size());
    761   EXPECT_EQ(kHttpsUrl, observer.preconnected_urls_[0]);
    762   EXPECT_EQ(kSubresourceUrl, observer.preconnected_urls_[1]);
    763 
    764   predictor.Shutdown();
    765 }
    766 
    767 #if defined(OS_ANDROID) || defined(OS_IOS)
    768 // Tests for the predictor with a proxy advisor
    769 
    770 class TestProxyAdvisor : public ProxyAdvisor {
    771  public:
    772   TestProxyAdvisor()
    773       : ProxyAdvisor(NULL, NULL),
    774         would_proxy_(false),
    775         advise_count_(0),
    776         would_proxy_count_(0) {
    777   }
    778 
    779   virtual ~TestProxyAdvisor() {}
    780 
    781   virtual void Advise(const GURL& url,
    782                       UrlInfo::ResolutionMotivation motivation,
    783                       bool is_preconnect) OVERRIDE {
    784     ++advise_count_;
    785   }
    786 
    787   virtual bool WouldProxyURL(const GURL& url) OVERRIDE {
    788     ++would_proxy_count_;
    789     return would_proxy_;
    790   }
    791 
    792   bool would_proxy_;
    793   int advise_count_;
    794   int would_proxy_count_;
    795 };
    796 
    797 TEST_F(PredictorTest, SingleLookupTestWithDisabledAdvisor) {
    798   Predictor testing_master(true);
    799   TestProxyAdvisor* advisor = new TestProxyAdvisor();
    800   testing_master.SetHostResolver(host_resolver_.get());
    801   testing_master.proxy_advisor_.reset(advisor);
    802 
    803   GURL goog("http://www.google.com:80");
    804 
    805   advisor->would_proxy_ = false;
    806 
    807   UrlList names;
    808   names.push_back(goog);
    809   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
    810 
    811   WaitForResolution(&testing_master, names);
    812   EXPECT_TRUE(testing_master.WasFound(goog));
    813   EXPECT_EQ(advisor->would_proxy_count_, 1);
    814   EXPECT_EQ(advisor->advise_count_, 1);
    815 
    816   base::MessageLoop::current()->RunUntilIdle();
    817 
    818   testing_master.Shutdown();
    819 }
    820 
    821 TEST_F(PredictorTest, SingleLookupTestWithEnabledAdvisor) {
    822   Predictor testing_master(true);
    823   testing_master.SetHostResolver(host_resolver_.get());
    824   TestProxyAdvisor* advisor = new TestProxyAdvisor();
    825   testing_master.proxy_advisor_.reset(advisor);
    826 
    827   GURL goog("http://www.google.com:80");
    828 
    829   advisor->would_proxy_ = true;
    830 
    831   UrlList names;
    832   names.push_back(goog);
    833 
    834   testing_master.ResolveList(names, UrlInfo::PAGE_SCAN_MOTIVATED);
    835 
    836   // Attempt to resolve a few times.
    837   WaitForResolutionWithLimit(&testing_master, names, 10);
    838 
    839   // Because the advisor indicated that the url would be proxied,
    840   // no resolution should have occurred.
    841   EXPECT_FALSE(testing_master.WasFound(goog));
    842   EXPECT_EQ(advisor->would_proxy_count_, 1);
    843   EXPECT_EQ(advisor->advise_count_, 1);
    844 
    845   base::MessageLoop::current()->RunUntilIdle();
    846 
    847   testing_master.Shutdown();
    848 }
    849 
    850 TEST_F(PredictorTest, TestSimplePreconnectAdvisor) {
    851   Predictor testing_master(true);
    852   testing_master.SetHostResolver(host_resolver_.get());
    853   TestProxyAdvisor* advisor = new TestProxyAdvisor();
    854   testing_master.proxy_advisor_.reset(advisor);
    855 
    856   GURL goog("http://www.google.com:80");
    857 
    858   testing_master.PreconnectUrl(goog, goog, UrlInfo::OMNIBOX_MOTIVATED, 2);
    859 
    860   EXPECT_EQ(advisor->would_proxy_count_, 0);
    861   EXPECT_EQ(advisor->advise_count_, 1);
    862 
    863   testing_master.Shutdown();
    864 }
    865 
    866 #endif  // defined(OS_ANDROID) || defined(OS_IOS)
    867 
    868 }  // namespace chrome_browser_net
    869