Home | History | Annotate | Download | only in safe_browsing 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 //
      5 // Unit tests for the SafeBrowsing storage system.
      6 
      7 #include "base/file_util.h"
      8 #include "base/files/scoped_temp_dir.h"
      9 #include "base/logging.h"
     10 #include "base/message_loop/message_loop.h"
     11 #include "base/sha1.h"
     12 #include "base/time/time.h"
     13 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
     14 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
     15 #include "chrome/browser/safe_browsing/safe_browsing_store_unittest_helper.h"
     16 #include "content/public/test/test_browser_thread_bundle.h"
     17 #include "crypto/sha2.h"
     18 #include "net/base/net_util.h"
     19 #include "sql/connection.h"
     20 #include "sql/statement.h"
     21 #include "testing/gtest/include/gtest/gtest.h"
     22 #include "testing/platform_test.h"
     23 #include "url/gurl.h"
     24 
     25 using base::Time;
     26 
     27 namespace {
     28 
     29 SBPrefix Sha256Prefix(const std::string& str) {
     30   SBPrefix prefix;
     31   crypto::SHA256HashString(str, &prefix, sizeof(prefix));
     32   return prefix;
     33 }
     34 
     35 SBFullHash Sha256Hash(const std::string& str) {
     36   SBFullHash hash;
     37   crypto::SHA256HashString(str, &hash, sizeof(hash));
     38   return hash;
     39 }
     40 
     41 std::string HashedIpPrefix(const std::string& ip_prefix, size_t prefix_size) {
     42   net::IPAddressNumber ip_number;
     43   EXPECT_TRUE(net::ParseIPLiteralToNumber(ip_prefix, &ip_number));
     44   EXPECT_EQ(net::kIPv6AddressSize, ip_number.size());
     45   const std::string hashed_ip_prefix = base::SHA1HashString(
     46       net::IPAddressToPackedString(ip_number));
     47   std::string hash(crypto::kSHA256Length, '\0');
     48   hash.replace(0, hashed_ip_prefix.size(), hashed_ip_prefix);
     49   hash[base::kSHA1Length] = static_cast<char>(prefix_size);
     50   return hash;
     51 }
     52 
     53 // Same as InsertAddChunkHostPrefixUrl, but with pre-computed
     54 // prefix values.
     55 void InsertAddChunkHostPrefixValue(SBChunk* chunk,
     56                                    int chunk_number,
     57                                    const SBPrefix& host_prefix,
     58                                    const SBPrefix& url_prefix) {
     59   chunk->chunk_number = chunk_number;
     60   chunk->is_add = true;
     61   SBChunkHost host;
     62   host.host = host_prefix;
     63   host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 1);
     64   host.entry->set_chunk_id(chunk->chunk_number);
     65   host.entry->SetPrefixAt(0, url_prefix);
     66   chunk->hosts.push_back(host);
     67 }
     68 
     69 // A helper function that appends one AddChunkHost to chunk with
     70 // one url for prefix.
     71 void InsertAddChunkHostPrefixUrl(SBChunk* chunk,
     72                                  int chunk_number,
     73                                  const std::string& host_name,
     74                                  const std::string& url) {
     75   InsertAddChunkHostPrefixValue(chunk, chunk_number,
     76                                 Sha256Prefix(host_name),
     77                                 Sha256Prefix(url));
     78 }
     79 
     80 // Same as InsertAddChunkHostPrefixUrl, but with full hashes.
     81 void InsertAddChunkHostFullHashes(SBChunk* chunk,
     82                                   int chunk_number,
     83                                   const std::string& host_name,
     84                                   const std::string& url) {
     85   chunk->chunk_number = chunk_number;
     86   chunk->is_add = true;
     87   SBChunkHost host;
     88   host.host = Sha256Prefix(host_name);
     89   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
     90   host.entry->set_chunk_id(chunk->chunk_number);
     91   host.entry->SetFullHashAt(0, Sha256Hash(url));
     92   chunk->hosts.push_back(host);
     93 }
     94 
     95 void InsertAddChunkFullHash(SBChunk* chunk,
     96                             int chunk_number,
     97                             const std::string& ip_str,
     98                             size_t prefix_size) {
     99   const std::string full_hash_str = HashedIpPrefix(ip_str, prefix_size);
    100   EXPECT_EQ(sizeof(SBFullHash), full_hash_str.size());
    101   SBFullHash full_hash;
    102   std::memcpy(&(full_hash.full_hash), full_hash_str.data(), sizeof(SBFullHash));
    103 
    104   chunk->chunk_number = chunk_number;
    105   chunk->is_add = true;
    106   SBChunkHost host;
    107   host.host = full_hash.prefix;
    108   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1);
    109   host.entry->set_chunk_id(chunk->chunk_number);
    110   host.entry->SetFullHashAt(0, full_hash);
    111   chunk->hosts.push_back(host);
    112 }
    113 
    114 // Same as InsertAddChunkHostPrefixUrl, but with two urls for prefixes.
    115 void InsertAddChunkHost2PrefixUrls(SBChunk* chunk,
    116                                    int chunk_number,
    117                                    const std::string& host_name,
    118                                    const std::string& url1,
    119                                    const std::string& url2) {
    120   chunk->chunk_number = chunk_number;
    121   chunk->is_add = true;
    122   SBChunkHost host;
    123   host.host = Sha256Prefix(host_name);
    124   host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 2);
    125   host.entry->set_chunk_id(chunk->chunk_number);
    126   host.entry->SetPrefixAt(0, Sha256Prefix(url1));
    127   host.entry->SetPrefixAt(1, Sha256Prefix(url2));
    128   chunk->hosts.push_back(host);
    129 }
    130 
    131 // Same as InsertAddChunkHost2PrefixUrls, but with full hashes.
    132 void InsertAddChunkHost2FullHashes(SBChunk* chunk,
    133                                    int chunk_number,
    134                                    const std::string& host_name,
    135                                    const std::string& url1,
    136                                    const std::string& url2) {
    137   chunk->chunk_number = chunk_number;
    138   chunk->is_add = true;
    139   SBChunkHost host;
    140   host.host = Sha256Prefix(host_name);
    141   host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 2);
    142   host.entry->set_chunk_id(chunk->chunk_number);
    143   host.entry->SetFullHashAt(0, Sha256Hash(url1));
    144   host.entry->SetFullHashAt(1, Sha256Hash(url2));
    145   chunk->hosts.push_back(host);
    146 }
    147 
    148 // Same as InsertSubChunkHostPrefixUrl, but with pre-computed
    149 // prefix values.
    150 void InsertSubChunkHostPrefixValue(SBChunk* chunk,
    151                                    int chunk_number,
    152                                    int chunk_id_to_sub,
    153                                    const SBPrefix& host_prefix,
    154                                    const SBPrefix& url_prefix) {
    155   chunk->chunk_number = chunk_number;
    156   chunk->is_add = false;
    157   SBChunkHost host;
    158   host.host = host_prefix;
    159   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
    160   host.entry->set_chunk_id(chunk->chunk_number);
    161   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
    162   host.entry->SetPrefixAt(0, url_prefix);
    163   chunk->hosts.push_back(host);
    164 }
    165 
    166 // A helper function that adds one SubChunkHost to chunk with
    167 // one url for prefix.
    168 void InsertSubChunkHostPrefixUrl(SBChunk* chunk,
    169                                  int chunk_number,
    170                                  int chunk_id_to_sub,
    171                                  const std::string& host_name,
    172                                  const std::string& url) {
    173   InsertSubChunkHostPrefixValue(chunk, chunk_number,
    174                                 chunk_id_to_sub,
    175                                 Sha256Prefix(host_name),
    176                                 Sha256Prefix(url));
    177 }
    178 
    179 // Same as InsertSubChunkHostPrefixUrl, but with two urls for prefixes.
    180 void InsertSubChunkHost2PrefixUrls(SBChunk* chunk,
    181                                    int chunk_number,
    182                                    int chunk_id_to_sub,
    183                                    const std::string& host_name,
    184                                    const std::string& url1,
    185                                    const std::string& url2) {
    186   chunk->chunk_number = chunk_number;
    187   chunk->is_add = false;
    188   SBChunkHost host;
    189   host.host = Sha256Prefix(host_name);
    190   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 2);
    191   host.entry->set_chunk_id(chunk->chunk_number);
    192   host.entry->SetPrefixAt(0, Sha256Prefix(url1));
    193   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
    194   host.entry->SetPrefixAt(1, Sha256Prefix(url2));
    195   host.entry->SetChunkIdAtPrefix(1, chunk_id_to_sub);
    196   chunk->hosts.push_back(host);
    197 }
    198 
    199 // Same as InsertSubChunkHost2PrefixUrls, but with full hashes.
    200 void InsertSubChunkHostFullHash(SBChunk* chunk,
    201                                 int chunk_number,
    202                                 int chunk_id_to_sub,
    203                                 const std::string& host_name,
    204                                 const std::string& url) {
    205   chunk->chunk_number = chunk_number;
    206   chunk->is_add = false;
    207   SBChunkHost host;
    208   host.host = Sha256Prefix(host_name);
    209   host.entry = SBEntry::Create(SBEntry::SUB_FULL_HASH, 2);
    210   host.entry->set_chunk_id(chunk->chunk_number);
    211   host.entry->SetFullHashAt(0, Sha256Hash(url));
    212   host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub);
    213   chunk->hosts.push_back(host);
    214 }
    215 
    216 // Prevent DCHECK from killing tests.
    217 // TODO(shess): Pawel disputes the use of this, so the test which uses
    218 // it is DISABLED.  http://crbug.com/56448
    219 class ScopedLogMessageIgnorer {
    220  public:
    221   ScopedLogMessageIgnorer() {
    222     logging::SetLogMessageHandler(&LogMessageIgnorer);
    223   }
    224   ~ScopedLogMessageIgnorer() {
    225     // TODO(shess): Would be better to verify whether anyone else
    226     // changed it, and then restore it to the previous value.
    227     logging::SetLogMessageHandler(NULL);
    228   }
    229 
    230  private:
    231   static bool LogMessageIgnorer(int severity, const char* file, int line,
    232       size_t message_start, const std::string& str) {
    233     // Intercept FATAL, strip the stack backtrace, and log it without
    234     // the crash part.
    235     if (severity == logging::LOG_FATAL) {
    236       size_t newline = str.find('\n');
    237       if (newline != std::string::npos) {
    238         const std::string msg = str.substr(0, newline + 1);
    239         fprintf(stderr, "%s", msg.c_str());
    240         fflush(stderr);
    241       }
    242       return true;
    243     }
    244 
    245     return false;
    246   }
    247 };
    248 
    249 }  // namespace
    250 
    251 class SafeBrowsingDatabaseTest : public PlatformTest {
    252  public:
    253   virtual void SetUp() {
    254     PlatformTest::SetUp();
    255 
    256     // Setup a database in a temporary directory.
    257     ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
    258     database_.reset(new SafeBrowsingDatabaseNew);
    259     database_filename_ =
    260         temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase");
    261     database_->Init(database_filename_);
    262   }
    263 
    264   virtual void TearDown() {
    265     database_.reset();
    266 
    267     PlatformTest::TearDown();
    268   }
    269 
    270   void GetListsInfo(std::vector<SBListChunkRanges>* lists) {
    271     lists->clear();
    272     EXPECT_TRUE(database_->UpdateStarted(lists));
    273     database_->UpdateFinished(true);
    274   }
    275 
    276   // Helper function to do an AddDel or SubDel command.
    277   void DelChunk(const std::string& list,
    278                 int chunk_id,
    279                 bool is_sub_del) {
    280     std::vector<SBChunkDelete> deletes;
    281     SBChunkDelete chunk_delete;
    282     chunk_delete.list_name = list;
    283     chunk_delete.is_sub_del = is_sub_del;
    284     chunk_delete.chunk_del.push_back(ChunkRange(chunk_id));
    285     deletes.push_back(chunk_delete);
    286     database_->DeleteChunks(deletes);
    287   }
    288 
    289   void AddDelChunk(const std::string& list, int chunk_id) {
    290     DelChunk(list, chunk_id, false);
    291   }
    292 
    293   void SubDelChunk(const std::string& list, int chunk_id) {
    294     DelChunk(list, chunk_id, true);
    295   }
    296 
    297   // Utility function for setting up the database for the caching test.
    298   void PopulateDatabaseForCacheTest();
    299 
    300   scoped_ptr<SafeBrowsingDatabaseNew> database_;
    301   base::FilePath database_filename_;
    302   base::ScopedTempDir temp_dir_;
    303 };
    304 
    305 // Tests retrieving list name information.
    306 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) {
    307   SBChunkList chunks;
    308   SBChunk chunk;
    309 
    310   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
    311                               "www.evil.com/malware.html");
    312   chunks.clear();
    313   chunks.push_back(chunk);
    314   std::vector<SBListChunkRanges> lists;
    315   EXPECT_TRUE(database_->UpdateStarted(&lists));
    316   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    317 
    318   chunk.hosts.clear();
    319   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
    320                               "www.foo.com/malware.html");
    321   chunks.clear();
    322   chunks.push_back(chunk);
    323   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    324 
    325   chunk.hosts.clear();
    326   InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
    327                               "www.whatever.com/malware.html");
    328   chunks.clear();
    329   chunks.push_back(chunk);
    330   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    331   database_->UpdateFinished(true);
    332 
    333   GetListsInfo(&lists);
    334   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    335   EXPECT_EQ(lists[0].adds, "1-3");
    336   EXPECT_TRUE(lists[0].subs.empty());
    337 
    338   // Insert a malware sub chunk.
    339   chunk.hosts.clear();
    340   InsertSubChunkHostPrefixUrl(&chunk, 7, 19, "www.subbed.com/",
    341                               "www.subbed.com/noteveil1.html");
    342   chunks.clear();
    343   chunks.push_back(chunk);
    344 
    345   EXPECT_TRUE(database_->UpdateStarted(&lists));
    346   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    347   database_->UpdateFinished(true);
    348 
    349   GetListsInfo(&lists);
    350   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    351   EXPECT_EQ(lists[0].adds, "1-3");
    352   EXPECT_EQ(lists[0].subs, "7");
    353   if (lists.size() == 2) {
    354     // Old style database won't have the second entry since it creates the lists
    355     // when it receives an update containing that list. The filter-based
    356     // database has these values hard coded.
    357     EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
    358     EXPECT_TRUE(lists[1].adds.empty());
    359     EXPECT_TRUE(lists[1].subs.empty());
    360   }
    361 
    362   // Add a phishing add chunk.
    363   chunk.hosts.clear();
    364   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
    365                               "www.evil.com/phishing.html");
    366   chunks.clear();
    367   chunks.push_back(chunk);
    368   EXPECT_TRUE(database_->UpdateStarted(&lists));
    369   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
    370 
    371   // Insert some phishing sub chunks.
    372   chunk.hosts.clear();
    373   InsertSubChunkHostPrefixUrl(&chunk, 200, 1999, "www.phishy.com/",
    374                               "www.phishy.com/notevil1.html");
    375   chunks.clear();
    376   chunks.push_back(chunk);
    377   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
    378 
    379   chunk.hosts.clear();
    380   InsertSubChunkHostPrefixUrl(&chunk, 201, 1999, "www.phishy2.com/",
    381                               "www.phishy2.com/notevil1.html");
    382   chunks.clear();
    383   chunks.push_back(chunk);
    384   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
    385   database_->UpdateFinished(true);
    386 
    387   GetListsInfo(&lists);
    388   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    389   EXPECT_EQ(lists[0].adds, "1-3");
    390   EXPECT_EQ(lists[0].subs, "7");
    391   EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
    392   EXPECT_EQ(lists[1].adds, "47");
    393   EXPECT_EQ(lists[1].subs, "200-201");
    394 }
    395 
    396 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) {
    397   database_.reset();
    398   base::MessageLoop loop(base::MessageLoop::TYPE_DEFAULT);
    399   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
    400   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
    401   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
    402   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
    403   SafeBrowsingStoreFile* extension_blacklist_store =
    404       new SafeBrowsingStoreFile();
    405   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
    406   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
    407                                               download_store,
    408                                               csd_whitelist_store,
    409                                               download_whitelist_store,
    410                                               extension_blacklist_store,
    411                                               NULL,
    412                                               ip_blacklist_store));
    413   database_->Init(database_filename_);
    414 
    415   SBChunkList chunks;
    416   SBChunk chunk;
    417 
    418   // Insert malware, phish, binurl and bindownload add chunks.
    419   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
    420                               "www.evil.com/malware.html");
    421   chunks.push_back(chunk);
    422   std::vector<SBListChunkRanges> lists;
    423   EXPECT_TRUE(database_->UpdateStarted(&lists));
    424   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    425 
    426   chunk.hosts.clear();
    427   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
    428                               "www.foo.com/malware.html");
    429   chunks.clear();
    430   chunks.push_back(chunk);
    431   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
    432 
    433   chunk.hosts.clear();
    434   InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/",
    435                               "www.whatever.com/download.html");
    436   chunks.clear();
    437   chunks.push_back(chunk);
    438   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
    439 
    440   chunk.hosts.clear();
    441   InsertAddChunkHostPrefixUrl(&chunk, 4, "www.forhash.com/",
    442                               "www.forhash.com/download.html");
    443   chunks.clear();
    444   chunks.push_back(chunk);
    445   database_->InsertChunks(safe_browsing_util::kBinHashList, chunks);
    446 
    447   chunk.hosts.clear();
    448   InsertAddChunkHostFullHashes(&chunk, 5, "www.forwhitelist.com/",
    449                                "www.forwhitelist.com/a.html");
    450   chunks.clear();
    451   chunks.push_back(chunk);
    452   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks);
    453 
    454   chunk.hosts.clear();
    455   InsertAddChunkHostFullHashes(&chunk, 6, "www.download.com/",
    456                                "www.download.com/");
    457 
    458   chunks.clear();
    459   chunks.push_back(chunk);
    460   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList, chunks);
    461 
    462   chunk.hosts.clear();
    463   InsertAddChunkHostFullHashes(&chunk, 8,
    464                                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
    465                                "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
    466 
    467   chunks.clear();
    468   chunks.push_back(chunk);
    469   database_->InsertChunks(safe_browsing_util::kExtensionBlacklist, chunks);
    470 
    471   chunk.hosts.clear();
    472   InsertAddChunkFullHash(&chunk, 9, "::ffff:192.168.1.0", 120);
    473 
    474   chunks.clear();
    475   chunks.push_back(chunk);
    476   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
    477 
    478   database_->UpdateFinished(true);
    479 
    480   GetListsInfo(&lists);
    481   ASSERT_EQ(7U, lists.size());
    482   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    483   EXPECT_EQ(lists[0].adds, "1");
    484   EXPECT_TRUE(lists[0].subs.empty());
    485   EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList);
    486   EXPECT_EQ(lists[1].adds, "2");
    487   EXPECT_TRUE(lists[1].subs.empty());
    488   EXPECT_TRUE(lists[2].name == safe_browsing_util::kBinUrlList);
    489   EXPECT_EQ(lists[2].adds, "3");
    490   EXPECT_TRUE(lists[2].subs.empty());
    491   // kBinHashList is ignored.  (http://crbug.com/108130)
    492   EXPECT_TRUE(lists[3].name == safe_browsing_util::kCsdWhiteList);
    493   EXPECT_EQ(lists[3].adds, "5");
    494   EXPECT_TRUE(lists[3].subs.empty());
    495   EXPECT_TRUE(lists[4].name == safe_browsing_util::kDownloadWhiteList);
    496   EXPECT_EQ(lists[4].adds, "6");
    497   EXPECT_TRUE(lists[4].subs.empty());
    498   EXPECT_TRUE(lists[5].name == safe_browsing_util::kExtensionBlacklist);
    499   EXPECT_EQ(lists[5].adds, "8");
    500   EXPECT_TRUE(lists[5].subs.empty());
    501   EXPECT_TRUE(lists[6].name == safe_browsing_util::kIPBlacklist);
    502   EXPECT_EQ(lists[6].adds, "9");
    503   EXPECT_TRUE(lists[6].subs.empty());
    504 
    505   database_.reset();
    506 }
    507 
    508 // Checks database reading and writing for browse.
    509 TEST_F(SafeBrowsingDatabaseTest, BrowseDatabase) {
    510   SBChunkList chunks;
    511   SBChunk chunk;
    512 
    513   // Add a simple chunk with one hostkey.
    514   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
    515                                 "www.evil.com/phishing.html",
    516                                 "www.evil.com/malware.html");
    517   chunks.push_back(chunk);
    518   std::vector<SBListChunkRanges> lists;
    519   EXPECT_TRUE(database_->UpdateStarted(&lists));
    520   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    521 
    522   chunk.hosts.clear();
    523   InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.evil.com/",
    524                                 "www.evil.com/notevil1.html",
    525                                 "www.evil.com/notevil2.html");
    526   InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.good.com/",
    527                                 "www.good.com/good1.html",
    528                                 "www.good.com/good2.html");
    529   chunks.clear();
    530   chunks.push_back(chunk);
    531   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    532 
    533   // and a chunk with an IP-based host
    534   chunk.hosts.clear();
    535   InsertAddChunkHostPrefixUrl(&chunk, 3, "192.168.0.1/",
    536                               "192.168.0.1/malware.html");
    537   chunks.clear();
    538   chunks.push_back(chunk);
    539   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    540   database_->UpdateFinished(true);
    541 
    542   // Make sure they were added correctly.
    543   GetListsInfo(&lists);
    544   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    545   EXPECT_EQ(lists[0].adds, "1-3");
    546   EXPECT_TRUE(lists[0].subs.empty());
    547 
    548   const Time now = Time::Now();
    549   std::vector<SBFullHashResult> full_hashes;
    550   std::vector<SBPrefix> prefix_hits;
    551   std::string matching_list;
    552   EXPECT_TRUE(database_->ContainsBrowseUrl(
    553       GURL("http://www.evil.com/phishing.html"),
    554       &matching_list, &prefix_hits,
    555       &full_hashes, now));
    556   EXPECT_EQ(prefix_hits[0], Sha256Prefix("www.evil.com/phishing.html"));
    557   EXPECT_EQ(prefix_hits.size(), 1U);
    558 
    559   EXPECT_TRUE(database_->ContainsBrowseUrl(
    560       GURL("http://www.evil.com/malware.html"),
    561       &matching_list, &prefix_hits,
    562       &full_hashes, now));
    563 
    564   EXPECT_TRUE(database_->ContainsBrowseUrl(
    565       GURL("http://www.evil.com/notevil1.html"),
    566       &matching_list, &prefix_hits,
    567       &full_hashes, now));
    568 
    569   EXPECT_TRUE(database_->ContainsBrowseUrl(
    570       GURL("http://www.evil.com/notevil2.html"),
    571       &matching_list, &prefix_hits,
    572       &full_hashes, now));
    573 
    574   EXPECT_TRUE(database_->ContainsBrowseUrl(
    575       GURL("http://www.good.com/good1.html"),
    576       &matching_list, &prefix_hits,
    577       &full_hashes, now));
    578 
    579   EXPECT_TRUE(database_->ContainsBrowseUrl(
    580       GURL("http://www.good.com/good2.html"),
    581       &matching_list, &prefix_hits,
    582       &full_hashes, now));
    583 
    584   EXPECT_TRUE(database_->ContainsBrowseUrl(
    585       GURL("http://192.168.0.1/malware.html"),
    586       &matching_list, &prefix_hits,
    587       &full_hashes, now));
    588 
    589   EXPECT_FALSE(database_->ContainsBrowseUrl(
    590       GURL("http://www.evil.com/"),
    591       &matching_list, &prefix_hits,
    592       &full_hashes, now));
    593   EXPECT_TRUE(prefix_hits.empty());
    594 
    595   EXPECT_FALSE(database_->ContainsBrowseUrl(
    596       GURL("http://www.evil.com/robots.txt"),
    597       &matching_list, &prefix_hits,
    598       &full_hashes, now));
    599 
    600   // Attempt to re-add the first chunk (should be a no-op).
    601   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
    602   chunk.hosts.clear();
    603   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
    604                                 "www.evil.com/phishing.html",
    605                                 "www.evil.com/malware.html");
    606   chunks.clear();
    607   chunks.push_back(chunk);
    608   EXPECT_TRUE(database_->UpdateStarted(&lists));
    609   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    610   database_->UpdateFinished(true);
    611 
    612   GetListsInfo(&lists);
    613   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    614   EXPECT_EQ(lists[0].adds, "1-3");
    615   EXPECT_TRUE(lists[0].subs.empty());
    616 
    617   // Test removing a single prefix from the add chunk.
    618   chunk.hosts.clear();
    619   InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
    620                               "www.evil.com/notevil1.html");
    621   chunks.clear();
    622   chunks.push_back(chunk);
    623   EXPECT_TRUE(database_->UpdateStarted(&lists));
    624   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    625 
    626   database_->UpdateFinished(true);
    627 
    628   EXPECT_TRUE(database_->ContainsBrowseUrl(
    629       GURL("http://www.evil.com/phishing.html"),
    630       &matching_list, &prefix_hits,
    631       &full_hashes, now));
    632   EXPECT_EQ(prefix_hits[0], Sha256Prefix("www.evil.com/phishing.html"));
    633   EXPECT_EQ(prefix_hits.size(), 1U);
    634 
    635   EXPECT_FALSE(database_->ContainsBrowseUrl(
    636       GURL("http://www.evil.com/notevil1.html"),
    637       &matching_list, &prefix_hits,
    638       &full_hashes, now));
    639   EXPECT_TRUE(prefix_hits.empty());
    640 
    641   EXPECT_TRUE(database_->ContainsBrowseUrl(
    642       GURL("http://www.evil.com/notevil2.html"),
    643       &matching_list, &prefix_hits,
    644       &full_hashes, now));
    645 
    646   EXPECT_TRUE(database_->ContainsBrowseUrl(
    647       GURL("http://www.good.com/good1.html"),
    648       &matching_list, &prefix_hits,
    649       &full_hashes, now));
    650 
    651   EXPECT_TRUE(database_->ContainsBrowseUrl(
    652       GURL("http://www.good.com/good2.html"),
    653       &matching_list, &prefix_hits,
    654       &full_hashes, now));
    655 
    656   GetListsInfo(&lists);
    657   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    658   EXPECT_EQ(lists[0].subs, "4");
    659 
    660   // Test the same sub chunk again.  This should be a no-op.
    661   // see bug: http://code.google.com/p/chromium/issues/detail?id=4522
    662   chunk.hosts.clear();
    663   InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/",
    664                               "www.evil.com/notevil1.html");
    665   chunks.clear();
    666   chunks.push_back(chunk);
    667 
    668   EXPECT_TRUE(database_->UpdateStarted(&lists));
    669   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    670   database_->UpdateFinished(true);
    671 
    672   GetListsInfo(&lists);
    673   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    674   EXPECT_EQ(lists[0].subs, "4");
    675 
    676   // Test removing all the prefixes from an add chunk.
    677   EXPECT_TRUE(database_->UpdateStarted(&lists));
    678   AddDelChunk(safe_browsing_util::kMalwareList, 2);
    679   database_->UpdateFinished(true);
    680 
    681   EXPECT_FALSE(database_->ContainsBrowseUrl(
    682       GURL("http://www.evil.com/notevil2.html"),
    683       &matching_list, &prefix_hits,
    684       &full_hashes, now));
    685 
    686   EXPECT_FALSE(database_->ContainsBrowseUrl(
    687       GURL("http://www.good.com/good1.html"),
    688       &matching_list, &prefix_hits,
    689       &full_hashes, now));
    690 
    691   EXPECT_FALSE(database_->ContainsBrowseUrl(
    692       GURL("http://www.good.com/good2.html"),
    693       &matching_list, &prefix_hits,
    694       &full_hashes, now));
    695 
    696   GetListsInfo(&lists);
    697   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    698   EXPECT_EQ(lists[0].adds, "1,3");
    699   EXPECT_EQ(lists[0].subs, "4");
    700 
    701   // The adddel command exposed a bug in the transaction code where any
    702   // transaction after it would fail.  Add a dummy entry and remove it to
    703   // make sure the transcation works fine.
    704   chunk.hosts.clear();
    705   InsertAddChunkHostPrefixUrl(&chunk, 44, "www.redherring.com/",
    706                               "www.redherring.com/index.html");
    707   chunks.clear();
    708   chunks.push_back(chunk);
    709   EXPECT_TRUE(database_->UpdateStarted(&lists));
    710   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    711 
    712   // Now remove the dummy entry.  If there are any problems with the
    713   // transactions, asserts will fire.
    714   AddDelChunk(safe_browsing_util::kMalwareList, 44);
    715 
    716   // Test the subdel command.
    717   SubDelChunk(safe_browsing_util::kMalwareList, 4);
    718   database_->UpdateFinished(true);
    719 
    720   GetListsInfo(&lists);
    721   EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList);
    722   EXPECT_EQ(lists[0].adds, "1,3");
    723   EXPECT_EQ(lists[0].subs, "");
    724 
    725   // Test a sub command coming in before the add.
    726   chunk.hosts.clear();
    727   InsertSubChunkHost2PrefixUrls(&chunk, 5, 10,
    728                                 "www.notevilanymore.com/",
    729                                 "www.notevilanymore.com/index.html",
    730                                 "www.notevilanymore.com/good.html");
    731   chunks.clear();
    732   chunks.push_back(chunk);
    733   EXPECT_TRUE(database_->UpdateStarted(&lists));
    734   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    735   database_->UpdateFinished(true);
    736 
    737   EXPECT_FALSE(database_->ContainsBrowseUrl(
    738       GURL("http://www.notevilanymore.com/index.html"),
    739       &matching_list, &prefix_hits, &full_hashes, now));
    740 
    741   // Now insert the tardy add chunk and we don't expect them to appear
    742   // in database because of the previous sub chunk.
    743   chunk.hosts.clear();
    744   InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.notevilanymore.com/",
    745                                 "www.notevilanymore.com/index.html",
    746                                 "www.notevilanymore.com/good.html");
    747   chunks.clear();
    748   chunks.push_back(chunk);
    749   EXPECT_TRUE(database_->UpdateStarted(&lists));
    750   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    751   database_->UpdateFinished(true);
    752 
    753   EXPECT_FALSE(database_->ContainsBrowseUrl(
    754       GURL("http://www.notevilanymore.com/index.html"),
    755       &matching_list, &prefix_hits, &full_hashes, now));
    756 
    757   EXPECT_FALSE(database_->ContainsBrowseUrl(
    758       GURL("http://www.notevilanymore.com/good.html"),
    759       &matching_list, &prefix_hits, &full_hashes, now));
    760 }
    761 
    762 
    763 // Test adding zero length chunks to the database.
    764 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) {
    765   SBChunkList chunks;
    766   SBChunk chunk;
    767 
    768   // Populate with a couple of normal chunks.
    769   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.test.com/",
    770                                 "www.test.com/test1.html",
    771                                 "www.test.com/test2.html");
    772   chunks.clear();
    773   chunks.push_back(chunk);
    774 
    775   chunk.hosts.clear();
    776   InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.random.com/",
    777                                 "www.random.com/random1.html",
    778                                 "www.random.com/random2.html");
    779   chunks.push_back(chunk);
    780 
    781   std::vector<SBListChunkRanges> lists;
    782   EXPECT_TRUE(database_->UpdateStarted(&lists));
    783   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    784   database_->UpdateFinished(true);
    785 
    786   // Add an empty ADD and SUB chunk.
    787   GetListsInfo(&lists);
    788   EXPECT_EQ(lists[0].adds, "1,10");
    789 
    790   SBChunk empty_chunk;
    791   empty_chunk.chunk_number = 19;
    792   empty_chunk.is_add = true;
    793   chunks.clear();
    794   chunks.push_back(empty_chunk);
    795   EXPECT_TRUE(database_->UpdateStarted(&lists));
    796   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    797   chunks.clear();
    798   empty_chunk.chunk_number = 7;
    799   empty_chunk.is_add = false;
    800   chunks.push_back(empty_chunk);
    801   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    802   database_->UpdateFinished(true);
    803 
    804   GetListsInfo(&lists);
    805   EXPECT_EQ(lists[0].adds, "1,10,19");
    806   EXPECT_EQ(lists[0].subs, "7");
    807 
    808   // Add an empty chunk along with a couple that contain data. This should
    809   // result in the chunk range being reduced in size.
    810   empty_chunk.hosts.clear();
    811   InsertAddChunkHostPrefixUrl(&empty_chunk, 20, "www.notempy.com/",
    812                               "www.notempty.com/full1.html");
    813   chunks.clear();
    814   chunks.push_back(empty_chunk);
    815 
    816   empty_chunk.chunk_number = 21;
    817   empty_chunk.is_add = true;
    818   empty_chunk.hosts.clear();
    819   chunks.push_back(empty_chunk);
    820 
    821   empty_chunk.hosts.clear();
    822   InsertAddChunkHostPrefixUrl(&empty_chunk, 22, "www.notempy.com/",
    823                               "www.notempty.com/full2.html");
    824   chunks.push_back(empty_chunk);
    825 
    826   EXPECT_TRUE(database_->UpdateStarted(&lists));
    827   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    828   database_->UpdateFinished(true);
    829 
    830   const Time now = Time::Now();
    831   std::vector<SBFullHashResult> full_hashes;
    832   std::vector<SBPrefix> prefix_hits;
    833   std::string matching_list;
    834   EXPECT_TRUE(database_->ContainsBrowseUrl(
    835       GURL("http://www.notempty.com/full1.html"),
    836       &matching_list, &prefix_hits,
    837       &full_hashes, now));
    838   EXPECT_TRUE(database_->ContainsBrowseUrl(
    839       GURL("http://www.notempty.com/full2.html"),
    840       &matching_list, &prefix_hits,
    841       &full_hashes, now));
    842 
    843   GetListsInfo(&lists);
    844   EXPECT_EQ(lists[0].adds, "1,10,19-22");
    845   EXPECT_EQ(lists[0].subs, "7");
    846 
    847   // Handle AddDel and SubDel commands for empty chunks.
    848   EXPECT_TRUE(database_->UpdateStarted(&lists));
    849   AddDelChunk(safe_browsing_util::kMalwareList, 21);
    850   database_->UpdateFinished(true);
    851 
    852   GetListsInfo(&lists);
    853   EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
    854   EXPECT_EQ(lists[0].subs, "7");
    855 
    856   EXPECT_TRUE(database_->UpdateStarted(&lists));
    857   SubDelChunk(safe_browsing_util::kMalwareList, 7);
    858   database_->UpdateFinished(true);
    859 
    860   GetListsInfo(&lists);
    861   EXPECT_EQ(lists[0].adds, "1,10,19-20,22");
    862   EXPECT_EQ(lists[0].subs, "");
    863 }
    864 
    865 // Utility function for setting up the database for the caching test.
    866 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() {
    867   SBChunkList chunks;
    868   SBChunk chunk;
    869   // Add a simple chunk with one hostkey and cache it.
    870   InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/",
    871                                 "www.evil.com/phishing.html",
    872                                 "www.evil.com/malware.html");
    873   chunks.push_back(chunk);
    874 
    875   std::vector<SBListChunkRanges> lists;
    876   EXPECT_TRUE(database_->UpdateStarted(&lists));
    877   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    878   database_->UpdateFinished(true);
    879 
    880   // Add the GetHash results to the cache.
    881   SBFullHashResult full_hash;
    882   full_hash.hash = Sha256Hash("www.evil.com/phishing.html");
    883   full_hash.list_name = safe_browsing_util::kMalwareList;
    884   full_hash.add_chunk_id = 1;
    885 
    886   std::vector<SBFullHashResult> results;
    887   results.push_back(full_hash);
    888 
    889   full_hash.hash = Sha256Hash("www.evil.com/malware.html");
    890   results.push_back(full_hash);
    891 
    892   std::vector<SBPrefix> prefixes;
    893   database_->CacheHashResults(prefixes, results);
    894 }
    895 
    896 TEST_F(SafeBrowsingDatabaseTest, HashCaching) {
    897   PopulateDatabaseForCacheTest();
    898 
    899   // We should have both full hashes in the cache.
    900   EXPECT_EQ(database_->pending_browse_hashes_.size(), 2U);
    901 
    902   // Test the cache lookup for the first prefix.
    903   std::string listname;
    904   std::vector<SBPrefix> prefixes;
    905   std::vector<SBFullHashResult> full_hashes;
    906   database_->ContainsBrowseUrl(
    907       GURL("http://www.evil.com/phishing.html"),
    908       &listname, &prefixes, &full_hashes, Time::Now());
    909   EXPECT_EQ(full_hashes.size(), 1U);
    910   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
    911                            Sha256Hash("www.evil.com/phishing.html")));
    912 
    913   prefixes.clear();
    914   full_hashes.clear();
    915 
    916   // Test the cache lookup for the second prefix.
    917   database_->ContainsBrowseUrl(
    918       GURL("http://www.evil.com/malware.html"),
    919       &listname, &prefixes, &full_hashes, Time::Now());
    920   EXPECT_EQ(full_hashes.size(), 1U);
    921   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
    922                            Sha256Hash("www.evil.com/malware.html")));
    923 
    924   prefixes.clear();
    925   full_hashes.clear();
    926 
    927   // Test removing a prefix via a sub chunk.
    928   SBChunk chunk;
    929   SBChunkList chunks;
    930   InsertSubChunkHostPrefixUrl(&chunk, 2, 1, "www.evil.com/",
    931                               "www.evil.com/phishing.html");
    932   chunks.push_back(chunk);
    933 
    934   std::vector<SBListChunkRanges> lists;
    935   EXPECT_TRUE(database_->UpdateStarted(&lists));
    936   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
    937   database_->UpdateFinished(true);
    938 
    939   // This prefix should still be there.
    940   database_->ContainsBrowseUrl(
    941       GURL("http://www.evil.com/malware.html"),
    942       &listname, &prefixes, &full_hashes, Time::Now());
    943   EXPECT_EQ(full_hashes.size(), 1U);
    944   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
    945                            Sha256Hash("www.evil.com/malware.html")));
    946   prefixes.clear();
    947   full_hashes.clear();
    948 
    949   // This prefix should be gone.
    950   database_->ContainsBrowseUrl(
    951       GURL("http://www.evil.com/phishing.html"),
    952       &listname, &prefixes, &full_hashes, Time::Now());
    953   EXPECT_TRUE(full_hashes.empty());
    954 
    955   prefixes.clear();
    956   full_hashes.clear();
    957 
    958   // Test that an AddDel for the original chunk removes the last cached entry.
    959   EXPECT_TRUE(database_->UpdateStarted(&lists));
    960   AddDelChunk(safe_browsing_util::kMalwareList, 1);
    961   database_->UpdateFinished(true);
    962   database_->ContainsBrowseUrl(
    963       GURL("http://www.evil.com/malware.html"),
    964       &listname, &prefixes, &full_hashes, Time::Now());
    965   EXPECT_TRUE(full_hashes.empty());
    966   EXPECT_TRUE(database_->full_browse_hashes_.empty());
    967   EXPECT_TRUE(database_->pending_browse_hashes_.empty());
    968 
    969   prefixes.clear();
    970   full_hashes.clear();
    971 
    972   // Test that the cache won't return expired values. First we have to adjust
    973   // the cached entries' received time to make them older, since the database
    974   // cache insert uses Time::Now(). First, store some entries.
    975   PopulateDatabaseForCacheTest();
    976 
    977   std::vector<SBAddFullHash>* hash_cache = &database_->pending_browse_hashes_;
    978   EXPECT_EQ(hash_cache->size(), 2U);
    979 
    980   // Now adjust one of the entries times to be in the past.
    981   base::Time expired = base::Time::Now() - base::TimeDelta::FromMinutes(60);
    982   const SBPrefix key = Sha256Prefix("www.evil.com/malware.html");
    983   std::vector<SBAddFullHash>::iterator iter;
    984   for (iter = hash_cache->begin(); iter != hash_cache->end(); ++iter) {
    985     if (iter->full_hash.prefix == key) {
    986       iter->received = static_cast<int32>(expired.ToTimeT());
    987       break;
    988     }
    989   }
    990   EXPECT_TRUE(iter != hash_cache->end());
    991 
    992   database_->ContainsBrowseUrl(
    993       GURL("http://www.evil.com/malware.html"),
    994       &listname, &prefixes, &full_hashes, expired);
    995   EXPECT_TRUE(full_hashes.empty());
    996 
    997   // This entry should still exist.
    998   database_->ContainsBrowseUrl(
    999       GURL("http://www.evil.com/phishing.html"),
   1000       &listname, &prefixes, &full_hashes, expired);
   1001   EXPECT_EQ(full_hashes.size(), 1U);
   1002 
   1003 
   1004   // Testing prefix miss caching. First, we clear out the existing database,
   1005   // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate
   1006   // chunks.
   1007   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1008   AddDelChunk(safe_browsing_util::kMalwareList, 1);
   1009   database_->UpdateFinished(true);
   1010 
   1011   std::vector<SBPrefix> prefix_misses;
   1012   std::vector<SBFullHashResult> empty_full_hash;
   1013   prefix_misses.push_back(Sha256Prefix("http://www.bad.com/malware.html"));
   1014   prefix_misses.push_back(Sha256Prefix("http://www.bad.com/phishing.html"));
   1015   database_->CacheHashResults(prefix_misses, empty_full_hash);
   1016 
   1017   // Prefixes with no full results are misses.
   1018   EXPECT_EQ(database_->prefix_miss_cache_.size(), 2U);
   1019 
   1020   // Update the database.
   1021   PopulateDatabaseForCacheTest();
   1022 
   1023   // Prefix miss cache should be cleared.
   1024   EXPECT_TRUE(database_->prefix_miss_cache_.empty());
   1025 
   1026   // Cache a GetHash miss for a particular prefix, and even though the prefix is
   1027   // in the database, it is flagged as a miss so looking up the associated URL
   1028   // will not succeed.
   1029   prefixes.clear();
   1030   full_hashes.clear();
   1031   prefix_misses.clear();
   1032   empty_full_hash.clear();
   1033   prefix_misses.push_back(Sha256Prefix("www.evil.com/phishing.html"));
   1034   database_->CacheHashResults(prefix_misses, empty_full_hash);
   1035   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1036       GURL("http://www.evil.com/phishing.html"),
   1037       &listname, &prefixes,
   1038       &full_hashes, Time::Now()));
   1039 
   1040   prefixes.clear();
   1041   full_hashes.clear();
   1042 
   1043   // Test receiving a full add chunk.
   1044   chunk.hosts.clear();
   1045   InsertAddChunkHost2FullHashes(&chunk, 20, "www.fullevil.com/",
   1046                                 "www.fullevil.com/bad1.html",
   1047                                 "www.fullevil.com/bad2.html");
   1048   chunks.clear();
   1049   chunks.push_back(chunk);
   1050   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1051   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1052   database_->UpdateFinished(true);
   1053 
   1054   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1055       GURL("http://www.fullevil.com/bad1.html"),
   1056       &listname, &prefixes, &full_hashes,
   1057       Time::Now()));
   1058   EXPECT_EQ(full_hashes.size(), 1U);
   1059   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
   1060                            Sha256Hash("www.fullevil.com/bad1.html")));
   1061   prefixes.clear();
   1062   full_hashes.clear();
   1063 
   1064   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1065       GURL("http://www.fullevil.com/bad2.html"),
   1066       &listname, &prefixes, &full_hashes,
   1067       Time::Now()));
   1068   EXPECT_EQ(full_hashes.size(), 1U);
   1069   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
   1070                            Sha256Hash("www.fullevil.com/bad2.html")));
   1071   prefixes.clear();
   1072   full_hashes.clear();
   1073 
   1074   // Test receiving a full sub chunk, which will remove one of the full adds.
   1075   chunk.hosts.clear();
   1076   InsertSubChunkHostFullHash(&chunk, 200, 20,
   1077                              "www.fullevil.com/",
   1078                              "www.fullevil.com/bad1.html");
   1079   chunks.clear();
   1080   chunks.push_back(chunk);
   1081   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1082   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1083   database_->UpdateFinished(true);
   1084 
   1085   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1086       GURL("http://www.fullevil.com/bad1.html"),
   1087       &listname, &prefixes, &full_hashes,
   1088       Time::Now()));
   1089   EXPECT_TRUE(full_hashes.empty());
   1090 
   1091   // There should be one remaining full add.
   1092   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1093       GURL("http://www.fullevil.com/bad2.html"),
   1094       &listname, &prefixes, &full_hashes,
   1095       Time::Now()));
   1096   EXPECT_EQ(full_hashes.size(), 1U);
   1097   EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash,
   1098                            Sha256Hash("www.fullevil.com/bad2.html")));
   1099   prefixes.clear();
   1100   full_hashes.clear();
   1101 
   1102   // Now test an AddDel for the remaining full add.
   1103   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1104   AddDelChunk(safe_browsing_util::kMalwareList, 20);
   1105   database_->UpdateFinished(true);
   1106 
   1107   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1108       GURL("http://www.fullevil.com/bad1.html"),
   1109       &listname, &prefixes, &full_hashes,
   1110       Time::Now()));
   1111   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1112       GURL("http://www.fullevil.com/bad2.html"),
   1113       &listname, &prefixes, &full_hashes,
   1114       Time::Now()));
   1115 }
   1116 
   1117 // Test that corrupt databases are appropriately handled, even if the
   1118 // corruption is detected in the midst of the update.
   1119 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved.
   1120 // http://crbug.com/56448
   1121 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) {
   1122   // Re-create the database in a captive message loop so that we can
   1123   // influence task-posting.  Database specifically needs to the
   1124   // file-backed.
   1125   database_.reset();
   1126   base::MessageLoop loop(base::MessageLoop::TYPE_DEFAULT);
   1127   SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile();
   1128   database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL, NULL, NULL,
   1129                                               NULL, NULL));
   1130   database_->Init(database_filename_);
   1131 
   1132   // This will cause an empty database to be created.
   1133   std::vector<SBListChunkRanges> lists;
   1134   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1135   database_->UpdateFinished(true);
   1136 
   1137   // Create a sub chunk to insert.
   1138   SBChunkList chunks;
   1139   SBChunk chunk;
   1140   SBChunkHost host;
   1141   host.host = Sha256Prefix("www.subbed.com/");
   1142   host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1);
   1143   host.entry->set_chunk_id(7);
   1144   host.entry->SetChunkIdAtPrefix(0, 19);
   1145   host.entry->SetPrefixAt(0, Sha256Prefix("www.subbed.com/notevil1.html"));
   1146   chunk.chunk_number = 7;
   1147   chunk.is_add = false;
   1148   chunk.hosts.clear();
   1149   chunk.hosts.push_back(host);
   1150   chunks.clear();
   1151   chunks.push_back(chunk);
   1152 
   1153   // Corrupt the file by corrupting the checksum, which is not checked
   1154   // until the entire table is read in |UpdateFinished()|.
   1155   FILE* fp = base::OpenFile(database_filename_, "r+");
   1156   ASSERT_TRUE(fp);
   1157   ASSERT_NE(-1, fseek(fp, -8, SEEK_END));
   1158   for (size_t i = 0; i < 8; ++i) {
   1159     fputc('!', fp);
   1160   }
   1161   fclose(fp);
   1162 
   1163   {
   1164     // The following code will cause DCHECKs, so suppress the crashes.
   1165     ScopedLogMessageIgnorer ignorer;
   1166 
   1167     // Start an update.  The insert will fail due to corruption.
   1168     EXPECT_TRUE(database_->UpdateStarted(&lists));
   1169     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1170     database_->UpdateFinished(true);
   1171 
   1172     // Database file still exists until the corruption handler has run.
   1173     EXPECT_TRUE(base::PathExists(database_filename_));
   1174 
   1175     // Flush through the corruption-handler task.
   1176     VLOG(1) << "Expect failed check on: SafeBrowsing database reset";
   1177     base::MessageLoop::current()->RunUntilIdle();
   1178   }
   1179 
   1180   // Database file should not exist.
   1181   EXPECT_FALSE(base::PathExists(database_filename_));
   1182 
   1183   // Run the update again successfully.
   1184   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1185   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1186   database_->UpdateFinished(true);
   1187   EXPECT_TRUE(base::PathExists(database_filename_));
   1188 
   1189   database_.reset();
   1190 }
   1191 
   1192 // Checks database reading and writing.
   1193 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) {
   1194   database_.reset();
   1195   base::MessageLoop loop(base::MessageLoop::TYPE_DEFAULT);
   1196   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
   1197   SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile();
   1198   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
   1199   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
   1200                                               download_store,
   1201                                               csd_whitelist_store,
   1202                                               NULL,
   1203                                               NULL,
   1204                                               NULL,
   1205                                               NULL));
   1206   database_->Init(database_filename_);
   1207 
   1208   const char kEvil1Host[] = "www.evil1.com/";
   1209   const char kEvil1Url1[] = "www.evil1.com/download1/";
   1210   const char kEvil1Url2[] = "www.evil1.com/download2.html";
   1211 
   1212   SBChunkList chunks;
   1213   SBChunk chunk;
   1214   // Add a simple chunk with one hostkey for download url list.
   1215   InsertAddChunkHost2PrefixUrls(&chunk, 1, kEvil1Host,
   1216                                 kEvil1Url1, kEvil1Url2);
   1217   chunks.push_back(chunk);
   1218   std::vector<SBListChunkRanges> lists;
   1219   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1220   database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks);
   1221   database_->UpdateFinished(true);
   1222 
   1223   std::vector<SBPrefix> prefix_hits;
   1224   std::vector<GURL> urls(1);
   1225 
   1226   urls[0] = GURL(std::string("http://") + kEvil1Url1);
   1227   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1228   ASSERT_EQ(prefix_hits.size(), 1U);
   1229   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
   1230 
   1231   urls[0] = GURL(std::string("http://") + kEvil1Url2);
   1232   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1233   ASSERT_EQ(prefix_hits.size(), 1U);
   1234   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
   1235 
   1236   urls[0] = GURL(std::string("https://") + kEvil1Url2);
   1237   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1238   ASSERT_EQ(prefix_hits.size(), 1U);
   1239   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
   1240 
   1241   urls[0] = GURL(std::string("ftp://") + kEvil1Url2);
   1242   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1243   ASSERT_EQ(prefix_hits.size(), 1U);
   1244   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
   1245 
   1246   urls[0] = GURL("http://www.randomevil.com");
   1247   EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1248 
   1249   // Should match with query args stripped.
   1250   urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah");
   1251   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1252   ASSERT_EQ(prefix_hits.size(), 1U);
   1253   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2));
   1254 
   1255   // Should match with extra path stuff and query args stripped.
   1256   urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah");
   1257   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1258   ASSERT_EQ(prefix_hits.size(), 1U);
   1259   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
   1260 
   1261   // First hit in redirect chain is malware.
   1262   urls.clear();
   1263   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
   1264   urls.push_back(GURL("http://www.randomevil.com"));
   1265   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1266   ASSERT_EQ(prefix_hits.size(), 1U);
   1267   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
   1268 
   1269   // Middle hit in redirect chain is malware.
   1270   urls.clear();
   1271   urls.push_back(GURL("http://www.randomevil.com"));
   1272   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
   1273   urls.push_back(GURL("http://www.randomevil2.com"));
   1274   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1275   ASSERT_EQ(prefix_hits.size(), 1U);
   1276   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
   1277 
   1278   // Final hit in redirect chain is malware.
   1279   urls.clear();
   1280   urls.push_back(GURL("http://www.randomevil.com"));
   1281   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
   1282   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1283   ASSERT_EQ(prefix_hits.size(), 1U);
   1284   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
   1285 
   1286   // Multiple hits in redirect chain are in malware list.
   1287   urls.clear();
   1288   urls.push_back(GURL(std::string("http://") + kEvil1Url1));
   1289   urls.push_back(GURL(std::string("https://") + kEvil1Url2));
   1290   EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits));
   1291   ASSERT_EQ(prefix_hits.size(), 2U);
   1292   EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1));
   1293   EXPECT_EQ(prefix_hits[1], Sha256Prefix(kEvil1Url2));
   1294   database_.reset();
   1295 }
   1296 
   1297 // Checks that the whitelists are handled properly.
   1298 TEST_F(SafeBrowsingDatabaseTest, Whitelists) {
   1299   database_.reset();
   1300   // We expect all calls to ContainsCsdWhitelistedUrl in particular to be made
   1301   // from the IO thread.  In general the whitelist lookups are thread-safe.
   1302   content::TestBrowserThreadBundle thread_bundle_;
   1303 
   1304   // If the whitelist is disabled everything should match the whitelist.
   1305   database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(),
   1306                                               NULL, NULL, NULL, NULL, NULL,
   1307                                               NULL));
   1308   database_->Init(database_filename_);
   1309   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1310       GURL(std::string("http://www.phishing.com/"))));
   1311   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1312       GURL(std::string("http://www.phishing.com/"))));
   1313   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
   1314 
   1315   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
   1316   SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile();
   1317   SafeBrowsingStoreFile* download_whitelist_store = new SafeBrowsingStoreFile();
   1318   SafeBrowsingStoreFile* extension_blacklist_store =
   1319       new SafeBrowsingStoreFile();
   1320   database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL,
   1321                                               csd_whitelist_store,
   1322                                               download_whitelist_store,
   1323                                               extension_blacklist_store,
   1324                                               NULL, NULL));
   1325   database_->Init(database_filename_);
   1326 
   1327   const char kGood1Host[] = "www.good1.com/";
   1328   const char kGood1Url1[] = "www.good1.com/a/b.html";
   1329   const char kGood1Url2[] = "www.good1.com/b/";
   1330 
   1331   const char kGood2Host[] = "www.good2.com/";
   1332   const char kGood2Url1[] = "www.good2.com/c";  // Should match '/c/bla'.
   1333 
   1334   // good3.com/a/b/c/d/e/f/g/ should match because it's a whitelist.
   1335   const char kGood3Host[] = "good3.com/";
   1336   const char kGood3Url1[] = "good3.com/";
   1337 
   1338   const char kGoodString[] = "good_string";
   1339 
   1340   SBChunkList download_chunks, csd_chunks;
   1341   SBChunk chunk;
   1342   // Add two simple chunks to the csd whitelist.
   1343   InsertAddChunkHost2FullHashes(&chunk, 1, kGood1Host,
   1344                                 kGood1Url1, kGood1Url2);
   1345   csd_chunks.push_back(chunk);
   1346 
   1347   chunk.hosts.clear();
   1348   InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
   1349   csd_chunks.push_back(chunk);
   1350 
   1351   chunk.hosts.clear();
   1352   InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1);
   1353   download_chunks.push_back(chunk);
   1354 
   1355   chunk.hosts.clear();
   1356   InsertAddChunkHostFullHashes(&chunk, 3, kGoodString, kGoodString);
   1357   download_chunks.push_back(chunk);
   1358 
   1359   chunk.hosts.clear();
   1360   InsertAddChunkHostFullHashes(&chunk, 4, kGood3Host, kGood3Url1);
   1361   download_chunks.push_back(chunk);
   1362 
   1363   std::vector<SBListChunkRanges> lists;
   1364   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1365   database_->InsertChunks(safe_browsing_util::kCsdWhiteList,
   1366                           csd_chunks);
   1367   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
   1368                           download_chunks);
   1369   database_->UpdateFinished(true);
   1370 
   1371   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
   1372       GURL(std::string("http://") + kGood1Host)));
   1373 
   1374   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1375       GURL(std::string("http://") + kGood1Url1)));
   1376   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1377       GURL(std::string("http://") + kGood1Url1 + "?a=b")));
   1378 
   1379   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1380       GURL(std::string("http://") + kGood1Url2)));
   1381   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1382       GURL(std::string("http://") + kGood1Url2 + "/c.html")));
   1383 
   1384   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1385       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
   1386 
   1387   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1388       GURL(std::string("http://") + kGood2Url1 + "/c")));
   1389   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1390       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
   1391   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1392       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
   1393 
   1394   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
   1395       GURL(std::string("http://www.google.com/"))));
   1396 
   1397   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1398       GURL(std::string("http://") + kGood2Url1 + "/c")));
   1399   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1400       GURL(std::string("http://") + kGood2Url1 + "/c?bla")));
   1401   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1402       GURL(std::string("http://") + kGood2Url1 + "/c/bla")));
   1403 
   1404   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1405       GURL(std::string("http://good3.com/a/b/c/d/e/f/g/"))));
   1406   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1407       GURL(std::string("http://a.b.good3.com/"))));
   1408 
   1409   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
   1410   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
   1411 
   1412   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
   1413       GURL(std::string("http://www.google.com/"))));
   1414 
   1415   // Test only add the malware IP killswitch
   1416   csd_chunks.clear();
   1417   chunk.hosts.clear();
   1418   InsertAddChunkHostFullHashes(
   1419       &chunk, 15, "sb-ssl.google.com/",
   1420       "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
   1421   csd_chunks.push_back(chunk);
   1422   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1423   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
   1424   database_->UpdateFinished(true);
   1425 
   1426   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
   1427 
   1428   // Test that the kill-switch works as intended.
   1429   csd_chunks.clear();
   1430   download_chunks.clear();
   1431   lists.clear();
   1432   chunk.hosts.clear();
   1433   InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
   1434                                "sb-ssl.google.com/safebrowsing/csd/killswitch");
   1435   csd_chunks.push_back(chunk);
   1436   chunk.hosts.clear();
   1437   InsertAddChunkHostFullHashes(&chunk, 5, "sb-ssl.google.com/",
   1438                                "sb-ssl.google.com/safebrowsing/csd/killswitch");
   1439   download_chunks.push_back(chunk);
   1440 
   1441   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1442   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
   1443   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
   1444                           download_chunks);
   1445   database_->UpdateFinished(true);
   1446 
   1447   EXPECT_TRUE(database_->IsMalwareIPMatchKillSwitchOn());
   1448   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1449       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
   1450   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1451       GURL(std::string("http://www.google.com/"))));
   1452   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1453       GURL(std::string("http://www.phishing_url.com/"))));
   1454 
   1455   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1456       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
   1457   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1458       GURL(std::string("http://www.google.com/"))));
   1459   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1460       GURL(std::string("http://www.phishing_url.com/"))));
   1461 
   1462   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString("asdf"));
   1463   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
   1464 
   1465   // Remove the kill-switch and verify that we can recover.
   1466   csd_chunks.clear();
   1467   download_chunks.clear();
   1468   lists.clear();
   1469   SBChunk sub_chunk;
   1470   InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
   1471                              "sb-ssl.google.com/",
   1472                              "sb-ssl.google.com/safebrowsing/csd/killswitch");
   1473   csd_chunks.push_back(sub_chunk);
   1474 
   1475   sub_chunk.hosts.clear();
   1476   InsertSubChunkHostFullHash(
   1477       &sub_chunk, 10, 15, "sb-ssl.google.com/",
   1478       "sb-ssl.google.com/safebrowsing/csd/killswitch_malware");
   1479   csd_chunks.push_back(sub_chunk);
   1480 
   1481   sub_chunk.hosts.clear();
   1482   InsertSubChunkHostFullHash(&sub_chunk, 1, 5,
   1483                              "sb-ssl.google.com/",
   1484                              "sb-ssl.google.com/safebrowsing/csd/killswitch");
   1485   download_chunks.push_back(sub_chunk);
   1486 
   1487   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1488   database_->InsertChunks(safe_browsing_util::kCsdWhiteList, csd_chunks);
   1489   database_->InsertChunks(safe_browsing_util::kDownloadWhiteList,
   1490                           download_chunks);
   1491   database_->UpdateFinished(true);
   1492 
   1493   EXPECT_FALSE(database_->IsMalwareIPMatchKillSwitchOn());
   1494   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1495       GURL(std::string("https://") + kGood1Url2 + "/c.html")));
   1496   EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl(
   1497       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
   1498   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
   1499       GURL(std::string("http://www.google.com/"))));
   1500   EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl(
   1501       GURL(std::string("http://www.phishing_url.com/"))));
   1502 
   1503   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1504       GURL(std::string("https://") + kGood2Url1 + "/c/bla")));
   1505   EXPECT_TRUE(database_->ContainsDownloadWhitelistedUrl(
   1506       GURL(std::string("https://good3.com/"))));
   1507   EXPECT_TRUE(database_->ContainsDownloadWhitelistedString(kGoodString));
   1508   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
   1509       GURL(std::string("http://www.google.com/"))));
   1510   EXPECT_FALSE(database_->ContainsDownloadWhitelistedUrl(
   1511       GURL(std::string("http://www.phishing_url.com/"))));
   1512   EXPECT_FALSE(database_->ContainsDownloadWhitelistedString("asdf"));
   1513 
   1514   database_.reset();
   1515 }
   1516 
   1517 // Test to make sure we could insert chunk list that
   1518 // contains entries for the same host.
   1519 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) {
   1520   SBChunk chunk;
   1521 
   1522   // Add a malware add chunk with two entries of the same host.
   1523   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
   1524                               "www.evil.com/malware1.html");
   1525   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
   1526                               "www.evil.com/malware2.html");
   1527   SBChunkList chunks;
   1528   chunks.push_back(chunk);
   1529 
   1530   // Insert the testing chunks into database.
   1531   std::vector<SBListChunkRanges> lists;
   1532   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1533   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1534   database_->UpdateFinished(true);
   1535 
   1536   GetListsInfo(&lists);
   1537   EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
   1538   EXPECT_EQ("1", lists[0].adds);
   1539   EXPECT_TRUE(lists[0].subs.empty());
   1540 
   1541   // Add a phishing add chunk with two entries of the same host.
   1542   chunk.hosts.clear();
   1543   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
   1544                               "www.evil.com/phishing1.html");
   1545   InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/",
   1546                               "www.evil.com/phishing2.html");
   1547   chunks.clear();
   1548   chunks.push_back(chunk);
   1549 
   1550   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1551   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
   1552   database_->UpdateFinished(true);
   1553 
   1554   GetListsInfo(&lists);
   1555   EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name);
   1556   EXPECT_EQ("1", lists[0].adds);
   1557   EXPECT_EQ(std::string(safe_browsing_util::kPhishingList), lists[1].name);
   1558   EXPECT_EQ("47", lists[1].adds);
   1559 
   1560   const Time now = Time::Now();
   1561   std::vector<SBPrefix> prefixes;
   1562   std::vector<SBFullHashResult> full_hashes;
   1563   std::vector<SBPrefix> prefix_hits;
   1564   std::string matching_list;
   1565   std::string listname;
   1566 
   1567   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1568       GURL("http://www.evil.com/malware1.html"),
   1569       &listname, &prefixes, &full_hashes, now));
   1570   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1571       GURL("http://www.evil.com/malware2.html"),
   1572       &listname, &prefixes, &full_hashes, now));
   1573   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1574       GURL("http://www.evil.com/phishing1.html"),
   1575       &listname, &prefixes, &full_hashes, now));
   1576   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1577       GURL("http://www.evil.com/phishing2.html"),
   1578       &listname, &prefixes, &full_hashes, now));
   1579 
   1580   // Test removing a single prefix from the add chunk.
   1581   // Remove the prefix that added first.
   1582   chunk.hosts.clear();
   1583   InsertSubChunkHostPrefixUrl(&chunk, 4, 1, "www.evil.com/",
   1584                               "www.evil.com/malware1.html");
   1585   chunks.clear();
   1586   chunks.push_back(chunk);
   1587   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1588   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1589   database_->UpdateFinished(true);
   1590 
   1591   // Remove the prefix that added last.
   1592   chunk.hosts.clear();
   1593   InsertSubChunkHostPrefixUrl(&chunk, 5, 47, "www.evil.com/",
   1594                               "www.evil.com/phishing2.html");
   1595   chunks.clear();
   1596   chunks.push_back(chunk);
   1597   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1598   database_->InsertChunks(safe_browsing_util::kPhishingList, chunks);
   1599   database_->UpdateFinished(true);
   1600 
   1601   // Verify that the database contains urls expected.
   1602   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1603       GURL("http://www.evil.com/malware1.html"),
   1604       &listname, &prefixes, &full_hashes, now));
   1605   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1606       GURL("http://www.evil.com/malware2.html"),
   1607       &listname, &prefixes, &full_hashes, now));
   1608   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1609       GURL("http://www.evil.com/phishing1.html"),
   1610       &listname, &prefixes, &full_hashes, now));
   1611   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1612       GURL("http://www.evil.com/phishing2.html"),
   1613       &listname, &prefixes, &full_hashes, now));
   1614 }
   1615 
   1616 // Test that an empty update doesn't actually update the database.
   1617 // This isn't a functionality requirement, but it is a useful
   1618 // optimization.
   1619 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) {
   1620   SBChunkList chunks;
   1621   SBChunk chunk;
   1622 
   1623   base::FilePath filename = database_->BrowseDBFilename(database_filename_);
   1624 
   1625   // Prime the database.
   1626   std::vector<SBListChunkRanges> lists;
   1627   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1628 
   1629   InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
   1630                               "www.evil.com/malware.html");
   1631   chunks.clear();
   1632   chunks.push_back(chunk);
   1633   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1634   database_->UpdateFinished(true);
   1635 
   1636   // Get an older time to reset the lastmod time for detecting whether
   1637   // the file has been updated.
   1638   base::PlatformFileInfo before_info, after_info;
   1639   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
   1640   const base::Time old_last_modified =
   1641       before_info.last_modified - base::TimeDelta::FromSeconds(10);
   1642 
   1643   // Inserting another chunk updates the database file.  The sleep is
   1644   // needed because otherwise the entire test can finish w/in the
   1645   // resolution of the lastmod time.
   1646   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
   1647   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
   1648   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1649   chunk.hosts.clear();
   1650   InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/",
   1651                               "www.foo.com/malware.html");
   1652   chunks.clear();
   1653   chunks.push_back(chunk);
   1654   database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1655   database_->UpdateFinished(true);
   1656   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
   1657   EXPECT_LT(before_info.last_modified, after_info.last_modified);
   1658 
   1659   // Deleting a chunk updates the database file.
   1660   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
   1661   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
   1662   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1663   AddDelChunk(safe_browsing_util::kMalwareList, chunk.chunk_number);
   1664   database_->UpdateFinished(true);
   1665   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
   1666   EXPECT_LT(before_info.last_modified, after_info.last_modified);
   1667 
   1668   // Simply calling |UpdateStarted()| then |UpdateFinished()| does not
   1669   // update the database file.
   1670   ASSERT_TRUE(base::TouchFile(filename, old_last_modified, old_last_modified));
   1671   ASSERT_TRUE(base::GetFileInfo(filename, &before_info));
   1672   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1673   database_->UpdateFinished(true);
   1674   ASSERT_TRUE(base::GetFileInfo(filename, &after_info));
   1675   EXPECT_EQ(before_info.last_modified, after_info.last_modified);
   1676 }
   1677 
   1678 // Test that a filter file is written out during update and read back
   1679 // in during setup.
   1680 TEST_F(SafeBrowsingDatabaseTest, FilterFile) {
   1681   // Create a database with trivial example data and write it out.
   1682   {
   1683     SBChunkList chunks;
   1684     SBChunk chunk;
   1685 
   1686     // Prime the database.
   1687     std::vector<SBListChunkRanges> lists;
   1688     EXPECT_TRUE(database_->UpdateStarted(&lists));
   1689 
   1690     InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/",
   1691                                 "www.evil.com/malware.html");
   1692     chunks.clear();
   1693     chunks.push_back(chunk);
   1694     database_->InsertChunks(safe_browsing_util::kMalwareList, chunks);
   1695     database_->UpdateFinished(true);
   1696   }
   1697 
   1698   // Find the malware url in the database, don't find a good url.
   1699   const Time now = Time::Now();
   1700   std::vector<SBFullHashResult> full_hashes;
   1701   std::vector<SBPrefix> prefix_hits;
   1702   std::string matching_list;
   1703   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1704       GURL("http://www.evil.com/malware.html"),
   1705       &matching_list, &prefix_hits, &full_hashes, now));
   1706   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1707       GURL("http://www.good.com/goodware.html"),
   1708       &matching_list, &prefix_hits, &full_hashes, now));
   1709 
   1710   base::FilePath filter_file = database_->PrefixSetForFilename(
   1711       database_->BrowseDBFilename(database_filename_));
   1712 
   1713   // After re-creating the database, it should have a filter read from
   1714   // a file, so it should find the same results.
   1715   ASSERT_TRUE(base::PathExists(filter_file));
   1716   database_.reset(new SafeBrowsingDatabaseNew);
   1717   database_->Init(database_filename_);
   1718   EXPECT_TRUE(database_->ContainsBrowseUrl(
   1719       GURL("http://www.evil.com/malware.html"),
   1720       &matching_list, &prefix_hits, &full_hashes, now));
   1721   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1722       GURL("http://www.good.com/goodware.html"),
   1723       &matching_list, &prefix_hits, &full_hashes, now));
   1724 
   1725   // If there is no filter file, the database cannot find malware urls.
   1726   base::DeleteFile(filter_file, false);
   1727   ASSERT_FALSE(base::PathExists(filter_file));
   1728   database_.reset(new SafeBrowsingDatabaseNew);
   1729   database_->Init(database_filename_);
   1730   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1731       GURL("http://www.evil.com/malware.html"),
   1732       &matching_list, &prefix_hits, &full_hashes, now));
   1733   EXPECT_FALSE(database_->ContainsBrowseUrl(
   1734       GURL("http://www.good.com/goodware.html"),
   1735       &matching_list, &prefix_hits, &full_hashes, now));
   1736 }
   1737 
   1738 TEST_F(SafeBrowsingDatabaseTest, MalwareIpBlacklist) {
   1739   database_.reset();
   1740   SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile();
   1741   SafeBrowsingStoreFile* ip_blacklist_store = new SafeBrowsingStoreFile();
   1742   database_.reset(new SafeBrowsingDatabaseNew(browse_store,
   1743                                               NULL,
   1744                                               NULL,
   1745                                               NULL,
   1746                                               NULL,
   1747                                               NULL,
   1748                                               ip_blacklist_store));
   1749   database_->Init(database_filename_);
   1750   std::vector<SBListChunkRanges> lists;
   1751   EXPECT_TRUE(database_->UpdateStarted(&lists));
   1752 
   1753   // IPv4 prefix match for ::ffff:192.168.1.0/120.
   1754   SBChunkList chunks;
   1755   SBChunk chunk;
   1756   InsertAddChunkFullHash(&chunk, 1, "::ffff:192.168.1.0", 120);
   1757   chunks.push_back(chunk);
   1758   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
   1759 
   1760   // IPv4 exact match for ::ffff:192.1.1.1.
   1761   chunks.clear();
   1762   chunk.hosts.clear();
   1763   InsertAddChunkFullHash(&chunk, 2, "::ffff:192.1.1.1", 128);
   1764   chunks.push_back(chunk);
   1765   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
   1766 
   1767   // IPv6 exact match for: fe80::31a:a0ff:fe10:786e/128.
   1768   chunks.clear();
   1769   chunk.hosts.clear();
   1770   InsertAddChunkFullHash(&chunk, 3, "fe80::31a:a0ff:fe10:786e", 128);
   1771   chunks.push_back(chunk);
   1772   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
   1773 
   1774   // IPv6 prefix match for: 2620:0:1000:3103::/64.
   1775   chunks.clear();
   1776   chunk.hosts.clear();
   1777   InsertAddChunkFullHash(&chunk, 4, "2620:0:1000:3103::", 64);
   1778   chunks.push_back(chunk);
   1779   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
   1780 
   1781   // IPv4 prefix match for ::ffff:192.1.122.0/119.
   1782   chunks.clear();
   1783   chunk.hosts.clear();
   1784   InsertAddChunkFullHash(&chunk, 5, "::ffff:192.1.122.0", 119);
   1785   chunks.push_back(chunk);
   1786   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
   1787 
   1788   // IPv4 prefix match for ::ffff:192.1.128.0/113.
   1789   chunks.clear();
   1790   chunk.hosts.clear();
   1791   InsertAddChunkFullHash(&chunk, 6, "::ffff:192.1.128.0", 113);
   1792   chunks.push_back(chunk);
   1793   database_->InsertChunks(safe_browsing_util::kIPBlacklist, chunks);
   1794 
   1795   database_->UpdateFinished(true);
   1796 
   1797   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.0.255"));
   1798   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.0"));
   1799   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.255"));
   1800   EXPECT_TRUE(database_->ContainsMalwareIP("192.168.1.10"));
   1801   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.168.1.2"));
   1802   EXPECT_FALSE(database_->ContainsMalwareIP("192.168.2.0"));
   1803 
   1804   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.0"));
   1805   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.1.1"));
   1806   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.1.2"));
   1807 
   1808   EXPECT_FALSE(database_->ContainsMalwareIP(
   1809       "2620:0:1000:3102:ffff:ffff:ffff:ffff"));
   1810   EXPECT_TRUE(database_->ContainsMalwareIP("2620:0:1000:3103::"));
   1811   EXPECT_TRUE(database_->ContainsMalwareIP(
   1812       "2620:0:1000:3103:ffff:ffff:ffff:ffff"));
   1813   EXPECT_FALSE(database_->ContainsMalwareIP("2620:0:1000:3104::"));
   1814 
   1815   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786d"));
   1816   EXPECT_TRUE(database_->ContainsMalwareIP("fe80::31a:a0ff:fe10:786e"));
   1817   EXPECT_FALSE(database_->ContainsMalwareIP("fe80::21a:a0ff:fe10:786f"));
   1818 
   1819   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.121.255"));
   1820   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.0"));
   1821   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.122.1"));
   1822   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.122.255"));
   1823   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.0"));
   1824   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.123.255"));
   1825   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.124.0"));
   1826 
   1827   EXPECT_FALSE(database_->ContainsMalwareIP("192.1.127.255"));
   1828   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.0"));
   1829   EXPECT_TRUE(database_->ContainsMalwareIP("::ffff:192.1.128.1"));
   1830   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.128.255"));
   1831   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.0"));
   1832   EXPECT_TRUE(database_->ContainsMalwareIP("192.1.255.255"));
   1833   EXPECT_FALSE(database_->ContainsMalwareIP("192.2.0.0"));
   1834 }
   1835