1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 // 5 // Unit tests for the SafeBrowsing storage system. 6 7 #include "app/sql/connection.h" 8 #include "app/sql/statement.h" 9 #include "base/file_util.h" 10 #include "base/logging.h" 11 #include "base/memory/scoped_temp_dir.h" 12 #include "base/message_loop.h" 13 #include "base/time.h" 14 #include "crypto/sha2.h" 15 #include "chrome/browser/safe_browsing/safe_browsing_database.h" 16 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" 17 #include "chrome/browser/safe_browsing/safe_browsing_store_unittest_helper.h" 18 #include "content/browser/browser_thread.h" 19 #include "googleurl/src/gurl.h" 20 #include "testing/gtest/include/gtest/gtest.h" 21 #include "testing/platform_test.h" 22 23 using base::Time; 24 25 namespace { 26 27 SBPrefix Sha256Prefix(const std::string& str) { 28 SBPrefix prefix; 29 crypto::SHA256HashString(str, &prefix, sizeof(prefix)); 30 return prefix; 31 } 32 33 SBFullHash Sha256Hash(const std::string& str) { 34 SBFullHash hash; 35 crypto::SHA256HashString(str, &hash, sizeof(hash)); 36 return hash; 37 } 38 39 // Same as InsertAddChunkHostPrefixUrl, but with pre-computed 40 // prefix values. 41 void InsertAddChunkHostPrefixValue(SBChunk* chunk, 42 int chunk_number, 43 const SBPrefix& host_prefix, 44 const SBPrefix& url_prefix) { 45 chunk->chunk_number = chunk_number; 46 chunk->is_add = true; 47 SBChunkHost host; 48 host.host = host_prefix; 49 host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 1); 50 host.entry->set_chunk_id(chunk->chunk_number); 51 host.entry->SetPrefixAt(0, url_prefix); 52 chunk->hosts.push_back(host); 53 } 54 55 // A helper function that appends one AddChunkHost to chunk with 56 // one url for prefix. 57 void InsertAddChunkHostPrefixUrl(SBChunk* chunk, 58 int chunk_number, 59 const std::string& host_name, 60 const std::string& url) { 61 InsertAddChunkHostPrefixValue(chunk, chunk_number, 62 Sha256Prefix(host_name), 63 Sha256Prefix(url)); 64 } 65 66 // Same as InsertAddChunkHostPrefixUrl, but with full hashes. 67 void InsertAddChunkHostFullHashes(SBChunk* chunk, 68 int chunk_number, 69 const std::string& host_name, 70 const std::string& url) { 71 chunk->chunk_number = chunk_number; 72 chunk->is_add = true; 73 SBChunkHost host; 74 host.host = Sha256Prefix(host_name); 75 host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 1); 76 host.entry->set_chunk_id(chunk->chunk_number); 77 host.entry->SetFullHashAt(0, Sha256Hash(url)); 78 chunk->hosts.push_back(host); 79 } 80 81 // Same as InsertAddChunkHostPrefixUrl, but with two urls for prefixes. 82 void InsertAddChunkHost2PrefixUrls(SBChunk* chunk, 83 int chunk_number, 84 const std::string& host_name, 85 const std::string& url1, 86 const std::string& url2) { 87 chunk->chunk_number = chunk_number; 88 chunk->is_add = true; 89 SBChunkHost host; 90 host.host = Sha256Prefix(host_name); 91 host.entry = SBEntry::Create(SBEntry::ADD_PREFIX, 2); 92 host.entry->set_chunk_id(chunk->chunk_number); 93 host.entry->SetPrefixAt(0, Sha256Prefix(url1)); 94 host.entry->SetPrefixAt(1, Sha256Prefix(url2)); 95 chunk->hosts.push_back(host); 96 } 97 98 // Same as InsertAddChunkHost2PrefixUrls, but with full hashes. 99 void InsertAddChunkHost2FullHashes(SBChunk* chunk, 100 int chunk_number, 101 const std::string& host_name, 102 const std::string& url1, 103 const std::string& url2) { 104 chunk->chunk_number = chunk_number; 105 chunk->is_add = true; 106 SBChunkHost host; 107 host.host = Sha256Prefix(host_name); 108 host.entry = SBEntry::Create(SBEntry::ADD_FULL_HASH, 2); 109 host.entry->set_chunk_id(chunk->chunk_number); 110 host.entry->SetFullHashAt(0, Sha256Hash(url1)); 111 host.entry->SetFullHashAt(1, Sha256Hash(url2)); 112 chunk->hosts.push_back(host); 113 } 114 115 // Same as InsertSubChunkHostPrefixUrl, but with pre-computed 116 // prefix values. 117 void InsertSubChunkHostPrefixValue(SBChunk* chunk, 118 int chunk_number, 119 int chunk_id_to_sub, 120 const SBPrefix& host_prefix, 121 const SBPrefix& url_prefix) { 122 chunk->chunk_number = chunk_number; 123 chunk->is_add = false; 124 SBChunkHost host; 125 host.host = host_prefix; 126 host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1); 127 host.entry->set_chunk_id(chunk->chunk_number); 128 host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub); 129 host.entry->SetPrefixAt(0, url_prefix); 130 chunk->hosts.push_back(host); 131 } 132 133 // A helper function that adds one SubChunkHost to chunk with 134 // one url for prefix. 135 void InsertSubChunkHostPrefixUrl(SBChunk* chunk, 136 int chunk_number, 137 int chunk_id_to_sub, 138 const std::string& host_name, 139 const std::string& url) { 140 InsertSubChunkHostPrefixValue(chunk, chunk_number, 141 chunk_id_to_sub, 142 Sha256Prefix(host_name), 143 Sha256Prefix(url)); 144 } 145 146 // Same as InsertSubChunkHostPrefixUrl, but with two urls for prefixes. 147 void InsertSubChunkHost2PrefixUrls(SBChunk* chunk, 148 int chunk_number, 149 int chunk_id_to_sub, 150 const std::string& host_name, 151 const std::string& url1, 152 const std::string& url2) { 153 chunk->chunk_number = chunk_number; 154 chunk->is_add = false; 155 SBChunkHost host; 156 host.host = Sha256Prefix(host_name); 157 host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 2); 158 host.entry->set_chunk_id(chunk->chunk_number); 159 host.entry->SetPrefixAt(0, Sha256Prefix(url1)); 160 host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub); 161 host.entry->SetPrefixAt(1, Sha256Prefix(url2)); 162 host.entry->SetChunkIdAtPrefix(1, chunk_id_to_sub); 163 chunk->hosts.push_back(host); 164 } 165 166 // Same as InsertSubChunkHost2PrefixUrls, but with full hashes. 167 void InsertSubChunkHost2FullHashes(SBChunk* chunk, 168 int chunk_number, 169 int chunk_id_to_sub, 170 const std::string& host_name, 171 const std::string& url1, 172 const std::string& url2) { 173 chunk->chunk_number = chunk_number; 174 chunk->is_add = false; 175 SBChunkHost host; 176 host.host = Sha256Prefix(host_name); 177 host.entry = SBEntry::Create(SBEntry::SUB_FULL_HASH, 2); 178 host.entry->set_chunk_id(chunk->chunk_number); 179 host.entry->SetFullHashAt(0, Sha256Hash(url1)); 180 host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub); 181 host.entry->SetFullHashAt(1, Sha256Hash(url2)); 182 host.entry->SetChunkIdAtPrefix(1, chunk_id_to_sub); 183 chunk->hosts.push_back(host); 184 } 185 186 // Same as InsertSubChunkHost2PrefixUrls, but with full hashes. 187 void InsertSubChunkHostFullHash(SBChunk* chunk, 188 int chunk_number, 189 int chunk_id_to_sub, 190 const std::string& host_name, 191 const std::string& url) { 192 chunk->chunk_number = chunk_number; 193 chunk->is_add = false; 194 SBChunkHost host; 195 host.host = Sha256Prefix(host_name); 196 host.entry = SBEntry::Create(SBEntry::SUB_FULL_HASH, 2); 197 host.entry->set_chunk_id(chunk->chunk_number); 198 host.entry->SetFullHashAt(0, Sha256Hash(url)); 199 host.entry->SetChunkIdAtPrefix(0, chunk_id_to_sub); 200 chunk->hosts.push_back(host); 201 } 202 203 // Prevent DCHECK from killing tests. 204 // TODO(shess): Pawel disputes the use of this, so the test which uses 205 // it is DISABLED. http://crbug.com/56448 206 class ScopedLogMessageIgnorer { 207 public: 208 ScopedLogMessageIgnorer() { 209 logging::SetLogMessageHandler(&LogMessageIgnorer); 210 } 211 ~ScopedLogMessageIgnorer() { 212 // TODO(shess): Would be better to verify whether anyone else 213 // changed it, and then restore it to the previous value. 214 logging::SetLogMessageHandler(NULL); 215 } 216 217 private: 218 static bool LogMessageIgnorer(int severity, const char* file, int line, 219 size_t message_start, const std::string& str) { 220 // Intercept FATAL, strip the stack backtrace, and log it without 221 // the crash part. 222 if (severity == logging::LOG_FATAL) { 223 size_t newline = str.find('\n'); 224 if (newline != std::string::npos) { 225 const std::string msg = str.substr(0, newline + 1); 226 fprintf(stderr, "%s", msg.c_str()); 227 fflush(stderr); 228 } 229 return true; 230 } 231 232 return false; 233 } 234 }; 235 236 } // namespace 237 238 class SafeBrowsingDatabaseTest : public PlatformTest { 239 public: 240 virtual void SetUp() { 241 PlatformTest::SetUp(); 242 243 // Setup a database in a temporary directory. 244 ASSERT_TRUE(temp_dir_.CreateUniqueTempDir()); 245 database_.reset(new SafeBrowsingDatabaseNew); 246 database_filename_ = 247 temp_dir_.path().AppendASCII("SafeBrowsingTestDatabase"); 248 database_->Init(database_filename_); 249 } 250 251 virtual void TearDown() { 252 database_.reset(); 253 254 PlatformTest::TearDown(); 255 } 256 257 void GetListsInfo(std::vector<SBListChunkRanges>* lists) { 258 lists->clear(); 259 EXPECT_TRUE(database_->UpdateStarted(lists)); 260 database_->UpdateFinished(true); 261 } 262 263 // Helper function to do an AddDel or SubDel command. 264 void DelChunk(const std::string& list, 265 int chunk_id, 266 bool is_sub_del) { 267 std::vector<SBChunkDelete> deletes; 268 SBChunkDelete chunk_delete; 269 chunk_delete.list_name = list; 270 chunk_delete.is_sub_del = is_sub_del; 271 chunk_delete.chunk_del.push_back(ChunkRange(chunk_id)); 272 deletes.push_back(chunk_delete); 273 database_->DeleteChunks(deletes); 274 } 275 276 void AddDelChunk(const std::string& list, int chunk_id) { 277 DelChunk(list, chunk_id, false); 278 } 279 280 void SubDelChunk(const std::string& list, int chunk_id) { 281 DelChunk(list, chunk_id, true); 282 } 283 284 // Utility function for setting up the database for the caching test. 285 void PopulateDatabaseForCacheTest(); 286 287 scoped_ptr<SafeBrowsingDatabaseNew> database_; 288 FilePath database_filename_; 289 ScopedTempDir temp_dir_; 290 }; 291 292 // Tests retrieving list name information. 293 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowse) { 294 SBChunkList chunks; 295 SBChunk chunk; 296 297 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/", 298 "www.evil.com/malware.html"); 299 chunks.clear(); 300 chunks.push_back(chunk); 301 std::vector<SBListChunkRanges> lists; 302 EXPECT_TRUE(database_->UpdateStarted(&lists)); 303 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 304 305 chunk.hosts.clear(); 306 InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/", 307 "www.foo.com/malware.html"); 308 chunks.clear(); 309 chunks.push_back(chunk); 310 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 311 312 chunk.hosts.clear(); 313 InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/", 314 "www.whatever.com/malware.html"); 315 chunks.clear(); 316 chunks.push_back(chunk); 317 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 318 database_->UpdateFinished(true); 319 320 GetListsInfo(&lists); 321 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 322 EXPECT_EQ(lists[0].adds, "1-3"); 323 EXPECT_TRUE(lists[0].subs.empty()); 324 325 // Insert a malware sub chunk. 326 chunk.hosts.clear(); 327 InsertSubChunkHostPrefixUrl(&chunk, 7, 19, "www.subbed.com/", 328 "www.subbed.com/noteveil1.html"); 329 chunks.clear(); 330 chunks.push_back(chunk); 331 332 EXPECT_TRUE(database_->UpdateStarted(&lists)); 333 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 334 database_->UpdateFinished(true); 335 336 GetListsInfo(&lists); 337 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 338 EXPECT_EQ(lists[0].adds, "1-3"); 339 EXPECT_EQ(lists[0].subs, "7"); 340 if (lists.size() == 2) { 341 // Old style database won't have the second entry since it creates the lists 342 // when it receives an update containing that list. The new bloom filter 343 // based database has these values hard coded. 344 EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList); 345 EXPECT_TRUE(lists[1].adds.empty()); 346 EXPECT_TRUE(lists[1].subs.empty()); 347 } 348 349 // Add a phishing add chunk. 350 chunk.hosts.clear(); 351 InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/", 352 "www.evil.com/phishing.html"); 353 chunks.clear(); 354 chunks.push_back(chunk); 355 EXPECT_TRUE(database_->UpdateStarted(&lists)); 356 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks); 357 358 // Insert some phishing sub chunks. 359 chunk.hosts.clear(); 360 InsertSubChunkHostPrefixUrl(&chunk, 200, 1999, "www.phishy.com/", 361 "www.phishy.com/notevil1.html"); 362 chunks.clear(); 363 chunks.push_back(chunk); 364 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks); 365 366 chunk.hosts.clear(); 367 InsertSubChunkHostPrefixUrl(&chunk, 201, 1999, "www.phishy2.com/", 368 "www.phishy2.com/notevil1.html"); 369 chunks.clear(); 370 chunks.push_back(chunk); 371 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks); 372 database_->UpdateFinished(true); 373 374 GetListsInfo(&lists); 375 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 376 EXPECT_EQ(lists[0].adds, "1-3"); 377 EXPECT_EQ(lists[0].subs, "7"); 378 EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList); 379 EXPECT_EQ(lists[1].adds, "47"); 380 EXPECT_EQ(lists[1].subs, "200-201"); 381 } 382 383 TEST_F(SafeBrowsingDatabaseTest, ListNameForBrowseAndDownload) { 384 database_.reset(); 385 MessageLoop loop(MessageLoop::TYPE_DEFAULT); 386 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile(); 387 SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile(); 388 SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile(); 389 database_.reset(new SafeBrowsingDatabaseNew(browse_store, 390 download_store, 391 csd_whitelist_store)); 392 database_->Init(database_filename_); 393 394 SBChunkList chunks; 395 SBChunk chunk; 396 397 // Insert malware, phish, binurl and bindownload add chunks. 398 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/", 399 "www.evil.com/malware.html"); 400 chunks.push_back(chunk); 401 std::vector<SBListChunkRanges> lists; 402 EXPECT_TRUE(database_->UpdateStarted(&lists)); 403 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 404 405 chunk.hosts.clear(); 406 InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/", 407 "www.foo.com/malware.html"); 408 chunks.clear(); 409 chunks.push_back(chunk); 410 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks); 411 412 chunk.hosts.clear(); 413 InsertAddChunkHostPrefixUrl(&chunk, 3, "www.whatever.com/", 414 "www.whatever.com/download.html"); 415 chunks.clear(); 416 chunks.push_back(chunk); 417 database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks); 418 419 chunk.hosts.clear(); 420 InsertAddChunkHostPrefixUrl(&chunk, 4, "www.forhash.com/", 421 "www.forhash.com/download.html"); 422 chunks.clear(); 423 chunks.push_back(chunk); 424 database_->InsertChunks(safe_browsing_util::kBinHashList, chunks); 425 426 chunk.hosts.clear(); 427 InsertAddChunkHostFullHashes(&chunk, 5, "www.forwhitelist.com/", 428 "www.forwhitelist.com/a.html"); 429 chunks.clear(); 430 chunks.push_back(chunk); 431 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); 432 433 database_->UpdateFinished(true); 434 435 GetListsInfo(&lists); 436 EXPECT_EQ(5U, lists.size()); 437 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 438 EXPECT_EQ(lists[0].adds, "1"); 439 EXPECT_TRUE(lists[0].subs.empty()); 440 EXPECT_TRUE(lists[1].name == safe_browsing_util::kPhishingList); 441 EXPECT_EQ(lists[1].adds, "2"); 442 EXPECT_TRUE(lists[1].subs.empty()); 443 EXPECT_TRUE(lists[2].name == safe_browsing_util::kBinUrlList); 444 EXPECT_EQ(lists[2].adds, "3"); 445 EXPECT_TRUE(lists[2].subs.empty()); 446 EXPECT_TRUE(lists[3].name == safe_browsing_util::kBinHashList); 447 EXPECT_EQ(lists[3].adds, "4"); 448 EXPECT_TRUE(lists[3].subs.empty()); 449 EXPECT_TRUE(lists[4].name == safe_browsing_util::kCsdWhiteList); 450 EXPECT_EQ(lists[4].adds, "5"); 451 EXPECT_TRUE(lists[4].subs.empty()); 452 database_.reset(); 453 } 454 455 // Checks database reading and writing for browse. 456 TEST_F(SafeBrowsingDatabaseTest, BrowseDatabase) { 457 SBChunkList chunks; 458 SBChunk chunk; 459 460 // Add a simple chunk with one hostkey. 461 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/", 462 "www.evil.com/phishing.html", 463 "www.evil.com/malware.html"); 464 chunks.push_back(chunk); 465 std::vector<SBListChunkRanges> lists; 466 EXPECT_TRUE(database_->UpdateStarted(&lists)); 467 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 468 469 chunk.hosts.clear(); 470 InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.evil.com/", 471 "www.evil.com/notevil1.html", 472 "www.evil.com/notevil2.html"); 473 InsertAddChunkHost2PrefixUrls(&chunk, 2, "www.good.com/", 474 "www.good.com/good1.html", 475 "www.good.com/good2.html"); 476 chunks.clear(); 477 chunks.push_back(chunk); 478 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 479 480 // and a chunk with an IP-based host 481 chunk.hosts.clear(); 482 InsertAddChunkHostPrefixUrl(&chunk, 3, "192.168.0.1/", 483 "192.168.0.1/malware.html"); 484 chunks.clear(); 485 chunks.push_back(chunk); 486 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 487 database_->UpdateFinished(true); 488 489 // Make sure they were added correctly. 490 GetListsInfo(&lists); 491 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 492 EXPECT_EQ(lists[0].adds, "1-3"); 493 EXPECT_TRUE(lists[0].subs.empty()); 494 495 const Time now = Time::Now(); 496 std::vector<SBFullHashResult> full_hashes; 497 std::vector<SBPrefix> prefix_hits; 498 std::string matching_list; 499 EXPECT_TRUE(database_->ContainsBrowseUrl( 500 GURL("http://www.evil.com/phishing.html"), 501 &matching_list, &prefix_hits, 502 &full_hashes, now)); 503 EXPECT_EQ(prefix_hits[0], Sha256Prefix("www.evil.com/phishing.html")); 504 EXPECT_EQ(prefix_hits.size(), 1U); 505 506 EXPECT_TRUE(database_->ContainsBrowseUrl( 507 GURL("http://www.evil.com/malware.html"), 508 &matching_list, &prefix_hits, 509 &full_hashes, now)); 510 511 EXPECT_TRUE(database_->ContainsBrowseUrl( 512 GURL("http://www.evil.com/notevil1.html"), 513 &matching_list, &prefix_hits, 514 &full_hashes, now)); 515 516 EXPECT_TRUE(database_->ContainsBrowseUrl( 517 GURL("http://www.evil.com/notevil2.html"), 518 &matching_list, &prefix_hits, 519 &full_hashes, now)); 520 521 EXPECT_TRUE(database_->ContainsBrowseUrl( 522 GURL("http://www.good.com/good1.html"), 523 &matching_list, &prefix_hits, 524 &full_hashes, now)); 525 526 EXPECT_TRUE(database_->ContainsBrowseUrl( 527 GURL("http://www.good.com/good2.html"), 528 &matching_list, &prefix_hits, 529 &full_hashes, now)); 530 531 EXPECT_TRUE(database_->ContainsBrowseUrl( 532 GURL("http://192.168.0.1/malware.html"), 533 &matching_list, &prefix_hits, 534 &full_hashes, now)); 535 536 EXPECT_FALSE(database_->ContainsBrowseUrl( 537 GURL("http://www.evil.com/"), 538 &matching_list, &prefix_hits, 539 &full_hashes, now)); 540 EXPECT_TRUE(prefix_hits.empty()); 541 542 EXPECT_FALSE(database_->ContainsBrowseUrl( 543 GURL("http://www.evil.com/robots.txt"), 544 &matching_list, &prefix_hits, 545 &full_hashes, now)); 546 547 // Attempt to re-add the first chunk (should be a no-op). 548 // see bug: http://code.google.com/p/chromium/issues/detail?id=4522 549 chunk.hosts.clear(); 550 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/", 551 "www.evil.com/phishing.html", 552 "www.evil.com/malware.html"); 553 chunks.clear(); 554 chunks.push_back(chunk); 555 EXPECT_TRUE(database_->UpdateStarted(&lists)); 556 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 557 database_->UpdateFinished(true); 558 559 GetListsInfo(&lists); 560 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 561 EXPECT_EQ(lists[0].adds, "1-3"); 562 EXPECT_TRUE(lists[0].subs.empty()); 563 564 // Test removing a single prefix from the add chunk. 565 chunk.hosts.clear(); 566 InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/", 567 "www.evil.com/notevil1.html"); 568 chunks.clear(); 569 chunks.push_back(chunk); 570 EXPECT_TRUE(database_->UpdateStarted(&lists)); 571 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 572 573 database_->UpdateFinished(true); 574 575 EXPECT_TRUE(database_->ContainsBrowseUrl( 576 GURL("http://www.evil.com/phishing.html"), 577 &matching_list, &prefix_hits, 578 &full_hashes, now)); 579 EXPECT_EQ(prefix_hits[0], Sha256Prefix("www.evil.com/phishing.html")); 580 EXPECT_EQ(prefix_hits.size(), 1U); 581 582 EXPECT_FALSE(database_->ContainsBrowseUrl( 583 GURL("http://www.evil.com/notevil1.html"), 584 &matching_list, &prefix_hits, 585 &full_hashes, now)); 586 EXPECT_TRUE(prefix_hits.empty()); 587 588 EXPECT_TRUE(database_->ContainsBrowseUrl( 589 GURL("http://www.evil.com/notevil2.html"), 590 &matching_list, &prefix_hits, 591 &full_hashes, now)); 592 593 EXPECT_TRUE(database_->ContainsBrowseUrl( 594 GURL("http://www.good.com/good1.html"), 595 &matching_list, &prefix_hits, 596 &full_hashes, now)); 597 598 EXPECT_TRUE(database_->ContainsBrowseUrl( 599 GURL("http://www.good.com/good2.html"), 600 &matching_list, &prefix_hits, 601 &full_hashes, now)); 602 603 GetListsInfo(&lists); 604 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 605 EXPECT_EQ(lists[0].subs, "4"); 606 607 // Test the same sub chunk again. This should be a no-op. 608 // see bug: http://code.google.com/p/chromium/issues/detail?id=4522 609 chunk.hosts.clear(); 610 InsertSubChunkHostPrefixUrl(&chunk, 4, 2, "www.evil.com/", 611 "www.evil.com/notevil1.html"); 612 chunks.clear(); 613 chunks.push_back(chunk); 614 615 EXPECT_TRUE(database_->UpdateStarted(&lists)); 616 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 617 database_->UpdateFinished(true); 618 619 GetListsInfo(&lists); 620 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 621 EXPECT_EQ(lists[0].subs, "4"); 622 623 // Test removing all the prefixes from an add chunk. 624 EXPECT_TRUE(database_->UpdateStarted(&lists)); 625 AddDelChunk(safe_browsing_util::kMalwareList, 2); 626 database_->UpdateFinished(true); 627 628 EXPECT_FALSE(database_->ContainsBrowseUrl( 629 GURL("http://www.evil.com/notevil2.html"), 630 &matching_list, &prefix_hits, 631 &full_hashes, now)); 632 633 EXPECT_FALSE(database_->ContainsBrowseUrl( 634 GURL("http://www.good.com/good1.html"), 635 &matching_list, &prefix_hits, 636 &full_hashes, now)); 637 638 EXPECT_FALSE(database_->ContainsBrowseUrl( 639 GURL("http://www.good.com/good2.html"), 640 &matching_list, &prefix_hits, 641 &full_hashes, now)); 642 643 GetListsInfo(&lists); 644 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 645 EXPECT_EQ(lists[0].adds, "1,3"); 646 EXPECT_EQ(lists[0].subs, "4"); 647 648 // The adddel command exposed a bug in the transaction code where any 649 // transaction after it would fail. Add a dummy entry and remove it to 650 // make sure the transcation works fine. 651 chunk.hosts.clear(); 652 InsertAddChunkHostPrefixUrl(&chunk, 44, "www.redherring.com/", 653 "www.redherring.com/index.html"); 654 chunks.clear(); 655 chunks.push_back(chunk); 656 EXPECT_TRUE(database_->UpdateStarted(&lists)); 657 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 658 659 // Now remove the dummy entry. If there are any problems with the 660 // transactions, asserts will fire. 661 AddDelChunk(safe_browsing_util::kMalwareList, 44); 662 663 // Test the subdel command. 664 SubDelChunk(safe_browsing_util::kMalwareList, 4); 665 database_->UpdateFinished(true); 666 667 GetListsInfo(&lists); 668 EXPECT_TRUE(lists[0].name == safe_browsing_util::kMalwareList); 669 EXPECT_EQ(lists[0].adds, "1,3"); 670 EXPECT_EQ(lists[0].subs, ""); 671 672 // Test a sub command coming in before the add. 673 chunk.hosts.clear(); 674 InsertSubChunkHost2PrefixUrls(&chunk, 5, 10, 675 "www.notevilanymore.com/", 676 "www.notevilanymore.com/index.html", 677 "www.notevilanymore.com/good.html"); 678 chunks.clear(); 679 chunks.push_back(chunk); 680 EXPECT_TRUE(database_->UpdateStarted(&lists)); 681 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 682 database_->UpdateFinished(true); 683 684 EXPECT_FALSE(database_->ContainsBrowseUrl( 685 GURL("http://www.notevilanymore.com/index.html"), 686 &matching_list, &prefix_hits, &full_hashes, now)); 687 688 // Now insert the tardy add chunk and we don't expect them to appear 689 // in database because of the previous sub chunk. 690 chunk.hosts.clear(); 691 InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.notevilanymore.com/", 692 "www.notevilanymore.com/index.html", 693 "www.notevilanymore.com/good.html"); 694 chunks.clear(); 695 chunks.push_back(chunk); 696 EXPECT_TRUE(database_->UpdateStarted(&lists)); 697 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 698 database_->UpdateFinished(true); 699 700 EXPECT_FALSE(database_->ContainsBrowseUrl( 701 GURL("http://www.notevilanymore.com/index.html"), 702 &matching_list, &prefix_hits, &full_hashes, now)); 703 704 EXPECT_FALSE(database_->ContainsBrowseUrl( 705 GURL("http://www.notevilanymore.com/good.html"), 706 &matching_list, &prefix_hits, &full_hashes, now)); 707 } 708 709 710 // Test adding zero length chunks to the database. 711 TEST_F(SafeBrowsingDatabaseTest, ZeroSizeChunk) { 712 SBChunkList chunks; 713 SBChunk chunk; 714 715 // Populate with a couple of normal chunks. 716 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.test.com/", 717 "www.test.com/test1.html", 718 "www.test.com/test2.html"); 719 chunks.clear(); 720 chunks.push_back(chunk); 721 722 chunk.hosts.clear(); 723 InsertAddChunkHost2PrefixUrls(&chunk, 10, "www.random.com/", 724 "www.random.com/random1.html", 725 "www.random.com/random2.html"); 726 chunks.push_back(chunk); 727 728 std::vector<SBListChunkRanges> lists; 729 EXPECT_TRUE(database_->UpdateStarted(&lists)); 730 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 731 database_->UpdateFinished(true); 732 733 // Add an empty ADD and SUB chunk. 734 GetListsInfo(&lists); 735 EXPECT_EQ(lists[0].adds, "1,10"); 736 737 SBChunk empty_chunk; 738 empty_chunk.chunk_number = 19; 739 empty_chunk.is_add = true; 740 chunks.clear(); 741 chunks.push_back(empty_chunk); 742 EXPECT_TRUE(database_->UpdateStarted(&lists)); 743 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 744 chunks.clear(); 745 empty_chunk.chunk_number = 7; 746 empty_chunk.is_add = false; 747 chunks.push_back(empty_chunk); 748 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 749 database_->UpdateFinished(true); 750 751 GetListsInfo(&lists); 752 EXPECT_EQ(lists[0].adds, "1,10,19"); 753 EXPECT_EQ(lists[0].subs, "7"); 754 755 // Add an empty chunk along with a couple that contain data. This should 756 // result in the chunk range being reduced in size. 757 empty_chunk.hosts.clear(); 758 InsertAddChunkHostPrefixUrl(&empty_chunk, 20, "www.notempy.com/", 759 "www.notempty.com/full1.html"); 760 chunks.clear(); 761 chunks.push_back(empty_chunk); 762 763 empty_chunk.chunk_number = 21; 764 empty_chunk.is_add = true; 765 empty_chunk.hosts.clear(); 766 chunks.push_back(empty_chunk); 767 768 empty_chunk.hosts.clear(); 769 InsertAddChunkHostPrefixUrl(&empty_chunk, 22, "www.notempy.com/", 770 "www.notempty.com/full2.html"); 771 chunks.push_back(empty_chunk); 772 773 EXPECT_TRUE(database_->UpdateStarted(&lists)); 774 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 775 database_->UpdateFinished(true); 776 777 const Time now = Time::Now(); 778 std::vector<SBFullHashResult> full_hashes; 779 std::vector<SBPrefix> prefix_hits; 780 std::string matching_list; 781 EXPECT_TRUE(database_->ContainsBrowseUrl( 782 GURL("http://www.notempty.com/full1.html"), 783 &matching_list, &prefix_hits, 784 &full_hashes, now)); 785 EXPECT_TRUE(database_->ContainsBrowseUrl( 786 GURL("http://www.notempty.com/full2.html"), 787 &matching_list, &prefix_hits, 788 &full_hashes, now)); 789 790 GetListsInfo(&lists); 791 EXPECT_EQ(lists[0].adds, "1,10,19-22"); 792 EXPECT_EQ(lists[0].subs, "7"); 793 794 // Handle AddDel and SubDel commands for empty chunks. 795 EXPECT_TRUE(database_->UpdateStarted(&lists)); 796 AddDelChunk(safe_browsing_util::kMalwareList, 21); 797 database_->UpdateFinished(true); 798 799 GetListsInfo(&lists); 800 EXPECT_EQ(lists[0].adds, "1,10,19-20,22"); 801 EXPECT_EQ(lists[0].subs, "7"); 802 803 EXPECT_TRUE(database_->UpdateStarted(&lists)); 804 SubDelChunk(safe_browsing_util::kMalwareList, 7); 805 database_->UpdateFinished(true); 806 807 GetListsInfo(&lists); 808 EXPECT_EQ(lists[0].adds, "1,10,19-20,22"); 809 EXPECT_EQ(lists[0].subs, ""); 810 } 811 812 // Utility function for setting up the database for the caching test. 813 void SafeBrowsingDatabaseTest::PopulateDatabaseForCacheTest() { 814 SBChunkList chunks; 815 SBChunk chunk; 816 // Add a simple chunk with one hostkey and cache it. 817 InsertAddChunkHost2PrefixUrls(&chunk, 1, "www.evil.com/", 818 "www.evil.com/phishing.html", 819 "www.evil.com/malware.html"); 820 chunks.push_back(chunk); 821 822 std::vector<SBListChunkRanges> lists; 823 EXPECT_TRUE(database_->UpdateStarted(&lists)); 824 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 825 database_->UpdateFinished(true); 826 827 // Add the GetHash results to the cache. 828 SBFullHashResult full_hash; 829 full_hash.hash = Sha256Hash("www.evil.com/phishing.html"); 830 full_hash.list_name = safe_browsing_util::kMalwareList; 831 full_hash.add_chunk_id = 1; 832 833 std::vector<SBFullHashResult> results; 834 results.push_back(full_hash); 835 836 full_hash.hash = Sha256Hash("www.evil.com/malware.html"); 837 results.push_back(full_hash); 838 839 std::vector<SBPrefix> prefixes; 840 database_->CacheHashResults(prefixes, results); 841 } 842 843 TEST_F(SafeBrowsingDatabaseTest, HashCaching) { 844 PopulateDatabaseForCacheTest(); 845 846 // We should have both full hashes in the cache. 847 EXPECT_EQ(database_->pending_browse_hashes_.size(), 2U); 848 849 // Test the cache lookup for the first prefix. 850 std::string listname; 851 std::vector<SBPrefix> prefixes; 852 std::vector<SBFullHashResult> full_hashes; 853 database_->ContainsBrowseUrl( 854 GURL("http://www.evil.com/phishing.html"), 855 &listname, &prefixes, &full_hashes, Time::Now()); 856 EXPECT_EQ(full_hashes.size(), 1U); 857 EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash, 858 Sha256Hash("www.evil.com/phishing.html"))); 859 860 prefixes.clear(); 861 full_hashes.clear(); 862 863 // Test the cache lookup for the second prefix. 864 database_->ContainsBrowseUrl( 865 GURL("http://www.evil.com/malware.html"), 866 &listname, &prefixes, &full_hashes, Time::Now()); 867 EXPECT_EQ(full_hashes.size(), 1U); 868 EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash, 869 Sha256Hash("www.evil.com/malware.html"))); 870 871 prefixes.clear(); 872 full_hashes.clear(); 873 874 // Test removing a prefix via a sub chunk. 875 SBChunk chunk; 876 SBChunkList chunks; 877 InsertSubChunkHostPrefixUrl(&chunk, 2, 1, "www.evil.com/", 878 "www.evil.com/phishing.html"); 879 chunks.push_back(chunk); 880 881 std::vector<SBListChunkRanges> lists; 882 EXPECT_TRUE(database_->UpdateStarted(&lists)); 883 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 884 database_->UpdateFinished(true); 885 886 // This prefix should still be there. 887 database_->ContainsBrowseUrl( 888 GURL("http://www.evil.com/malware.html"), 889 &listname, &prefixes, &full_hashes, Time::Now()); 890 EXPECT_EQ(full_hashes.size(), 1U); 891 EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash, 892 Sha256Hash("www.evil.com/malware.html"))); 893 prefixes.clear(); 894 full_hashes.clear(); 895 896 // This prefix should be gone. 897 database_->ContainsBrowseUrl( 898 GURL("http://www.evil.com/phishing.html"), 899 &listname, &prefixes, &full_hashes, Time::Now()); 900 EXPECT_TRUE(full_hashes.empty()); 901 902 prefixes.clear(); 903 full_hashes.clear(); 904 905 // Test that an AddDel for the original chunk removes the last cached entry. 906 EXPECT_TRUE(database_->UpdateStarted(&lists)); 907 AddDelChunk(safe_browsing_util::kMalwareList, 1); 908 database_->UpdateFinished(true); 909 database_->ContainsBrowseUrl( 910 GURL("http://www.evil.com/malware.html"), 911 &listname, &prefixes, &full_hashes, Time::Now()); 912 EXPECT_TRUE(full_hashes.empty()); 913 EXPECT_TRUE(database_->full_browse_hashes_.empty()); 914 EXPECT_TRUE(database_->pending_browse_hashes_.empty()); 915 916 prefixes.clear(); 917 full_hashes.clear(); 918 919 // Test that the cache won't return expired values. First we have to adjust 920 // the cached entries' received time to make them older, since the database 921 // cache insert uses Time::Now(). First, store some entries. 922 PopulateDatabaseForCacheTest(); 923 924 std::vector<SBAddFullHash>* hash_cache = &database_->pending_browse_hashes_; 925 EXPECT_EQ(hash_cache->size(), 2U); 926 927 // Now adjust one of the entries times to be in the past. 928 base::Time expired = base::Time::Now() - base::TimeDelta::FromMinutes(60); 929 const SBPrefix key = Sha256Prefix("www.evil.com/malware.html"); 930 std::vector<SBAddFullHash>::iterator iter; 931 for (iter = hash_cache->begin(); iter != hash_cache->end(); ++iter) { 932 if (iter->full_hash.prefix == key) { 933 iter->received = static_cast<int32>(expired.ToTimeT()); 934 break; 935 } 936 } 937 EXPECT_TRUE(iter != hash_cache->end()); 938 939 database_->ContainsBrowseUrl( 940 GURL("http://www.evil.com/malware.html"), 941 &listname, &prefixes, &full_hashes, expired); 942 EXPECT_TRUE(full_hashes.empty()); 943 944 // This entry should still exist. 945 database_->ContainsBrowseUrl( 946 GURL("http://www.evil.com/phishing.html"), 947 &listname, &prefixes, &full_hashes, expired); 948 EXPECT_EQ(full_hashes.size(), 1U); 949 950 951 // Testing prefix miss caching. First, we clear out the existing database, 952 // Since PopulateDatabaseForCacheTest() doesn't handle adding duplicate 953 // chunks. 954 EXPECT_TRUE(database_->UpdateStarted(&lists)); 955 AddDelChunk(safe_browsing_util::kMalwareList, 1); 956 database_->UpdateFinished(true); 957 958 std::vector<SBPrefix> prefix_misses; 959 std::vector<SBFullHashResult> empty_full_hash; 960 prefix_misses.push_back(Sha256Prefix("http://www.bad.com/malware.html")); 961 prefix_misses.push_back(Sha256Prefix("http://www.bad.com/phishing.html")); 962 database_->CacheHashResults(prefix_misses, empty_full_hash); 963 964 // Prefixes with no full results are misses. 965 EXPECT_EQ(database_->prefix_miss_cache_.size(), 2U); 966 967 // Update the database. 968 PopulateDatabaseForCacheTest(); 969 970 // Prefix miss cache should be cleared. 971 EXPECT_TRUE(database_->prefix_miss_cache_.empty()); 972 973 // Cache a GetHash miss for a particular prefix, and even though the prefix is 974 // in the database, it is flagged as a miss so looking up the associated URL 975 // will not succeed. 976 prefixes.clear(); 977 full_hashes.clear(); 978 prefix_misses.clear(); 979 empty_full_hash.clear(); 980 prefix_misses.push_back(Sha256Prefix("www.evil.com/phishing.html")); 981 database_->CacheHashResults(prefix_misses, empty_full_hash); 982 EXPECT_FALSE(database_->ContainsBrowseUrl( 983 GURL("http://www.evil.com/phishing.html"), 984 &listname, &prefixes, 985 &full_hashes, Time::Now())); 986 987 prefixes.clear(); 988 full_hashes.clear(); 989 990 // Test receiving a full add chunk. 991 chunk.hosts.clear(); 992 InsertAddChunkHost2FullHashes(&chunk, 20, "www.fullevil.com/", 993 "www.fullevil.com/bad1.html", 994 "www.fullevil.com/bad2.html"); 995 chunks.clear(); 996 chunks.push_back(chunk); 997 EXPECT_TRUE(database_->UpdateStarted(&lists)); 998 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 999 database_->UpdateFinished(true); 1000 1001 EXPECT_TRUE(database_->ContainsBrowseUrl( 1002 GURL("http://www.fullevil.com/bad1.html"), 1003 &listname, &prefixes, &full_hashes, 1004 Time::Now())); 1005 EXPECT_EQ(full_hashes.size(), 1U); 1006 EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash, 1007 Sha256Hash("www.fullevil.com/bad1.html"))); 1008 prefixes.clear(); 1009 full_hashes.clear(); 1010 1011 EXPECT_TRUE(database_->ContainsBrowseUrl( 1012 GURL("http://www.fullevil.com/bad2.html"), 1013 &listname, &prefixes, &full_hashes, 1014 Time::Now())); 1015 EXPECT_EQ(full_hashes.size(), 1U); 1016 EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash, 1017 Sha256Hash("www.fullevil.com/bad2.html"))); 1018 prefixes.clear(); 1019 full_hashes.clear(); 1020 1021 // Test receiving a full sub chunk, which will remove one of the full adds. 1022 chunk.hosts.clear(); 1023 InsertSubChunkHostFullHash(&chunk, 200, 20, 1024 "www.fullevil.com/", 1025 "www.fullevil.com/bad1.html"); 1026 chunks.clear(); 1027 chunks.push_back(chunk); 1028 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1029 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 1030 database_->UpdateFinished(true); 1031 1032 EXPECT_FALSE(database_->ContainsBrowseUrl( 1033 GURL("http://www.fullevil.com/bad1.html"), 1034 &listname, &prefixes, &full_hashes, 1035 Time::Now())); 1036 EXPECT_TRUE(full_hashes.empty()); 1037 1038 // There should be one remaining full add. 1039 EXPECT_TRUE(database_->ContainsBrowseUrl( 1040 GURL("http://www.fullevil.com/bad2.html"), 1041 &listname, &prefixes, &full_hashes, 1042 Time::Now())); 1043 EXPECT_EQ(full_hashes.size(), 1U); 1044 EXPECT_TRUE(SBFullHashEq(full_hashes[0].hash, 1045 Sha256Hash("www.fullevil.com/bad2.html"))); 1046 prefixes.clear(); 1047 full_hashes.clear(); 1048 1049 // Now test an AddDel for the remaining full add. 1050 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1051 AddDelChunk(safe_browsing_util::kMalwareList, 20); 1052 database_->UpdateFinished(true); 1053 1054 EXPECT_FALSE(database_->ContainsBrowseUrl( 1055 GURL("http://www.fullevil.com/bad1.html"), 1056 &listname, &prefixes, &full_hashes, 1057 Time::Now())); 1058 EXPECT_FALSE(database_->ContainsBrowseUrl( 1059 GURL("http://www.fullevil.com/bad2.html"), 1060 &listname, &prefixes, &full_hashes, 1061 Time::Now())); 1062 } 1063 1064 // Test that corrupt databases are appropriately handled, even if the 1065 // corruption is detected in the midst of the update. 1066 // TODO(shess): Disabled until ScopedLogMessageIgnorer resolved. 1067 // http://crbug.com/56448 1068 TEST_F(SafeBrowsingDatabaseTest, DISABLED_FileCorruptionHandling) { 1069 // Re-create the database in a captive message loop so that we can 1070 // influence task-posting. Database specifically needs to the 1071 // file-backed. 1072 database_.reset(); 1073 MessageLoop loop(MessageLoop::TYPE_DEFAULT); 1074 SafeBrowsingStoreFile* store = new SafeBrowsingStoreFile(); 1075 database_.reset(new SafeBrowsingDatabaseNew(store, NULL, NULL)); 1076 database_->Init(database_filename_); 1077 1078 // This will cause an empty database to be created. 1079 std::vector<SBListChunkRanges> lists; 1080 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1081 database_->UpdateFinished(true); 1082 1083 // Create a sub chunk to insert. 1084 SBChunkList chunks; 1085 SBChunk chunk; 1086 SBChunkHost host; 1087 host.host = Sha256Prefix("www.subbed.com/"); 1088 host.entry = SBEntry::Create(SBEntry::SUB_PREFIX, 1); 1089 host.entry->set_chunk_id(7); 1090 host.entry->SetChunkIdAtPrefix(0, 19); 1091 host.entry->SetPrefixAt(0, Sha256Prefix("www.subbed.com/notevil1.html")); 1092 chunk.chunk_number = 7; 1093 chunk.is_add = false; 1094 chunk.hosts.clear(); 1095 chunk.hosts.push_back(host); 1096 chunks.clear(); 1097 chunks.push_back(chunk); 1098 1099 // Corrupt the file by corrupting the checksum, which is not checked 1100 // until the entire table is read in |UpdateFinished()|. 1101 FILE* fp = file_util::OpenFile(database_filename_, "r+"); 1102 ASSERT_TRUE(fp); 1103 ASSERT_NE(-1, fseek(fp, -8, SEEK_END)); 1104 for (size_t i = 0; i < 8; ++i) { 1105 fputc('!', fp); 1106 } 1107 fclose(fp); 1108 1109 { 1110 // The following code will cause DCHECKs, so suppress the crashes. 1111 ScopedLogMessageIgnorer ignorer; 1112 1113 // Start an update. The insert will fail due to corruption. 1114 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1115 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 1116 database_->UpdateFinished(true); 1117 1118 // Database file still exists until the corruption handler has run. 1119 EXPECT_TRUE(file_util::PathExists(database_filename_)); 1120 1121 // Flush through the corruption-handler task. 1122 VLOG(1) << "Expect failed check on: SafeBrowsing database reset"; 1123 MessageLoop::current()->RunAllPending(); 1124 } 1125 1126 // Database file should not exist. 1127 EXPECT_FALSE(file_util::PathExists(database_filename_)); 1128 1129 // Run the update again successfully. 1130 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1131 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 1132 database_->UpdateFinished(true); 1133 EXPECT_TRUE(file_util::PathExists(database_filename_)); 1134 1135 database_.reset(); 1136 } 1137 1138 // Checks database reading and writing. 1139 TEST_F(SafeBrowsingDatabaseTest, ContainsDownloadUrl) { 1140 database_.reset(); 1141 MessageLoop loop(MessageLoop::TYPE_DEFAULT); 1142 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile(); 1143 SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile(); 1144 SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile(); 1145 database_.reset(new SafeBrowsingDatabaseNew(browse_store, 1146 download_store, 1147 csd_whitelist_store)); 1148 database_->Init(database_filename_); 1149 1150 const char kEvil1Host[] = "www.evil1.com/"; 1151 const char kEvil1Url1[] = "www.evil1.com/download1/"; 1152 const char kEvil1Url2[] = "www.evil1.com/download2.html"; 1153 1154 SBChunkList chunks; 1155 SBChunk chunk; 1156 // Add a simple chunk with one hostkey for download url list. 1157 InsertAddChunkHost2PrefixUrls(&chunk, 1, kEvil1Host, 1158 kEvil1Url1, kEvil1Url2); 1159 chunks.push_back(chunk); 1160 std::vector<SBListChunkRanges> lists; 1161 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1162 database_->InsertChunks(safe_browsing_util::kBinUrlList, chunks); 1163 database_->UpdateFinished(true); 1164 1165 std::vector<SBPrefix> prefix_hits; 1166 std::vector<GURL> urls(1); 1167 1168 urls[0] = GURL(std::string("http://") + kEvil1Url1); 1169 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1170 ASSERT_EQ(prefix_hits.size(), 1U); 1171 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1)); 1172 1173 urls[0] = GURL(std::string("http://") + kEvil1Url2); 1174 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1175 ASSERT_EQ(prefix_hits.size(), 1U); 1176 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2)); 1177 1178 urls[0] = GURL(std::string("https://") + kEvil1Url2); 1179 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1180 ASSERT_EQ(prefix_hits.size(), 1U); 1181 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2)); 1182 1183 urls[0] = GURL(std::string("ftp://") + kEvil1Url2); 1184 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1185 ASSERT_EQ(prefix_hits.size(), 1U); 1186 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2)); 1187 1188 urls[0] = GURL("http://www.randomevil.com"); 1189 EXPECT_FALSE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1190 1191 // Should match with query args stripped. 1192 urls[0] = GURL(std::string("http://") + kEvil1Url2 + "?blah"); 1193 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1194 ASSERT_EQ(prefix_hits.size(), 1U); 1195 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url2)); 1196 1197 // Should match with extra path stuff and query args stripped. 1198 urls[0] = GURL(std::string("http://") + kEvil1Url1 + "foo/bar?blah"); 1199 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1200 ASSERT_EQ(prefix_hits.size(), 1U); 1201 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1)); 1202 1203 // First hit in redirect chain is malware. 1204 urls.clear(); 1205 urls.push_back(GURL(std::string("http://") + kEvil1Url1)); 1206 urls.push_back(GURL("http://www.randomevil.com")); 1207 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1208 ASSERT_EQ(prefix_hits.size(), 1U); 1209 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1)); 1210 1211 // Middle hit in redirect chain is malware. 1212 urls.clear(); 1213 urls.push_back(GURL("http://www.randomevil.com")); 1214 urls.push_back(GURL(std::string("http://") + kEvil1Url1)); 1215 urls.push_back(GURL("http://www.randomevil2.com")); 1216 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1217 ASSERT_EQ(prefix_hits.size(), 1U); 1218 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1)); 1219 1220 // Final hit in redirect chain is malware. 1221 urls.clear(); 1222 urls.push_back(GURL("http://www.randomevil.com")); 1223 urls.push_back(GURL(std::string("http://") + kEvil1Url1)); 1224 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1225 ASSERT_EQ(prefix_hits.size(), 1U); 1226 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1)); 1227 1228 // Multiple hits in redirect chain are in malware list. 1229 urls.clear(); 1230 urls.push_back(GURL(std::string("http://") + kEvil1Url1)); 1231 urls.push_back(GURL(std::string("https://") + kEvil1Url2)); 1232 EXPECT_TRUE(database_->ContainsDownloadUrl(urls, &prefix_hits)); 1233 ASSERT_EQ(prefix_hits.size(), 2U); 1234 EXPECT_EQ(prefix_hits[0], Sha256Prefix(kEvil1Url1)); 1235 EXPECT_EQ(prefix_hits[1], Sha256Prefix(kEvil1Url2)); 1236 database_.reset(); 1237 } 1238 1239 // Checks that the csd-whitelist is handled properly. 1240 TEST_F(SafeBrowsingDatabaseTest, CsdWhitelist) { 1241 database_.reset(); 1242 MessageLoop loop(MessageLoop::TYPE_DEFAULT); 1243 // We expect all calls to ContainsCsdWhitelistedUrl to be made from the IO 1244 // thread. 1245 BrowserThread io_thread(BrowserThread::IO, &loop); 1246 1247 // If the whitelist is disabled everything should match the whitelist. 1248 database_.reset(new SafeBrowsingDatabaseNew(new SafeBrowsingStoreFile(), 1249 NULL, NULL)); 1250 database_->Init(database_filename_); 1251 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1252 GURL(std::string("http://www.phishig.com/")))); 1253 1254 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile(); 1255 SafeBrowsingStoreFile* csd_whitelist_store = new SafeBrowsingStoreFile(); 1256 database_.reset(new SafeBrowsingDatabaseNew(browse_store, NULL, 1257 csd_whitelist_store)); 1258 database_->Init(database_filename_); 1259 1260 const char kGood1Host[] = "www.good1.com/"; 1261 const char kGood1Url1[] = "www.good1.com/a/b.html"; 1262 const char kGood1Url2[] = "www.good1.com/b/"; 1263 1264 const char kGood2Host[] = "www.good2.com/"; 1265 const char kGood2Url1[] = "www.good2.com/c"; // Should match '/c/bla'. 1266 1267 SBChunkList chunks; 1268 SBChunk chunk; 1269 // Add two simple chunks to the csd whitelist. 1270 InsertAddChunkHost2FullHashes(&chunk, 1, kGood1Host, 1271 kGood1Url1, kGood1Url2); 1272 chunks.push_back(chunk); 1273 1274 chunk.hosts.clear(); 1275 InsertAddChunkHostFullHashes(&chunk, 2, kGood2Host, kGood2Url1); 1276 chunks.push_back(chunk); 1277 1278 std::vector<SBListChunkRanges> lists; 1279 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1280 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); 1281 database_->UpdateFinished(true); 1282 1283 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl( 1284 GURL(std::string("http://") + kGood1Host))); 1285 1286 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1287 GURL(std::string("http://") + kGood1Url1))); 1288 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1289 GURL(std::string("http://") + kGood1Url1 + "?a=b"))); 1290 1291 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1292 GURL(std::string("http://") + kGood1Url2))); 1293 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1294 GURL(std::string("http://") + kGood1Url2 + "/c.html"))); 1295 1296 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1297 GURL(std::string("https://") + kGood1Url2 + "/c.html"))); 1298 1299 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1300 GURL(std::string("http://") + kGood2Url1 + "/c"))); 1301 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1302 GURL(std::string("http://") + kGood2Url1 + "/c?bla"))); 1303 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1304 GURL(std::string("http://") + kGood2Url1 + "/c/bla"))); 1305 1306 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl( 1307 GURL(std::string("http://www.google.com/")))); 1308 1309 // Test that the kill-switch works as intended. 1310 chunks.clear(); 1311 lists.clear(); 1312 SBChunk chunk2; 1313 InsertAddChunkHostFullHashes(&chunk2, 3, "sb-ssl.google.com/", 1314 "sb-ssl.google.com/safebrowsing/csd/killswitch"); 1315 chunks.push_back(chunk2); 1316 1317 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1318 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); 1319 database_->UpdateFinished(true); 1320 1321 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1322 GURL(std::string("https://") + kGood1Url2 + "/c.html"))); 1323 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1324 GURL(std::string("http://www.google.com/")))); 1325 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1326 GURL(std::string("http://www.phishing_url.com/")))); 1327 1328 // Remove the kill-switch and verify that we can recover. 1329 chunks.clear(); 1330 lists.clear(); 1331 SBChunk sub_chunk; 1332 InsertSubChunkHostFullHash(&sub_chunk, 1, 3, 1333 "sb-ssl.google.com/", 1334 "sb-ssl.google.com/safebrowsing/csd/killswitch"); 1335 chunks.push_back(sub_chunk); 1336 1337 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1338 database_->InsertChunks(safe_browsing_util::kCsdWhiteList, chunks); 1339 database_->UpdateFinished(true); 1340 1341 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1342 GURL(std::string("https://") + kGood1Url2 + "/c.html"))); 1343 EXPECT_TRUE(database_->ContainsCsdWhitelistedUrl( 1344 GURL(std::string("https://") + kGood2Url1 + "/c/bla"))); 1345 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl( 1346 GURL(std::string("http://www.google.com/")))); 1347 EXPECT_FALSE(database_->ContainsCsdWhitelistedUrl( 1348 GURL(std::string("http://www.phishing_url.com/")))); 1349 1350 database_.reset(); 1351 } 1352 1353 // Test to make sure we could insert chunk list that 1354 // contains entries for the same host. 1355 TEST_F(SafeBrowsingDatabaseTest, SameHostEntriesOkay) { 1356 SBChunk chunk; 1357 1358 // Add a malware add chunk with two entries of the same host. 1359 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/", 1360 "www.evil.com/malware1.html"); 1361 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/", 1362 "www.evil.com/malware2.html"); 1363 SBChunkList chunks; 1364 chunks.push_back(chunk); 1365 1366 // Insert the testing chunks into database. 1367 std::vector<SBListChunkRanges> lists; 1368 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1369 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 1370 database_->UpdateFinished(true); 1371 1372 GetListsInfo(&lists); 1373 EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name); 1374 EXPECT_EQ("1", lists[0].adds); 1375 EXPECT_TRUE(lists[0].subs.empty()); 1376 1377 // Add a phishing add chunk with two entries of the same host. 1378 chunk.hosts.clear(); 1379 InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/", 1380 "www.evil.com/phishing1.html"); 1381 InsertAddChunkHostPrefixUrl(&chunk, 47, "www.evil.com/", 1382 "www.evil.com/phishing2.html"); 1383 chunks.clear(); 1384 chunks.push_back(chunk); 1385 1386 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1387 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks); 1388 database_->UpdateFinished(true); 1389 1390 GetListsInfo(&lists); 1391 EXPECT_EQ(std::string(safe_browsing_util::kMalwareList), lists[0].name); 1392 EXPECT_EQ("1", lists[0].adds); 1393 EXPECT_EQ(std::string(safe_browsing_util::kPhishingList), lists[1].name); 1394 EXPECT_EQ("47", lists[1].adds); 1395 1396 const Time now = Time::Now(); 1397 std::vector<SBPrefix> prefixes; 1398 std::vector<SBFullHashResult> full_hashes; 1399 std::vector<SBPrefix> prefix_hits; 1400 std::string matching_list; 1401 std::string listname; 1402 1403 EXPECT_TRUE(database_->ContainsBrowseUrl( 1404 GURL("http://www.evil.com/malware1.html"), 1405 &listname, &prefixes, &full_hashes, now)); 1406 EXPECT_TRUE(database_->ContainsBrowseUrl( 1407 GURL("http://www.evil.com/malware2.html"), 1408 &listname, &prefixes, &full_hashes, now)); 1409 EXPECT_TRUE(database_->ContainsBrowseUrl( 1410 GURL("http://www.evil.com/phishing1.html"), 1411 &listname, &prefixes, &full_hashes, now)); 1412 EXPECT_TRUE(database_->ContainsBrowseUrl( 1413 GURL("http://www.evil.com/phishing2.html"), 1414 &listname, &prefixes, &full_hashes, now)); 1415 1416 // Test removing a single prefix from the add chunk. 1417 // Remove the prefix that added first. 1418 chunk.hosts.clear(); 1419 InsertSubChunkHostPrefixUrl(&chunk, 4, 1, "www.evil.com/", 1420 "www.evil.com/malware1.html"); 1421 chunks.clear(); 1422 chunks.push_back(chunk); 1423 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1424 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 1425 database_->UpdateFinished(true); 1426 1427 // Remove the prefix that added last. 1428 chunk.hosts.clear(); 1429 InsertSubChunkHostPrefixUrl(&chunk, 5, 47, "www.evil.com/", 1430 "www.evil.com/phishing2.html"); 1431 chunks.clear(); 1432 chunks.push_back(chunk); 1433 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1434 database_->InsertChunks(safe_browsing_util::kPhishingList, chunks); 1435 database_->UpdateFinished(true); 1436 1437 // Verify that the database contains urls expected. 1438 EXPECT_FALSE(database_->ContainsBrowseUrl( 1439 GURL("http://www.evil.com/malware1.html"), 1440 &listname, &prefixes, &full_hashes, now)); 1441 EXPECT_TRUE(database_->ContainsBrowseUrl( 1442 GURL("http://www.evil.com/malware2.html"), 1443 &listname, &prefixes, &full_hashes, now)); 1444 EXPECT_TRUE(database_->ContainsBrowseUrl( 1445 GURL("http://www.evil.com/phishing1.html"), 1446 &listname, &prefixes, &full_hashes, now)); 1447 EXPECT_FALSE(database_->ContainsBrowseUrl( 1448 GURL("http://www.evil.com/phishing2.html"), 1449 &listname, &prefixes, &full_hashes, now)); 1450 } 1451 1452 TEST_F(SafeBrowsingDatabaseTest, BinHashInsertLookup) { 1453 const SBPrefix kPrefix1 = 0x31313131; 1454 const SBPrefix kPrefix2 = 0x32323232; 1455 const SBPrefix kPrefix3 = 0x33333333; 1456 database_.reset(); 1457 MessageLoop loop(MessageLoop::TYPE_DEFAULT); 1458 SafeBrowsingStoreFile* browse_store = new SafeBrowsingStoreFile(); 1459 SafeBrowsingStoreFile* download_store = new SafeBrowsingStoreFile(); 1460 database_.reset(new SafeBrowsingDatabaseNew(browse_store, 1461 download_store, 1462 NULL)); 1463 database_->Init(database_filename_); 1464 1465 SBChunkList chunks; 1466 SBChunk chunk; 1467 // Insert one host. 1468 InsertAddChunkHostPrefixValue(&chunk, 1, 0, kPrefix1); 1469 // Insert a second host, which has the same host prefix as the first one. 1470 InsertAddChunkHostPrefixValue(&chunk, 1, 0, kPrefix2); 1471 chunks.push_back(chunk); 1472 1473 // Insert the testing chunks into database. 1474 std::vector<SBListChunkRanges> lists; 1475 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1476 database_->InsertChunks(safe_browsing_util::kBinHashList, chunks); 1477 database_->UpdateFinished(true); 1478 1479 GetListsInfo(&lists); 1480 ASSERT_EQ(4U, lists.size()); 1481 EXPECT_EQ(std::string(safe_browsing_util::kBinHashList), lists[3].name); 1482 EXPECT_EQ("1", lists[3].adds); 1483 EXPECT_TRUE(lists[3].subs.empty()); 1484 1485 EXPECT_TRUE(database_->ContainsDownloadHashPrefix(kPrefix1)); 1486 EXPECT_TRUE(database_->ContainsDownloadHashPrefix(kPrefix2)); 1487 EXPECT_FALSE(database_->ContainsDownloadHashPrefix(kPrefix3)); 1488 database_.reset(); 1489 } 1490 1491 // Test that an empty update doesn't actually update the database. 1492 // This isn't a functionality requirement, but it is a useful 1493 // optimization. 1494 TEST_F(SafeBrowsingDatabaseTest, EmptyUpdate) { 1495 SBChunkList chunks; 1496 SBChunk chunk; 1497 1498 FilePath filename = database_->BrowseDBFilename(database_filename_); 1499 1500 // Prime the database. 1501 std::vector<SBListChunkRanges> lists; 1502 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1503 1504 InsertAddChunkHostPrefixUrl(&chunk, 1, "www.evil.com/", 1505 "www.evil.com/malware.html"); 1506 chunks.clear(); 1507 chunks.push_back(chunk); 1508 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 1509 database_->UpdateFinished(true); 1510 1511 // Get an older time to reset the lastmod time for detecting whether 1512 // the file has been updated. 1513 base::PlatformFileInfo before_info, after_info; 1514 ASSERT_TRUE(file_util::GetFileInfo(filename, &before_info)); 1515 const base::Time old_last_modified = 1516 before_info.last_modified - base::TimeDelta::FromSeconds(10); 1517 1518 // Inserting another chunk updates the database file. The sleep is 1519 // needed because otherwise the entire test can finish w/in the 1520 // resolution of the lastmod time. 1521 ASSERT_TRUE(file_util::SetLastModifiedTime(filename, old_last_modified)); 1522 ASSERT_TRUE(file_util::GetFileInfo(filename, &before_info)); 1523 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1524 chunk.hosts.clear(); 1525 InsertAddChunkHostPrefixUrl(&chunk, 2, "www.foo.com/", 1526 "www.foo.com/malware.html"); 1527 chunks.clear(); 1528 chunks.push_back(chunk); 1529 database_->InsertChunks(safe_browsing_util::kMalwareList, chunks); 1530 database_->UpdateFinished(true); 1531 ASSERT_TRUE(file_util::GetFileInfo(filename, &after_info)); 1532 EXPECT_LT(before_info.last_modified, after_info.last_modified); 1533 1534 // Deleting a chunk updates the database file. 1535 ASSERT_TRUE(file_util::SetLastModifiedTime(filename, old_last_modified)); 1536 ASSERT_TRUE(file_util::GetFileInfo(filename, &before_info)); 1537 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1538 AddDelChunk(safe_browsing_util::kMalwareList, chunk.chunk_number); 1539 database_->UpdateFinished(true); 1540 ASSERT_TRUE(file_util::GetFileInfo(filename, &after_info)); 1541 EXPECT_LT(before_info.last_modified, after_info.last_modified); 1542 1543 // Simply calling |UpdateStarted()| then |UpdateFinished()| does not 1544 // update the database file. 1545 ASSERT_TRUE(file_util::SetLastModifiedTime(filename, old_last_modified)); 1546 ASSERT_TRUE(file_util::GetFileInfo(filename, &before_info)); 1547 EXPECT_TRUE(database_->UpdateStarted(&lists)); 1548 database_->UpdateFinished(true); 1549 ASSERT_TRUE(file_util::GetFileInfo(filename, &after_info)); 1550 EXPECT_EQ(before_info.last_modified, after_info.last_modified); 1551 } 1552
