1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/safe_browsing/database_manager.h" 6 7 #include <algorithm> 8 9 #include "base/bind.h" 10 #include "base/bind_helpers.h" 11 #include "base/callback.h" 12 #include "base/command_line.h" 13 #include "base/debug/leak_tracker.h" 14 #include "base/path_service.h" 15 #include "base/stl_util.h" 16 #include "base/strings/string_util.h" 17 #include "base/threading/thread.h" 18 #include "base/threading/thread_restrictions.h" 19 #include "chrome/browser/browser_process.h" 20 #include "chrome/browser/chrome_notification_types.h" 21 #include "chrome/browser/prerender/prerender_field_trial.h" 22 #include "chrome/browser/safe_browsing/client_side_detection_service.h" 23 #include "chrome/browser/safe_browsing/download_protection_service.h" 24 #include "chrome/browser/safe_browsing/malware_details.h" 25 #include "chrome/browser/safe_browsing/protocol_manager.h" 26 #include "chrome/browser/safe_browsing/safe_browsing_database.h" 27 #include "chrome/browser/safe_browsing/safe_browsing_service.h" 28 #include "chrome/browser/safe_browsing/ui_manager.h" 29 #include "chrome/common/chrome_constants.h" 30 #include "chrome/common/chrome_paths.h" 31 #include "chrome/common/chrome_switches.h" 32 #include "components/metrics/metrics_service.h" 33 #include "components/startup_metric_utils/startup_metric_utils.h" 34 #include "content/public/browser/browser_thread.h" 35 #include "content/public/browser/notification_service.h" 36 #include "url/url_constants.h" 37 38 using content::BrowserThread; 39 40 namespace { 41 42 // Timeout for match checks, e.g. download URLs, hashes. 43 const int kCheckTimeoutMs = 10000; 44 45 // Records disposition information about the check. |hit| should be 46 // |true| if there were any prefix hits in |full_hashes|. 47 void RecordGetHashCheckStatus( 48 bool hit, 49 safe_browsing_util::ListType check_type, 50 const std::vector<SBFullHashResult>& full_hashes) { 51 SafeBrowsingProtocolManager::ResultType result; 52 if (full_hashes.empty()) { 53 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY; 54 } else if (hit) { 55 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT; 56 } else { 57 result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS; 58 } 59 bool is_download = check_type == safe_browsing_util::BINURL; 60 SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result); 61 } 62 63 bool IsExpectedThreat( 64 const SBThreatType threat_type, 65 const std::vector<SBThreatType>& expected_threats) { 66 return expected_threats.end() != std::find(expected_threats.begin(), 67 expected_threats.end(), 68 threat_type); 69 } 70 71 // Return the list id from the first result in |full_hashes| which matches 72 // |hash|, or INVALID if none match. 73 safe_browsing_util::ListType GetHashThreatListType( 74 const SBFullHash& hash, 75 const std::vector<SBFullHashResult>& full_hashes) { 76 for (size_t i = 0; i < full_hashes.size(); ++i) { 77 if (SBFullHashEqual(hash, full_hashes[i].hash)) 78 return static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id); 79 } 80 return safe_browsing_util::INVALID; 81 } 82 83 // Given a URL, compare all the possible host + path full hashes to the set of 84 // provided full hashes. Returns the list id of the a matching result from 85 // |full_hashes|, or INVALID if none match. 86 safe_browsing_util::ListType GetUrlThreatListType( 87 const GURL& url, 88 const std::vector<SBFullHashResult>& full_hashes) { 89 if (full_hashes.empty()) 90 return safe_browsing_util::INVALID; 91 92 std::vector<std::string> patterns; 93 safe_browsing_util::GeneratePatternsToCheck(url, &patterns); 94 95 for (size_t i = 0; i < patterns.size(); ++i) { 96 safe_browsing_util::ListType threat = 97 GetHashThreatListType(SBFullHashForString(patterns[i]), full_hashes); 98 if (threat != safe_browsing_util::INVALID) 99 return threat; 100 } 101 return safe_browsing_util::INVALID; 102 } 103 104 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) { 105 switch (list_type) { 106 case safe_browsing_util::PHISH: 107 return SB_THREAT_TYPE_URL_PHISHING; 108 case safe_browsing_util::MALWARE: 109 return SB_THREAT_TYPE_URL_MALWARE; 110 case safe_browsing_util::BINURL: 111 return SB_THREAT_TYPE_BINARY_MALWARE_URL; 112 case safe_browsing_util::EXTENSIONBLACKLIST: 113 return SB_THREAT_TYPE_EXTENSION; 114 default: 115 DVLOG(1) << "Unknown safe browsing list id " << list_type; 116 return SB_THREAT_TYPE_SAFE; 117 } 118 } 119 120 } // namespace 121 122 // static 123 SBThreatType SafeBrowsingDatabaseManager::GetHashThreatType( 124 const SBFullHash& hash, 125 const std::vector<SBFullHashResult>& full_hashes) { 126 return GetThreatTypeFromListType(GetHashThreatListType(hash, full_hashes)); 127 } 128 129 // static 130 SBThreatType SafeBrowsingDatabaseManager::GetUrlThreatType( 131 const GURL& url, 132 const std::vector<SBFullHashResult>& full_hashes) { 133 return GetThreatTypeFromListType(GetUrlThreatListType(url, full_hashes)); 134 } 135 136 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck( 137 const std::vector<GURL>& urls, 138 const std::vector<SBFullHash>& full_hashes, 139 Client* client, 140 safe_browsing_util::ListType check_type, 141 const std::vector<SBThreatType>& expected_threats) 142 : urls(urls), 143 url_results(urls.size(), SB_THREAT_TYPE_SAFE), 144 full_hashes(full_hashes), 145 full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE), 146 client(client), 147 need_get_hash(false), 148 check_type(check_type), 149 expected_threats(expected_threats) { 150 DCHECK_EQ(urls.empty(), !full_hashes.empty()) 151 << "Exactly one of urls and full_hashes must be set"; 152 } 153 154 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {} 155 156 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult( 157 const SafeBrowsingCheck& check) { 158 DCHECK_EQ(check.urls.size(), check.url_results.size()); 159 DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size()); 160 if (!check.urls.empty()) { 161 DCHECK(check.full_hashes.empty()); 162 switch (check.check_type) { 163 case safe_browsing_util::MALWARE: 164 case safe_browsing_util::PHISH: 165 DCHECK_EQ(1u, check.urls.size()); 166 OnCheckBrowseUrlResult(check.urls[0], check.url_results[0]); 167 break; 168 case safe_browsing_util::BINURL: 169 DCHECK_EQ(check.urls.size(), check.url_results.size()); 170 OnCheckDownloadUrlResult( 171 check.urls, 172 *std::max_element(check.url_results.begin(), 173 check.url_results.end())); 174 break; 175 default: 176 NOTREACHED(); 177 } 178 } else if (!check.full_hashes.empty()) { 179 switch (check.check_type) { 180 case safe_browsing_util::EXTENSIONBLACKLIST: { 181 std::set<std::string> unsafe_extension_ids; 182 for (size_t i = 0; i < check.full_hashes.size(); ++i) { 183 std::string extension_id = 184 safe_browsing_util::SBFullHashToString(check.full_hashes[i]); 185 if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION) 186 unsafe_extension_ids.insert(extension_id); 187 } 188 OnCheckExtensionsResult(unsafe_extension_ids); 189 break; 190 } 191 default: 192 NOTREACHED(); 193 } 194 } else { 195 NOTREACHED(); 196 } 197 } 198 199 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager( 200 const scoped_refptr<SafeBrowsingService>& service) 201 : sb_service_(service), 202 database_(NULL), 203 enabled_(false), 204 enable_download_protection_(false), 205 enable_csd_whitelist_(false), 206 enable_download_whitelist_(false), 207 enable_extension_blacklist_(false), 208 enable_side_effect_free_whitelist_(false), 209 enable_ip_blacklist_(false), 210 update_in_progress_(false), 211 database_update_in_progress_(false), 212 closing_database_(false), 213 check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) { 214 DCHECK(sb_service_.get() != NULL); 215 216 CommandLine* cmdline = CommandLine::ForCurrentProcess(); 217 enable_download_protection_ = 218 !cmdline->HasSwitch(switches::kSbDisableDownloadProtection); 219 220 // We only download the csd-whitelist if client-side phishing detection is 221 // enabled. 222 enable_csd_whitelist_ = 223 !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection); 224 225 // TODO(noelutz): remove this boolean variable since it should always be true 226 // if SafeBrowsing is enabled. Unfortunately, we have no test data for this 227 // list right now. This means that we need to be able to disable this list 228 // for the SafeBrowsing test to pass. 229 enable_download_whitelist_ = enable_csd_whitelist_; 230 231 // TODO(kalman): there really shouldn't be a flag for this. 232 enable_extension_blacklist_ = 233 !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist); 234 235 enable_side_effect_free_whitelist_ = 236 prerender::IsSideEffectFreeWhitelistEnabled() && 237 !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist); 238 239 // The client-side IP blacklist feature is tightly integrated with client-side 240 // phishing protection for now. 241 enable_ip_blacklist_ = enable_csd_whitelist_; 242 243 enum SideEffectFreeWhitelistStatus { 244 SIDE_EFFECT_FREE_WHITELIST_ENABLED, 245 SIDE_EFFECT_FREE_WHITELIST_DISABLED, 246 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX 247 }; 248 249 SideEffectFreeWhitelistStatus side_effect_free_whitelist_status = 250 enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED : 251 SIDE_EFFECT_FREE_WHITELIST_DISABLED; 252 253 UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus", 254 side_effect_free_whitelist_status, 255 SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX); 256 } 257 258 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() { 259 // We should have already been shut down. If we're still enabled, then the 260 // database isn't going to be closed properly, which could lead to corruption. 261 DCHECK(!enabled_); 262 } 263 264 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const { 265 return url.SchemeIs(url::kFtpScheme) || 266 url.SchemeIs(url::kHttpScheme) || 267 url.SchemeIs(url::kHttpsScheme); 268 } 269 270 bool SafeBrowsingDatabaseManager::CheckDownloadUrl( 271 const std::vector<GURL>& url_chain, 272 Client* client) { 273 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 274 if (!enabled_ || !enable_download_protection_) 275 return true; 276 277 // We need to check the database for url prefix, and later may fetch the url 278 // from the safebrowsing backends. These need to be asynchronous. 279 SafeBrowsingCheck* check = 280 new SafeBrowsingCheck(url_chain, 281 std::vector<SBFullHash>(), 282 client, 283 safe_browsing_util::BINURL, 284 std::vector<SBThreatType>(1, 285 SB_THREAT_TYPE_BINARY_MALWARE_URL)); 286 StartSafeBrowsingCheck( 287 check, 288 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this, 289 check)); 290 return false; 291 } 292 293 bool SafeBrowsingDatabaseManager::CheckExtensionIDs( 294 const std::set<std::string>& extension_ids, 295 Client* client) { 296 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 297 298 if (!enabled_ || !enable_extension_blacklist_) 299 return true; 300 301 std::vector<SBFullHash> extension_id_hashes; 302 std::transform(extension_ids.begin(), extension_ids.end(), 303 std::back_inserter(extension_id_hashes), 304 safe_browsing_util::StringToSBFullHash); 305 306 SafeBrowsingCheck* check = new SafeBrowsingCheck( 307 std::vector<GURL>(), 308 extension_id_hashes, 309 client, 310 safe_browsing_util::EXTENSIONBLACKLIST, 311 std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION)); 312 313 StartSafeBrowsingCheck( 314 check, 315 base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread, 316 this, 317 check)); 318 return false; 319 } 320 321 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl( 322 const GURL& url) { 323 if (!enabled_) 324 return false; 325 326 if (!CanCheckUrl(url)) 327 return false; 328 329 return database_->ContainsSideEffectFreeWhitelistUrl(url); 330 } 331 332 bool SafeBrowsingDatabaseManager::MatchMalwareIP( 333 const std::string& ip_address) { 334 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 335 if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) { 336 return false; // Fail open. 337 } 338 return database_->ContainsMalwareIP(ip_address); 339 } 340 341 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) { 342 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 343 if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) { 344 // There is something funky going on here -- for example, perhaps the user 345 // has not restarted since enabling metrics reporting, so we haven't 346 // enabled the csd whitelist yet. Just to be safe we return true in this 347 // case. 348 return true; 349 } 350 return database_->ContainsCsdWhitelistedUrl(url); 351 } 352 353 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) { 354 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 355 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) { 356 return true; 357 } 358 return database_->ContainsDownloadWhitelistedUrl(url); 359 } 360 361 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString( 362 const std::string& str) { 363 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 364 if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) { 365 return true; 366 } 367 return database_->ContainsDownloadWhitelistedString(str); 368 } 369 370 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() { 371 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 372 if (!enabled_ || !MakeDatabaseAvailable()) { 373 return true; 374 } 375 return database_->IsMalwareIPMatchKillSwitchOn(); 376 } 377 378 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() { 379 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 380 if (!enabled_ || !MakeDatabaseAvailable()) { 381 return true; 382 } 383 return database_->IsCsdWhitelistKillSwitchOn(); 384 } 385 386 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url, 387 Client* client) { 388 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 389 if (!enabled_) 390 return true; 391 392 if (!CanCheckUrl(url)) 393 return true; 394 395 std::vector<SBThreatType> expected_threats; 396 expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE); 397 expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING); 398 399 const base::TimeTicks start = base::TimeTicks::Now(); 400 if (!MakeDatabaseAvailable()) { 401 QueuedCheck queued_check(safe_browsing_util::MALWARE, // or PHISH 402 client, 403 url, 404 expected_threats, 405 start); 406 queued_checks_.push_back(queued_check); 407 return false; 408 } 409 410 std::vector<SBPrefix> prefix_hits; 411 std::vector<SBFullHashResult> cache_hits; 412 413 bool prefix_match = 414 database_->ContainsBrowseUrl(url, &prefix_hits, &cache_hits); 415 416 UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start); 417 418 if (!prefix_match) 419 return true; // URL is okay. 420 421 // Needs to be asynchronous, since we could be in the constructor of a 422 // ResourceDispatcherHost event handler which can't pause there. 423 SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url), 424 std::vector<SBFullHash>(), 425 client, 426 safe_browsing_util::MALWARE, 427 expected_threats); 428 check->need_get_hash = cache_hits.empty(); 429 check->prefix_hits.swap(prefix_hits); 430 check->cache_hits.swap(cache_hits); 431 checks_.insert(check); 432 433 BrowserThread::PostTask( 434 BrowserThread::IO, FROM_HERE, 435 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check)); 436 437 return false; 438 } 439 440 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) { 441 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 442 for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) { 443 // We can't delete matching checks here because the db thread has a copy of 444 // the pointer. Instead, we simply NULL out the client, and when the db 445 // thread calls us back, we'll clean up the check. 446 if ((*i)->client == client) 447 (*i)->client = NULL; 448 } 449 450 // Scan the queued clients store. Clients may be here if they requested a URL 451 // check before the database has finished loading. 452 for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin()); 453 it != queued_checks_.end(); ) { 454 // In this case it's safe to delete matches entirely since nothing has a 455 // pointer to them. 456 if (it->client == client) 457 it = queued_checks_.erase(it); 458 else 459 ++it; 460 } 461 } 462 463 void SafeBrowsingDatabaseManager::HandleGetHashResults( 464 SafeBrowsingCheck* check, 465 const std::vector<SBFullHashResult>& full_hashes, 466 const base::TimeDelta& cache_lifetime) { 467 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 468 469 if (!enabled_) 470 return; 471 472 // If the service has been shut down, |check| should have been deleted. 473 DCHECK(checks_.find(check) != checks_.end()); 474 475 // |start| is set before calling |GetFullHash()|, which should be 476 // the only path which gets to here. 477 DCHECK(!check->start.is_null()); 478 UMA_HISTOGRAM_LONG_TIMES("SB2.Network", 479 base::TimeTicks::Now() - check->start); 480 481 std::vector<SBPrefix> prefixes = check->prefix_hits; 482 OnHandleGetHashResults(check, full_hashes); // 'check' is deleted here. 483 484 // Cache the GetHash results. 485 if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable()) 486 database_->CacheHashResults(prefixes, full_hashes, cache_lifetime); 487 } 488 489 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) { 490 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 491 DCHECK(enabled_); 492 DCHECK(!callback.is_null()); 493 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind( 494 &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback)); 495 } 496 497 void SafeBrowsingDatabaseManager::AddChunks( 498 const std::string& list, 499 scoped_ptr<ScopedVector<SBChunkData> > chunks, 500 AddChunksCallback callback) { 501 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 502 DCHECK(enabled_); 503 DCHECK(!callback.is_null()); 504 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind( 505 &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list, 506 base::Passed(&chunks), callback)); 507 } 508 509 void SafeBrowsingDatabaseManager::DeleteChunks( 510 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) { 511 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 512 DCHECK(enabled_); 513 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind( 514 &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this, 515 base::Passed(&chunk_deletes))); 516 } 517 518 void SafeBrowsingDatabaseManager::UpdateStarted() { 519 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 520 DCHECK(enabled_); 521 DCHECK(!update_in_progress_); 522 update_in_progress_ = true; 523 } 524 525 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) { 526 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 527 DCHECK(enabled_); 528 if (update_in_progress_) { 529 update_in_progress_ = false; 530 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, 531 base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished, 532 this, update_succeeded)); 533 } 534 } 535 536 void SafeBrowsingDatabaseManager::ResetDatabase() { 537 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 538 DCHECK(enabled_); 539 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind( 540 &SafeBrowsingDatabaseManager::OnResetDatabase, this)); 541 } 542 543 void SafeBrowsingDatabaseManager::LogPauseDelay(base::TimeDelta time) { 544 UMA_HISTOGRAM_LONG_TIMES("SB2.Delay", time); 545 } 546 547 void SafeBrowsingDatabaseManager::StartOnIOThread() { 548 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 549 if (enabled_) 550 return; 551 552 DCHECK(!safe_browsing_thread_.get()); 553 safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread")); 554 if (!safe_browsing_thread_->Start()) 555 return; 556 enabled_ = true; 557 558 MakeDatabaseAvailable(); 559 } 560 561 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) { 562 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 563 564 DoStopOnIOThread(); 565 if (shutdown) { 566 sb_service_ = NULL; 567 } 568 } 569 570 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished( 571 bool update_succeeded) { 572 content::NotificationService::current()->Notify( 573 chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE, 574 content::Source<SafeBrowsingDatabaseManager>(this), 575 content::Details<bool>(&update_succeeded)); 576 } 577 578 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck( 579 const safe_browsing_util::ListType check_type, 580 Client* client, 581 const GURL& url, 582 const std::vector<SBThreatType>& expected_threats, 583 const base::TimeTicks& start) 584 : check_type(check_type), 585 client(client), 586 url(url), 587 expected_threats(expected_threats), 588 start(start) { 589 } 590 591 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() { 592 } 593 594 void SafeBrowsingDatabaseManager::DoStopOnIOThread() { 595 if (!enabled_) 596 return; 597 598 enabled_ = false; 599 600 // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'. 601 while (!queued_checks_.empty()) { 602 QueuedCheck queued = queued_checks_.front(); 603 if (queued.client) { 604 SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url), 605 std::vector<SBFullHash>(), 606 queued.client, 607 queued.check_type, 608 queued.expected_threats); 609 queued.client->OnSafeBrowsingResult(sb_check); 610 } 611 queued_checks_.pop_front(); 612 } 613 614 // Close the database. Cases to avoid: 615 // * If |closing_database_| is true, continuing will queue up a second 616 // request, |closing_database_| will be reset after handling the first 617 // request, and if any functions on the db thread recreate the database, we 618 // could start using it on the IO thread and then have the second request 619 // handler delete it out from under us. 620 // * If |database_| is NULL, then either no creation request is in flight, in 621 // which case we don't need to do anything, or one is in flight, in which 622 // case the database will be recreated before our deletion request is 623 // handled, and could be used on the IO thread in that time period, leading 624 // to the same problem as above. 625 // Checking DatabaseAvailable() avoids both of these. 626 if (DatabaseAvailable()) { 627 closing_database_ = true; 628 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, 629 base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this)); 630 } 631 632 // Flush the database thread. Any in-progress database check results will be 633 // ignored and cleaned up below. 634 // 635 // Note that to avoid leaking the database, we rely on the fact that no new 636 // tasks will be added to the db thread between the call above and this one. 637 // See comments on the declaration of |safe_browsing_thread_|. 638 { 639 // A ScopedAllowIO object is required to join the thread when calling Stop. 640 // See http://crbug.com/72696. Note that we call Stop() first to clear out 641 // any remaining tasks before clearing safe_browsing_thread_. 642 base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join; 643 safe_browsing_thread_->Stop(); 644 safe_browsing_thread_.reset(); 645 } 646 647 // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'. 648 // We have to do this after the db thread returns because methods on it can 649 // have copies of these pointers, so deleting them might lead to accessing 650 // garbage. 651 for (CurrentChecks::iterator it = checks_.begin(); 652 it != checks_.end(); ++it) { 653 SafeBrowsingCheck* check = *it; 654 if (check->client) 655 check->client->OnSafeBrowsingResult(*check); 656 } 657 STLDeleteElements(&checks_); 658 659 gethash_requests_.clear(); 660 } 661 662 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const { 663 base::AutoLock lock(database_lock_); 664 return !closing_database_ && (database_ != NULL); 665 } 666 667 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() { 668 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 669 DCHECK(enabled_); 670 if (DatabaseAvailable()) 671 return true; 672 safe_browsing_thread_->message_loop()->PostTask( 673 FROM_HERE, 674 base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase), 675 this)); 676 return false; 677 } 678 679 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() { 680 DCHECK_EQ(base::MessageLoop::current(), 681 safe_browsing_thread_->message_loop()); 682 if (database_) 683 return database_; 684 startup_metric_utils::ScopedSlowStartupUMA 685 scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase"); 686 const base::TimeTicks before = base::TimeTicks::Now(); 687 688 SafeBrowsingDatabase* database = 689 SafeBrowsingDatabase::Create(enable_download_protection_, 690 enable_csd_whitelist_, 691 enable_download_whitelist_, 692 enable_extension_blacklist_, 693 enable_side_effect_free_whitelist_, 694 enable_ip_blacklist_); 695 696 database->Init(SafeBrowsingService::GetBaseFilename()); 697 { 698 // Acquiring the lock here guarantees correct ordering between the writes to 699 // the new database object above, and the setting of |databse_| below. 700 base::AutoLock lock(database_lock_); 701 database_ = database; 702 } 703 704 BrowserThread::PostTask( 705 BrowserThread::IO, FROM_HERE, 706 base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this)); 707 708 UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before); 709 return database_; 710 } 711 712 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) { 713 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 714 715 if (!enabled_) 716 return; 717 718 // If the service has been shut down, |check| should have been deleted. 719 DCHECK(checks_.find(check) != checks_.end()); 720 721 if (check->client && check->need_get_hash) { 722 // We have a partial match so we need to query Google for the full hash. 723 // Clean up will happen in HandleGetHashResults. 724 725 // See if we have a GetHash request already in progress for this particular 726 // prefix. If so, we just append ourselves to the list of interested parties 727 // when the results arrive. We only do this for checks involving one prefix, 728 // since that is the common case (multiple prefixes will issue the request 729 // as normal). 730 if (check->prefix_hits.size() == 1) { 731 SBPrefix prefix = check->prefix_hits[0]; 732 GetHashRequests::iterator it = gethash_requests_.find(prefix); 733 if (it != gethash_requests_.end()) { 734 // There's already a request in progress. 735 it->second.push_back(check); 736 return; 737 } 738 739 // No request in progress, so we're the first for this prefix. 740 GetHashRequestors requestors; 741 requestors.push_back(check); 742 gethash_requests_[prefix] = requestors; 743 } 744 745 // Reset the start time so that we can measure the network time without the 746 // database time. 747 check->start = base::TimeTicks::Now(); 748 // Note: If |this| is deleted or stopped, the protocol_manager will 749 // be destroyed as well - hence it's OK to do unretained in this case. 750 bool is_download = check->check_type == safe_browsing_util::BINURL; 751 sb_service_->protocol_manager()->GetFullHash( 752 check->prefix_hits, 753 base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults, 754 base::Unretained(this), 755 check), 756 is_download); 757 } else { 758 // We may have cached results for previous GetHash queries. Since 759 // this data comes from cache, don't histogram hits. 760 HandleOneCheck(check, check->cache_hits); 761 } 762 } 763 764 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase( 765 GetChunksCallback callback) { 766 DCHECK_EQ(base::MessageLoop::current(), 767 safe_browsing_thread_->message_loop()); 768 769 bool database_error = true; 770 std::vector<SBListChunkRanges> lists; 771 DCHECK(!database_update_in_progress_); 772 database_update_in_progress_ = true; 773 GetDatabase(); // This guarantees that |database_| is non-NULL. 774 if (database_->UpdateStarted(&lists)) { 775 database_error = false; 776 } else { 777 database_->UpdateFinished(false); 778 } 779 780 BrowserThread::PostTask( 781 BrowserThread::IO, FROM_HERE, 782 base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase, 783 this, lists, database_error, callback)); 784 } 785 786 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase( 787 const std::vector<SBListChunkRanges>& lists, bool database_error, 788 GetChunksCallback callback) { 789 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 790 if (enabled_) 791 callback.Run(lists, database_error); 792 } 793 794 void SafeBrowsingDatabaseManager::OnAddChunksComplete( 795 AddChunksCallback callback) { 796 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 797 if (enabled_) 798 callback.Run(); 799 } 800 801 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() { 802 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 803 if (!enabled_) 804 return; 805 806 HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size()); 807 if (queued_checks_.empty()) 808 return; 809 810 // If the database isn't already available, calling CheckUrl() in the loop 811 // below will add the check back to the queue, and we'll infinite-loop. 812 DCHECK(DatabaseAvailable()); 813 while (!queued_checks_.empty()) { 814 QueuedCheck check = queued_checks_.front(); 815 DCHECK(!check.start.is_null()); 816 HISTOGRAM_TIMES("SB.QueueDelay", base::TimeTicks::Now() - check.start); 817 // If CheckUrl() determines the URL is safe immediately, it doesn't call the 818 // client's handler function (because normally it's being directly called by 819 // the client). Since we're not the client, we have to convey this result. 820 if (check.client && CheckBrowseUrl(check.url, check.client)) { 821 SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url), 822 std::vector<SBFullHash>(), 823 check.client, 824 check.check_type, 825 check.expected_threats); 826 check.client->OnSafeBrowsingResult(sb_check); 827 } 828 queued_checks_.pop_front(); 829 } 830 } 831 832 void SafeBrowsingDatabaseManager::AddDatabaseChunks( 833 const std::string& list_name, 834 scoped_ptr<ScopedVector<SBChunkData> > chunks, 835 AddChunksCallback callback) { 836 DCHECK_EQ(base::MessageLoop::current(), 837 safe_browsing_thread_->message_loop()); 838 if (chunks) 839 GetDatabase()->InsertChunks(list_name, chunks->get()); 840 BrowserThread::PostTask( 841 BrowserThread::IO, FROM_HERE, 842 base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this, 843 callback)); 844 } 845 846 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks( 847 scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) { 848 DCHECK_EQ(base::MessageLoop::current(), 849 safe_browsing_thread_->message_loop()); 850 if (chunk_deletes) 851 GetDatabase()->DeleteChunks(*chunk_deletes); 852 } 853 854 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished( 855 bool update_succeeded) { 856 DCHECK_EQ(base::MessageLoop::current(), 857 safe_browsing_thread_->message_loop()); 858 GetDatabase()->UpdateFinished(update_succeeded); 859 DCHECK(database_update_in_progress_); 860 database_update_in_progress_ = false; 861 BrowserThread::PostTask( 862 BrowserThread::UI, FROM_HERE, 863 base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished, 864 this, update_succeeded)); 865 } 866 867 void SafeBrowsingDatabaseManager::OnCloseDatabase() { 868 DCHECK_EQ(base::MessageLoop::current(), 869 safe_browsing_thread_->message_loop()); 870 DCHECK(closing_database_); 871 872 // Because |closing_database_| is true, nothing on the IO thread will be 873 // accessing the database, so it's safe to delete and then NULL the pointer. 874 delete database_; 875 database_ = NULL; 876 877 // Acquiring the lock here guarantees correct ordering between the resetting 878 // of |database_| above and of |closing_database_| below, which ensures there 879 // won't be a window during which the IO thread falsely believes the database 880 // is available. 881 base::AutoLock lock(database_lock_); 882 closing_database_ = false; 883 } 884 885 void SafeBrowsingDatabaseManager::OnResetDatabase() { 886 DCHECK_EQ(base::MessageLoop::current(), 887 safe_browsing_thread_->message_loop()); 888 GetDatabase()->ResetDatabase(); 889 } 890 891 void SafeBrowsingDatabaseManager::OnHandleGetHashResults( 892 SafeBrowsingCheck* check, 893 const std::vector<SBFullHashResult>& full_hashes) { 894 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 895 safe_browsing_util::ListType check_type = check->check_type; 896 SBPrefix prefix = check->prefix_hits[0]; 897 GetHashRequests::iterator it = gethash_requests_.find(prefix); 898 if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) { 899 const bool hit = HandleOneCheck(check, full_hashes); 900 RecordGetHashCheckStatus(hit, check_type, full_hashes); 901 return; 902 } 903 904 // Call back all interested parties, noting if any has a hit. 905 GetHashRequestors& requestors = it->second; 906 bool hit = false; 907 for (GetHashRequestors::iterator r = requestors.begin(); 908 r != requestors.end(); ++r) { 909 if (HandleOneCheck(*r, full_hashes)) 910 hit = true; 911 } 912 RecordGetHashCheckStatus(hit, check_type, full_hashes); 913 914 gethash_requests_.erase(it); 915 } 916 917 bool SafeBrowsingDatabaseManager::HandleOneCheck( 918 SafeBrowsingCheck* check, 919 const std::vector<SBFullHashResult>& full_hashes) { 920 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 921 DCHECK(check); 922 923 bool is_threat = false; 924 925 // TODO(shess): GetHashThreadListType() contains a loop, 926 // GetUrlThreatListType() a loop around that loop. Having another loop out 927 // here concerns me. It is likely that SAFE is an expected outcome, which 928 // means all of those loops run to completion. Refactoring this to generate a 929 // set of sorted items to compare in sequence would probably improve things. 930 // 931 // Additionally, the set of patterns generated from the urls is very similar 932 // to the patterns generated in ContainsBrowseUrl() and other database checks, 933 // which are called from this code. Refactoring that across the checks could 934 // interact well with batching the checks here. 935 936 for (size_t i = 0; i < check->urls.size(); ++i) { 937 SBThreatType threat = GetUrlThreatType(check->urls[i], full_hashes); 938 if (threat != SB_THREAT_TYPE_SAFE && 939 IsExpectedThreat(threat, check->expected_threats)) { 940 check->url_results[i] = threat; 941 is_threat = true; 942 } 943 } 944 945 for (size_t i = 0; i < check->full_hashes.size(); ++i) { 946 SBThreatType threat = GetHashThreatType(check->full_hashes[i], full_hashes); 947 if (threat != SB_THREAT_TYPE_SAFE && 948 IsExpectedThreat(threat, check->expected_threats)) { 949 check->full_hash_results[i] = threat; 950 is_threat = true; 951 } 952 } 953 954 SafeBrowsingCheckDone(check); 955 return is_threat; 956 } 957 958 void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread( 959 SafeBrowsingCheck* check) { 960 DCHECK_EQ(base::MessageLoop::current(), 961 safe_browsing_thread_->message_loop()); 962 DCHECK(enable_download_protection_); 963 964 std::vector<SBPrefix> prefix_hits; 965 966 if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) { 967 // Good, we don't have hash for this url prefix. 968 BrowserThread::PostTask( 969 BrowserThread::IO, FROM_HERE, 970 base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this, 971 check)); 972 return; 973 } 974 975 check->need_get_hash = true; 976 check->prefix_hits.clear(); 977 check->prefix_hits = prefix_hits; 978 BrowserThread::PostTask( 979 BrowserThread::IO, FROM_HERE, 980 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check)); 981 } 982 983 void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread( 984 SafeBrowsingCheck* check) { 985 DCHECK_EQ(base::MessageLoop::current(), 986 safe_browsing_thread_->message_loop()); 987 988 std::vector<SBPrefix> prefixes; 989 for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin(); 990 it != check->full_hashes.end(); ++it) { 991 prefixes.push_back((*it).prefix); 992 } 993 database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits); 994 995 if (check->prefix_hits.empty()) { 996 // No matches for any extensions. 997 BrowserThread::PostTask( 998 BrowserThread::IO, 999 FROM_HERE, 1000 base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this, 1001 check)); 1002 } else { 1003 // Some prefixes matched, we need to ask Google whether they're legit. 1004 check->need_get_hash = true; 1005 BrowserThread::PostTask( 1006 BrowserThread::IO, 1007 FROM_HERE, 1008 base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check)); 1009 } 1010 } 1011 1012 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) { 1013 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 1014 DCHECK(check); 1015 1016 if (!enabled_) 1017 return; 1018 1019 DCHECK(checks_.find(check) != checks_.end()); 1020 if (check->client) { 1021 check->client->OnSafeBrowsingResult(*check); 1022 check->client = NULL; 1023 } 1024 } 1025 1026 void SafeBrowsingDatabaseManager::CheckDownloadUrlDone( 1027 SafeBrowsingCheck* check) { 1028 DCHECK(enable_download_protection_); 1029 SafeBrowsingCheckDone(check); 1030 } 1031 1032 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone( 1033 SafeBrowsingCheck* check) { 1034 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 1035 DCHECK(check); 1036 1037 if (!enabled_) 1038 return; 1039 1040 VLOG(1) << "SafeBrowsingCheckDone"; 1041 DCHECK(checks_.find(check) != checks_.end()); 1042 if (check->client) 1043 check->client->OnSafeBrowsingResult(*check); 1044 checks_.erase(check); 1045 delete check; 1046 } 1047 1048 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck( 1049 SafeBrowsingCheck* check, 1050 const base::Closure& task) { 1051 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); 1052 check->timeout_factory_.reset( 1053 new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this)); 1054 checks_.insert(check); 1055 1056 safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task); 1057 1058 base::MessageLoop::current()->PostDelayedTask(FROM_HERE, 1059 base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback, 1060 check->timeout_factory_->GetWeakPtr(), check), 1061 check_timeout_); 1062 } 1063