Home | History | Annotate | Download | only in safe_browsing 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/safe_browsing/database_manager.h"
      6 
      7 #include <algorithm>
      8 
      9 #include "base/bind.h"
     10 #include "base/bind_helpers.h"
     11 #include "base/callback.h"
     12 #include "base/command_line.h"
     13 #include "base/debug/leak_tracker.h"
     14 #include "base/path_service.h"
     15 #include "base/stl_util.h"
     16 #include "base/strings/string_util.h"
     17 #include "base/threading/thread.h"
     18 #include "base/threading/thread_restrictions.h"
     19 #include "chrome/browser/browser_process.h"
     20 #include "chrome/browser/chrome_notification_types.h"
     21 #include "chrome/browser/prerender/prerender_field_trial.h"
     22 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
     23 #include "chrome/browser/safe_browsing/download_protection_service.h"
     24 #include "chrome/browser/safe_browsing/malware_details.h"
     25 #include "chrome/browser/safe_browsing/protocol_manager.h"
     26 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
     27 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
     28 #include "chrome/browser/safe_browsing/ui_manager.h"
     29 #include "chrome/common/chrome_constants.h"
     30 #include "chrome/common/chrome_paths.h"
     31 #include "chrome/common/chrome_switches.h"
     32 #include "components/metrics/metrics_service.h"
     33 #include "components/startup_metric_utils/startup_metric_utils.h"
     34 #include "content/public/browser/browser_thread.h"
     35 #include "content/public/browser/notification_service.h"
     36 #include "url/url_constants.h"
     37 
     38 using content::BrowserThread;
     39 
     40 namespace {
     41 
     42 // Timeout for match checks, e.g. download URLs, hashes.
     43 const int kCheckTimeoutMs = 10000;
     44 
     45 // Records disposition information about the check.  |hit| should be
     46 // |true| if there were any prefix hits in |full_hashes|.
     47 void RecordGetHashCheckStatus(
     48     bool hit,
     49     safe_browsing_util::ListType check_type,
     50     const std::vector<SBFullHashResult>& full_hashes) {
     51   SafeBrowsingProtocolManager::ResultType result;
     52   if (full_hashes.empty()) {
     53     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
     54   } else if (hit) {
     55     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
     56   } else {
     57     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
     58   }
     59   bool is_download = check_type == safe_browsing_util::BINURL;
     60   SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
     61 }
     62 
     63 bool IsExpectedThreat(
     64     const SBThreatType threat_type,
     65     const std::vector<SBThreatType>& expected_threats) {
     66   return expected_threats.end() != std::find(expected_threats.begin(),
     67                                              expected_threats.end(),
     68                                              threat_type);
     69 }
     70 
     71 // Return the list id from the first result in |full_hashes| which matches
     72 // |hash|, or INVALID if none match.
     73 safe_browsing_util::ListType GetHashThreatListType(
     74     const SBFullHash& hash,
     75     const std::vector<SBFullHashResult>& full_hashes) {
     76   for (size_t i = 0; i < full_hashes.size(); ++i) {
     77     if (SBFullHashEqual(hash, full_hashes[i].hash))
     78       return static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
     79   }
     80   return safe_browsing_util::INVALID;
     81 }
     82 
     83 // Given a URL, compare all the possible host + path full hashes to the set of
     84 // provided full hashes.  Returns the list id of the a matching result from
     85 // |full_hashes|, or INVALID if none match.
     86 safe_browsing_util::ListType GetUrlThreatListType(
     87     const GURL& url,
     88     const std::vector<SBFullHashResult>& full_hashes) {
     89   if (full_hashes.empty())
     90     return safe_browsing_util::INVALID;
     91 
     92   std::vector<std::string> patterns;
     93   safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
     94 
     95   for (size_t i = 0; i < patterns.size(); ++i) {
     96     safe_browsing_util::ListType threat =
     97         GetHashThreatListType(SBFullHashForString(patterns[i]), full_hashes);
     98     if (threat != safe_browsing_util::INVALID)
     99       return threat;
    100   }
    101   return safe_browsing_util::INVALID;
    102 }
    103 
    104 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
    105   switch (list_type) {
    106     case safe_browsing_util::PHISH:
    107       return SB_THREAT_TYPE_URL_PHISHING;
    108     case safe_browsing_util::MALWARE:
    109       return SB_THREAT_TYPE_URL_MALWARE;
    110     case safe_browsing_util::BINURL:
    111       return SB_THREAT_TYPE_BINARY_MALWARE_URL;
    112     case safe_browsing_util::EXTENSIONBLACKLIST:
    113       return SB_THREAT_TYPE_EXTENSION;
    114     default:
    115       DVLOG(1) << "Unknown safe browsing list id " << list_type;
    116       return SB_THREAT_TYPE_SAFE;
    117   }
    118 }
    119 
    120 }  // namespace
    121 
    122 // static
    123 SBThreatType SafeBrowsingDatabaseManager::GetHashThreatType(
    124     const SBFullHash& hash,
    125     const std::vector<SBFullHashResult>& full_hashes) {
    126   return GetThreatTypeFromListType(GetHashThreatListType(hash, full_hashes));
    127 }
    128 
    129 // static
    130 SBThreatType SafeBrowsingDatabaseManager::GetUrlThreatType(
    131     const GURL& url,
    132     const std::vector<SBFullHashResult>& full_hashes) {
    133   return GetThreatTypeFromListType(GetUrlThreatListType(url, full_hashes));
    134 }
    135 
    136 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
    137     const std::vector<GURL>& urls,
    138     const std::vector<SBFullHash>& full_hashes,
    139     Client* client,
    140     safe_browsing_util::ListType check_type,
    141     const std::vector<SBThreatType>& expected_threats)
    142     : urls(urls),
    143       url_results(urls.size(), SB_THREAT_TYPE_SAFE),
    144       full_hashes(full_hashes),
    145       full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
    146       client(client),
    147       need_get_hash(false),
    148       check_type(check_type),
    149       expected_threats(expected_threats) {
    150   DCHECK_EQ(urls.empty(), !full_hashes.empty())
    151       << "Exactly one of urls and full_hashes must be set";
    152 }
    153 
    154 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
    155 
    156 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
    157     const SafeBrowsingCheck& check) {
    158   DCHECK_EQ(check.urls.size(), check.url_results.size());
    159   DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
    160   if (!check.urls.empty()) {
    161     DCHECK(check.full_hashes.empty());
    162     switch (check.check_type) {
    163       case safe_browsing_util::MALWARE:
    164       case safe_browsing_util::PHISH:
    165         DCHECK_EQ(1u, check.urls.size());
    166         OnCheckBrowseUrlResult(check.urls[0], check.url_results[0]);
    167         break;
    168       case safe_browsing_util::BINURL:
    169         DCHECK_EQ(check.urls.size(), check.url_results.size());
    170         OnCheckDownloadUrlResult(
    171             check.urls,
    172             *std::max_element(check.url_results.begin(),
    173                               check.url_results.end()));
    174         break;
    175       default:
    176         NOTREACHED();
    177     }
    178   } else if (!check.full_hashes.empty()) {
    179     switch (check.check_type) {
    180       case safe_browsing_util::EXTENSIONBLACKLIST: {
    181         std::set<std::string> unsafe_extension_ids;
    182         for (size_t i = 0; i < check.full_hashes.size(); ++i) {
    183           std::string extension_id =
    184               safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
    185           if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
    186             unsafe_extension_ids.insert(extension_id);
    187         }
    188         OnCheckExtensionsResult(unsafe_extension_ids);
    189         break;
    190       }
    191       default:
    192         NOTREACHED();
    193     }
    194   } else {
    195     NOTREACHED();
    196   }
    197 }
    198 
    199 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
    200     const scoped_refptr<SafeBrowsingService>& service)
    201     : sb_service_(service),
    202       database_(NULL),
    203       enabled_(false),
    204       enable_download_protection_(false),
    205       enable_csd_whitelist_(false),
    206       enable_download_whitelist_(false),
    207       enable_extension_blacklist_(false),
    208       enable_side_effect_free_whitelist_(false),
    209       enable_ip_blacklist_(false),
    210       update_in_progress_(false),
    211       database_update_in_progress_(false),
    212       closing_database_(false),
    213       check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
    214   DCHECK(sb_service_.get() != NULL);
    215 
    216   CommandLine* cmdline = CommandLine::ForCurrentProcess();
    217   enable_download_protection_ =
    218       !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
    219 
    220   // We only download the csd-whitelist if client-side phishing detection is
    221   // enabled.
    222   enable_csd_whitelist_ =
    223       !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
    224 
    225   // TODO(noelutz): remove this boolean variable since it should always be true
    226   // if SafeBrowsing is enabled.  Unfortunately, we have no test data for this
    227   // list right now.  This means that we need to be able to disable this list
    228   // for the SafeBrowsing test to pass.
    229   enable_download_whitelist_ = enable_csd_whitelist_;
    230 
    231   // TODO(kalman): there really shouldn't be a flag for this.
    232   enable_extension_blacklist_ =
    233       !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
    234 
    235   enable_side_effect_free_whitelist_ =
    236       prerender::IsSideEffectFreeWhitelistEnabled() &&
    237       !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
    238 
    239   // The client-side IP blacklist feature is tightly integrated with client-side
    240   // phishing protection for now.
    241   enable_ip_blacklist_ = enable_csd_whitelist_;
    242 
    243   enum SideEffectFreeWhitelistStatus {
    244     SIDE_EFFECT_FREE_WHITELIST_ENABLED,
    245     SIDE_EFFECT_FREE_WHITELIST_DISABLED,
    246     SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
    247   };
    248 
    249   SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
    250       enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
    251       SIDE_EFFECT_FREE_WHITELIST_DISABLED;
    252 
    253   UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
    254                             side_effect_free_whitelist_status,
    255                             SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
    256 }
    257 
    258 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
    259   // We should have already been shut down. If we're still enabled, then the
    260   // database isn't going to be closed properly, which could lead to corruption.
    261   DCHECK(!enabled_);
    262 }
    263 
    264 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
    265   return url.SchemeIs(url::kFtpScheme) ||
    266          url.SchemeIs(url::kHttpScheme) ||
    267          url.SchemeIs(url::kHttpsScheme);
    268 }
    269 
    270 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
    271     const std::vector<GURL>& url_chain,
    272     Client* client) {
    273   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    274   if (!enabled_ || !enable_download_protection_)
    275     return true;
    276 
    277   // We need to check the database for url prefix, and later may fetch the url
    278   // from the safebrowsing backends. These need to be asynchronous.
    279   SafeBrowsingCheck* check =
    280       new SafeBrowsingCheck(url_chain,
    281                             std::vector<SBFullHash>(),
    282                             client,
    283                             safe_browsing_util::BINURL,
    284                             std::vector<SBThreatType>(1,
    285                                 SB_THREAT_TYPE_BINARY_MALWARE_URL));
    286   StartSafeBrowsingCheck(
    287       check,
    288       base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
    289                  check));
    290   return false;
    291 }
    292 
    293 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
    294     const std::set<std::string>& extension_ids,
    295     Client* client) {
    296   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    297 
    298   if (!enabled_ || !enable_extension_blacklist_)
    299     return true;
    300 
    301   std::vector<SBFullHash> extension_id_hashes;
    302   std::transform(extension_ids.begin(), extension_ids.end(),
    303                  std::back_inserter(extension_id_hashes),
    304                  safe_browsing_util::StringToSBFullHash);
    305 
    306   SafeBrowsingCheck* check = new SafeBrowsingCheck(
    307       std::vector<GURL>(),
    308       extension_id_hashes,
    309       client,
    310       safe_browsing_util::EXTENSIONBLACKLIST,
    311       std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
    312 
    313   StartSafeBrowsingCheck(
    314       check,
    315       base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
    316                  this,
    317                  check));
    318   return false;
    319 }
    320 
    321 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
    322     const GURL& url) {
    323   if (!enabled_)
    324     return false;
    325 
    326   if (!CanCheckUrl(url))
    327     return false;
    328 
    329   return database_->ContainsSideEffectFreeWhitelistUrl(url);
    330 }
    331 
    332 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
    333     const std::string& ip_address) {
    334   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    335   if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
    336     return false;  // Fail open.
    337   }
    338   return database_->ContainsMalwareIP(ip_address);
    339 }
    340 
    341 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
    342   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    343   if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
    344     // There is something funky going on here -- for example, perhaps the user
    345     // has not restarted since enabling metrics reporting, so we haven't
    346     // enabled the csd whitelist yet.  Just to be safe we return true in this
    347     // case.
    348     return true;
    349   }
    350   return database_->ContainsCsdWhitelistedUrl(url);
    351 }
    352 
    353 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
    354   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    355   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
    356     return true;
    357   }
    358   return database_->ContainsDownloadWhitelistedUrl(url);
    359 }
    360 
    361 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
    362     const std::string& str) {
    363   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    364   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
    365     return true;
    366   }
    367   return database_->ContainsDownloadWhitelistedString(str);
    368 }
    369 
    370 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
    371   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    372   if (!enabled_ || !MakeDatabaseAvailable()) {
    373     return true;
    374   }
    375   return database_->IsMalwareIPMatchKillSwitchOn();
    376 }
    377 
    378 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
    379   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    380   if (!enabled_ || !MakeDatabaseAvailable()) {
    381     return true;
    382   }
    383   return database_->IsCsdWhitelistKillSwitchOn();
    384 }
    385 
    386 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
    387                                                  Client* client) {
    388   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    389   if (!enabled_)
    390     return true;
    391 
    392   if (!CanCheckUrl(url))
    393     return true;
    394 
    395   std::vector<SBThreatType> expected_threats;
    396   expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
    397   expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
    398 
    399   const base::TimeTicks start = base::TimeTicks::Now();
    400   if (!MakeDatabaseAvailable()) {
    401     QueuedCheck queued_check(safe_browsing_util::MALWARE,  // or PHISH
    402                              client,
    403                              url,
    404                              expected_threats,
    405                              start);
    406     queued_checks_.push_back(queued_check);
    407     return false;
    408   }
    409 
    410   std::vector<SBPrefix> prefix_hits;
    411   std::vector<SBFullHashResult> cache_hits;
    412 
    413   bool prefix_match =
    414       database_->ContainsBrowseUrl(url, &prefix_hits, &cache_hits);
    415 
    416   UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
    417 
    418   if (!prefix_match)
    419     return true;  // URL is okay.
    420 
    421   // Needs to be asynchronous, since we could be in the constructor of a
    422   // ResourceDispatcherHost event handler which can't pause there.
    423   SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
    424                                                    std::vector<SBFullHash>(),
    425                                                    client,
    426                                                    safe_browsing_util::MALWARE,
    427                                                    expected_threats);
    428   check->need_get_hash = cache_hits.empty();
    429   check->prefix_hits.swap(prefix_hits);
    430   check->cache_hits.swap(cache_hits);
    431   checks_.insert(check);
    432 
    433   BrowserThread::PostTask(
    434       BrowserThread::IO, FROM_HERE,
    435       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
    436 
    437   return false;
    438 }
    439 
    440 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
    441   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    442   for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
    443     // We can't delete matching checks here because the db thread has a copy of
    444     // the pointer.  Instead, we simply NULL out the client, and when the db
    445     // thread calls us back, we'll clean up the check.
    446     if ((*i)->client == client)
    447       (*i)->client = NULL;
    448   }
    449 
    450   // Scan the queued clients store. Clients may be here if they requested a URL
    451   // check before the database has finished loading.
    452   for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
    453        it != queued_checks_.end(); ) {
    454     // In this case it's safe to delete matches entirely since nothing has a
    455     // pointer to them.
    456     if (it->client == client)
    457       it = queued_checks_.erase(it);
    458     else
    459       ++it;
    460   }
    461 }
    462 
    463 void SafeBrowsingDatabaseManager::HandleGetHashResults(
    464     SafeBrowsingCheck* check,
    465     const std::vector<SBFullHashResult>& full_hashes,
    466     const base::TimeDelta& cache_lifetime) {
    467   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    468 
    469   if (!enabled_)
    470     return;
    471 
    472   // If the service has been shut down, |check| should have been deleted.
    473   DCHECK(checks_.find(check) != checks_.end());
    474 
    475   // |start| is set before calling |GetFullHash()|, which should be
    476   // the only path which gets to here.
    477   DCHECK(!check->start.is_null());
    478   UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
    479                            base::TimeTicks::Now() - check->start);
    480 
    481   std::vector<SBPrefix> prefixes = check->prefix_hits;
    482   OnHandleGetHashResults(check, full_hashes);  // 'check' is deleted here.
    483 
    484   // Cache the GetHash results.
    485   if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
    486     database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
    487 }
    488 
    489 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
    490   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    491   DCHECK(enabled_);
    492   DCHECK(!callback.is_null());
    493   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    494       &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback));
    495 }
    496 
    497 void SafeBrowsingDatabaseManager::AddChunks(
    498     const std::string& list,
    499     scoped_ptr<ScopedVector<SBChunkData> > chunks,
    500     AddChunksCallback callback) {
    501   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    502   DCHECK(enabled_);
    503   DCHECK(!callback.is_null());
    504   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    505       &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list,
    506       base::Passed(&chunks), callback));
    507 }
    508 
    509 void SafeBrowsingDatabaseManager::DeleteChunks(
    510     scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
    511   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    512   DCHECK(enabled_);
    513   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    514       &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this,
    515       base::Passed(&chunk_deletes)));
    516 }
    517 
    518 void SafeBrowsingDatabaseManager::UpdateStarted() {
    519   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    520   DCHECK(enabled_);
    521   DCHECK(!update_in_progress_);
    522   update_in_progress_ = true;
    523 }
    524 
    525 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
    526   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    527   DCHECK(enabled_);
    528   if (update_in_progress_) {
    529     update_in_progress_ = false;
    530     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
    531       base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished,
    532                  this, update_succeeded));
    533   }
    534 }
    535 
    536 void SafeBrowsingDatabaseManager::ResetDatabase() {
    537   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    538   DCHECK(enabled_);
    539   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    540       &SafeBrowsingDatabaseManager::OnResetDatabase, this));
    541 }
    542 
    543 void SafeBrowsingDatabaseManager::LogPauseDelay(base::TimeDelta time) {
    544   UMA_HISTOGRAM_LONG_TIMES("SB2.Delay", time);
    545 }
    546 
    547 void SafeBrowsingDatabaseManager::StartOnIOThread() {
    548   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    549   if (enabled_)
    550     return;
    551 
    552   DCHECK(!safe_browsing_thread_.get());
    553   safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread"));
    554   if (!safe_browsing_thread_->Start())
    555     return;
    556   enabled_ = true;
    557 
    558   MakeDatabaseAvailable();
    559 }
    560 
    561 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
    562   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    563 
    564   DoStopOnIOThread();
    565   if (shutdown) {
    566     sb_service_ = NULL;
    567   }
    568 }
    569 
    570 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
    571     bool update_succeeded) {
    572   content::NotificationService::current()->Notify(
    573       chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
    574       content::Source<SafeBrowsingDatabaseManager>(this),
    575       content::Details<bool>(&update_succeeded));
    576 }
    577 
    578 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
    579     const safe_browsing_util::ListType check_type,
    580     Client* client,
    581     const GURL& url,
    582     const std::vector<SBThreatType>& expected_threats,
    583     const base::TimeTicks& start)
    584     : check_type(check_type),
    585       client(client),
    586       url(url),
    587       expected_threats(expected_threats),
    588       start(start) {
    589 }
    590 
    591 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
    592 }
    593 
    594 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
    595   if (!enabled_)
    596     return;
    597 
    598   enabled_ = false;
    599 
    600   // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
    601   while (!queued_checks_.empty()) {
    602     QueuedCheck queued = queued_checks_.front();
    603     if (queued.client) {
    604       SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
    605                                  std::vector<SBFullHash>(),
    606                                  queued.client,
    607                                  queued.check_type,
    608                                  queued.expected_threats);
    609       queued.client->OnSafeBrowsingResult(sb_check);
    610     }
    611     queued_checks_.pop_front();
    612   }
    613 
    614   // Close the database.  Cases to avoid:
    615   //  * If |closing_database_| is true, continuing will queue up a second
    616   //    request, |closing_database_| will be reset after handling the first
    617   //    request, and if any functions on the db thread recreate the database, we
    618   //    could start using it on the IO thread and then have the second request
    619   //    handler delete it out from under us.
    620   //  * If |database_| is NULL, then either no creation request is in flight, in
    621   //    which case we don't need to do anything, or one is in flight, in which
    622   //    case the database will be recreated before our deletion request is
    623   //    handled, and could be used on the IO thread in that time period, leading
    624   //    to the same problem as above.
    625   // Checking DatabaseAvailable() avoids both of these.
    626   if (DatabaseAvailable()) {
    627     closing_database_ = true;
    628     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
    629         base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
    630   }
    631 
    632   // Flush the database thread. Any in-progress database check results will be
    633   // ignored and cleaned up below.
    634   //
    635   // Note that to avoid leaking the database, we rely on the fact that no new
    636   // tasks will be added to the db thread between the call above and this one.
    637   // See comments on the declaration of |safe_browsing_thread_|.
    638   {
    639     // A ScopedAllowIO object is required to join the thread when calling Stop.
    640     // See http://crbug.com/72696. Note that we call Stop() first to clear out
    641     // any remaining tasks before clearing safe_browsing_thread_.
    642     base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join;
    643     safe_browsing_thread_->Stop();
    644     safe_browsing_thread_.reset();
    645   }
    646 
    647   // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
    648   // We have to do this after the db thread returns because methods on it can
    649   // have copies of these pointers, so deleting them might lead to accessing
    650   // garbage.
    651   for (CurrentChecks::iterator it = checks_.begin();
    652        it != checks_.end(); ++it) {
    653     SafeBrowsingCheck* check = *it;
    654     if (check->client)
    655       check->client->OnSafeBrowsingResult(*check);
    656   }
    657   STLDeleteElements(&checks_);
    658 
    659   gethash_requests_.clear();
    660 }
    661 
    662 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
    663   base::AutoLock lock(database_lock_);
    664   return !closing_database_ && (database_ != NULL);
    665 }
    666 
    667 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
    668   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    669   DCHECK(enabled_);
    670   if (DatabaseAvailable())
    671     return true;
    672   safe_browsing_thread_->message_loop()->PostTask(
    673       FROM_HERE,
    674       base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
    675                  this));
    676   return false;
    677 }
    678 
    679 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
    680   DCHECK_EQ(base::MessageLoop::current(),
    681             safe_browsing_thread_->message_loop());
    682   if (database_)
    683     return database_;
    684   startup_metric_utils::ScopedSlowStartupUMA
    685       scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
    686   const base::TimeTicks before = base::TimeTicks::Now();
    687 
    688   SafeBrowsingDatabase* database =
    689       SafeBrowsingDatabase::Create(enable_download_protection_,
    690                                    enable_csd_whitelist_,
    691                                    enable_download_whitelist_,
    692                                    enable_extension_blacklist_,
    693                                    enable_side_effect_free_whitelist_,
    694                                    enable_ip_blacklist_);
    695 
    696   database->Init(SafeBrowsingService::GetBaseFilename());
    697   {
    698     // Acquiring the lock here guarantees correct ordering between the writes to
    699     // the new database object above, and the setting of |databse_| below.
    700     base::AutoLock lock(database_lock_);
    701     database_ = database;
    702   }
    703 
    704   BrowserThread::PostTask(
    705       BrowserThread::IO, FROM_HERE,
    706       base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
    707 
    708   UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
    709   return database_;
    710 }
    711 
    712 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
    713   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    714 
    715   if (!enabled_)
    716     return;
    717 
    718   // If the service has been shut down, |check| should have been deleted.
    719   DCHECK(checks_.find(check) != checks_.end());
    720 
    721   if (check->client && check->need_get_hash) {
    722     // We have a partial match so we need to query Google for the full hash.
    723     // Clean up will happen in HandleGetHashResults.
    724 
    725     // See if we have a GetHash request already in progress for this particular
    726     // prefix. If so, we just append ourselves to the list of interested parties
    727     // when the results arrive. We only do this for checks involving one prefix,
    728     // since that is the common case (multiple prefixes will issue the request
    729     // as normal).
    730     if (check->prefix_hits.size() == 1) {
    731       SBPrefix prefix = check->prefix_hits[0];
    732       GetHashRequests::iterator it = gethash_requests_.find(prefix);
    733       if (it != gethash_requests_.end()) {
    734         // There's already a request in progress.
    735         it->second.push_back(check);
    736         return;
    737       }
    738 
    739       // No request in progress, so we're the first for this prefix.
    740       GetHashRequestors requestors;
    741       requestors.push_back(check);
    742       gethash_requests_[prefix] = requestors;
    743     }
    744 
    745     // Reset the start time so that we can measure the network time without the
    746     // database time.
    747     check->start = base::TimeTicks::Now();
    748     // Note: If |this| is deleted or stopped, the protocol_manager will
    749     // be destroyed as well - hence it's OK to do unretained in this case.
    750     bool is_download = check->check_type == safe_browsing_util::BINURL;
    751     sb_service_->protocol_manager()->GetFullHash(
    752         check->prefix_hits,
    753         base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
    754                    base::Unretained(this),
    755                    check),
    756         is_download);
    757   } else {
    758     // We may have cached results for previous GetHash queries.  Since
    759     // this data comes from cache, don't histogram hits.
    760     HandleOneCheck(check, check->cache_hits);
    761   }
    762 }
    763 
    764 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
    765     GetChunksCallback callback) {
    766   DCHECK_EQ(base::MessageLoop::current(),
    767             safe_browsing_thread_->message_loop());
    768 
    769   bool database_error = true;
    770   std::vector<SBListChunkRanges> lists;
    771   DCHECK(!database_update_in_progress_);
    772   database_update_in_progress_ = true;
    773   GetDatabase();  // This guarantees that |database_| is non-NULL.
    774   if (database_->UpdateStarted(&lists)) {
    775     database_error = false;
    776   } else {
    777     database_->UpdateFinished(false);
    778   }
    779 
    780   BrowserThread::PostTask(
    781       BrowserThread::IO, FROM_HERE,
    782       base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
    783                  this, lists, database_error, callback));
    784 }
    785 
    786 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
    787     const std::vector<SBListChunkRanges>& lists, bool database_error,
    788     GetChunksCallback callback) {
    789   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    790   if (enabled_)
    791     callback.Run(lists, database_error);
    792 }
    793 
    794 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
    795     AddChunksCallback callback) {
    796   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    797   if (enabled_)
    798     callback.Run();
    799 }
    800 
    801 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
    802   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    803   if (!enabled_)
    804     return;
    805 
    806   HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
    807   if (queued_checks_.empty())
    808     return;
    809 
    810   // If the database isn't already available, calling CheckUrl() in the loop
    811   // below will add the check back to the queue, and we'll infinite-loop.
    812   DCHECK(DatabaseAvailable());
    813   while (!queued_checks_.empty()) {
    814     QueuedCheck check = queued_checks_.front();
    815     DCHECK(!check.start.is_null());
    816     HISTOGRAM_TIMES("SB.QueueDelay", base::TimeTicks::Now() - check.start);
    817     // If CheckUrl() determines the URL is safe immediately, it doesn't call the
    818     // client's handler function (because normally it's being directly called by
    819     // the client).  Since we're not the client, we have to convey this result.
    820     if (check.client && CheckBrowseUrl(check.url, check.client)) {
    821       SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
    822                                  std::vector<SBFullHash>(),
    823                                  check.client,
    824                                  check.check_type,
    825                                  check.expected_threats);
    826       check.client->OnSafeBrowsingResult(sb_check);
    827     }
    828     queued_checks_.pop_front();
    829   }
    830 }
    831 
    832 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
    833     const std::string& list_name,
    834     scoped_ptr<ScopedVector<SBChunkData> > chunks,
    835     AddChunksCallback callback) {
    836   DCHECK_EQ(base::MessageLoop::current(),
    837             safe_browsing_thread_->message_loop());
    838   if (chunks)
    839     GetDatabase()->InsertChunks(list_name, chunks->get());
    840   BrowserThread::PostTask(
    841       BrowserThread::IO, FROM_HERE,
    842       base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
    843                  callback));
    844 }
    845 
    846 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
    847     scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
    848   DCHECK_EQ(base::MessageLoop::current(),
    849             safe_browsing_thread_->message_loop());
    850   if (chunk_deletes)
    851     GetDatabase()->DeleteChunks(*chunk_deletes);
    852 }
    853 
    854 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
    855     bool update_succeeded) {
    856   DCHECK_EQ(base::MessageLoop::current(),
    857             safe_browsing_thread_->message_loop());
    858   GetDatabase()->UpdateFinished(update_succeeded);
    859   DCHECK(database_update_in_progress_);
    860   database_update_in_progress_ = false;
    861   BrowserThread::PostTask(
    862       BrowserThread::UI, FROM_HERE,
    863       base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
    864                  this, update_succeeded));
    865 }
    866 
    867 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
    868   DCHECK_EQ(base::MessageLoop::current(),
    869             safe_browsing_thread_->message_loop());
    870   DCHECK(closing_database_);
    871 
    872   // Because |closing_database_| is true, nothing on the IO thread will be
    873   // accessing the database, so it's safe to delete and then NULL the pointer.
    874   delete database_;
    875   database_ = NULL;
    876 
    877   // Acquiring the lock here guarantees correct ordering between the resetting
    878   // of |database_| above and of |closing_database_| below, which ensures there
    879   // won't be a window during which the IO thread falsely believes the database
    880   // is available.
    881   base::AutoLock lock(database_lock_);
    882   closing_database_ = false;
    883 }
    884 
    885 void SafeBrowsingDatabaseManager::OnResetDatabase() {
    886   DCHECK_EQ(base::MessageLoop::current(),
    887             safe_browsing_thread_->message_loop());
    888   GetDatabase()->ResetDatabase();
    889 }
    890 
    891 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
    892     SafeBrowsingCheck* check,
    893     const std::vector<SBFullHashResult>& full_hashes) {
    894   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    895   safe_browsing_util::ListType check_type = check->check_type;
    896   SBPrefix prefix = check->prefix_hits[0];
    897   GetHashRequests::iterator it = gethash_requests_.find(prefix);
    898   if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
    899     const bool hit = HandleOneCheck(check, full_hashes);
    900     RecordGetHashCheckStatus(hit, check_type, full_hashes);
    901     return;
    902   }
    903 
    904   // Call back all interested parties, noting if any has a hit.
    905   GetHashRequestors& requestors = it->second;
    906   bool hit = false;
    907   for (GetHashRequestors::iterator r = requestors.begin();
    908        r != requestors.end(); ++r) {
    909     if (HandleOneCheck(*r, full_hashes))
    910       hit = true;
    911   }
    912   RecordGetHashCheckStatus(hit, check_type, full_hashes);
    913 
    914   gethash_requests_.erase(it);
    915 }
    916 
    917 bool SafeBrowsingDatabaseManager::HandleOneCheck(
    918     SafeBrowsingCheck* check,
    919     const std::vector<SBFullHashResult>& full_hashes) {
    920   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    921   DCHECK(check);
    922 
    923   bool is_threat = false;
    924 
    925   // TODO(shess): GetHashThreadListType() contains a loop,
    926   // GetUrlThreatListType() a loop around that loop.  Having another loop out
    927   // here concerns me.  It is likely that SAFE is an expected outcome, which
    928   // means all of those loops run to completion.  Refactoring this to generate a
    929   // set of sorted items to compare in sequence would probably improve things.
    930   //
    931   // Additionally, the set of patterns generated from the urls is very similar
    932   // to the patterns generated in ContainsBrowseUrl() and other database checks,
    933   // which are called from this code.  Refactoring that across the checks could
    934   // interact well with batching the checks here.
    935 
    936   for (size_t i = 0; i < check->urls.size(); ++i) {
    937     SBThreatType threat = GetUrlThreatType(check->urls[i], full_hashes);
    938     if (threat != SB_THREAT_TYPE_SAFE &&
    939         IsExpectedThreat(threat, check->expected_threats)) {
    940       check->url_results[i] = threat;
    941       is_threat = true;
    942     }
    943   }
    944 
    945   for (size_t i = 0; i < check->full_hashes.size(); ++i) {
    946     SBThreatType threat = GetHashThreatType(check->full_hashes[i], full_hashes);
    947     if (threat != SB_THREAT_TYPE_SAFE &&
    948         IsExpectedThreat(threat, check->expected_threats)) {
    949       check->full_hash_results[i] = threat;
    950       is_threat = true;
    951     }
    952   }
    953 
    954   SafeBrowsingCheckDone(check);
    955   return is_threat;
    956 }
    957 
    958 void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
    959     SafeBrowsingCheck* check) {
    960   DCHECK_EQ(base::MessageLoop::current(),
    961             safe_browsing_thread_->message_loop());
    962   DCHECK(enable_download_protection_);
    963 
    964   std::vector<SBPrefix> prefix_hits;
    965 
    966   if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) {
    967     // Good, we don't have hash for this url prefix.
    968     BrowserThread::PostTask(
    969         BrowserThread::IO, FROM_HERE,
    970         base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this,
    971                    check));
    972     return;
    973   }
    974 
    975   check->need_get_hash = true;
    976   check->prefix_hits.clear();
    977   check->prefix_hits = prefix_hits;
    978   BrowserThread::PostTask(
    979       BrowserThread::IO, FROM_HERE,
    980       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
    981 }
    982 
    983 void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
    984     SafeBrowsingCheck* check) {
    985   DCHECK_EQ(base::MessageLoop::current(),
    986             safe_browsing_thread_->message_loop());
    987 
    988   std::vector<SBPrefix> prefixes;
    989   for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin();
    990        it != check->full_hashes.end(); ++it) {
    991     prefixes.push_back((*it).prefix);
    992   }
    993   database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits);
    994 
    995   if (check->prefix_hits.empty()) {
    996     // No matches for any extensions.
    997     BrowserThread::PostTask(
    998         BrowserThread::IO,
    999         FROM_HERE,
   1000         base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this,
   1001                    check));
   1002   } else {
   1003     // Some prefixes matched, we need to ask Google whether they're legit.
   1004     check->need_get_hash = true;
   1005     BrowserThread::PostTask(
   1006         BrowserThread::IO,
   1007         FROM_HERE,
   1008         base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
   1009   }
   1010 }
   1011 
   1012 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
   1013   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   1014   DCHECK(check);
   1015 
   1016   if (!enabled_)
   1017     return;
   1018 
   1019   DCHECK(checks_.find(check) != checks_.end());
   1020   if (check->client) {
   1021     check->client->OnSafeBrowsingResult(*check);
   1022     check->client = NULL;
   1023   }
   1024 }
   1025 
   1026 void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
   1027     SafeBrowsingCheck* check) {
   1028   DCHECK(enable_download_protection_);
   1029   SafeBrowsingCheckDone(check);
   1030 }
   1031 
   1032 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
   1033     SafeBrowsingCheck* check) {
   1034   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   1035   DCHECK(check);
   1036 
   1037   if (!enabled_)
   1038     return;
   1039 
   1040   VLOG(1) << "SafeBrowsingCheckDone";
   1041   DCHECK(checks_.find(check) != checks_.end());
   1042   if (check->client)
   1043     check->client->OnSafeBrowsingResult(*check);
   1044   checks_.erase(check);
   1045   delete check;
   1046 }
   1047 
   1048 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
   1049     SafeBrowsingCheck* check,
   1050     const base::Closure& task) {
   1051   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   1052   check->timeout_factory_.reset(
   1053       new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
   1054   checks_.insert(check);
   1055 
   1056   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task);
   1057 
   1058   base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
   1059       base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
   1060                  check->timeout_factory_->GetWeakPtr(), check),
   1061       check_timeout_);
   1062 }
   1063