Home | History | Annotate | Download | only in safe_browsing 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/safe_browsing/database_manager.h"
      6 
      7 #include <algorithm>
      8 
      9 #include "base/bind.h"
     10 #include "base/bind_helpers.h"
     11 #include "base/callback.h"
     12 #include "base/command_line.h"
     13 #include "base/debug/leak_tracker.h"
     14 #include "base/path_service.h"
     15 #include "base/stl_util.h"
     16 #include "base/strings/string_util.h"
     17 #include "base/threading/thread.h"
     18 #include "base/threading/thread_restrictions.h"
     19 #include "chrome/browser/browser_process.h"
     20 #include "chrome/browser/chrome_notification_types.h"
     21 #include "chrome/browser/prerender/prerender_field_trial.h"
     22 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
     23 #include "chrome/browser/safe_browsing/download_protection_service.h"
     24 #include "chrome/browser/safe_browsing/malware_details.h"
     25 #include "chrome/browser/safe_browsing/protocol_manager.h"
     26 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
     27 #include "chrome/browser/safe_browsing/safe_browsing_service.h"
     28 #include "chrome/browser/safe_browsing/ui_manager.h"
     29 #include "chrome/common/chrome_constants.h"
     30 #include "chrome/common/chrome_paths.h"
     31 #include "chrome/common/chrome_switches.h"
     32 #include "components/metrics/metrics_service.h"
     33 #include "components/startup_metric_utils/startup_metric_utils.h"
     34 #include "content/public/browser/browser_thread.h"
     35 #include "content/public/browser/notification_service.h"
     36 #include "url/url_constants.h"
     37 
     38 using content::BrowserThread;
     39 
     40 namespace {
     41 
     42 // Timeout for match checks, e.g. download URLs, hashes.
     43 const int kCheckTimeoutMs = 10000;
     44 
     45 // Records disposition information about the check.  |hit| should be
     46 // |true| if there were any prefix hits in |full_hashes|.
     47 void RecordGetHashCheckStatus(
     48     bool hit,
     49     safe_browsing_util::ListType check_type,
     50     const std::vector<SBFullHashResult>& full_hashes) {
     51   SafeBrowsingProtocolManager::ResultType result;
     52   if (full_hashes.empty()) {
     53     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_EMPTY;
     54   } else if (hit) {
     55     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_HIT;
     56   } else {
     57     result = SafeBrowsingProtocolManager::GET_HASH_FULL_HASH_MISS;
     58   }
     59   bool is_download = check_type == safe_browsing_util::BINURL;
     60   SafeBrowsingProtocolManager::RecordGetHashResult(is_download, result);
     61 }
     62 
     63 bool IsExpectedThreat(
     64     const SBThreatType threat_type,
     65     const std::vector<SBThreatType>& expected_threats) {
     66   return expected_threats.end() != std::find(expected_threats.begin(),
     67                                              expected_threats.end(),
     68                                              threat_type);
     69 }
     70 
     71 // Return the list id from the first result in |full_hashes| which matches
     72 // |hash|, or INVALID if none match.
     73 safe_browsing_util::ListType GetHashThreatListType(
     74     const SBFullHash& hash,
     75     const std::vector<SBFullHashResult>& full_hashes,
     76     size_t* index) {
     77   for (size_t i = 0; i < full_hashes.size(); ++i) {
     78     if (SBFullHashEqual(hash, full_hashes[i].hash)) {
     79       if (index)
     80         *index = i;
     81       return static_cast<safe_browsing_util::ListType>(full_hashes[i].list_id);
     82     }
     83   }
     84   return safe_browsing_util::INVALID;
     85 }
     86 
     87 // Given a URL, compare all the possible host + path full hashes to the set of
     88 // provided full hashes.  Returns the list id of the a matching result from
     89 // |full_hashes|, or INVALID if none match.
     90 safe_browsing_util::ListType GetUrlThreatListType(
     91     const GURL& url,
     92     const std::vector<SBFullHashResult>& full_hashes,
     93     size_t* index) {
     94   if (full_hashes.empty())
     95     return safe_browsing_util::INVALID;
     96 
     97   std::vector<std::string> patterns;
     98   safe_browsing_util::GeneratePatternsToCheck(url, &patterns);
     99 
    100   for (size_t i = 0; i < patterns.size(); ++i) {
    101     safe_browsing_util::ListType threat = GetHashThreatListType(
    102         SBFullHashForString(patterns[i]), full_hashes, index);
    103     if (threat != safe_browsing_util::INVALID)
    104       return threat;
    105   }
    106   return safe_browsing_util::INVALID;
    107 }
    108 
    109 SBThreatType GetThreatTypeFromListType(safe_browsing_util::ListType list_type) {
    110   switch (list_type) {
    111     case safe_browsing_util::PHISH:
    112       return SB_THREAT_TYPE_URL_PHISHING;
    113     case safe_browsing_util::MALWARE:
    114       return SB_THREAT_TYPE_URL_MALWARE;
    115     case safe_browsing_util::BINURL:
    116       return SB_THREAT_TYPE_BINARY_MALWARE_URL;
    117     case safe_browsing_util::EXTENSIONBLACKLIST:
    118       return SB_THREAT_TYPE_EXTENSION;
    119     default:
    120       DVLOG(1) << "Unknown safe browsing list id " << list_type;
    121       return SB_THREAT_TYPE_SAFE;
    122   }
    123 }
    124 
    125 }  // namespace
    126 
    127 // static
    128 SBThreatType SafeBrowsingDatabaseManager::GetHashThreatType(
    129     const SBFullHash& hash,
    130     const std::vector<SBFullHashResult>& full_hashes) {
    131   return GetThreatTypeFromListType(
    132       GetHashThreatListType(hash, full_hashes, NULL));
    133 }
    134 
    135 // static
    136 SBThreatType SafeBrowsingDatabaseManager::GetUrlThreatType(
    137     const GURL& url,
    138     const std::vector<SBFullHashResult>& full_hashes,
    139     size_t* index) {
    140   return GetThreatTypeFromListType(
    141       GetUrlThreatListType(url, full_hashes, index));
    142 }
    143 
    144 SafeBrowsingDatabaseManager::SafeBrowsingCheck::SafeBrowsingCheck(
    145     const std::vector<GURL>& urls,
    146     const std::vector<SBFullHash>& full_hashes,
    147     Client* client,
    148     safe_browsing_util::ListType check_type,
    149     const std::vector<SBThreatType>& expected_threats)
    150     : urls(urls),
    151       url_results(urls.size(), SB_THREAT_TYPE_SAFE),
    152       url_metadata(urls.size()),
    153       full_hashes(full_hashes),
    154       full_hash_results(full_hashes.size(), SB_THREAT_TYPE_SAFE),
    155       client(client),
    156       need_get_hash(false),
    157       check_type(check_type),
    158       expected_threats(expected_threats) {
    159   DCHECK_EQ(urls.empty(), !full_hashes.empty())
    160       << "Exactly one of urls and full_hashes must be set";
    161 }
    162 
    163 SafeBrowsingDatabaseManager::SafeBrowsingCheck::~SafeBrowsingCheck() {}
    164 
    165 void SafeBrowsingDatabaseManager::Client::OnSafeBrowsingResult(
    166     const SafeBrowsingCheck& check) {
    167   DCHECK_EQ(check.urls.size(), check.url_results.size());
    168   DCHECK_EQ(check.full_hashes.size(), check.full_hash_results.size());
    169   if (!check.urls.empty()) {
    170     DCHECK(check.full_hashes.empty());
    171     switch (check.check_type) {
    172       case safe_browsing_util::MALWARE:
    173       case safe_browsing_util::PHISH:
    174         DCHECK_EQ(1u, check.urls.size());
    175         OnCheckBrowseUrlResult(
    176             check.urls[0], check.url_results[0], check.url_metadata[0]);
    177         break;
    178       case safe_browsing_util::BINURL:
    179         DCHECK_EQ(check.urls.size(), check.url_results.size());
    180         OnCheckDownloadUrlResult(
    181             check.urls,
    182             *std::max_element(check.url_results.begin(),
    183                               check.url_results.end()));
    184         break;
    185       default:
    186         NOTREACHED();
    187     }
    188   } else if (!check.full_hashes.empty()) {
    189     switch (check.check_type) {
    190       case safe_browsing_util::EXTENSIONBLACKLIST: {
    191         std::set<std::string> unsafe_extension_ids;
    192         for (size_t i = 0; i < check.full_hashes.size(); ++i) {
    193           std::string extension_id =
    194               safe_browsing_util::SBFullHashToString(check.full_hashes[i]);
    195           if (check.full_hash_results[i] == SB_THREAT_TYPE_EXTENSION)
    196             unsafe_extension_ids.insert(extension_id);
    197         }
    198         OnCheckExtensionsResult(unsafe_extension_ids);
    199         break;
    200       }
    201       default:
    202         NOTREACHED();
    203     }
    204   } else {
    205     NOTREACHED();
    206   }
    207 }
    208 
    209 SafeBrowsingDatabaseManager::SafeBrowsingDatabaseManager(
    210     const scoped_refptr<SafeBrowsingService>& service)
    211     : sb_service_(service),
    212       database_(NULL),
    213       enabled_(false),
    214       enable_download_protection_(false),
    215       enable_csd_whitelist_(false),
    216       enable_download_whitelist_(false),
    217       enable_extension_blacklist_(false),
    218       enable_side_effect_free_whitelist_(false),
    219       enable_ip_blacklist_(false),
    220       update_in_progress_(false),
    221       database_update_in_progress_(false),
    222       closing_database_(false),
    223       check_timeout_(base::TimeDelta::FromMilliseconds(kCheckTimeoutMs)) {
    224   DCHECK(sb_service_.get() != NULL);
    225 
    226   // Android only supports a subset of FULL_SAFE_BROWSING.
    227   // TODO(shess): This shouldn't be OS-driven <http://crbug.com/394379>
    228 #if !defined(OS_ANDROID)
    229   CommandLine* cmdline = CommandLine::ForCurrentProcess();
    230   enable_download_protection_ =
    231       !cmdline->HasSwitch(switches::kSbDisableDownloadProtection);
    232 
    233   // We only download the csd-whitelist if client-side phishing detection is
    234   // enabled.
    235   enable_csd_whitelist_ =
    236       !cmdline->HasSwitch(switches::kDisableClientSidePhishingDetection);
    237 
    238   // TODO(noelutz): remove this boolean variable since it should always be true
    239   // if SafeBrowsing is enabled.  Unfortunately, we have no test data for this
    240   // list right now.  This means that we need to be able to disable this list
    241   // for the SafeBrowsing test to pass.
    242   enable_download_whitelist_ = enable_csd_whitelist_;
    243 
    244   // TODO(kalman): there really shouldn't be a flag for this.
    245   enable_extension_blacklist_ =
    246       !cmdline->HasSwitch(switches::kSbDisableExtensionBlacklist);
    247 
    248   enable_side_effect_free_whitelist_ =
    249       prerender::IsSideEffectFreeWhitelistEnabled() &&
    250       !cmdline->HasSwitch(switches::kSbDisableSideEffectFreeWhitelist);
    251 
    252   // The client-side IP blacklist feature is tightly integrated with client-side
    253   // phishing protection for now.
    254   enable_ip_blacklist_ = enable_csd_whitelist_;
    255 
    256   enum SideEffectFreeWhitelistStatus {
    257     SIDE_EFFECT_FREE_WHITELIST_ENABLED,
    258     SIDE_EFFECT_FREE_WHITELIST_DISABLED,
    259     SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX
    260   };
    261 
    262   SideEffectFreeWhitelistStatus side_effect_free_whitelist_status =
    263       enable_side_effect_free_whitelist_ ? SIDE_EFFECT_FREE_WHITELIST_ENABLED :
    264       SIDE_EFFECT_FREE_WHITELIST_DISABLED;
    265 
    266   UMA_HISTOGRAM_ENUMERATION("SB2.SideEffectFreeWhitelistStatus",
    267                             side_effect_free_whitelist_status,
    268                             SIDE_EFFECT_FREE_WHITELIST_STATUS_MAX);
    269 #endif
    270 }
    271 
    272 SafeBrowsingDatabaseManager::~SafeBrowsingDatabaseManager() {
    273   // We should have already been shut down. If we're still enabled, then the
    274   // database isn't going to be closed properly, which could lead to corruption.
    275   DCHECK(!enabled_);
    276 }
    277 
    278 bool SafeBrowsingDatabaseManager::CanCheckUrl(const GURL& url) const {
    279   return url.SchemeIs(url::kFtpScheme) ||
    280          url.SchemeIs(url::kHttpScheme) ||
    281          url.SchemeIs(url::kHttpsScheme);
    282 }
    283 
    284 bool SafeBrowsingDatabaseManager::CheckDownloadUrl(
    285     const std::vector<GURL>& url_chain,
    286     Client* client) {
    287   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    288   if (!enabled_ || !enable_download_protection_)
    289     return true;
    290 
    291   // We need to check the database for url prefix, and later may fetch the url
    292   // from the safebrowsing backends. These need to be asynchronous.
    293   SafeBrowsingCheck* check =
    294       new SafeBrowsingCheck(url_chain,
    295                             std::vector<SBFullHash>(),
    296                             client,
    297                             safe_browsing_util::BINURL,
    298                             std::vector<SBThreatType>(1,
    299                                 SB_THREAT_TYPE_BINARY_MALWARE_URL));
    300   StartSafeBrowsingCheck(
    301       check,
    302       base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread, this,
    303                  check));
    304   return false;
    305 }
    306 
    307 bool SafeBrowsingDatabaseManager::CheckExtensionIDs(
    308     const std::set<std::string>& extension_ids,
    309     Client* client) {
    310   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    311 
    312   if (!enabled_ || !enable_extension_blacklist_)
    313     return true;
    314 
    315   std::vector<SBFullHash> extension_id_hashes;
    316   std::transform(extension_ids.begin(), extension_ids.end(),
    317                  std::back_inserter(extension_id_hashes),
    318                  safe_browsing_util::StringToSBFullHash);
    319 
    320   SafeBrowsingCheck* check = new SafeBrowsingCheck(
    321       std::vector<GURL>(),
    322       extension_id_hashes,
    323       client,
    324       safe_browsing_util::EXTENSIONBLACKLIST,
    325       std::vector<SBThreatType>(1, SB_THREAT_TYPE_EXTENSION));
    326 
    327   StartSafeBrowsingCheck(
    328       check,
    329       base::Bind(&SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread,
    330                  this,
    331                  check));
    332   return false;
    333 }
    334 
    335 bool SafeBrowsingDatabaseManager::CheckSideEffectFreeWhitelistUrl(
    336     const GURL& url) {
    337   if (!enabled_)
    338     return false;
    339 
    340   if (!CanCheckUrl(url))
    341     return false;
    342 
    343   return database_->ContainsSideEffectFreeWhitelistUrl(url);
    344 }
    345 
    346 bool SafeBrowsingDatabaseManager::MatchMalwareIP(
    347     const std::string& ip_address) {
    348   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    349   if (!enabled_ || !enable_ip_blacklist_ || !MakeDatabaseAvailable()) {
    350     return false;  // Fail open.
    351   }
    352   return database_->ContainsMalwareIP(ip_address);
    353 }
    354 
    355 bool SafeBrowsingDatabaseManager::MatchCsdWhitelistUrl(const GURL& url) {
    356   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    357   if (!enabled_ || !enable_csd_whitelist_ || !MakeDatabaseAvailable()) {
    358     // There is something funky going on here -- for example, perhaps the user
    359     // has not restarted since enabling metrics reporting, so we haven't
    360     // enabled the csd whitelist yet.  Just to be safe we return true in this
    361     // case.
    362     return true;
    363   }
    364   return database_->ContainsCsdWhitelistedUrl(url);
    365 }
    366 
    367 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistUrl(const GURL& url) {
    368   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    369   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
    370     return true;
    371   }
    372   return database_->ContainsDownloadWhitelistedUrl(url);
    373 }
    374 
    375 bool SafeBrowsingDatabaseManager::MatchDownloadWhitelistString(
    376     const std::string& str) {
    377   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    378   if (!enabled_ || !enable_download_whitelist_ || !MakeDatabaseAvailable()) {
    379     return true;
    380   }
    381   return database_->ContainsDownloadWhitelistedString(str);
    382 }
    383 
    384 bool SafeBrowsingDatabaseManager::IsMalwareKillSwitchOn() {
    385   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    386   if (!enabled_ || !MakeDatabaseAvailable()) {
    387     return true;
    388   }
    389   return database_->IsMalwareIPMatchKillSwitchOn();
    390 }
    391 
    392 bool SafeBrowsingDatabaseManager::IsCsdWhitelistKillSwitchOn() {
    393   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    394   if (!enabled_ || !MakeDatabaseAvailable()) {
    395     return true;
    396   }
    397   return database_->IsCsdWhitelistKillSwitchOn();
    398 }
    399 
    400 bool SafeBrowsingDatabaseManager::CheckBrowseUrl(const GURL& url,
    401                                                  Client* client) {
    402   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    403   if (!enabled_)
    404     return true;
    405 
    406   if (!CanCheckUrl(url))
    407     return true;
    408 
    409   std::vector<SBThreatType> expected_threats;
    410   expected_threats.push_back(SB_THREAT_TYPE_URL_MALWARE);
    411   expected_threats.push_back(SB_THREAT_TYPE_URL_PHISHING);
    412 
    413   const base::TimeTicks start = base::TimeTicks::Now();
    414   if (!MakeDatabaseAvailable()) {
    415     QueuedCheck queued_check(safe_browsing_util::MALWARE,  // or PHISH
    416                              client,
    417                              url,
    418                              expected_threats,
    419                              start);
    420     queued_checks_.push_back(queued_check);
    421     return false;
    422   }
    423 
    424   std::vector<SBPrefix> prefix_hits;
    425   std::vector<SBFullHashResult> cache_hits;
    426 
    427   bool prefix_match =
    428       database_->ContainsBrowseUrl(url, &prefix_hits, &cache_hits);
    429 
    430   UMA_HISTOGRAM_TIMES("SB2.FilterCheck", base::TimeTicks::Now() - start);
    431 
    432   if (!prefix_match)
    433     return true;  // URL is okay.
    434 
    435   // Needs to be asynchronous, since we could be in the constructor of a
    436   // ResourceDispatcherHost event handler which can't pause there.
    437   SafeBrowsingCheck* check = new SafeBrowsingCheck(std::vector<GURL>(1, url),
    438                                                    std::vector<SBFullHash>(),
    439                                                    client,
    440                                                    safe_browsing_util::MALWARE,
    441                                                    expected_threats);
    442   check->need_get_hash = cache_hits.empty();
    443   check->prefix_hits.swap(prefix_hits);
    444   check->cache_hits.swap(cache_hits);
    445   checks_.insert(check);
    446 
    447   BrowserThread::PostTask(
    448       BrowserThread::IO, FROM_HERE,
    449       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
    450 
    451   return false;
    452 }
    453 
    454 void SafeBrowsingDatabaseManager::CancelCheck(Client* client) {
    455   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    456   for (CurrentChecks::iterator i = checks_.begin(); i != checks_.end(); ++i) {
    457     // We can't delete matching checks here because the db thread has a copy of
    458     // the pointer.  Instead, we simply NULL out the client, and when the db
    459     // thread calls us back, we'll clean up the check.
    460     if ((*i)->client == client)
    461       (*i)->client = NULL;
    462   }
    463 
    464   // Scan the queued clients store. Clients may be here if they requested a URL
    465   // check before the database has finished loading.
    466   for (std::deque<QueuedCheck>::iterator it(queued_checks_.begin());
    467        it != queued_checks_.end(); ) {
    468     // In this case it's safe to delete matches entirely since nothing has a
    469     // pointer to them.
    470     if (it->client == client)
    471       it = queued_checks_.erase(it);
    472     else
    473       ++it;
    474   }
    475 }
    476 
    477 void SafeBrowsingDatabaseManager::HandleGetHashResults(
    478     SafeBrowsingCheck* check,
    479     const std::vector<SBFullHashResult>& full_hashes,
    480     const base::TimeDelta& cache_lifetime) {
    481   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    482 
    483   if (!enabled_)
    484     return;
    485 
    486   // If the service has been shut down, |check| should have been deleted.
    487   DCHECK(checks_.find(check) != checks_.end());
    488 
    489   // |start| is set before calling |GetFullHash()|, which should be
    490   // the only path which gets to here.
    491   DCHECK(!check->start.is_null());
    492   UMA_HISTOGRAM_LONG_TIMES("SB2.Network",
    493                            base::TimeTicks::Now() - check->start);
    494 
    495   std::vector<SBPrefix> prefixes = check->prefix_hits;
    496   OnHandleGetHashResults(check, full_hashes);  // 'check' is deleted here.
    497 
    498   // Cache the GetHash results.
    499   if (cache_lifetime != base::TimeDelta() && MakeDatabaseAvailable())
    500     database_->CacheHashResults(prefixes, full_hashes, cache_lifetime);
    501 }
    502 
    503 void SafeBrowsingDatabaseManager::GetChunks(GetChunksCallback callback) {
    504   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    505   DCHECK(enabled_);
    506   DCHECK(!callback.is_null());
    507   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    508       &SafeBrowsingDatabaseManager::GetAllChunksFromDatabase, this, callback));
    509 }
    510 
    511 void SafeBrowsingDatabaseManager::AddChunks(
    512     const std::string& list,
    513     scoped_ptr<ScopedVector<SBChunkData> > chunks,
    514     AddChunksCallback callback) {
    515   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    516   DCHECK(enabled_);
    517   DCHECK(!callback.is_null());
    518   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    519       &SafeBrowsingDatabaseManager::AddDatabaseChunks, this, list,
    520       base::Passed(&chunks), callback));
    521 }
    522 
    523 void SafeBrowsingDatabaseManager::DeleteChunks(
    524     scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
    525   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    526   DCHECK(enabled_);
    527   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    528       &SafeBrowsingDatabaseManager::DeleteDatabaseChunks, this,
    529       base::Passed(&chunk_deletes)));
    530 }
    531 
    532 void SafeBrowsingDatabaseManager::UpdateStarted() {
    533   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    534   DCHECK(enabled_);
    535   DCHECK(!update_in_progress_);
    536   update_in_progress_ = true;
    537 }
    538 
    539 void SafeBrowsingDatabaseManager::UpdateFinished(bool update_succeeded) {
    540   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    541   DCHECK(enabled_);
    542   if (update_in_progress_) {
    543     update_in_progress_ = false;
    544     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
    545       base::Bind(&SafeBrowsingDatabaseManager::DatabaseUpdateFinished,
    546                  this, update_succeeded));
    547   }
    548 }
    549 
    550 void SafeBrowsingDatabaseManager::ResetDatabase() {
    551   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    552   DCHECK(enabled_);
    553   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
    554       &SafeBrowsingDatabaseManager::OnResetDatabase, this));
    555 }
    556 
    557 void SafeBrowsingDatabaseManager::LogPauseDelay(base::TimeDelta time) {
    558   UMA_HISTOGRAM_LONG_TIMES("SB2.Delay", time);
    559 }
    560 
    561 void SafeBrowsingDatabaseManager::StartOnIOThread() {
    562   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    563   if (enabled_)
    564     return;
    565 
    566   DCHECK(!safe_browsing_thread_.get());
    567   safe_browsing_thread_.reset(new base::Thread("Chrome_SafeBrowsingThread"));
    568   if (!safe_browsing_thread_->Start())
    569     return;
    570   enabled_ = true;
    571 
    572   MakeDatabaseAvailable();
    573 }
    574 
    575 void SafeBrowsingDatabaseManager::StopOnIOThread(bool shutdown) {
    576   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    577 
    578   DoStopOnIOThread();
    579   if (shutdown) {
    580     sb_service_ = NULL;
    581   }
    582 }
    583 
    584 void SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished(
    585     bool update_succeeded) {
    586   content::NotificationService::current()->Notify(
    587       chrome::NOTIFICATION_SAFE_BROWSING_UPDATE_COMPLETE,
    588       content::Source<SafeBrowsingDatabaseManager>(this),
    589       content::Details<bool>(&update_succeeded));
    590 }
    591 
    592 SafeBrowsingDatabaseManager::QueuedCheck::QueuedCheck(
    593     const safe_browsing_util::ListType check_type,
    594     Client* client,
    595     const GURL& url,
    596     const std::vector<SBThreatType>& expected_threats,
    597     const base::TimeTicks& start)
    598     : check_type(check_type),
    599       client(client),
    600       url(url),
    601       expected_threats(expected_threats),
    602       start(start) {
    603 }
    604 
    605 SafeBrowsingDatabaseManager::QueuedCheck::~QueuedCheck() {
    606 }
    607 
    608 void SafeBrowsingDatabaseManager::DoStopOnIOThread() {
    609   if (!enabled_)
    610     return;
    611 
    612   enabled_ = false;
    613 
    614   // Delete queued checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
    615   while (!queued_checks_.empty()) {
    616     QueuedCheck queued = queued_checks_.front();
    617     if (queued.client) {
    618       SafeBrowsingCheck sb_check(std::vector<GURL>(1, queued.url),
    619                                  std::vector<SBFullHash>(),
    620                                  queued.client,
    621                                  queued.check_type,
    622                                  queued.expected_threats);
    623       queued.client->OnSafeBrowsingResult(sb_check);
    624     }
    625     queued_checks_.pop_front();
    626   }
    627 
    628   // Close the database.  Cases to avoid:
    629   //  * If |closing_database_| is true, continuing will queue up a second
    630   //    request, |closing_database_| will be reset after handling the first
    631   //    request, and if any functions on the db thread recreate the database, we
    632   //    could start using it on the IO thread and then have the second request
    633   //    handler delete it out from under us.
    634   //  * If |database_| is NULL, then either no creation request is in flight, in
    635   //    which case we don't need to do anything, or one is in flight, in which
    636   //    case the database will be recreated before our deletion request is
    637   //    handled, and could be used on the IO thread in that time period, leading
    638   //    to the same problem as above.
    639   // Checking DatabaseAvailable() avoids both of these.
    640   if (DatabaseAvailable()) {
    641     closing_database_ = true;
    642     safe_browsing_thread_->message_loop()->PostTask(FROM_HERE,
    643         base::Bind(&SafeBrowsingDatabaseManager::OnCloseDatabase, this));
    644   }
    645 
    646   // Flush the database thread. Any in-progress database check results will be
    647   // ignored and cleaned up below.
    648   //
    649   // Note that to avoid leaking the database, we rely on the fact that no new
    650   // tasks will be added to the db thread between the call above and this one.
    651   // See comments on the declaration of |safe_browsing_thread_|.
    652   {
    653     // A ScopedAllowIO object is required to join the thread when calling Stop.
    654     // See http://crbug.com/72696. Note that we call Stop() first to clear out
    655     // any remaining tasks before clearing safe_browsing_thread_.
    656     base::ThreadRestrictions::ScopedAllowIO allow_io_for_thread_join;
    657     safe_browsing_thread_->Stop();
    658     safe_browsing_thread_.reset();
    659   }
    660 
    661   // Delete pending checks, calling back any clients with 'SB_THREAT_TYPE_SAFE'.
    662   // We have to do this after the db thread returns because methods on it can
    663   // have copies of these pointers, so deleting them might lead to accessing
    664   // garbage.
    665   for (CurrentChecks::iterator it = checks_.begin();
    666        it != checks_.end(); ++it) {
    667     SafeBrowsingCheck* check = *it;
    668     if (check->client)
    669       check->client->OnSafeBrowsingResult(*check);
    670   }
    671   STLDeleteElements(&checks_);
    672 
    673   gethash_requests_.clear();
    674 }
    675 
    676 bool SafeBrowsingDatabaseManager::DatabaseAvailable() const {
    677   base::AutoLock lock(database_lock_);
    678   return !closing_database_ && (database_ != NULL);
    679 }
    680 
    681 bool SafeBrowsingDatabaseManager::MakeDatabaseAvailable() {
    682   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    683   DCHECK(enabled_);
    684   if (DatabaseAvailable())
    685     return true;
    686   safe_browsing_thread_->message_loop()->PostTask(
    687       FROM_HERE,
    688       base::Bind(base::IgnoreResult(&SafeBrowsingDatabaseManager::GetDatabase),
    689                  this));
    690   return false;
    691 }
    692 
    693 SafeBrowsingDatabase* SafeBrowsingDatabaseManager::GetDatabase() {
    694   DCHECK_EQ(base::MessageLoop::current(),
    695             safe_browsing_thread_->message_loop());
    696   if (database_)
    697     return database_;
    698   startup_metric_utils::ScopedSlowStartupUMA
    699       scoped_timer("Startup.SlowStartupSafeBrowsingGetDatabase");
    700   const base::TimeTicks before = base::TimeTicks::Now();
    701 
    702   SafeBrowsingDatabase* database =
    703       SafeBrowsingDatabase::Create(enable_download_protection_,
    704                                    enable_csd_whitelist_,
    705                                    enable_download_whitelist_,
    706                                    enable_extension_blacklist_,
    707                                    enable_side_effect_free_whitelist_,
    708                                    enable_ip_blacklist_);
    709 
    710   database->Init(SafeBrowsingService::GetBaseFilename());
    711   {
    712     // Acquiring the lock here guarantees correct ordering between the writes to
    713     // the new database object above, and the setting of |databse_| below.
    714     base::AutoLock lock(database_lock_);
    715     database_ = database;
    716   }
    717 
    718   BrowserThread::PostTask(
    719       BrowserThread::IO, FROM_HERE,
    720       base::Bind(&SafeBrowsingDatabaseManager::DatabaseLoadComplete, this));
    721 
    722   UMA_HISTOGRAM_TIMES("SB2.DatabaseOpen", base::TimeTicks::Now() - before);
    723   return database_;
    724 }
    725 
    726 void SafeBrowsingDatabaseManager::OnCheckDone(SafeBrowsingCheck* check) {
    727   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    728 
    729   if (!enabled_)
    730     return;
    731 
    732   // If the service has been shut down, |check| should have been deleted.
    733   DCHECK(checks_.find(check) != checks_.end());
    734 
    735   if (check->client && check->need_get_hash) {
    736     // We have a partial match so we need to query Google for the full hash.
    737     // Clean up will happen in HandleGetHashResults.
    738 
    739     // See if we have a GetHash request already in progress for this particular
    740     // prefix. If so, we just append ourselves to the list of interested parties
    741     // when the results arrive. We only do this for checks involving one prefix,
    742     // since that is the common case (multiple prefixes will issue the request
    743     // as normal).
    744     if (check->prefix_hits.size() == 1) {
    745       SBPrefix prefix = check->prefix_hits[0];
    746       GetHashRequests::iterator it = gethash_requests_.find(prefix);
    747       if (it != gethash_requests_.end()) {
    748         // There's already a request in progress.
    749         it->second.push_back(check);
    750         return;
    751       }
    752 
    753       // No request in progress, so we're the first for this prefix.
    754       GetHashRequestors requestors;
    755       requestors.push_back(check);
    756       gethash_requests_[prefix] = requestors;
    757     }
    758 
    759     // Reset the start time so that we can measure the network time without the
    760     // database time.
    761     check->start = base::TimeTicks::Now();
    762     // Note: If |this| is deleted or stopped, the protocol_manager will
    763     // be destroyed as well - hence it's OK to do unretained in this case.
    764     bool is_download = check->check_type == safe_browsing_util::BINURL;
    765     sb_service_->protocol_manager()->GetFullHash(
    766         check->prefix_hits,
    767         base::Bind(&SafeBrowsingDatabaseManager::HandleGetHashResults,
    768                    base::Unretained(this),
    769                    check),
    770         is_download);
    771   } else {
    772     // We may have cached results for previous GetHash queries.  Since
    773     // this data comes from cache, don't histogram hits.
    774     bool is_threat = HandleOneCheck(check, check->cache_hits);
    775     // cache_hits should only contain hits for a fullhash we searched for, so if
    776     // we got to this point it should always result in a threat match.
    777     DCHECK(is_threat);
    778   }
    779 }
    780 
    781 void SafeBrowsingDatabaseManager::GetAllChunksFromDatabase(
    782     GetChunksCallback callback) {
    783   DCHECK_EQ(base::MessageLoop::current(),
    784             safe_browsing_thread_->message_loop());
    785 
    786   bool database_error = true;
    787   std::vector<SBListChunkRanges> lists;
    788   DCHECK(!database_update_in_progress_);
    789   database_update_in_progress_ = true;
    790   GetDatabase();  // This guarantees that |database_| is non-NULL.
    791   if (database_->UpdateStarted(&lists)) {
    792     database_error = false;
    793   } else {
    794     database_->UpdateFinished(false);
    795   }
    796 
    797   BrowserThread::PostTask(
    798       BrowserThread::IO, FROM_HERE,
    799       base::Bind(&SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase,
    800                  this, lists, database_error, callback));
    801 }
    802 
    803 void SafeBrowsingDatabaseManager::OnGetAllChunksFromDatabase(
    804     const std::vector<SBListChunkRanges>& lists, bool database_error,
    805     GetChunksCallback callback) {
    806   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    807   if (enabled_)
    808     callback.Run(lists, database_error);
    809 }
    810 
    811 void SafeBrowsingDatabaseManager::OnAddChunksComplete(
    812     AddChunksCallback callback) {
    813   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    814   if (enabled_)
    815     callback.Run();
    816 }
    817 
    818 void SafeBrowsingDatabaseManager::DatabaseLoadComplete() {
    819   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    820   if (!enabled_)
    821     return;
    822 
    823   LOCAL_HISTOGRAM_COUNTS("SB.QueueDepth", queued_checks_.size());
    824   if (queued_checks_.empty())
    825     return;
    826 
    827   // If the database isn't already available, calling CheckUrl() in the loop
    828   // below will add the check back to the queue, and we'll infinite-loop.
    829   DCHECK(DatabaseAvailable());
    830   while (!queued_checks_.empty()) {
    831     QueuedCheck check = queued_checks_.front();
    832     DCHECK(!check.start.is_null());
    833     LOCAL_HISTOGRAM_TIMES("SB.QueueDelay",
    834                           base::TimeTicks::Now() - check.start);
    835     // If CheckUrl() determines the URL is safe immediately, it doesn't call the
    836     // client's handler function (because normally it's being directly called by
    837     // the client).  Since we're not the client, we have to convey this result.
    838     if (check.client && CheckBrowseUrl(check.url, check.client)) {
    839       SafeBrowsingCheck sb_check(std::vector<GURL>(1, check.url),
    840                                  std::vector<SBFullHash>(),
    841                                  check.client,
    842                                  check.check_type,
    843                                  check.expected_threats);
    844       check.client->OnSafeBrowsingResult(sb_check);
    845     }
    846     queued_checks_.pop_front();
    847   }
    848 }
    849 
    850 void SafeBrowsingDatabaseManager::AddDatabaseChunks(
    851     const std::string& list_name,
    852     scoped_ptr<ScopedVector<SBChunkData> > chunks,
    853     AddChunksCallback callback) {
    854   DCHECK_EQ(base::MessageLoop::current(),
    855             safe_browsing_thread_->message_loop());
    856   if (chunks)
    857     GetDatabase()->InsertChunks(list_name, chunks->get());
    858   BrowserThread::PostTask(
    859       BrowserThread::IO, FROM_HERE,
    860       base::Bind(&SafeBrowsingDatabaseManager::OnAddChunksComplete, this,
    861                  callback));
    862 }
    863 
    864 void SafeBrowsingDatabaseManager::DeleteDatabaseChunks(
    865     scoped_ptr<std::vector<SBChunkDelete> > chunk_deletes) {
    866   DCHECK_EQ(base::MessageLoop::current(),
    867             safe_browsing_thread_->message_loop());
    868   if (chunk_deletes)
    869     GetDatabase()->DeleteChunks(*chunk_deletes);
    870 }
    871 
    872 void SafeBrowsingDatabaseManager::DatabaseUpdateFinished(
    873     bool update_succeeded) {
    874   DCHECK_EQ(base::MessageLoop::current(),
    875             safe_browsing_thread_->message_loop());
    876   GetDatabase()->UpdateFinished(update_succeeded);
    877   DCHECK(database_update_in_progress_);
    878   database_update_in_progress_ = false;
    879   BrowserThread::PostTask(
    880       BrowserThread::UI, FROM_HERE,
    881       base::Bind(&SafeBrowsingDatabaseManager::NotifyDatabaseUpdateFinished,
    882                  this, update_succeeded));
    883 }
    884 
    885 void SafeBrowsingDatabaseManager::OnCloseDatabase() {
    886   DCHECK_EQ(base::MessageLoop::current(),
    887             safe_browsing_thread_->message_loop());
    888   DCHECK(closing_database_);
    889 
    890   // Because |closing_database_| is true, nothing on the IO thread will be
    891   // accessing the database, so it's safe to delete and then NULL the pointer.
    892   delete database_;
    893   database_ = NULL;
    894 
    895   // Acquiring the lock here guarantees correct ordering between the resetting
    896   // of |database_| above and of |closing_database_| below, which ensures there
    897   // won't be a window during which the IO thread falsely believes the database
    898   // is available.
    899   base::AutoLock lock(database_lock_);
    900   closing_database_ = false;
    901 }
    902 
    903 void SafeBrowsingDatabaseManager::OnResetDatabase() {
    904   DCHECK_EQ(base::MessageLoop::current(),
    905             safe_browsing_thread_->message_loop());
    906   GetDatabase()->ResetDatabase();
    907 }
    908 
    909 void SafeBrowsingDatabaseManager::OnHandleGetHashResults(
    910     SafeBrowsingCheck* check,
    911     const std::vector<SBFullHashResult>& full_hashes) {
    912   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    913   safe_browsing_util::ListType check_type = check->check_type;
    914   SBPrefix prefix = check->prefix_hits[0];
    915   GetHashRequests::iterator it = gethash_requests_.find(prefix);
    916   if (check->prefix_hits.size() > 1 || it == gethash_requests_.end()) {
    917     const bool hit = HandleOneCheck(check, full_hashes);
    918     RecordGetHashCheckStatus(hit, check_type, full_hashes);
    919     return;
    920   }
    921 
    922   // Call back all interested parties, noting if any has a hit.
    923   GetHashRequestors& requestors = it->second;
    924   bool hit = false;
    925   for (GetHashRequestors::iterator r = requestors.begin();
    926        r != requestors.end(); ++r) {
    927     if (HandleOneCheck(*r, full_hashes))
    928       hit = true;
    929   }
    930   RecordGetHashCheckStatus(hit, check_type, full_hashes);
    931 
    932   gethash_requests_.erase(it);
    933 }
    934 
    935 bool SafeBrowsingDatabaseManager::HandleOneCheck(
    936     SafeBrowsingCheck* check,
    937     const std::vector<SBFullHashResult>& full_hashes) {
    938   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    939   DCHECK(check);
    940 
    941   bool is_threat = false;
    942 
    943   // TODO(shess): GetHashThreadListType() contains a loop,
    944   // GetUrlThreatListType() a loop around that loop.  Having another loop out
    945   // here concerns me.  It is likely that SAFE is an expected outcome, which
    946   // means all of those loops run to completion.  Refactoring this to generate a
    947   // set of sorted items to compare in sequence would probably improve things.
    948   //
    949   // Additionally, the set of patterns generated from the urls is very similar
    950   // to the patterns generated in ContainsBrowseUrl() and other database checks,
    951   // which are called from this code.  Refactoring that across the checks could
    952   // interact well with batching the checks here.
    953 
    954   for (size_t i = 0; i < check->urls.size(); ++i) {
    955     size_t threat_index;
    956     SBThreatType threat =
    957         GetUrlThreatType(check->urls[i], full_hashes, &threat_index);
    958     if (threat != SB_THREAT_TYPE_SAFE &&
    959         IsExpectedThreat(threat, check->expected_threats)) {
    960       check->url_results[i] = threat;
    961       check->url_metadata[i] = full_hashes[threat_index].metadata;
    962       is_threat = true;
    963     }
    964   }
    965 
    966   for (size_t i = 0; i < check->full_hashes.size(); ++i) {
    967     SBThreatType threat = GetHashThreatType(check->full_hashes[i], full_hashes);
    968     if (threat != SB_THREAT_TYPE_SAFE &&
    969         IsExpectedThreat(threat, check->expected_threats)) {
    970       check->full_hash_results[i] = threat;
    971       is_threat = true;
    972     }
    973   }
    974 
    975   SafeBrowsingCheckDone(check);
    976   return is_threat;
    977 }
    978 
    979 void SafeBrowsingDatabaseManager::CheckDownloadUrlOnSBThread(
    980     SafeBrowsingCheck* check) {
    981   DCHECK_EQ(base::MessageLoop::current(),
    982             safe_browsing_thread_->message_loop());
    983   DCHECK(enable_download_protection_);
    984 
    985   std::vector<SBPrefix> prefix_hits;
    986 
    987   if (!database_->ContainsDownloadUrl(check->urls, &prefix_hits)) {
    988     // Good, we don't have hash for this url prefix.
    989     BrowserThread::PostTask(
    990         BrowserThread::IO, FROM_HERE,
    991         base::Bind(&SafeBrowsingDatabaseManager::CheckDownloadUrlDone, this,
    992                    check));
    993     return;
    994   }
    995 
    996   check->need_get_hash = true;
    997   check->prefix_hits.clear();
    998   check->prefix_hits = prefix_hits;
    999   BrowserThread::PostTask(
   1000       BrowserThread::IO, FROM_HERE,
   1001       base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
   1002 }
   1003 
   1004 void SafeBrowsingDatabaseManager::CheckExtensionIDsOnSBThread(
   1005     SafeBrowsingCheck* check) {
   1006   DCHECK_EQ(base::MessageLoop::current(),
   1007             safe_browsing_thread_->message_loop());
   1008 
   1009   std::vector<SBPrefix> prefixes;
   1010   for (std::vector<SBFullHash>::iterator it = check->full_hashes.begin();
   1011        it != check->full_hashes.end(); ++it) {
   1012     prefixes.push_back((*it).prefix);
   1013   }
   1014   database_->ContainsExtensionPrefixes(prefixes, &check->prefix_hits);
   1015 
   1016   if (check->prefix_hits.empty()) {
   1017     // No matches for any extensions.
   1018     BrowserThread::PostTask(
   1019         BrowserThread::IO,
   1020         FROM_HERE,
   1021         base::Bind(&SafeBrowsingDatabaseManager::SafeBrowsingCheckDone, this,
   1022                    check));
   1023   } else {
   1024     // Some prefixes matched, we need to ask Google whether they're legit.
   1025     check->need_get_hash = true;
   1026     BrowserThread::PostTask(
   1027         BrowserThread::IO,
   1028         FROM_HERE,
   1029         base::Bind(&SafeBrowsingDatabaseManager::OnCheckDone, this, check));
   1030   }
   1031 }
   1032 
   1033 void SafeBrowsingDatabaseManager::TimeoutCallback(SafeBrowsingCheck* check) {
   1034   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   1035   DCHECK(check);
   1036 
   1037   if (!enabled_)
   1038     return;
   1039 
   1040   DCHECK(checks_.find(check) != checks_.end());
   1041   if (check->client) {
   1042     check->client->OnSafeBrowsingResult(*check);
   1043     check->client = NULL;
   1044   }
   1045 }
   1046 
   1047 void SafeBrowsingDatabaseManager::CheckDownloadUrlDone(
   1048     SafeBrowsingCheck* check) {
   1049   DCHECK(enable_download_protection_);
   1050   SafeBrowsingCheckDone(check);
   1051 }
   1052 
   1053 void SafeBrowsingDatabaseManager::SafeBrowsingCheckDone(
   1054     SafeBrowsingCheck* check) {
   1055   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   1056   DCHECK(check);
   1057 
   1058   if (!enabled_)
   1059     return;
   1060 
   1061   VLOG(1) << "SafeBrowsingCheckDone";
   1062   DCHECK(checks_.find(check) != checks_.end());
   1063   if (check->client)
   1064     check->client->OnSafeBrowsingResult(*check);
   1065   checks_.erase(check);
   1066   delete check;
   1067 }
   1068 
   1069 void SafeBrowsingDatabaseManager::StartSafeBrowsingCheck(
   1070     SafeBrowsingCheck* check,
   1071     const base::Closure& task) {
   1072   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   1073   check->timeout_factory_.reset(
   1074       new base::WeakPtrFactory<SafeBrowsingDatabaseManager>(this));
   1075   checks_.insert(check);
   1076 
   1077   safe_browsing_thread_->message_loop()->PostTask(FROM_HERE, task);
   1078 
   1079   base::MessageLoop::current()->PostDelayedTask(FROM_HERE,
   1080       base::Bind(&SafeBrowsingDatabaseManager::TimeoutCallback,
   1081                  check->timeout_factory_->GetWeakPtr(), check),
   1082       check_timeout_);
   1083 }
   1084