Home | History | Annotate | Download | only in safe_browsing 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/safe_browsing/client_side_detection_service.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/command_line.h"
      9 #include "base/logging.h"
     10 #include "base/memory/scoped_ptr.h"
     11 #include "base/message_loop/message_loop.h"
     12 #include "base/metrics/histogram.h"
     13 #include "base/metrics/sparse_histogram.h"
     14 #include "base/prefs/pref_service.h"
     15 #include "base/stl_util.h"
     16 #include "base/strings/string_util.h"
     17 #include "base/time/time.h"
     18 #include "chrome/browser/browser_process.h"
     19 #include "chrome/browser/profiles/profile.h"
     20 #include "chrome/common/chrome_switches.h"
     21 #include "chrome/common/pref_names.h"
     22 #include "chrome/common/safe_browsing/client_model.pb.h"
     23 #include "chrome/common/safe_browsing/csd.pb.h"
     24 #include "chrome/common/safe_browsing/safebrowsing_messages.h"
     25 #include "content/public/browser/browser_thread.h"
     26 #include "content/public/browser/notification_service.h"
     27 #include "content/public/browser/notification_types.h"
     28 #include "content/public/browser/render_process_host.h"
     29 #include "crypto/sha2.h"
     30 #include "google_apis/google_api_keys.h"
     31 #include "net/base/escape.h"
     32 #include "net/base/load_flags.h"
     33 #include "net/http/http_response_headers.h"
     34 #include "net/http/http_status_code.h"
     35 #include "net/url_request/url_fetcher.h"
     36 #include "net/url_request/url_request_context_getter.h"
     37 #include "net/url_request/url_request_status.h"
     38 #include "url/gurl.h"
     39 
     40 using content::BrowserThread;
     41 
     42 namespace safe_browsing {
     43 
     44 namespace {
     45 
     46   // malware report type for UMA histogram counting.
     47   enum MalwareReportTypes {
     48     REPORT_SENT,
     49     REPORT_HIT_LIMIT,
     50     REPORT_FAILED_SERIALIZATION,
     51 
     52     // Always at the end
     53     REPORT_RESULT_MAX
     54   };
     55 
     56   void UpdateEnumUMAHistogram(MalwareReportTypes report_type) {
     57     DCHECK(report_type >= 0 && report_type < REPORT_RESULT_MAX);
     58     UMA_HISTOGRAM_ENUMERATION("SBClientMalware.SentReports",
     59                               report_type, REPORT_RESULT_MAX);
     60   }
     61 
     62 }  // namespace
     63 
     64 const size_t ClientSideDetectionService::kMaxModelSizeBytes = 90 * 1024;
     65 const int ClientSideDetectionService::kMaxReportsPerInterval = 3;
     66 // TODO(noelutz): once we know this mechanism works as intended we should fetch
     67 // the model much more frequently.  E.g., every 5 minutes or so.
     68 const int ClientSideDetectionService::kClientModelFetchIntervalMs = 3600 * 1000;
     69 const int ClientSideDetectionService::kInitialClientModelFetchDelayMs = 10000;
     70 
     71 const int ClientSideDetectionService::kReportsIntervalDays = 1;
     72 const int ClientSideDetectionService::kNegativeCacheIntervalDays = 1;
     73 const int ClientSideDetectionService::kPositiveCacheIntervalMinutes = 30;
     74 
     75 const char ClientSideDetectionService::kClientReportPhishingUrl[] =
     76     "https://sb-ssl.google.com/safebrowsing/clientreport/phishing";
     77 const char ClientSideDetectionService::kClientReportMalwareUrl[] =
     78     "https://sb-ssl.google.com/safebrowsing/clientreport/malware-check";
     79 const char ClientSideDetectionService::kClientModelUrl[] =
     80     "https://ssl.gstatic.com/safebrowsing/csd/client_model_v5.pb";
     81 
     82 struct ClientSideDetectionService::ClientReportInfo {
     83   ClientReportPhishingRequestCallback callback;
     84   GURL phishing_url;
     85 };
     86 
     87 struct ClientSideDetectionService::ClientMalwareReportInfo {
     88   ClientReportMalwareRequestCallback callback;
     89   // This is the original landing url, may not be the malware url.
     90   GURL original_url;
     91 };
     92 
     93 ClientSideDetectionService::CacheState::CacheState(bool phish, base::Time time)
     94     : is_phishing(phish),
     95       timestamp(time) {}
     96 
     97 ClientSideDetectionService::ClientSideDetectionService(
     98     net::URLRequestContextGetter* request_context_getter)
     99     : enabled_(false),
    100       weak_factory_(this),
    101       request_context_getter_(request_context_getter) {
    102   registrar_.Add(this, content::NOTIFICATION_RENDERER_PROCESS_CREATED,
    103                  content::NotificationService::AllBrowserContextsAndSources());
    104 }
    105 
    106 ClientSideDetectionService::~ClientSideDetectionService() {
    107   weak_factory_.InvalidateWeakPtrs();
    108   STLDeleteContainerPairPointers(client_phishing_reports_.begin(),
    109                                  client_phishing_reports_.end());
    110   client_phishing_reports_.clear();
    111   STLDeleteContainerPairPointers(client_malware_reports_.begin(),
    112                                  client_malware_reports_.end());
    113   client_malware_reports_.clear();
    114 }
    115 
    116 // static
    117 ClientSideDetectionService* ClientSideDetectionService::Create(
    118     net::URLRequestContextGetter* request_context_getter) {
    119   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    120   scoped_ptr<ClientSideDetectionService> service(
    121       new ClientSideDetectionService(request_context_getter));
    122   if (!service->InitializePrivateNetworks()) {
    123     UMA_HISTOGRAM_COUNTS("SBClientPhishing.InitPrivateNetworksFailed", 1);
    124     return NULL;
    125   }
    126   return service.release();
    127 }
    128 
    129 void ClientSideDetectionService::SetEnabledAndRefreshState(bool enabled) {
    130   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    131   SendModelToRenderers();  // always refresh the renderer state
    132   if (enabled == enabled_)
    133     return;
    134   enabled_ = enabled;
    135   if (enabled_) {
    136     // Refresh the model when the service is enabled.  This can happen when the
    137     // preference is toggled, or early during startup if the preference is
    138     // already enabled. In a lot of cases the model will be in the cache so it
    139     // won't actually be fetched from the network.
    140     // We delay the first model fetch to avoid slowing down browser startup.
    141     ScheduleFetchModel(kInitialClientModelFetchDelayMs);
    142   } else {
    143     // Cancel pending requests.
    144     model_fetcher_.reset();
    145     // Invoke pending callbacks with a false verdict.
    146     for (std::map<const net::URLFetcher*, ClientReportInfo*>::iterator it =
    147              client_phishing_reports_.begin();
    148          it != client_phishing_reports_.end(); ++it) {
    149       ClientReportInfo* info = it->second;
    150       if (!info->callback.is_null())
    151         info->callback.Run(info->phishing_url, false);
    152     }
    153     STLDeleteContainerPairPointers(client_phishing_reports_.begin(),
    154                                    client_phishing_reports_.end());
    155     client_phishing_reports_.clear();
    156     for (std::map<const net::URLFetcher*, ClientMalwareReportInfo*>::iterator it
    157              = client_malware_reports_.begin();
    158          it != client_malware_reports_.end(); ++it) {
    159       ClientMalwareReportInfo* info = it->second;
    160       if (!info->callback.is_null())
    161         info->callback.Run(info->original_url, info->original_url, false);
    162     }
    163     STLDeleteContainerPairPointers(client_malware_reports_.begin(),
    164                                    client_malware_reports_.end());
    165     client_malware_reports_.clear();
    166     cache_.clear();
    167   }
    168 }
    169 
    170 void ClientSideDetectionService::SendClientReportPhishingRequest(
    171     ClientPhishingRequest* verdict,
    172     const ClientReportPhishingRequestCallback& callback) {
    173   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    174   base::MessageLoop::current()->PostTask(
    175       FROM_HERE,
    176       base::Bind(&ClientSideDetectionService::StartClientReportPhishingRequest,
    177                  weak_factory_.GetWeakPtr(), verdict, callback));
    178 }
    179 
    180 void ClientSideDetectionService::SendClientReportMalwareRequest(
    181     ClientMalwareRequest* verdict,
    182     const ClientReportMalwareRequestCallback& callback) {
    183   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    184   base::MessageLoop::current()->PostTask(
    185       FROM_HERE,
    186       base::Bind(&ClientSideDetectionService::StartClientReportMalwareRequest,
    187                  weak_factory_.GetWeakPtr(), verdict, callback));
    188 }
    189 
    190 bool ClientSideDetectionService::IsPrivateIPAddress(
    191     const std::string& ip_address) const {
    192   net::IPAddressNumber ip_number;
    193   if (!net::ParseIPLiteralToNumber(ip_address, &ip_number)) {
    194     VLOG(2) << "Unable to parse IP address: '" << ip_address << "'";
    195     // Err on the side of safety and assume this might be private.
    196     return true;
    197   }
    198 
    199   for (std::vector<AddressRange>::const_iterator it =
    200            private_networks_.begin();
    201        it != private_networks_.end(); ++it) {
    202     if (net::IPNumberMatchesPrefix(ip_number, it->first, it->second)) {
    203       return true;
    204     }
    205   }
    206   return false;
    207 }
    208 
    209 void ClientSideDetectionService::OnURLFetchComplete(
    210     const net::URLFetcher* source) {
    211   std::string data;
    212   source->GetResponseAsString(&data);
    213   if (source == model_fetcher_.get()) {
    214     HandleModelResponse(
    215         source, source->GetURL(), source->GetStatus(),
    216         source->GetResponseCode(), source->GetCookies(), data);
    217   } else if (client_phishing_reports_.find(source) !=
    218              client_phishing_reports_.end()) {
    219     HandlePhishingVerdict(
    220         source, source->GetURL(), source->GetStatus(),
    221         source->GetResponseCode(), source->GetCookies(), data);
    222   } else if (client_malware_reports_.find(source) !=
    223              client_malware_reports_.end()) {
    224     HandleMalwareVerdict(
    225         source, source->GetURL(), source->GetStatus(),
    226         source->GetResponseCode(), source->GetCookies(), data);
    227   } else {
    228     NOTREACHED();
    229   }
    230 }
    231 
    232 void ClientSideDetectionService::Observe(
    233     int type,
    234     const content::NotificationSource& source,
    235     const content::NotificationDetails& details) {
    236   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    237   DCHECK(type == content::NOTIFICATION_RENDERER_PROCESS_CREATED);
    238   if (!model_.get()) {
    239     // Model might not be ready or maybe there was an error.
    240     return;
    241   }
    242   SendModelToProcess(
    243       content::Source<content::RenderProcessHost>(source).ptr());
    244 }
    245 
    246 void ClientSideDetectionService::SendModelToProcess(
    247     content::RenderProcessHost* process) {
    248   // The ClientSideDetectionService is enabled if _any_ active profile has
    249   // SafeBrowsing turned on.  Here we check the profile for each renderer
    250   // process and only send the model to those that have SafeBrowsing enabled.
    251   Profile* profile = Profile::FromBrowserContext(process->GetBrowserContext());
    252   std::string model;
    253   if (profile->GetPrefs()->GetBoolean(prefs::kSafeBrowsingEnabled)) {
    254     VLOG(2) << "Sending phishing model to RenderProcessHost @" << process;
    255     model = model_str_;
    256   } else {
    257     VLOG(2) << "Disabling client-side phishing detection for "
    258             << "RenderProcessHost @" << process;
    259   }
    260   process->Send(new SafeBrowsingMsg_SetPhishingModel(model));
    261 }
    262 
    263 void ClientSideDetectionService::SendModelToRenderers() {
    264   for (content::RenderProcessHost::iterator i(
    265           content::RenderProcessHost::AllHostsIterator());
    266        !i.IsAtEnd(); i.Advance()) {
    267     SendModelToProcess(i.GetCurrentValue());
    268   }
    269 }
    270 
    271 void ClientSideDetectionService::ScheduleFetchModel(int64 delay_ms) {
    272   if (CommandLine::ForCurrentProcess()->HasSwitch(
    273       switches::kSbDisableAutoUpdate))
    274     return;
    275   base::MessageLoop::current()->PostDelayedTask(
    276       FROM_HERE,
    277       base::Bind(&ClientSideDetectionService::StartFetchModel,
    278                  weak_factory_.GetWeakPtr()),
    279       base::TimeDelta::FromMilliseconds(delay_ms));
    280 }
    281 
    282 void ClientSideDetectionService::StartFetchModel() {
    283   if (enabled_) {
    284     // Start fetching the model either from the cache or possibly from the
    285     // network if the model isn't in the cache.
    286     model_fetcher_.reset(net::URLFetcher::Create(
    287         0 /* ID used for testing */, GURL(kClientModelUrl),
    288         net::URLFetcher::GET, this));
    289     model_fetcher_->SetRequestContext(request_context_getter_.get());
    290     model_fetcher_->Start();
    291   }
    292 }
    293 
    294 void ClientSideDetectionService::EndFetchModel(ClientModelStatus status) {
    295   UMA_HISTOGRAM_ENUMERATION("SBClientPhishing.ClientModelStatus",
    296                             status,
    297                             MODEL_STATUS_MAX);
    298   if (status == MODEL_SUCCESS) {
    299     SetBadSubnets(*model_, &bad_subnets_);
    300     SendModelToRenderers();
    301   }
    302   int delay_ms = kClientModelFetchIntervalMs;
    303   // If the most recently fetched model had a valid max-age and the model was
    304   // valid we're scheduling the next model update for after the max-age expired.
    305   if (model_max_age_.get() &&
    306       (status == MODEL_SUCCESS || status == MODEL_NOT_CHANGED)) {
    307     // We're adding 60s of additional delay to make sure we're past
    308     // the model's age.
    309     *model_max_age_ += base::TimeDelta::FromMinutes(1);
    310     delay_ms = model_max_age_->InMilliseconds();
    311   }
    312   model_max_age_.reset();
    313 
    314   // Schedule the next model reload.
    315   ScheduleFetchModel(delay_ms);
    316 }
    317 
    318 void ClientSideDetectionService::StartClientReportPhishingRequest(
    319     ClientPhishingRequest* verdict,
    320     const ClientReportPhishingRequestCallback& callback) {
    321   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    322   scoped_ptr<ClientPhishingRequest> request(verdict);
    323 
    324   if (!enabled_) {
    325     if (!callback.is_null())
    326       callback.Run(GURL(request->url()), false);
    327     return;
    328   }
    329 
    330   std::string request_data;
    331   if (!request->SerializeToString(&request_data)) {
    332     UMA_HISTOGRAM_COUNTS("SBClientPhishing.RequestNotSerialized", 1);
    333     VLOG(1) << "Unable to serialize the CSD request. Proto file changed?";
    334     if (!callback.is_null())
    335       callback.Run(GURL(request->url()), false);
    336     return;
    337   }
    338 
    339   net::URLFetcher* fetcher = net::URLFetcher::Create(
    340       0 /* ID used for testing */,
    341       GetClientReportUrl(kClientReportPhishingUrl),
    342       net::URLFetcher::POST, this);
    343 
    344   // Remember which callback and URL correspond to the current fetcher object.
    345   ClientReportInfo* info = new ClientReportInfo;
    346   info->callback = callback;
    347   info->phishing_url = GURL(request->url());
    348   client_phishing_reports_[fetcher] = info;
    349 
    350   fetcher->SetLoadFlags(net::LOAD_DISABLE_CACHE);
    351   fetcher->SetRequestContext(request_context_getter_.get());
    352   fetcher->SetUploadData("application/octet-stream", request_data);
    353   fetcher->Start();
    354 
    355   // Record that we made a request
    356   phishing_report_times_.push(base::Time::Now());
    357 }
    358 
    359 void ClientSideDetectionService::StartClientReportMalwareRequest(
    360     ClientMalwareRequest* verdict,
    361     const ClientReportMalwareRequestCallback& callback) {
    362   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    363   scoped_ptr<ClientMalwareRequest> request(verdict);
    364 
    365   if (!enabled_) {
    366     if (!callback.is_null())
    367       callback.Run(GURL(request->url()), GURL(request->url()), false);
    368     return;
    369   }
    370 
    371   if (OverMalwareReportLimit()) {
    372     UpdateEnumUMAHistogram(REPORT_HIT_LIMIT);
    373     DVLOG(1) << "Too many malware report requests sent recently."
    374              << "Skip sending malware report for " << GURL(request->url());
    375     if (!callback.is_null())
    376       callback.Run(GURL(request->url()), GURL(request->url()), false);
    377     return;
    378   }
    379 
    380   std::string request_data;
    381   if (!request->SerializeToString(&request_data)) {
    382     UpdateEnumUMAHistogram(REPORT_FAILED_SERIALIZATION);
    383     DVLOG(1) << "Unable to serialize the CSD request. Proto file changed?";
    384     if (!callback.is_null())
    385       callback.Run(GURL(request->url()), GURL(request->url()), false);
    386     return;
    387   }
    388 
    389   net::URLFetcher* fetcher = net::URLFetcher::Create(
    390       0 /* ID used for testing */,
    391       GetClientReportUrl(kClientReportMalwareUrl),
    392       net::URLFetcher::POST, this);
    393 
    394   // Remember which callback and URL correspond to the current fetcher object.
    395   ClientMalwareReportInfo* info = new ClientMalwareReportInfo;
    396   info->callback = callback;
    397   info->original_url = GURL(request->url());
    398   client_malware_reports_[fetcher] = info;
    399 
    400   fetcher->SetLoadFlags(net::LOAD_DISABLE_CACHE);
    401   fetcher->SetRequestContext(request_context_getter_.get());
    402   fetcher->SetUploadData("application/octet-stream", request_data);
    403   fetcher->Start();
    404 
    405   UMA_HISTOGRAM_ENUMERATION("SBClientMalware.SentReports",
    406                             REPORT_SENT, REPORT_RESULT_MAX);
    407 
    408   UMA_HISTOGRAM_COUNTS("SBClientMalware.IPBlacklistRequestPayloadSize",
    409                        request_data.size());
    410 
    411   // Record that we made a malware request
    412   malware_report_times_.push(base::Time::Now());
    413 }
    414 
    415 void ClientSideDetectionService::HandleModelResponse(
    416     const net::URLFetcher* source,
    417     const GURL& url,
    418     const net::URLRequestStatus& status,
    419     int response_code,
    420     const net::ResponseCookies& cookies,
    421     const std::string& data) {
    422   base::TimeDelta max_age;
    423   if (status.is_success() && net::HTTP_OK == response_code &&
    424       source->GetResponseHeaders() &&
    425       source->GetResponseHeaders()->GetMaxAgeValue(&max_age)) {
    426     model_max_age_.reset(new base::TimeDelta(max_age));
    427   }
    428   scoped_ptr<ClientSideModel> model(new ClientSideModel());
    429   ClientModelStatus model_status;
    430   if (!status.is_success() || net::HTTP_OK != response_code) {
    431     model_status = MODEL_FETCH_FAILED;
    432   } else if (data.empty()) {
    433     model_status = MODEL_EMPTY;
    434   } else if (data.size() > kMaxModelSizeBytes) {
    435     model_status = MODEL_TOO_LARGE;
    436   } else if (!model->ParseFromString(data)) {
    437     model_status = MODEL_PARSE_ERROR;
    438   } else if (!model->IsInitialized() || !model->has_version()) {
    439     model_status = MODEL_MISSING_FIELDS;
    440   } else if (!ModelHasValidHashIds(*model)) {
    441     model_status = MODEL_BAD_HASH_IDS;
    442   } else if (model->version() < 0 ||
    443              (model_.get() && model->version() < model_->version())) {
    444     model_status = MODEL_INVALID_VERSION_NUMBER;
    445   } else if (model_.get() && model->version() == model_->version()) {
    446     model_status = MODEL_NOT_CHANGED;
    447   } else {
    448     // The model is valid => replace the existing model with the new one.
    449     model_str_.assign(data);
    450     model_.swap(model);
    451     model_status = MODEL_SUCCESS;
    452   }
    453   EndFetchModel(model_status);
    454 }
    455 
    456 void ClientSideDetectionService::HandlePhishingVerdict(
    457     const net::URLFetcher* source,
    458     const GURL& url,
    459     const net::URLRequestStatus& status,
    460     int response_code,
    461     const net::ResponseCookies& cookies,
    462     const std::string& data) {
    463   ClientPhishingResponse response;
    464   scoped_ptr<ClientReportInfo> info(client_phishing_reports_[source]);
    465   bool is_phishing = false;
    466   if (status.is_success() && net::HTTP_OK == response_code &&
    467       response.ParseFromString(data)) {
    468     // Cache response, possibly flushing an old one.
    469     cache_[info->phishing_url] =
    470         make_linked_ptr(new CacheState(response.phishy(), base::Time::Now()));
    471     is_phishing = response.phishy();
    472   } else {
    473     DLOG(ERROR) << "Unable to get the server verdict for URL: "
    474                 << info->phishing_url << " status: " << status.status() << " "
    475                 << "response_code:" << response_code;
    476   }
    477   if (!info->callback.is_null())
    478     info->callback.Run(info->phishing_url, is_phishing);
    479   client_phishing_reports_.erase(source);
    480   delete source;
    481 }
    482 
    483 void ClientSideDetectionService::HandleMalwareVerdict(
    484     const net::URLFetcher* source,
    485     const GURL& url,
    486     const net::URLRequestStatus& status,
    487     int response_code,
    488     const net::ResponseCookies& cookies,
    489     const std::string& data) {
    490   if (status.is_success()) {
    491     UMA_HISTOGRAM_SPARSE_SLOWLY(
    492         "SBClientMalware.IPBlacklistRequestResponseCode", response_code);
    493   }
    494   // status error is negative, so we put - in front of it.
    495   UMA_HISTOGRAM_SPARSE_SLOWLY(
    496       "SBClientMalware.IPBlacklistRequestNetError", -status.error());
    497 
    498   ClientMalwareResponse response;
    499   scoped_ptr<ClientMalwareReportInfo> info(client_malware_reports_[source]);
    500   bool should_blacklist = false;
    501   if (status.is_success() && net::HTTP_OK == response_code &&
    502       response.ParseFromString(data)) {
    503     should_blacklist = response.blacklist();
    504   } else {
    505     DLOG(ERROR) << "Unable to get the server verdict for URL: "
    506                 << info->original_url << " status: " << status.status() << " "
    507                 << "response_code:" << response_code;
    508   }
    509 
    510   if (!info->callback.is_null()) {
    511     if (response.has_bad_url())
    512       info->callback.Run(info->original_url, GURL(response.bad_url()),
    513                          should_blacklist);
    514     else
    515       info->callback.Run(info->original_url, info->original_url, false);
    516   }
    517 
    518   client_malware_reports_.erase(source);
    519   delete source;
    520 }
    521 
    522 bool ClientSideDetectionService::IsInCache(const GURL& url) {
    523   UpdateCache();
    524 
    525   return cache_.find(url) != cache_.end();
    526 }
    527 
    528 bool ClientSideDetectionService::GetValidCachedResult(const GURL& url,
    529                                                       bool* is_phishing) {
    530   UpdateCache();
    531 
    532   PhishingCache::iterator it = cache_.find(url);
    533   if (it == cache_.end()) {
    534     return false;
    535   }
    536 
    537   // We still need to check if the result is valid.
    538   const CacheState& cache_state = *it->second;
    539   if (cache_state.is_phishing ?
    540       cache_state.timestamp > base::Time::Now() -
    541           base::TimeDelta::FromMinutes(kPositiveCacheIntervalMinutes) :
    542       cache_state.timestamp > base::Time::Now() -
    543           base::TimeDelta::FromDays(kNegativeCacheIntervalDays)) {
    544     *is_phishing = cache_state.is_phishing;
    545     return true;
    546   }
    547   return false;
    548 }
    549 
    550 void ClientSideDetectionService::UpdateCache() {
    551   // Since we limit the number of requests but allow pass-through for cache
    552   // refreshes, we don't want to remove elements from the cache if they
    553   // could be used for this purpose even if we will not use the entry to
    554   // satisfy the request from the cache.
    555   base::TimeDelta positive_cache_interval =
    556       std::max(base::TimeDelta::FromMinutes(kPositiveCacheIntervalMinutes),
    557                base::TimeDelta::FromDays(kReportsIntervalDays));
    558   base::TimeDelta negative_cache_interval =
    559       std::max(base::TimeDelta::FromDays(kNegativeCacheIntervalDays),
    560                base::TimeDelta::FromDays(kReportsIntervalDays));
    561 
    562   // Remove elements from the cache that will no longer be used.
    563   for (PhishingCache::iterator it = cache_.begin(); it != cache_.end();) {
    564     const CacheState& cache_state = *it->second;
    565     if (cache_state.is_phishing ?
    566         cache_state.timestamp > base::Time::Now() - positive_cache_interval :
    567         cache_state.timestamp > base::Time::Now() - negative_cache_interval) {
    568       ++it;
    569     } else {
    570       cache_.erase(it++);
    571     }
    572   }
    573 }
    574 
    575 bool ClientSideDetectionService::OverMalwareReportLimit() {
    576   return GetMalwareNumReports() > kMaxReportsPerInterval;
    577 }
    578 
    579 bool ClientSideDetectionService::OverPhishingReportLimit() {
    580   return GetPhishingNumReports() > kMaxReportsPerInterval;
    581 }
    582 
    583 int ClientSideDetectionService::GetMalwareNumReports() {
    584   return GetNumReports(&malware_report_times_);
    585 }
    586 
    587 int ClientSideDetectionService::GetPhishingNumReports() {
    588   return GetNumReports(&phishing_report_times_);
    589 }
    590 
    591 int ClientSideDetectionService::GetNumReports(
    592     std::queue<base::Time>* report_times) {
    593   base::Time cutoff =
    594       base::Time::Now() - base::TimeDelta::FromDays(kReportsIntervalDays);
    595 
    596   // Erase items older than cutoff because we will never care about them again.
    597   while (!report_times->empty() &&
    598          report_times->front() < cutoff) {
    599     report_times->pop();
    600   }
    601 
    602   // Return the number of elements that are above the cutoff.
    603   return report_times->size();
    604 }
    605 
    606 bool ClientSideDetectionService::InitializePrivateNetworks() {
    607   static const char* const kPrivateNetworks[] = {
    608     "10.0.0.0/8",
    609     "127.0.0.0/8",
    610     "172.16.0.0/12",
    611     "192.168.0.0/16",
    612     // IPv6 address ranges
    613     "fc00::/7",
    614     "fec0::/10",
    615     "::1/128",
    616   };
    617 
    618   for (size_t i = 0; i < arraysize(kPrivateNetworks); ++i) {
    619     net::IPAddressNumber ip_number;
    620     size_t prefix_length;
    621     if (net::ParseCIDRBlock(kPrivateNetworks[i], &ip_number, &prefix_length)) {
    622       private_networks_.push_back(std::make_pair(ip_number, prefix_length));
    623     } else {
    624       DLOG(FATAL) << "Unable to parse IP address range: "
    625                   << kPrivateNetworks[i];
    626       return false;
    627     }
    628   }
    629   return true;
    630 }
    631 
    632 // static
    633 void ClientSideDetectionService::SetBadSubnets(const ClientSideModel& model,
    634                                                BadSubnetMap* bad_subnets) {
    635   bad_subnets->clear();
    636   for (int i = 0; i < model.bad_subnet_size(); ++i) {
    637     int size = model.bad_subnet(i).size();
    638     if (size < 0 || size > static_cast<int>(net::kIPv6AddressSize) * 8) {
    639       DLOG(ERROR) << "Invalid bad subnet size: " << size;
    640       continue;
    641     }
    642     if (model.bad_subnet(i).prefix().size() != crypto::kSHA256Length) {
    643       DLOG(ERROR) << "Invalid bad subnet prefix length: "
    644                   << model.bad_subnet(i).prefix().size();
    645       continue;
    646     }
    647     // We precompute the mask for the given subnet size to speed up lookups.
    648     // Basically we need to create a 16B long string which has the highest
    649     // |size| bits sets to one.
    650     std::string mask(net::kIPv6AddressSize, '\x00');
    651     mask.replace(0, size / 8, size / 8, '\xFF');
    652     if (size % 8) {
    653       mask[size / 8] = 0xFF << (8 - (size % 8));
    654     }
    655     (*bad_subnets)[mask].insert(model.bad_subnet(i).prefix());
    656   }
    657 }
    658 
    659 // static
    660 bool ClientSideDetectionService::ModelHasValidHashIds(
    661     const ClientSideModel& model) {
    662   const int max_index = model.hashes_size() - 1;
    663   for (int i = 0; i < model.rule_size(); ++i) {
    664     for (int j = 0; j < model.rule(i).feature_size(); ++j) {
    665       if (model.rule(i).feature(j) < 0 ||
    666           model.rule(i).feature(j) > max_index) {
    667         return false;
    668       }
    669     }
    670   }
    671   for (int i = 0; i < model.page_term_size(); ++i) {
    672     if (model.page_term(i) < 0 || model.page_term(i) > max_index) {
    673       return false;
    674     }
    675   }
    676   return true;
    677 }
    678 
    679 // static
    680 GURL ClientSideDetectionService::GetClientReportUrl(
    681     const std::string& report_url) {
    682   GURL url(report_url);
    683   std::string api_key = google_apis::GetAPIKey();
    684   if (!api_key.empty())
    685     url = url.Resolve("?key=" + net::EscapeQueryParamValue(api_key, true));
    686 
    687   return url;
    688 }
    689 }  // namespace safe_browsing
    690