Home | History | Annotate | Download | only in core 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "components/precache/core/precache_fetcher.h"
      6 
      7 #include <string>
      8 
      9 #include "base/bind.h"
     10 #include "base/callback.h"
     11 #include "base/command_line.h"
     12 #include "base/compiler_specific.h"
     13 #include "base/containers/hash_tables.h"
     14 #include "components/precache/core/precache_switches.h"
     15 #include "components/precache/core/proto/precache.pb.h"
     16 #include "net/base/escape.h"
     17 #include "net/base/load_flags.h"
     18 #include "net/url_request/url_fetcher.h"
     19 #include "net/url_request/url_fetcher_delegate.h"
     20 #include "net/url_request/url_request_context_getter.h"
     21 #include "net/url_request/url_request_status.h"
     22 
     23 using net::URLFetcher;
     24 
     25 namespace precache {
     26 
     27 namespace {
     28 
     29 GURL GetConfigURL() {
     30   const CommandLine& command_line = *CommandLine::ForCurrentProcess();
     31   if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) {
     32     return GURL(
     33         command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL));
     34   }
     35 
     36 #if defined(PRECACHE_CONFIG_SETTINGS_URL)
     37   return GURL(PRECACHE_CONFIG_SETTINGS_URL);
     38 #else
     39   // The precache config settings URL could not be determined, so return an
     40   // empty, invalid GURL.
     41   return GURL();
     42 #endif
     43 }
     44 
     45 std::string GetManifestURLPrefix() {
     46   const CommandLine& command_line = *CommandLine::ForCurrentProcess();
     47   if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) {
     48     return command_line.GetSwitchValueASCII(
     49         switches::kPrecacheManifestURLPrefix);
     50   }
     51 
     52 #if defined(PRECACHE_MANIFEST_URL_PREFIX)
     53   return PRECACHE_MANIFEST_URL_PREFIX;
     54 #else
     55   // The precache manifest URL prefix could not be determined, so return an
     56   // empty string.
     57   return std::string();
     58 #endif
     59 }
     60 
     61 // Construct the URL of the precache manifest for the given starting URL.
     62 // The server is expecting a request for a URL consisting of the manifest URL
     63 // prefix followed by the doubly escaped starting URL.
     64 GURL ConstructManifestURL(const GURL& starting_url) {
     65   return GURL(
     66       GetManifestURLPrefix() +
     67       net::EscapeQueryParamValue(
     68           net::EscapeQueryParamValue(starting_url.spec(), false), false));
     69 }
     70 
     71 // Attempts to parse a protobuf message from the response string of a
     72 // URLFetcher. If parsing is successful, the message parameter will contain the
     73 // parsed protobuf and this function will return true. Otherwise, returns false.
     74 bool ParseProtoFromFetchResponse(const URLFetcher& source,
     75                                  ::google::protobuf::MessageLite* message) {
     76   std::string response_string;
     77 
     78   if (!source.GetStatus().is_success()) {
     79     DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec();
     80     return false;
     81   }
     82   if (!source.GetResponseAsString(&response_string)) {
     83     DLOG(WARNING) << "No response string present: "
     84                   << source.GetOriginalURL().spec();
     85     return false;
     86   }
     87   if (!message->ParseFromString(response_string)) {
     88     DLOG(WARNING) << "Unable to parse proto served from "
     89                   << source.GetOriginalURL().spec();
     90     return false;
     91   }
     92   return true;
     93 }
     94 
     95 }  // namespace
     96 
     97 // Class that fetches a URL, and runs the specified callback when the fetch is
     98 // complete. This class exists so that a different method can be run in
     99 // response to different kinds of fetches, e.g. OnConfigFetchComplete when
    100 // configuration settings are fetched, OnManifestFetchComplete when a manifest
    101 // is fetched, etc.
    102 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate {
    103  public:
    104   // Construct a new Fetcher. This will create and start a new URLFetcher for
    105   // the specified URL using the specified request context.
    106   Fetcher(net::URLRequestContextGetter* request_context, const GURL& url,
    107           const base::Callback<void(const URLFetcher&)>& callback);
    108   virtual ~Fetcher() {}
    109   virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE;
    110 
    111  private:
    112   const base::Callback<void(const URLFetcher&)> callback_;
    113   scoped_ptr<URLFetcher> url_fetcher_;
    114 
    115   DISALLOW_COPY_AND_ASSIGN(Fetcher);
    116 };
    117 
    118 PrecacheFetcher::Fetcher::Fetcher(
    119     net::URLRequestContextGetter* request_context, const GURL& url,
    120     const base::Callback<void(const URLFetcher&)>& callback)
    121     : callback_(callback) {
    122   url_fetcher_.reset(URLFetcher::Create(url, URLFetcher::GET, this));
    123   url_fetcher_->SetRequestContext(request_context);
    124   url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_PROMPT_FOR_LOGIN);
    125   url_fetcher_->Start();
    126 }
    127 
    128 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) {
    129   callback_.Run(*source);
    130 }
    131 
    132 PrecacheFetcher::PrecacheFetcher(
    133     const std::list<GURL>& starting_urls,
    134     net::URLRequestContextGetter* request_context,
    135     PrecacheFetcher::PrecacheDelegate* precache_delegate)
    136     : starting_urls_(starting_urls),
    137       request_context_(request_context),
    138       precache_delegate_(precache_delegate) {
    139   DCHECK(request_context_);    // Request context must be non-NULL.
    140   DCHECK(precache_delegate_);  // Precache delegate must be non-NULL.
    141 
    142   DCHECK_NE(GURL(), GetConfigURL())
    143       << "Could not determine the precache config settings URL.";
    144   DCHECK_NE(std::string(), GetManifestURLPrefix())
    145       << "Could not determine the precache manifest URL prefix.";
    146 }
    147 
    148 PrecacheFetcher::~PrecacheFetcher() {
    149 }
    150 
    151 void PrecacheFetcher::Start() {
    152   DCHECK(!fetcher_);  // Start shouldn't be called repeatedly.
    153 
    154   GURL config_url = GetConfigURL();
    155   DCHECK(config_url.is_valid());
    156 
    157   // Fetch the precache configuration settings from the server.
    158   fetcher_.reset(new Fetcher(request_context_, config_url,
    159                              base::Bind(&PrecacheFetcher::OnConfigFetchComplete,
    160                                         base::Unretained(this))));
    161 }
    162 
    163 void PrecacheFetcher::StartNextFetch() {
    164   if (!resource_urls_to_fetch_.empty()) {
    165     // Fetch the next resource URL.
    166     fetcher_.reset(
    167         new Fetcher(request_context_, resource_urls_to_fetch_.front(),
    168                     base::Bind(&PrecacheFetcher::OnResourceFetchComplete,
    169                                base::Unretained(this))));
    170 
    171     resource_urls_to_fetch_.pop_front();
    172     return;
    173   }
    174 
    175   if (!manifest_urls_to_fetch_.empty()) {
    176     // Fetch the next manifest URL.
    177     fetcher_.reset(
    178         new Fetcher(request_context_, manifest_urls_to_fetch_.front(),
    179                     base::Bind(&PrecacheFetcher::OnManifestFetchComplete,
    180                                base::Unretained(this))));
    181 
    182     manifest_urls_to_fetch_.pop_front();
    183     return;
    184   }
    185 
    186   // There are no more URLs to fetch, so end the precache cycle.
    187   precache_delegate_->OnDone();
    188   // OnDone may have deleted this PrecacheFetcher, so don't do anything after it
    189   // is called.
    190 }
    191 
    192 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) {
    193   PrecacheConfigurationSettings config;
    194 
    195   if (ParseProtoFromFetchResponse(source, &config)) {
    196     // Keep track of starting URLs that manifests are being fetched for, in
    197     // order to remove duplicates. This is a hash set on strings, and not GURLs,
    198     // because there is no hash function defined for GURL.
    199     base::hash_set<std::string> unique_starting_urls;
    200 
    201     // Attempt to fetch manifests for starting URLs up to the maximum top sites
    202     // count. If a manifest does not exist for a particular starting URL, then
    203     // the fetch will fail, and that starting URL will be ignored.
    204     int64 rank = 0;
    205     for (std::list<GURL>::const_iterator it = starting_urls_.begin();
    206          it != starting_urls_.end() && rank < config.top_sites_count();
    207          ++it, ++rank) {
    208       if (unique_starting_urls.find(it->spec()) == unique_starting_urls.end()) {
    209         // Only add a fetch for the manifest URL if this manifest isn't already
    210         // going to be fetched.
    211         manifest_urls_to_fetch_.push_back(ConstructManifestURL(*it));
    212         unique_starting_urls.insert(it->spec());
    213       }
    214     }
    215 
    216     for (int i = 0; i < config.forced_starting_url_size(); ++i) {
    217       // Convert the string URL into a GURL and take the spec() of it so that
    218       // the URL string gets canonicalized.
    219       GURL url(config.forced_starting_url(i));
    220       if (unique_starting_urls.find(url.spec()) == unique_starting_urls.end()) {
    221         // Only add a fetch for the manifest URL if this manifest isn't already
    222         // going to be fetched.
    223         manifest_urls_to_fetch_.push_back(ConstructManifestURL(url));
    224         unique_starting_urls.insert(url.spec());
    225       }
    226     }
    227   }
    228 
    229   StartNextFetch();
    230 }
    231 
    232 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) {
    233   PrecacheManifest manifest;
    234 
    235   if (ParseProtoFromFetchResponse(source, &manifest)) {
    236     for (int i = 0; i < manifest.resource_size(); ++i) {
    237       if (manifest.resource(i).has_url()) {
    238         resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url()));
    239       }
    240     }
    241   }
    242 
    243   StartNextFetch();
    244 }
    245 
    246 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) {
    247   // The resource has already been put in the cache during the fetch process, so
    248   // nothing more needs to be done for the resource.
    249   StartNextFetch();
    250 }
    251 
    252 }  // namespace precache
    253