1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "components/precache/core/precache_fetcher.h" 6 7 #include <string> 8 9 #include "base/bind.h" 10 #include "base/callback.h" 11 #include "base/command_line.h" 12 #include "base/compiler_specific.h" 13 #include "base/containers/hash_tables.h" 14 #include "components/precache/core/precache_switches.h" 15 #include "components/precache/core/proto/precache.pb.h" 16 #include "net/base/escape.h" 17 #include "net/base/load_flags.h" 18 #include "net/url_request/url_fetcher.h" 19 #include "net/url_request/url_fetcher_delegate.h" 20 #include "net/url_request/url_request_context_getter.h" 21 #include "net/url_request/url_request_status.h" 22 23 using net::URLFetcher; 24 25 namespace precache { 26 27 namespace { 28 29 GURL GetConfigURL() { 30 const CommandLine& command_line = *CommandLine::ForCurrentProcess(); 31 if (command_line.HasSwitch(switches::kPrecacheConfigSettingsURL)) { 32 return GURL( 33 command_line.GetSwitchValueASCII(switches::kPrecacheConfigSettingsURL)); 34 } 35 36 #if defined(PRECACHE_CONFIG_SETTINGS_URL) 37 return GURL(PRECACHE_CONFIG_SETTINGS_URL); 38 #else 39 // The precache config settings URL could not be determined, so return an 40 // empty, invalid GURL. 41 return GURL(); 42 #endif 43 } 44 45 std::string GetManifestURLPrefix() { 46 const CommandLine& command_line = *CommandLine::ForCurrentProcess(); 47 if (command_line.HasSwitch(switches::kPrecacheManifestURLPrefix)) { 48 return command_line.GetSwitchValueASCII( 49 switches::kPrecacheManifestURLPrefix); 50 } 51 52 #if defined(PRECACHE_MANIFEST_URL_PREFIX) 53 return PRECACHE_MANIFEST_URL_PREFIX; 54 #else 55 // The precache manifest URL prefix could not be determined, so return an 56 // empty string. 57 return std::string(); 58 #endif 59 } 60 61 // Construct the URL of the precache manifest for the given starting URL. 62 // The server is expecting a request for a URL consisting of the manifest URL 63 // prefix followed by the doubly escaped starting URL. 64 GURL ConstructManifestURL(const GURL& starting_url) { 65 return GURL( 66 GetManifestURLPrefix() + 67 net::EscapeQueryParamValue( 68 net::EscapeQueryParamValue(starting_url.spec(), false), false)); 69 } 70 71 // Attempts to parse a protobuf message from the response string of a 72 // URLFetcher. If parsing is successful, the message parameter will contain the 73 // parsed protobuf and this function will return true. Otherwise, returns false. 74 bool ParseProtoFromFetchResponse(const URLFetcher& source, 75 ::google::protobuf::MessageLite* message) { 76 std::string response_string; 77 78 if (!source.GetStatus().is_success()) { 79 DLOG(WARNING) << "Fetch failed: " << source.GetOriginalURL().spec(); 80 return false; 81 } 82 if (!source.GetResponseAsString(&response_string)) { 83 DLOG(WARNING) << "No response string present: " 84 << source.GetOriginalURL().spec(); 85 return false; 86 } 87 if (!message->ParseFromString(response_string)) { 88 DLOG(WARNING) << "Unable to parse proto served from " 89 << source.GetOriginalURL().spec(); 90 return false; 91 } 92 return true; 93 } 94 95 } // namespace 96 97 // Class that fetches a URL, and runs the specified callback when the fetch is 98 // complete. This class exists so that a different method can be run in 99 // response to different kinds of fetches, e.g. OnConfigFetchComplete when 100 // configuration settings are fetched, OnManifestFetchComplete when a manifest 101 // is fetched, etc. 102 class PrecacheFetcher::Fetcher : public net::URLFetcherDelegate { 103 public: 104 // Construct a new Fetcher. This will create and start a new URLFetcher for 105 // the specified URL using the specified request context. 106 Fetcher(net::URLRequestContextGetter* request_context, const GURL& url, 107 const base::Callback<void(const URLFetcher&)>& callback); 108 virtual ~Fetcher() {} 109 virtual void OnURLFetchComplete(const URLFetcher* source) OVERRIDE; 110 111 private: 112 const base::Callback<void(const URLFetcher&)> callback_; 113 scoped_ptr<URLFetcher> url_fetcher_; 114 115 DISALLOW_COPY_AND_ASSIGN(Fetcher); 116 }; 117 118 PrecacheFetcher::Fetcher::Fetcher( 119 net::URLRequestContextGetter* request_context, const GURL& url, 120 const base::Callback<void(const URLFetcher&)>& callback) 121 : callback_(callback) { 122 url_fetcher_.reset(URLFetcher::Create(url, URLFetcher::GET, this)); 123 url_fetcher_->SetRequestContext(request_context); 124 url_fetcher_->SetLoadFlags(net::LOAD_DO_NOT_PROMPT_FOR_LOGIN); 125 url_fetcher_->Start(); 126 } 127 128 void PrecacheFetcher::Fetcher::OnURLFetchComplete(const URLFetcher* source) { 129 callback_.Run(*source); 130 } 131 132 PrecacheFetcher::PrecacheFetcher( 133 const std::list<GURL>& starting_urls, 134 net::URLRequestContextGetter* request_context, 135 PrecacheFetcher::PrecacheDelegate* precache_delegate) 136 : starting_urls_(starting_urls), 137 request_context_(request_context), 138 precache_delegate_(precache_delegate) { 139 DCHECK(request_context_); // Request context must be non-NULL. 140 DCHECK(precache_delegate_); // Precache delegate must be non-NULL. 141 142 DCHECK_NE(GURL(), GetConfigURL()) 143 << "Could not determine the precache config settings URL."; 144 DCHECK_NE(std::string(), GetManifestURLPrefix()) 145 << "Could not determine the precache manifest URL prefix."; 146 } 147 148 PrecacheFetcher::~PrecacheFetcher() { 149 } 150 151 void PrecacheFetcher::Start() { 152 DCHECK(!fetcher_); // Start shouldn't be called repeatedly. 153 154 GURL config_url = GetConfigURL(); 155 DCHECK(config_url.is_valid()); 156 157 // Fetch the precache configuration settings from the server. 158 fetcher_.reset(new Fetcher(request_context_, config_url, 159 base::Bind(&PrecacheFetcher::OnConfigFetchComplete, 160 base::Unretained(this)))); 161 } 162 163 void PrecacheFetcher::StartNextFetch() { 164 if (!resource_urls_to_fetch_.empty()) { 165 // Fetch the next resource URL. 166 fetcher_.reset( 167 new Fetcher(request_context_, resource_urls_to_fetch_.front(), 168 base::Bind(&PrecacheFetcher::OnResourceFetchComplete, 169 base::Unretained(this)))); 170 171 resource_urls_to_fetch_.pop_front(); 172 return; 173 } 174 175 if (!manifest_urls_to_fetch_.empty()) { 176 // Fetch the next manifest URL. 177 fetcher_.reset( 178 new Fetcher(request_context_, manifest_urls_to_fetch_.front(), 179 base::Bind(&PrecacheFetcher::OnManifestFetchComplete, 180 base::Unretained(this)))); 181 182 manifest_urls_to_fetch_.pop_front(); 183 return; 184 } 185 186 // There are no more URLs to fetch, so end the precache cycle. 187 precache_delegate_->OnDone(); 188 // OnDone may have deleted this PrecacheFetcher, so don't do anything after it 189 // is called. 190 } 191 192 void PrecacheFetcher::OnConfigFetchComplete(const URLFetcher& source) { 193 PrecacheConfigurationSettings config; 194 195 if (ParseProtoFromFetchResponse(source, &config)) { 196 // Keep track of starting URLs that manifests are being fetched for, in 197 // order to remove duplicates. This is a hash set on strings, and not GURLs, 198 // because there is no hash function defined for GURL. 199 base::hash_set<std::string> unique_starting_urls; 200 201 // Attempt to fetch manifests for starting URLs up to the maximum top sites 202 // count. If a manifest does not exist for a particular starting URL, then 203 // the fetch will fail, and that starting URL will be ignored. 204 int64 rank = 0; 205 for (std::list<GURL>::const_iterator it = starting_urls_.begin(); 206 it != starting_urls_.end() && rank < config.top_sites_count(); 207 ++it, ++rank) { 208 if (unique_starting_urls.find(it->spec()) == unique_starting_urls.end()) { 209 // Only add a fetch for the manifest URL if this manifest isn't already 210 // going to be fetched. 211 manifest_urls_to_fetch_.push_back(ConstructManifestURL(*it)); 212 unique_starting_urls.insert(it->spec()); 213 } 214 } 215 216 for (int i = 0; i < config.forced_starting_url_size(); ++i) { 217 // Convert the string URL into a GURL and take the spec() of it so that 218 // the URL string gets canonicalized. 219 GURL url(config.forced_starting_url(i)); 220 if (unique_starting_urls.find(url.spec()) == unique_starting_urls.end()) { 221 // Only add a fetch for the manifest URL if this manifest isn't already 222 // going to be fetched. 223 manifest_urls_to_fetch_.push_back(ConstructManifestURL(url)); 224 unique_starting_urls.insert(url.spec()); 225 } 226 } 227 } 228 229 StartNextFetch(); 230 } 231 232 void PrecacheFetcher::OnManifestFetchComplete(const URLFetcher& source) { 233 PrecacheManifest manifest; 234 235 if (ParseProtoFromFetchResponse(source, &manifest)) { 236 for (int i = 0; i < manifest.resource_size(); ++i) { 237 if (manifest.resource(i).has_url()) { 238 resource_urls_to_fetch_.push_back(GURL(manifest.resource(i).url())); 239 } 240 } 241 } 242 243 StartNextFetch(); 244 } 245 246 void PrecacheFetcher::OnResourceFetchComplete(const URLFetcher& source) { 247 // The resource has already been put in the cache during the fetch process, so 248 // nothing more needs to be done for the resource. 249 StartNextFetch(); 250 } 251 252 } // namespace precache 253