1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "content/browser/appcache/appcache_update_job.h" 6 7 #include "base/bind.h" 8 #include "base/bind_helpers.h" 9 #include "base/compiler_specific.h" 10 #include "base/message_loop/message_loop.h" 11 #include "base/strings/string_util.h" 12 #include "base/strings/stringprintf.h" 13 #include "content/browser/appcache/appcache_group.h" 14 #include "content/browser/appcache/appcache_histograms.h" 15 #include "net/base/host_port_pair.h" 16 #include "net/base/io_buffer.h" 17 #include "net/base/load_flags.h" 18 #include "net/base/net_errors.h" 19 #include "net/base/request_priority.h" 20 #include "net/http/http_request_headers.h" 21 #include "net/http/http_response_headers.h" 22 #include "net/url_request/url_request_context.h" 23 24 namespace { 25 bool IsDataReductionProxy(const net::HostPortPair& proxy_server) { 26 return ( 27 proxy_server.Equals(net::HostPortPair("proxy.googlezip.net", 443)) || 28 proxy_server.Equals(net::HostPortPair("compress.googlezip.net", 80)) || 29 proxy_server.Equals(net::HostPortPair("proxy-dev.googlezip.net", 80))); 30 } 31 } // namspace 32 33 namespace content { 34 35 static const int kBufferSize = 32768; 36 static const size_t kMaxConcurrentUrlFetches = 2; 37 static const int kMax503Retries = 3; 38 39 static std::string FormatUrlErrorMessage( 40 const char* format, const GURL& url, 41 AppCacheUpdateJob::ResultType error, 42 int response_code) { 43 // Show the net response code if we have one. 44 int code = response_code; 45 if (error != AppCacheUpdateJob::SERVER_ERROR) 46 code = static_cast<int>(error); 47 return base::StringPrintf(format, code, url.spec().c_str()); 48 } 49 50 // Helper class for collecting hosts per frontend when sending notifications 51 // so that only one notification is sent for all hosts using the same frontend. 52 class HostNotifier { 53 public: 54 typedef std::vector<int> HostIds; 55 typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap; 56 57 // Caller is responsible for ensuring there will be no duplicate hosts. 58 void AddHost(AppCacheHost* host) { 59 std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert( 60 NotifyHostMap::value_type(host->frontend(), HostIds())); 61 ret.first->second.push_back(host->host_id()); 62 } 63 64 void AddHosts(const std::set<AppCacheHost*>& hosts) { 65 for (std::set<AppCacheHost*>::const_iterator it = hosts.begin(); 66 it != hosts.end(); ++it) { 67 AddHost(*it); 68 } 69 } 70 71 void SendNotifications(AppCacheEventID event_id) { 72 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 73 it != hosts_to_notify.end(); ++it) { 74 AppCacheFrontend* frontend = it->first; 75 frontend->OnEventRaised(it->second, event_id); 76 } 77 } 78 79 void SendProgressNotifications( 80 const GURL& url, int num_total, int num_complete) { 81 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 82 it != hosts_to_notify.end(); ++it) { 83 AppCacheFrontend* frontend = it->first; 84 frontend->OnProgressEventRaised(it->second, url, 85 num_total, num_complete); 86 } 87 } 88 89 void SendErrorNotifications(const AppCacheErrorDetails& details) { 90 DCHECK(!details.message.empty()); 91 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 92 it != hosts_to_notify.end(); ++it) { 93 AppCacheFrontend* frontend = it->first; 94 frontend->OnErrorEventRaised(it->second, details); 95 } 96 } 97 98 void SendLogMessage(const std::string& message) { 99 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 100 it != hosts_to_notify.end(); ++it) { 101 AppCacheFrontend* frontend = it->first; 102 for (HostIds::iterator id = it->second.begin(); 103 id != it->second.end(); ++id) { 104 frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message); 105 } 106 } 107 } 108 109 private: 110 NotifyHostMap hosts_to_notify; 111 }; 112 113 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url, 114 bool checked, 115 AppCacheResponseInfo* info) 116 : url(url), 117 storage_checked(checked), 118 existing_response_info(info) { 119 } 120 121 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() { 122 } 123 124 // Helper class to fetch resources. Depending on the fetch type, 125 // can either fetch to an in-memory string or write the response 126 // data out to the disk cache. 127 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url, 128 FetchType fetch_type, 129 AppCacheUpdateJob* job) 130 : url_(url), 131 job_(job), 132 fetch_type_(fetch_type), 133 retry_503_attempts_(0), 134 buffer_(new net::IOBuffer(kBufferSize)), 135 request_(job->service_->request_context() 136 ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)), 137 result_(UPDATE_OK), 138 redirect_response_code_(-1) {} 139 140 AppCacheUpdateJob::URLFetcher::~URLFetcher() { 141 } 142 143 void AppCacheUpdateJob::URLFetcher::Start() { 144 request_->set_first_party_for_cookies(job_->manifest_url_); 145 request_->SetLoadFlags(request_->load_flags() | 146 net::LOAD_DISABLE_INTERCEPT); 147 if (existing_response_headers_.get()) 148 AddConditionalHeaders(existing_response_headers_.get()); 149 request_->Start(); 150 } 151 152 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect( 153 net::URLRequest* request, 154 const net::RedirectInfo& redirect_info, 155 bool* defer_redirect) { 156 DCHECK(request_ == request); 157 // TODO(bengr): Remove this special case logic when crbug.com/429505 is 158 // resolved. Until then, the data reduction proxy client logic uses the 159 // redirect mechanism to resend requests over a direct connection when 160 // the proxy instructs it to do so. The redirect is to the same location 161 // as the original URL. 162 if ((request->load_flags() & net::LOAD_BYPASS_PROXY) && 163 IsDataReductionProxy(request->proxy_server())) { 164 DCHECK_EQ(request->original_url(), request->url()); 165 return; 166 } 167 // Redirect is not allowed by the update process. 168 job_->MadeProgress(); 169 redirect_response_code_ = request->GetResponseCode(); 170 request->Cancel(); 171 result_ = REDIRECT_ERROR; 172 OnResponseCompleted(); 173 } 174 175 void AppCacheUpdateJob::URLFetcher::OnResponseStarted( 176 net::URLRequest *request) { 177 DCHECK(request == request_); 178 int response_code = -1; 179 if (request->status().is_success()) { 180 response_code = request->GetResponseCode(); 181 job_->MadeProgress(); 182 } 183 if ((response_code / 100) == 2) { 184 185 // See http://code.google.com/p/chromium/issues/detail?id=69594 186 // We willfully violate the HTML5 spec at this point in order 187 // to support the appcaching of cross-origin HTTPS resources. 188 // We've opted for a milder constraint and allow caching unless 189 // the resource has a "no-store" header. A spec change has been 190 // requested on the whatwg list. 191 // TODO(michaeln): Consider doing this for cross-origin HTTP resources too. 192 if (url_.SchemeIsSecure() && 193 url_.GetOrigin() != job_->manifest_url_.GetOrigin()) { 194 if (request->response_headers()-> 195 HasHeaderValue("cache-control", "no-store")) { 196 DCHECK_EQ(-1, redirect_response_code_); 197 request->Cancel(); 198 result_ = SERVER_ERROR; // Not the best match? 199 OnResponseCompleted(); 200 return; 201 } 202 } 203 204 // Write response info to storage for URL fetches. Wait for async write 205 // completion before reading any response data. 206 if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) { 207 response_writer_.reset(job_->CreateResponseWriter()); 208 scoped_refptr<HttpResponseInfoIOBuffer> io_buffer( 209 new HttpResponseInfoIOBuffer( 210 new net::HttpResponseInfo(request->response_info()))); 211 response_writer_->WriteInfo( 212 io_buffer.get(), 213 base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this))); 214 } else { 215 ReadResponseData(); 216 } 217 } else { 218 if (response_code > 0) 219 result_ = SERVER_ERROR; 220 else 221 result_ = NETWORK_ERROR; 222 OnResponseCompleted(); 223 } 224 } 225 226 void AppCacheUpdateJob::URLFetcher::OnReadCompleted( 227 net::URLRequest* request, int bytes_read) { 228 DCHECK(request_ == request); 229 bool data_consumed = true; 230 if (request->status().is_success() && bytes_read > 0) { 231 job_->MadeProgress(); 232 data_consumed = ConsumeResponseData(bytes_read); 233 if (data_consumed) { 234 bytes_read = 0; 235 while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) { 236 if (bytes_read > 0) { 237 data_consumed = ConsumeResponseData(bytes_read); 238 if (!data_consumed) 239 break; // wait for async data processing, then read more 240 } else { 241 break; 242 } 243 } 244 } 245 } 246 if (data_consumed && !request->status().is_io_pending()) { 247 DCHECK_EQ(UPDATE_OK, result_); 248 OnResponseCompleted(); 249 } 250 } 251 252 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders( 253 const net::HttpResponseHeaders* headers) { 254 DCHECK(request_.get() && headers); 255 net::HttpRequestHeaders extra_headers; 256 257 // Add If-Modified-Since header if response info has Last-Modified header. 258 const std::string last_modified = "Last-Modified"; 259 std::string last_modified_value; 260 headers->EnumerateHeader(NULL, last_modified, &last_modified_value); 261 if (!last_modified_value.empty()) { 262 extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince, 263 last_modified_value); 264 } 265 266 // Add If-None-Match header if response info has ETag header. 267 const std::string etag = "ETag"; 268 std::string etag_value; 269 headers->EnumerateHeader(NULL, etag, &etag_value); 270 if (!etag_value.empty()) { 271 extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch, 272 etag_value); 273 } 274 if (!extra_headers.IsEmpty()) 275 request_->SetExtraRequestHeaders(extra_headers); 276 } 277 278 void AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) { 279 if (result < 0) { 280 request_->Cancel(); 281 result_ = DISKCACHE_ERROR; 282 OnResponseCompleted(); 283 return; 284 } 285 ReadResponseData(); 286 } 287 288 void AppCacheUpdateJob::URLFetcher::ReadResponseData() { 289 InternalUpdateState state = job_->internal_state_; 290 if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED) 291 return; 292 int bytes_read = 0; 293 request_->Read(buffer_.get(), kBufferSize, &bytes_read); 294 OnReadCompleted(request_.get(), bytes_read); 295 } 296 297 // Returns false if response data is processed asynchronously, in which 298 // case ReadResponseData will be invoked when it is safe to continue 299 // reading more response data from the request. 300 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) { 301 DCHECK_GT(bytes_read, 0); 302 switch (fetch_type_) { 303 case MANIFEST_FETCH: 304 case MANIFEST_REFETCH: 305 manifest_data_.append(buffer_->data(), bytes_read); 306 break; 307 case URL_FETCH: 308 case MASTER_ENTRY_FETCH: 309 DCHECK(response_writer_.get()); 310 response_writer_->WriteData( 311 buffer_.get(), 312 bytes_read, 313 base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this))); 314 return false; // wait for async write completion to continue reading 315 default: 316 NOTREACHED(); 317 } 318 return true; 319 } 320 321 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() { 322 if (request_->status().is_success()) 323 job_->MadeProgress(); 324 325 // Retry for 503s where retry-after is 0. 326 if (request_->status().is_success() && 327 request_->GetResponseCode() == 503 && 328 MaybeRetryRequest()) { 329 return; 330 } 331 332 switch (fetch_type_) { 333 case MANIFEST_FETCH: 334 job_->HandleManifestFetchCompleted(this); 335 break; 336 case URL_FETCH: 337 job_->HandleUrlFetchCompleted(this); 338 break; 339 case MASTER_ENTRY_FETCH: 340 job_->HandleMasterEntryFetchCompleted(this); 341 break; 342 case MANIFEST_REFETCH: 343 job_->HandleManifestRefetchCompleted(this); 344 break; 345 default: 346 NOTREACHED(); 347 } 348 349 delete this; 350 } 351 352 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() { 353 if (retry_503_attempts_ >= kMax503Retries || 354 !request_->response_headers()->HasHeaderValue("retry-after", "0")) { 355 return false; 356 } 357 ++retry_503_attempts_; 358 result_ = UPDATE_OK; 359 request_ = job_->service_->request_context()->CreateRequest( 360 url_, net::DEFAULT_PRIORITY, this, NULL); 361 Start(); 362 return true; 363 } 364 365 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service, 366 AppCacheGroup* group) 367 : service_(service), 368 manifest_url_(group->manifest_url()), 369 group_(group), 370 update_type_(UNKNOWN_TYPE), 371 internal_state_(FETCH_MANIFEST), 372 master_entries_completed_(0), 373 url_fetches_completed_(0), 374 manifest_fetcher_(NULL), 375 manifest_has_valid_mime_type_(false), 376 stored_state_(UNSTORED), 377 storage_(service->storage()) { 378 service_->AddObserver(this); 379 } 380 381 AppCacheUpdateJob::~AppCacheUpdateJob() { 382 if (service_) 383 service_->RemoveObserver(this); 384 if (internal_state_ != COMPLETED) 385 Cancel(); 386 387 DCHECK(!manifest_fetcher_); 388 DCHECK(pending_url_fetches_.empty()); 389 DCHECK(!inprogress_cache_.get()); 390 DCHECK(pending_master_entries_.empty()); 391 DCHECK(master_entry_fetches_.empty()); 392 393 if (group_) 394 group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE); 395 } 396 397 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host, 398 const GURL& new_master_resource) { 399 DCHECK(group_->update_job() == this); 400 DCHECK(!group_->is_obsolete()); 401 402 bool is_new_pending_master_entry = false; 403 if (!new_master_resource.is_empty()) { 404 DCHECK(new_master_resource == host->pending_master_entry_url()); 405 DCHECK(!new_master_resource.has_ref()); 406 DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin()); 407 408 // Cannot add more to this update if already terminating. 409 if (IsTerminating()) { 410 group_->QueueUpdate(host, new_master_resource); 411 return; 412 } 413 414 std::pair<PendingMasters::iterator, bool> ret = 415 pending_master_entries_.insert( 416 PendingMasters::value_type(new_master_resource, PendingHosts())); 417 is_new_pending_master_entry = ret.second; 418 ret.first->second.push_back(host); 419 host->AddObserver(this); 420 } 421 422 // Notify host (if any) if already checking or downloading. 423 AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status(); 424 if (update_status == AppCacheGroup::CHECKING || 425 update_status == AppCacheGroup::DOWNLOADING) { 426 if (host) { 427 NotifySingleHost(host, APPCACHE_CHECKING_EVENT); 428 if (update_status == AppCacheGroup::DOWNLOADING) 429 NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT); 430 431 // Add to fetch list or an existing entry if already fetched. 432 if (!new_master_resource.is_empty()) { 433 AddMasterEntryToFetchList(host, new_master_resource, 434 is_new_pending_master_entry); 435 } 436 } 437 return; 438 } 439 440 // Begin update process for the group. 441 MadeProgress(); 442 group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING); 443 if (group_->HasCache()) { 444 update_type_ = UPGRADE_ATTEMPT; 445 NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT); 446 } else { 447 update_type_ = CACHE_ATTEMPT; 448 DCHECK(host); 449 NotifySingleHost(host, APPCACHE_CHECKING_EVENT); 450 } 451 452 if (!new_master_resource.is_empty()) { 453 AddMasterEntryToFetchList(host, new_master_resource, 454 is_new_pending_master_entry); 455 } 456 457 FetchManifest(true); 458 } 459 460 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() { 461 AppCacheResponseWriter* writer = 462 storage_->CreateResponseWriter(manifest_url_, 463 group_->group_id()); 464 stored_response_ids_.push_back(writer->response_id()); 465 return writer; 466 } 467 468 void AppCacheUpdateJob::HandleCacheFailure( 469 const AppCacheErrorDetails& error_details, 470 ResultType result, 471 const GURL& failed_resource_url) { 472 // 6.9.4 cache failure steps 2-8. 473 DCHECK(internal_state_ != CACHE_FAILURE); 474 DCHECK(!error_details.message.empty()); 475 DCHECK(result != UPDATE_OK); 476 internal_state_ = CACHE_FAILURE; 477 LogHistogramStats(result, failed_resource_url); 478 CancelAllUrlFetches(); 479 CancelAllMasterEntryFetches(error_details); 480 NotifyAllError(error_details); 481 DiscardInprogressCache(); 482 internal_state_ = COMPLETED; 483 DeleteSoon(); // To unwind the stack prior to deletion. 484 } 485 486 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) { 487 DCHECK(!manifest_fetcher_); 488 manifest_fetcher_ = new URLFetcher( 489 manifest_url_, 490 is_first_fetch ? URLFetcher::MANIFEST_FETCH : 491 URLFetcher::MANIFEST_REFETCH, 492 this); 493 494 // Add any necessary Http headers before sending fetch request. 495 if (is_first_fetch) { 496 AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ? 497 group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL; 498 if (entry) { 499 // Asynchronously load response info for manifest from newest cache. 500 storage_->LoadResponseInfo(manifest_url_, group_->group_id(), 501 entry->response_id(), this); 502 } else { 503 manifest_fetcher_->Start(); 504 } 505 } else { 506 DCHECK(internal_state_ == REFETCH_MANIFEST); 507 DCHECK(manifest_response_info_.get()); 508 manifest_fetcher_->set_existing_response_headers( 509 manifest_response_info_->headers.get()); 510 manifest_fetcher_->Start(); 511 } 512 } 513 514 515 void AppCacheUpdateJob::HandleManifestFetchCompleted( 516 URLFetcher* fetcher) { 517 DCHECK_EQ(internal_state_, FETCH_MANIFEST); 518 DCHECK_EQ(manifest_fetcher_, fetcher); 519 manifest_fetcher_ = NULL; 520 521 net::URLRequest* request = fetcher->request(); 522 int response_code = -1; 523 bool is_valid_response_code = false; 524 if (request->status().is_success()) { 525 response_code = request->GetResponseCode(); 526 is_valid_response_code = (response_code / 100 == 2); 527 528 std::string mime_type; 529 request->GetMimeType(&mime_type); 530 manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest"); 531 } 532 533 if (is_valid_response_code) { 534 manifest_data_ = fetcher->manifest_data(); 535 manifest_response_info_.reset( 536 new net::HttpResponseInfo(request->response_info())); 537 if (update_type_ == UPGRADE_ATTEMPT) 538 CheckIfManifestChanged(); // continues asynchronously 539 else 540 ContinueHandleManifestFetchCompleted(true); 541 } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) { 542 ContinueHandleManifestFetchCompleted(false); 543 } else if ((response_code == 404 || response_code == 410) && 544 update_type_ == UPGRADE_ATTEMPT) { 545 storage_->MakeGroupObsolete(group_, this, response_code); // async 546 } else { 547 const char* kFormatString = "Manifest fetch failed (%d) %s"; 548 std::string message = FormatUrlErrorMessage( 549 kFormatString, manifest_url_, fetcher->result(), response_code); 550 HandleCacheFailure(AppCacheErrorDetails(message, 551 APPCACHE_MANIFEST_ERROR, 552 manifest_url_, 553 response_code, 554 false /*is_cross_origin*/), 555 fetcher->result(), 556 GURL()); 557 } 558 } 559 560 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group, 561 bool success, 562 int response_code) { 563 DCHECK(master_entry_fetches_.empty()); 564 CancelAllMasterEntryFetches(AppCacheErrorDetails( 565 "The cache has been made obsolete, " 566 "the manifest file returned 404 or 410", 567 APPCACHE_MANIFEST_ERROR, 568 GURL(), 569 response_code, 570 false /*is_cross_origin*/)); 571 if (success) { 572 DCHECK(group->is_obsolete()); 573 NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT); 574 internal_state_ = COMPLETED; 575 MaybeCompleteUpdate(); 576 } else { 577 // Treat failure to mark group obsolete as a cache failure. 578 HandleCacheFailure(AppCacheErrorDetails( 579 "Failed to mark the cache as obsolete", 580 APPCACHE_UNKNOWN_ERROR, 581 GURL(), 582 0, 583 false /*is_cross_origin*/), 584 DB_ERROR, 585 GURL()); 586 } 587 } 588 589 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) { 590 DCHECK(internal_state_ == FETCH_MANIFEST); 591 592 if (!changed) { 593 DCHECK(update_type_ == UPGRADE_ATTEMPT); 594 internal_state_ = NO_UPDATE; 595 596 // Wait for pending master entries to download. 597 FetchMasterEntries(); 598 MaybeCompleteUpdate(); // if not done, run async 6.9.4 step 7 substeps 599 return; 600 } 601 602 AppCacheManifest manifest; 603 if (!ParseManifest(manifest_url_, manifest_data_.data(), 604 manifest_data_.length(), 605 manifest_has_valid_mime_type_ ? 606 PARSE_MANIFEST_ALLOWING_INTERCEPTS : 607 PARSE_MANIFEST_PER_STANDARD, 608 manifest)) { 609 const char* kFormatString = "Failed to parse manifest %s"; 610 const std::string message = base::StringPrintf(kFormatString, 611 manifest_url_.spec().c_str()); 612 HandleCacheFailure( 613 AppCacheErrorDetails( 614 message, APPCACHE_SIGNATURE_ERROR, GURL(), 0, 615 false /*is_cross_origin*/), 616 MANIFEST_ERROR, 617 GURL()); 618 VLOG(1) << message; 619 return; 620 } 621 622 // Proceed with update process. Section 6.9.4 steps 8-20. 623 internal_state_ = DOWNLOADING; 624 inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId()); 625 BuildUrlFileList(manifest); 626 inprogress_cache_->InitializeWithManifest(&manifest); 627 628 // Associate all pending master hosts with the newly created cache. 629 for (PendingMasters::iterator it = pending_master_entries_.begin(); 630 it != pending_master_entries_.end(); ++it) { 631 PendingHosts& hosts = it->second; 632 for (PendingHosts::iterator host_it = hosts.begin(); 633 host_it != hosts.end(); ++host_it) { 634 (*host_it) 635 ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_); 636 } 637 } 638 639 if (manifest.did_ignore_intercept_namespaces) { 640 // Must be done after associating all pending master hosts. 641 std::string message( 642 "Ignoring the INTERCEPT section of the application cache manifest " 643 "because the content type is not text/cache-manifest"); 644 LogConsoleMessageToAll(message); 645 } 646 647 group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING); 648 NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT); 649 FetchUrls(); 650 FetchMasterEntries(); 651 MaybeCompleteUpdate(); // if not done, continues when async fetches complete 652 } 653 654 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) { 655 DCHECK(internal_state_ == DOWNLOADING); 656 657 net::URLRequest* request = fetcher->request(); 658 const GURL& url = request->original_url(); 659 pending_url_fetches_.erase(url); 660 NotifyAllProgress(url); 661 ++url_fetches_completed_; 662 663 int response_code = request->status().is_success() 664 ? request->GetResponseCode() 665 : fetcher->redirect_response_code(); 666 667 AppCacheEntry& entry = url_file_list_.find(url)->second; 668 669 if (response_code / 100 == 2) { 670 // Associate storage with the new entry. 671 DCHECK(fetcher->response_writer()); 672 entry.set_response_id(fetcher->response_writer()->response_id()); 673 entry.set_response_size(fetcher->response_writer()->amount_written()); 674 if (!inprogress_cache_->AddOrModifyEntry(url, entry)) 675 duplicate_response_ids_.push_back(entry.response_id()); 676 677 // TODO(michaeln): Check for <html manifest=xxx> 678 // See http://code.google.com/p/chromium/issues/detail?id=97930 679 // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept)) 680 // if (!manifestAttribute) skip it 681 682 // Foreign entries will be detected during cache selection. 683 // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML 684 // file whose root element is an html element with a manifest attribute 685 // whose value doesn't match the manifest url of the application cache 686 // being processed, mark the entry as being foreign. 687 } else { 688 VLOG(1) << "Request status: " << request->status().status() 689 << " error: " << request->status().error() 690 << " response code: " << response_code; 691 if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) { 692 if (response_code == 304 && fetcher->existing_entry().has_response_id()) { 693 // Keep the existing response. 694 entry.set_response_id(fetcher->existing_entry().response_id()); 695 entry.set_response_size(fetcher->existing_entry().response_size()); 696 inprogress_cache_->AddOrModifyEntry(url, entry); 697 } else { 698 const char* kFormatString = "Resource fetch failed (%d) %s"; 699 std::string message = FormatUrlErrorMessage( 700 kFormatString, url, fetcher->result(), response_code); 701 ResultType result = fetcher->result(); 702 bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin(); 703 switch (result) { 704 case DISKCACHE_ERROR: 705 HandleCacheFailure( 706 AppCacheErrorDetails( 707 message, APPCACHE_UNKNOWN_ERROR, GURL(), 0, 708 is_cross_origin), 709 result, 710 url); 711 break; 712 case NETWORK_ERROR: 713 HandleCacheFailure( 714 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0, 715 is_cross_origin), 716 result, 717 url); 718 break; 719 default: 720 HandleCacheFailure(AppCacheErrorDetails(message, 721 APPCACHE_RESOURCE_ERROR, 722 url, 723 response_code, 724 is_cross_origin), 725 result, 726 url); 727 break; 728 } 729 return; 730 } 731 } else if (response_code == 404 || response_code == 410) { 732 // Entry is skipped. They are dropped from the cache. 733 } else if (update_type_ == UPGRADE_ATTEMPT && 734 fetcher->existing_entry().has_response_id()) { 735 // Keep the existing response. 736 // TODO(michaeln): Not sure this is a good idea. This is spec compliant 737 // but the old resource may or may not be compatible with the new contents 738 // of the cache. Impossible to know one way or the other. 739 entry.set_response_id(fetcher->existing_entry().response_id()); 740 entry.set_response_size(fetcher->existing_entry().response_size()); 741 inprogress_cache_->AddOrModifyEntry(url, entry); 742 } 743 } 744 745 // Fetch another URL now that one request has completed. 746 DCHECK(internal_state_ != CACHE_FAILURE); 747 FetchUrls(); 748 MaybeCompleteUpdate(); 749 } 750 751 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted( 752 URLFetcher* fetcher) { 753 DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING); 754 755 // TODO(jennb): Handle downloads completing during cache failure when update 756 // no longer fetches master entries directly. For now, we cancel all pending 757 // master entry fetches when entering cache failure state so this will never 758 // be called in CACHE_FAILURE state. 759 760 net::URLRequest* request = fetcher->request(); 761 const GURL& url = request->original_url(); 762 master_entry_fetches_.erase(url); 763 ++master_entries_completed_; 764 765 int response_code = request->status().is_success() 766 ? request->GetResponseCode() : -1; 767 768 PendingMasters::iterator found = pending_master_entries_.find(url); 769 DCHECK(found != pending_master_entries_.end()); 770 PendingHosts& hosts = found->second; 771 772 // Section 6.9.4. No update case: step 7.3, else step 22. 773 if (response_code / 100 == 2) { 774 // Add fetched master entry to the appropriate cache. 775 AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get() 776 : group_->newest_complete_cache(); 777 DCHECK(fetcher->response_writer()); 778 AppCacheEntry master_entry(AppCacheEntry::MASTER, 779 fetcher->response_writer()->response_id(), 780 fetcher->response_writer()->amount_written()); 781 if (cache->AddOrModifyEntry(url, master_entry)) 782 added_master_entries_.push_back(url); 783 else 784 duplicate_response_ids_.push_back(master_entry.response_id()); 785 786 // In no-update case, associate host with the newest cache. 787 if (!inprogress_cache_.get()) { 788 // TODO(michaeln): defer until the updated cache has been stored 789 DCHECK(cache == group_->newest_complete_cache()); 790 for (PendingHosts::iterator host_it = hosts.begin(); 791 host_it != hosts.end(); ++host_it) { 792 (*host_it)->AssociateCompleteCache(cache); 793 } 794 } 795 } else { 796 HostNotifier host_notifier; 797 for (PendingHosts::iterator host_it = hosts.begin(); 798 host_it != hosts.end(); ++host_it) { 799 AppCacheHost* host = *host_it; 800 host_notifier.AddHost(host); 801 802 // In downloading case, disassociate host from inprogress cache. 803 if (inprogress_cache_.get()) 804 host->AssociateNoCache(GURL()); 805 806 host->RemoveObserver(this); 807 } 808 hosts.clear(); 809 810 const char* kFormatString = "Manifest fetch failed (%d) %s"; 811 std::string message = FormatUrlErrorMessage( 812 kFormatString, request->url(), fetcher->result(), response_code); 813 host_notifier.SendErrorNotifications( 814 AppCacheErrorDetails(message, 815 APPCACHE_MANIFEST_ERROR, 816 request->url(), 817 response_code, 818 false /*is_cross_origin*/)); 819 820 // In downloading case, update result is different if all master entries 821 // failed vs. only some failing. 822 if (inprogress_cache_.get()) { 823 // Only count successful downloads to know if all master entries failed. 824 pending_master_entries_.erase(found); 825 --master_entries_completed_; 826 827 // Section 6.9.4, step 22.3. 828 if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) { 829 HandleCacheFailure(AppCacheErrorDetails(message, 830 APPCACHE_MANIFEST_ERROR, 831 request->url(), 832 response_code, 833 false /*is_cross_origin*/), 834 fetcher->result(), 835 GURL()); 836 return; 837 } 838 } 839 } 840 841 DCHECK(internal_state_ != CACHE_FAILURE); 842 FetchMasterEntries(); 843 MaybeCompleteUpdate(); 844 } 845 846 void AppCacheUpdateJob::HandleManifestRefetchCompleted( 847 URLFetcher* fetcher) { 848 DCHECK(internal_state_ == REFETCH_MANIFEST); 849 DCHECK(manifest_fetcher_ == fetcher); 850 manifest_fetcher_ = NULL; 851 852 net::URLRequest* request = fetcher->request(); 853 int response_code = request->status().is_success() 854 ? request->GetResponseCode() : -1; 855 if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) { 856 // Only need to store response in storage if manifest is not already 857 // an entry in the cache. 858 AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_); 859 if (entry) { 860 entry->add_types(AppCacheEntry::MANIFEST); 861 StoreGroupAndCache(); 862 } else { 863 manifest_response_writer_.reset(CreateResponseWriter()); 864 scoped_refptr<HttpResponseInfoIOBuffer> io_buffer( 865 new HttpResponseInfoIOBuffer(manifest_response_info_.release())); 866 manifest_response_writer_->WriteInfo( 867 io_buffer.get(), 868 base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete, 869 base::Unretained(this))); 870 } 871 } else { 872 VLOG(1) << "Request status: " << request->status().status() 873 << " error: " << request->status().error() 874 << " response code: " << response_code; 875 ScheduleUpdateRetry(kRerunDelayMs); 876 if (response_code == 200) { 877 HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update", 878 APPCACHE_CHANGED_ERROR, 879 GURL(), 880 0, 881 false /*is_cross_origin*/), 882 MANIFEST_ERROR, 883 GURL()); 884 } else { 885 const char* kFormatString = "Manifest re-fetch failed (%d) %s"; 886 std::string message = FormatUrlErrorMessage( 887 kFormatString, manifest_url_, fetcher->result(), response_code); 888 HandleCacheFailure(AppCacheErrorDetails(message, 889 APPCACHE_MANIFEST_ERROR, 890 GURL(), 891 response_code, 892 false /*is_cross_origin*/), 893 fetcher->result(), 894 GURL()); 895 } 896 } 897 } 898 899 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) { 900 if (result > 0) { 901 scoped_refptr<net::StringIOBuffer> io_buffer( 902 new net::StringIOBuffer(manifest_data_)); 903 manifest_response_writer_->WriteData( 904 io_buffer.get(), 905 manifest_data_.length(), 906 base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete, 907 base::Unretained(this))); 908 } else { 909 HandleCacheFailure( 910 AppCacheErrorDetails("Failed to write the manifest headers to storage", 911 APPCACHE_UNKNOWN_ERROR, 912 GURL(), 913 0, 914 false /*is_cross_origin*/), 915 DISKCACHE_ERROR, 916 GURL()); 917 } 918 } 919 920 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) { 921 if (result > 0) { 922 AppCacheEntry entry(AppCacheEntry::MANIFEST, 923 manifest_response_writer_->response_id(), 924 manifest_response_writer_->amount_written()); 925 if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry)) 926 duplicate_response_ids_.push_back(entry.response_id()); 927 StoreGroupAndCache(); 928 } else { 929 HandleCacheFailure( 930 AppCacheErrorDetails("Failed to write the manifest data to storage", 931 APPCACHE_UNKNOWN_ERROR, 932 GURL(), 933 0, 934 false /*is_cross_origin*/), 935 DISKCACHE_ERROR, 936 GURL()); 937 } 938 } 939 940 void AppCacheUpdateJob::StoreGroupAndCache() { 941 DCHECK(stored_state_ == UNSTORED); 942 stored_state_ = STORING; 943 scoped_refptr<AppCache> newest_cache; 944 if (inprogress_cache_.get()) 945 newest_cache.swap(inprogress_cache_); 946 else 947 newest_cache = group_->newest_complete_cache(); 948 newest_cache->set_update_time(base::Time::Now()); 949 950 // TODO(michaeln): dcheck is fishing for clues to crbug/95101 951 DCHECK_EQ(manifest_url_, group_->manifest_url()); 952 storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this); 953 } 954 955 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group, 956 AppCache* newest_cache, 957 bool success, 958 bool would_exceed_quota) { 959 DCHECK(stored_state_ == STORING); 960 if (success) { 961 stored_state_ = STORED; 962 MaybeCompleteUpdate(); // will definitely complete 963 } else { 964 stored_state_ = UNSTORED; 965 966 // Restore inprogress_cache_ to get the proper events delivered 967 // and the proper cleanup to occur. 968 if (newest_cache != group->newest_complete_cache()) 969 inprogress_cache_ = newest_cache; 970 971 ResultType result = DB_ERROR; 972 AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR; 973 std::string message("Failed to commit new cache to storage"); 974 if (would_exceed_quota) { 975 message.append(", would exceed quota"); 976 result = QUOTA_ERROR; 977 reason = APPCACHE_QUOTA_ERROR; 978 } 979 HandleCacheFailure( 980 AppCacheErrorDetails(message, reason, GURL(), 0, 981 false /*is_cross_origin*/), 982 result, 983 GURL()); 984 } 985 } 986 987 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host, 988 AppCacheEventID event_id) { 989 std::vector<int> ids(1, host->host_id()); 990 host->frontend()->OnEventRaised(ids, event_id); 991 } 992 993 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) { 994 HostNotifier host_notifier; 995 AddAllAssociatedHostsToNotifier(&host_notifier); 996 host_notifier.SendNotifications(event_id); 997 } 998 999 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) { 1000 HostNotifier host_notifier; 1001 AddAllAssociatedHostsToNotifier(&host_notifier); 1002 host_notifier.SendProgressNotifications( 1003 url, url_file_list_.size(), url_fetches_completed_); 1004 } 1005 1006 void AppCacheUpdateJob::NotifyAllFinalProgress() { 1007 DCHECK(url_file_list_.size() == url_fetches_completed_); 1008 NotifyAllProgress(GURL()); 1009 } 1010 1011 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) { 1012 HostNotifier host_notifier; 1013 AddAllAssociatedHostsToNotifier(&host_notifier); 1014 host_notifier.SendErrorNotifications(details); 1015 } 1016 1017 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) { 1018 HostNotifier host_notifier; 1019 AddAllAssociatedHostsToNotifier(&host_notifier); 1020 host_notifier.SendLogMessage(message); 1021 } 1022 1023 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier( 1024 HostNotifier* host_notifier) { 1025 // Collect hosts so we only send one notification per frontend. 1026 // A host can only be associated with a single cache so no need to worry 1027 // about duplicate hosts being added to the notifier. 1028 if (inprogress_cache_.get()) { 1029 DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE); 1030 host_notifier->AddHosts(inprogress_cache_->associated_hosts()); 1031 } 1032 1033 AppCacheGroup::Caches old_caches = group_->old_caches(); 1034 for (AppCacheGroup::Caches::const_iterator it = old_caches.begin(); 1035 it != old_caches.end(); ++it) { 1036 host_notifier->AddHosts((*it)->associated_hosts()); 1037 } 1038 1039 AppCache* newest_cache = group_->newest_complete_cache(); 1040 if (newest_cache) 1041 host_notifier->AddHosts(newest_cache->associated_hosts()); 1042 } 1043 1044 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) { 1045 // The host is about to be deleted; remove from our collection. 1046 PendingMasters::iterator found = 1047 pending_master_entries_.find(host->pending_master_entry_url()); 1048 DCHECK(found != pending_master_entries_.end()); 1049 PendingHosts& hosts = found->second; 1050 PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host); 1051 DCHECK(it != hosts.end()); 1052 hosts.erase(it); 1053 } 1054 1055 void AppCacheUpdateJob::OnServiceReinitialized( 1056 AppCacheStorageReference* old_storage_ref) { 1057 // We continue to use the disabled instance, but arrange for its 1058 // deletion when its no longer needed. 1059 if (old_storage_ref->storage() == storage_) 1060 disabled_storage_reference_ = old_storage_ref; 1061 } 1062 1063 void AppCacheUpdateJob::CheckIfManifestChanged() { 1064 DCHECK(update_type_ == UPGRADE_ATTEMPT); 1065 AppCacheEntry* entry = NULL; 1066 if (group_->newest_complete_cache()) 1067 entry = group_->newest_complete_cache()->GetEntry(manifest_url_); 1068 if (!entry) { 1069 // TODO(michaeln): This is just a bandaid to avoid a crash. 1070 // http://code.google.com/p/chromium/issues/detail?id=95101 1071 if (service_->storage() == storage_) { 1072 // Use a local variable because service_ is reset in HandleCacheFailure. 1073 AppCacheServiceImpl* service = service_; 1074 HandleCacheFailure( 1075 AppCacheErrorDetails("Manifest entry not found in existing cache", 1076 APPCACHE_UNKNOWN_ERROR, 1077 GURL(), 1078 0, 1079 false /*is_cross_origin*/), 1080 DB_ERROR, 1081 GURL()); 1082 AppCacheHistograms::AddMissingManifestEntrySample(); 1083 service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback()); 1084 } 1085 return; 1086 } 1087 1088 // Load manifest data from storage to compare against fetched manifest. 1089 manifest_response_reader_.reset( 1090 storage_->CreateResponseReader(manifest_url_, 1091 group_->group_id(), 1092 entry->response_id())); 1093 read_manifest_buffer_ = new net::IOBuffer(kBufferSize); 1094 manifest_response_reader_->ReadData( 1095 read_manifest_buffer_.get(), 1096 kBufferSize, 1097 base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete, 1098 base::Unretained(this))); // async read 1099 } 1100 1101 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) { 1102 if (result > 0) { 1103 loaded_manifest_data_.append(read_manifest_buffer_->data(), result); 1104 manifest_response_reader_->ReadData( 1105 read_manifest_buffer_.get(), 1106 kBufferSize, 1107 base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete, 1108 base::Unretained(this))); // read more 1109 } else { 1110 read_manifest_buffer_ = NULL; 1111 manifest_response_reader_.reset(); 1112 ContinueHandleManifestFetchCompleted( 1113 result < 0 || manifest_data_ != loaded_manifest_data_); 1114 } 1115 } 1116 1117 void AppCacheUpdateJob::BuildUrlFileList(const AppCacheManifest& manifest) { 1118 for (base::hash_set<std::string>::const_iterator it = 1119 manifest.explicit_urls.begin(); 1120 it != manifest.explicit_urls.end(); ++it) { 1121 AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT); 1122 } 1123 1124 const std::vector<AppCacheNamespace>& intercepts = 1125 manifest.intercept_namespaces; 1126 for (std::vector<AppCacheNamespace>::const_iterator it = intercepts.begin(); 1127 it != intercepts.end(); ++it) { 1128 int flags = AppCacheEntry::INTERCEPT; 1129 if (it->is_executable) 1130 flags |= AppCacheEntry::EXECUTABLE; 1131 AddUrlToFileList(it->target_url, flags); 1132 } 1133 1134 const std::vector<AppCacheNamespace>& fallbacks = 1135 manifest.fallback_namespaces; 1136 for (std::vector<AppCacheNamespace>::const_iterator it = fallbacks.begin(); 1137 it != fallbacks.end(); ++it) { 1138 AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK); 1139 } 1140 1141 // Add all master entries from newest complete cache. 1142 if (update_type_ == UPGRADE_ATTEMPT) { 1143 const AppCache::EntryMap& entries = 1144 group_->newest_complete_cache()->entries(); 1145 for (AppCache::EntryMap::const_iterator it = entries.begin(); 1146 it != entries.end(); ++it) { 1147 const AppCacheEntry& entry = it->second; 1148 if (entry.IsMaster()) 1149 AddUrlToFileList(it->first, AppCacheEntry::MASTER); 1150 } 1151 } 1152 } 1153 1154 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) { 1155 std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert( 1156 AppCache::EntryMap::value_type(url, AppCacheEntry(type))); 1157 1158 if (ret.second) 1159 urls_to_fetch_.push_back(UrlToFetch(url, false, NULL)); 1160 else 1161 ret.first->second.add_types(type); // URL already exists. Merge types. 1162 } 1163 1164 void AppCacheUpdateJob::FetchUrls() { 1165 DCHECK(internal_state_ == DOWNLOADING); 1166 1167 // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3. 1168 // Fetch up to the concurrent limit. Other fetches will be triggered as each 1169 // each fetch completes. 1170 while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches && 1171 !urls_to_fetch_.empty()) { 1172 UrlToFetch url_to_fetch = urls_to_fetch_.front(); 1173 urls_to_fetch_.pop_front(); 1174 1175 AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url); 1176 DCHECK(it != url_file_list_.end()); 1177 AppCacheEntry& entry = it->second; 1178 if (ShouldSkipUrlFetch(entry)) { 1179 NotifyAllProgress(url_to_fetch.url); 1180 ++url_fetches_completed_; 1181 } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) { 1182 NotifyAllProgress(url_to_fetch.url); 1183 ++url_fetches_completed_; // saved a URL request 1184 } else if (!url_to_fetch.storage_checked && 1185 MaybeLoadFromNewestCache(url_to_fetch.url, entry)) { 1186 // Continues asynchronously after data is loaded from newest cache. 1187 } else { 1188 URLFetcher* fetcher = new URLFetcher( 1189 url_to_fetch.url, URLFetcher::URL_FETCH, this); 1190 if (url_to_fetch.existing_response_info.get()) { 1191 DCHECK(group_->newest_complete_cache()); 1192 AppCacheEntry* existing_entry = 1193 group_->newest_complete_cache()->GetEntry(url_to_fetch.url); 1194 DCHECK(existing_entry); 1195 DCHECK(existing_entry->response_id() == 1196 url_to_fetch.existing_response_info->response_id()); 1197 fetcher->set_existing_response_headers( 1198 url_to_fetch.existing_response_info->http_response_info()->headers 1199 .get()); 1200 fetcher->set_existing_entry(*existing_entry); 1201 } 1202 fetcher->Start(); 1203 pending_url_fetches_.insert( 1204 PendingUrlFetches::value_type(url_to_fetch.url, fetcher)); 1205 } 1206 } 1207 } 1208 1209 void AppCacheUpdateJob::CancelAllUrlFetches() { 1210 // Cancel any pending URL requests. 1211 for (PendingUrlFetches::iterator it = pending_url_fetches_.begin(); 1212 it != pending_url_fetches_.end(); ++it) { 1213 delete it->second; 1214 } 1215 1216 url_fetches_completed_ += 1217 pending_url_fetches_.size() + urls_to_fetch_.size(); 1218 pending_url_fetches_.clear(); 1219 urls_to_fetch_.clear(); 1220 } 1221 1222 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) { 1223 // 6.6.4 Step 17 1224 // If the resource URL being processed was flagged as neither an 1225 // "explicit entry" nor or a "fallback entry", then the user agent 1226 // may skip this URL. 1227 if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) 1228 return false; 1229 1230 // TODO(jennb): decide if entry should be skipped to expire it from cache 1231 return false; 1232 } 1233 1234 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url, 1235 int entry_type) { 1236 DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE); 1237 AppCacheEntry* existing = 1238 inprogress_cache_.get() ? inprogress_cache_->GetEntry(url) 1239 : group_->newest_complete_cache()->GetEntry(url); 1240 if (existing) { 1241 existing->add_types(entry_type); 1242 return true; 1243 } 1244 return false; 1245 } 1246 1247 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host, 1248 const GURL& url, 1249 bool is_new) { 1250 DCHECK(!IsTerminating()); 1251 1252 if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) { 1253 AppCache* cache; 1254 if (inprogress_cache_.get()) { 1255 // always associate 1256 host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_); 1257 cache = inprogress_cache_.get(); 1258 } else { 1259 cache = group_->newest_complete_cache(); 1260 } 1261 1262 // Update existing entry if it has already been fetched. 1263 AppCacheEntry* entry = cache->GetEntry(url); 1264 if (entry) { 1265 entry->add_types(AppCacheEntry::MASTER); 1266 if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) { 1267 // only associate if have entry 1268 host->AssociateCompleteCache(cache); 1269 } 1270 if (is_new) 1271 ++master_entries_completed_; // pretend fetching completed 1272 return; 1273 } 1274 } 1275 1276 // Add to fetch list if not already fetching. 1277 if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) { 1278 master_entries_to_fetch_.insert(url); 1279 if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) 1280 FetchMasterEntries(); 1281 } 1282 } 1283 1284 void AppCacheUpdateJob::FetchMasterEntries() { 1285 DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING); 1286 1287 // Fetch each master entry in the list, up to the concurrent limit. 1288 // Additional fetches will be triggered as each fetch completes. 1289 while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches && 1290 !master_entries_to_fetch_.empty()) { 1291 const GURL& url = *master_entries_to_fetch_.begin(); 1292 1293 if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) { 1294 ++master_entries_completed_; // saved a URL request 1295 1296 // In no update case, associate hosts to newest cache in group 1297 // now that master entry has been "successfully downloaded". 1298 if (internal_state_ == NO_UPDATE) { 1299 // TODO(michaeln): defer until the updated cache has been stored. 1300 DCHECK(!inprogress_cache_.get()); 1301 AppCache* cache = group_->newest_complete_cache(); 1302 PendingMasters::iterator found = pending_master_entries_.find(url); 1303 DCHECK(found != pending_master_entries_.end()); 1304 PendingHosts& hosts = found->second; 1305 for (PendingHosts::iterator host_it = hosts.begin(); 1306 host_it != hosts.end(); ++host_it) { 1307 (*host_it)->AssociateCompleteCache(cache); 1308 } 1309 } 1310 } else { 1311 URLFetcher* fetcher = new URLFetcher( 1312 url, URLFetcher::MASTER_ENTRY_FETCH, this); 1313 fetcher->Start(); 1314 master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher)); 1315 } 1316 1317 master_entries_to_fetch_.erase(master_entries_to_fetch_.begin()); 1318 } 1319 } 1320 1321 void AppCacheUpdateJob::CancelAllMasterEntryFetches( 1322 const AppCacheErrorDetails& error_details) { 1323 // For now, cancel all in-progress fetches for master entries and pretend 1324 // all master entries fetches have completed. 1325 // TODO(jennb): Delete this when update no longer fetches master entries 1326 // directly. 1327 1328 // Cancel all in-progress fetches. 1329 for (PendingUrlFetches::iterator it = master_entry_fetches_.begin(); 1330 it != master_entry_fetches_.end(); ++it) { 1331 delete it->second; 1332 master_entries_to_fetch_.insert(it->first); // back in unfetched list 1333 } 1334 master_entry_fetches_.clear(); 1335 1336 master_entries_completed_ += master_entries_to_fetch_.size(); 1337 1338 // Cache failure steps, step 2. 1339 // Pretend all master entries that have not yet been fetched have completed 1340 // downloading. Unassociate hosts from any appcache and send ERROR event. 1341 HostNotifier host_notifier; 1342 while (!master_entries_to_fetch_.empty()) { 1343 const GURL& url = *master_entries_to_fetch_.begin(); 1344 PendingMasters::iterator found = pending_master_entries_.find(url); 1345 DCHECK(found != pending_master_entries_.end()); 1346 PendingHosts& hosts = found->second; 1347 for (PendingHosts::iterator host_it = hosts.begin(); 1348 host_it != hosts.end(); ++host_it) { 1349 AppCacheHost* host = *host_it; 1350 host->AssociateNoCache(GURL()); 1351 host_notifier.AddHost(host); 1352 host->RemoveObserver(this); 1353 } 1354 hosts.clear(); 1355 1356 master_entries_to_fetch_.erase(master_entries_to_fetch_.begin()); 1357 } 1358 host_notifier.SendErrorNotifications(error_details); 1359 } 1360 1361 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url, 1362 AppCacheEntry& entry) { 1363 if (update_type_ != UPGRADE_ATTEMPT) 1364 return false; 1365 1366 AppCache* newest = group_->newest_complete_cache(); 1367 AppCacheEntry* copy_me = newest->GetEntry(url); 1368 if (!copy_me || !copy_me->has_response_id()) 1369 return false; 1370 1371 // Load HTTP headers for entry from newest cache. 1372 loading_responses_.insert( 1373 LoadingResponses::value_type(copy_me->response_id(), url)); 1374 storage_->LoadResponseInfo(manifest_url_, group_->group_id(), 1375 copy_me->response_id(), 1376 this); 1377 // Async: wait for OnResponseInfoLoaded to complete. 1378 return true; 1379 } 1380 1381 void AppCacheUpdateJob::OnResponseInfoLoaded( 1382 AppCacheResponseInfo* response_info, int64 response_id) { 1383 const net::HttpResponseInfo* http_info = response_info ? 1384 response_info->http_response_info() : NULL; 1385 1386 // Needed response info for a manifest fetch request. 1387 if (internal_state_ == FETCH_MANIFEST) { 1388 if (http_info) 1389 manifest_fetcher_->set_existing_response_headers( 1390 http_info->headers.get()); 1391 manifest_fetcher_->Start(); 1392 return; 1393 } 1394 1395 LoadingResponses::iterator found = loading_responses_.find(response_id); 1396 DCHECK(found != loading_responses_.end()); 1397 const GURL& url = found->second; 1398 1399 if (!http_info) { 1400 LoadFromNewestCacheFailed(url, NULL); // no response found 1401 } else { 1402 // Check if response can be re-used according to HTTP caching semantics. 1403 // Responses with a "vary" header get treated as expired. 1404 const std::string name = "vary"; 1405 std::string value; 1406 void* iter = NULL; 1407 if (!http_info->headers.get() || 1408 http_info->headers->RequiresValidation(http_info->request_time, 1409 http_info->response_time, 1410 base::Time::Now()) || 1411 http_info->headers->EnumerateHeader(&iter, name, &value)) { 1412 LoadFromNewestCacheFailed(url, response_info); 1413 } else { 1414 DCHECK(group_->newest_complete_cache()); 1415 AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url); 1416 DCHECK(copy_me); 1417 DCHECK(copy_me->response_id() == response_id); 1418 1419 AppCache::EntryMap::iterator it = url_file_list_.find(url); 1420 DCHECK(it != url_file_list_.end()); 1421 AppCacheEntry& entry = it->second; 1422 entry.set_response_id(response_id); 1423 entry.set_response_size(copy_me->response_size()); 1424 inprogress_cache_->AddOrModifyEntry(url, entry); 1425 NotifyAllProgress(url); 1426 ++url_fetches_completed_; 1427 } 1428 } 1429 loading_responses_.erase(found); 1430 1431 MaybeCompleteUpdate(); 1432 } 1433 1434 void AppCacheUpdateJob::LoadFromNewestCacheFailed( 1435 const GURL& url, AppCacheResponseInfo* response_info) { 1436 if (internal_state_ == CACHE_FAILURE) 1437 return; 1438 1439 // Re-insert url at front of fetch list. Indicate storage has been checked. 1440 urls_to_fetch_.push_front(UrlToFetch(url, true, response_info)); 1441 FetchUrls(); 1442 } 1443 1444 void AppCacheUpdateJob::MaybeCompleteUpdate() { 1445 DCHECK(internal_state_ != CACHE_FAILURE); 1446 1447 // Must wait for any pending master entries or url fetches to complete. 1448 if (master_entries_completed_ != pending_master_entries_.size() || 1449 url_fetches_completed_ != url_file_list_.size()) { 1450 DCHECK(internal_state_ != COMPLETED); 1451 return; 1452 } 1453 1454 switch (internal_state_) { 1455 case NO_UPDATE: 1456 if (master_entries_completed_ > 0) { 1457 switch (stored_state_) { 1458 case UNSTORED: 1459 StoreGroupAndCache(); 1460 return; 1461 case STORING: 1462 return; 1463 case STORED: 1464 break; 1465 } 1466 } 1467 // 6.9.4 steps 7.3-7.7. 1468 NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT); 1469 DiscardDuplicateResponses(); 1470 internal_state_ = COMPLETED; 1471 break; 1472 case DOWNLOADING: 1473 internal_state_ = REFETCH_MANIFEST; 1474 FetchManifest(false); 1475 break; 1476 case REFETCH_MANIFEST: 1477 DCHECK(stored_state_ == STORED); 1478 NotifyAllFinalProgress(); 1479 if (update_type_ == CACHE_ATTEMPT) 1480 NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT); 1481 else 1482 NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT); 1483 DiscardDuplicateResponses(); 1484 internal_state_ = COMPLETED; 1485 LogHistogramStats(UPDATE_OK, GURL()); 1486 break; 1487 case CACHE_FAILURE: 1488 NOTREACHED(); // See HandleCacheFailure 1489 break; 1490 default: 1491 break; 1492 } 1493 1494 // Let the stack unwind before deletion to make it less risky as this 1495 // method is called from multiple places in this file. 1496 if (internal_state_ == COMPLETED) 1497 DeleteSoon(); 1498 } 1499 1500 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) { 1501 // TODO(jennb): post a delayed task with the "same parameters" as this job 1502 // to retry the update at a later time. Need group, URLs of pending master 1503 // entries and their hosts. 1504 } 1505 1506 void AppCacheUpdateJob::Cancel() { 1507 internal_state_ = CANCELLED; 1508 1509 LogHistogramStats(CANCELLED_ERROR, GURL()); 1510 1511 if (manifest_fetcher_) { 1512 delete manifest_fetcher_; 1513 manifest_fetcher_ = NULL; 1514 } 1515 1516 for (PendingUrlFetches::iterator it = pending_url_fetches_.begin(); 1517 it != pending_url_fetches_.end(); ++it) { 1518 delete it->second; 1519 } 1520 pending_url_fetches_.clear(); 1521 1522 for (PendingUrlFetches::iterator it = master_entry_fetches_.begin(); 1523 it != master_entry_fetches_.end(); ++it) { 1524 delete it->second; 1525 } 1526 master_entry_fetches_.clear(); 1527 1528 ClearPendingMasterEntries(); 1529 DiscardInprogressCache(); 1530 1531 // Delete response writer to avoid any callbacks. 1532 if (manifest_response_writer_) 1533 manifest_response_writer_.reset(); 1534 1535 storage_->CancelDelegateCallbacks(this); 1536 } 1537 1538 void AppCacheUpdateJob::ClearPendingMasterEntries() { 1539 for (PendingMasters::iterator it = pending_master_entries_.begin(); 1540 it != pending_master_entries_.end(); ++it) { 1541 PendingHosts& hosts = it->second; 1542 for (PendingHosts::iterator host_it = hosts.begin(); 1543 host_it != hosts.end(); ++host_it) { 1544 (*host_it)->RemoveObserver(this); 1545 } 1546 } 1547 1548 pending_master_entries_.clear(); 1549 } 1550 1551 void AppCacheUpdateJob::DiscardInprogressCache() { 1552 if (stored_state_ == STORING) { 1553 // We can make no assumptions about whether the StoreGroupAndCacheTask 1554 // actually completed or not. This condition should only be reachable 1555 // during shutdown. Free things up and return to do no harm. 1556 inprogress_cache_ = NULL; 1557 added_master_entries_.clear(); 1558 return; 1559 } 1560 1561 storage_->DoomResponses(manifest_url_, stored_response_ids_); 1562 1563 if (!inprogress_cache_.get()) { 1564 // We have to undo the changes we made, if any, to the existing cache. 1565 if (group_ && group_->newest_complete_cache()) { 1566 for (std::vector<GURL>::iterator iter = added_master_entries_.begin(); 1567 iter != added_master_entries_.end(); ++iter) { 1568 group_->newest_complete_cache()->RemoveEntry(*iter); 1569 } 1570 } 1571 added_master_entries_.clear(); 1572 return; 1573 } 1574 1575 AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts(); 1576 while (!hosts.empty()) 1577 (*hosts.begin())->AssociateNoCache(GURL()); 1578 1579 inprogress_cache_ = NULL; 1580 added_master_entries_.clear(); 1581 } 1582 1583 void AppCacheUpdateJob::DiscardDuplicateResponses() { 1584 storage_->DoomResponses(manifest_url_, duplicate_response_ids_); 1585 } 1586 1587 void AppCacheUpdateJob::LogHistogramStats( 1588 ResultType result, const GURL& failed_resource_url) { 1589 AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin()); 1590 if (result == UPDATE_OK) 1591 return; 1592 1593 int percent_complete = 0; 1594 if (url_file_list_.size() > 0) { 1595 size_t actual_fetches_completed = url_fetches_completed_; 1596 if (!failed_resource_url.is_empty() && actual_fetches_completed) 1597 --actual_fetches_completed; 1598 percent_complete = (static_cast<double>(actual_fetches_completed) / 1599 static_cast<double>(url_file_list_.size())) * 100.0; 1600 percent_complete = std::min(percent_complete, 99); 1601 } 1602 1603 bool was_making_progress = 1604 base::Time::Now() - last_progress_time_ < 1605 base::TimeDelta::FromMinutes(5); 1606 1607 bool off_origin_resource_failure = 1608 !failed_resource_url.is_empty() && 1609 (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin()); 1610 1611 AppCacheHistograms::LogUpdateFailureStats( 1612 manifest_url_.GetOrigin(), 1613 percent_complete, 1614 was_making_progress, 1615 off_origin_resource_failure); 1616 } 1617 1618 void AppCacheUpdateJob::DeleteSoon() { 1619 ClearPendingMasterEntries(); 1620 manifest_response_writer_.reset(); 1621 storage_->CancelDelegateCallbacks(this); 1622 service_->RemoveObserver(this); 1623 service_ = NULL; 1624 1625 // Break the connection with the group so the group cannot call delete 1626 // on this object after we've posted a task to delete ourselves. 1627 group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE); 1628 group_ = NULL; 1629 1630 base::MessageLoop::current()->DeleteSoon(FROM_HERE, this); 1631 } 1632 1633 } // namespace content 1634
