1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "webkit/browser/appcache/appcache_update_job.h" 6 7 #include "base/bind.h" 8 #include "base/bind_helpers.h" 9 #include "base/compiler_specific.h" 10 #include "base/message_loop/message_loop.h" 11 #include "base/strings/string_util.h" 12 #include "base/strings/stringprintf.h" 13 #include "net/base/io_buffer.h" 14 #include "net/base/load_flags.h" 15 #include "net/base/net_errors.h" 16 #include "net/base/request_priority.h" 17 #include "net/http/http_request_headers.h" 18 #include "net/http/http_response_headers.h" 19 #include "net/url_request/url_request_context.h" 20 #include "webkit/browser/appcache/appcache_group.h" 21 #include "webkit/browser/appcache/appcache_histograms.h" 22 23 namespace appcache { 24 25 static const int kBufferSize = 32768; 26 static const size_t kMaxConcurrentUrlFetches = 2; 27 static const int kMax503Retries = 3; 28 29 static std::string FormatUrlErrorMessage( 30 const char* format, const GURL& url, 31 AppCacheUpdateJob::ResultType error, 32 int response_code) { 33 // Show the net response code if we have one. 34 int code = response_code; 35 if (error != AppCacheUpdateJob::SERVER_ERROR) 36 code = static_cast<int>(error); 37 return base::StringPrintf(format, code, url.spec().c_str()); 38 } 39 40 // Helper class for collecting hosts per frontend when sending notifications 41 // so that only one notification is sent for all hosts using the same frontend. 42 class HostNotifier { 43 public: 44 typedef std::vector<int> HostIds; 45 typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap; 46 47 // Caller is responsible for ensuring there will be no duplicate hosts. 48 void AddHost(AppCacheHost* host) { 49 std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert( 50 NotifyHostMap::value_type(host->frontend(), HostIds())); 51 ret.first->second.push_back(host->host_id()); 52 } 53 54 void AddHosts(const std::set<AppCacheHost*>& hosts) { 55 for (std::set<AppCacheHost*>::const_iterator it = hosts.begin(); 56 it != hosts.end(); ++it) { 57 AddHost(*it); 58 } 59 } 60 61 void SendNotifications(AppCacheEventID event_id) { 62 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 63 it != hosts_to_notify.end(); ++it) { 64 AppCacheFrontend* frontend = it->first; 65 frontend->OnEventRaised(it->second, event_id); 66 } 67 } 68 69 void SendProgressNotifications( 70 const GURL& url, int num_total, int num_complete) { 71 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 72 it != hosts_to_notify.end(); ++it) { 73 AppCacheFrontend* frontend = it->first; 74 frontend->OnProgressEventRaised(it->second, url, 75 num_total, num_complete); 76 } 77 } 78 79 void SendErrorNotifications(const AppCacheErrorDetails& details) { 80 DCHECK(!details.message.empty()); 81 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 82 it != hosts_to_notify.end(); ++it) { 83 AppCacheFrontend* frontend = it->first; 84 frontend->OnErrorEventRaised(it->second, details); 85 } 86 } 87 88 void SendLogMessage(const std::string& message) { 89 for (NotifyHostMap::iterator it = hosts_to_notify.begin(); 90 it != hosts_to_notify.end(); ++it) { 91 AppCacheFrontend* frontend = it->first; 92 for (HostIds::iterator id = it->second.begin(); 93 id != it->second.end(); ++id) { 94 frontend->OnLogMessage(*id, APPCACHE_LOG_WARNING, message); 95 } 96 } 97 } 98 99 private: 100 NotifyHostMap hosts_to_notify; 101 }; 102 103 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url, 104 bool checked, 105 AppCacheResponseInfo* info) 106 : url(url), 107 storage_checked(checked), 108 existing_response_info(info) { 109 } 110 111 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() { 112 } 113 114 // Helper class to fetch resources. Depending on the fetch type, 115 // can either fetch to an in-memory string or write the response 116 // data out to the disk cache. 117 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url, 118 FetchType fetch_type, 119 AppCacheUpdateJob* job) 120 : url_(url), 121 job_(job), 122 fetch_type_(fetch_type), 123 retry_503_attempts_(0), 124 buffer_(new net::IOBuffer(kBufferSize)), 125 request_(job->service_->request_context() 126 ->CreateRequest(url, net::DEFAULT_PRIORITY, this, NULL)), 127 result_(UPDATE_OK), 128 redirect_response_code_(-1) {} 129 130 AppCacheUpdateJob::URLFetcher::~URLFetcher() { 131 } 132 133 void AppCacheUpdateJob::URLFetcher::Start() { 134 request_->set_first_party_for_cookies(job_->manifest_url_); 135 request_->SetLoadFlags(request_->load_flags() | 136 net::LOAD_DISABLE_INTERCEPT); 137 if (existing_response_headers_.get()) 138 AddConditionalHeaders(existing_response_headers_.get()); 139 request_->Start(); 140 } 141 142 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect( 143 net::URLRequest* request, const GURL& new_url, bool* defer_redirect) { 144 DCHECK(request_ == request); 145 // Redirect is not allowed by the update process. 146 job_->MadeProgress(); 147 redirect_response_code_ = request->GetResponseCode(); 148 request->Cancel(); 149 result_ = REDIRECT_ERROR; 150 OnResponseCompleted(); 151 } 152 153 void AppCacheUpdateJob::URLFetcher::OnResponseStarted( 154 net::URLRequest *request) { 155 DCHECK(request == request_); 156 int response_code = -1; 157 if (request->status().is_success()) { 158 response_code = request->GetResponseCode(); 159 job_->MadeProgress(); 160 } 161 if ((response_code / 100) == 2) { 162 163 // See http://code.google.com/p/chromium/issues/detail?id=69594 164 // We willfully violate the HTML5 spec at this point in order 165 // to support the appcaching of cross-origin HTTPS resources. 166 // We've opted for a milder constraint and allow caching unless 167 // the resource has a "no-store" header. A spec change has been 168 // requested on the whatwg list. 169 // TODO(michaeln): Consider doing this for cross-origin HTTP resources too. 170 if (url_.SchemeIsSecure() && 171 url_.GetOrigin() != job_->manifest_url_.GetOrigin()) { 172 if (request->response_headers()-> 173 HasHeaderValue("cache-control", "no-store")) { 174 DCHECK_EQ(-1, redirect_response_code_); 175 request->Cancel(); 176 result_ = SERVER_ERROR; // Not the best match? 177 OnResponseCompleted(); 178 return; 179 } 180 } 181 182 // Write response info to storage for URL fetches. Wait for async write 183 // completion before reading any response data. 184 if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) { 185 response_writer_.reset(job_->CreateResponseWriter()); 186 scoped_refptr<HttpResponseInfoIOBuffer> io_buffer( 187 new HttpResponseInfoIOBuffer( 188 new net::HttpResponseInfo(request->response_info()))); 189 response_writer_->WriteInfo( 190 io_buffer.get(), 191 base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this))); 192 } else { 193 ReadResponseData(); 194 } 195 } else { 196 if (response_code > 0) 197 result_ = SERVER_ERROR; 198 else 199 result_ = NETWORK_ERROR; 200 OnResponseCompleted(); 201 } 202 } 203 204 void AppCacheUpdateJob::URLFetcher::OnReadCompleted( 205 net::URLRequest* request, int bytes_read) { 206 DCHECK(request_ == request); 207 bool data_consumed = true; 208 if (request->status().is_success() && bytes_read > 0) { 209 job_->MadeProgress(); 210 data_consumed = ConsumeResponseData(bytes_read); 211 if (data_consumed) { 212 bytes_read = 0; 213 while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) { 214 if (bytes_read > 0) { 215 data_consumed = ConsumeResponseData(bytes_read); 216 if (!data_consumed) 217 break; // wait for async data processing, then read more 218 } else { 219 break; 220 } 221 } 222 } 223 } 224 if (data_consumed && !request->status().is_io_pending()) { 225 DCHECK_EQ(UPDATE_OK, result_); 226 OnResponseCompleted(); 227 } 228 } 229 230 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders( 231 const net::HttpResponseHeaders* headers) { 232 DCHECK(request_.get() && headers); 233 net::HttpRequestHeaders extra_headers; 234 235 // Add If-Modified-Since header if response info has Last-Modified header. 236 const std::string last_modified = "Last-Modified"; 237 std::string last_modified_value; 238 headers->EnumerateHeader(NULL, last_modified, &last_modified_value); 239 if (!last_modified_value.empty()) { 240 extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince, 241 last_modified_value); 242 } 243 244 // Add If-None-Match header if response info has ETag header. 245 const std::string etag = "ETag"; 246 std::string etag_value; 247 headers->EnumerateHeader(NULL, etag, &etag_value); 248 if (!etag_value.empty()) { 249 extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch, 250 etag_value); 251 } 252 if (!extra_headers.IsEmpty()) 253 request_->SetExtraRequestHeaders(extra_headers); 254 } 255 256 void AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) { 257 if (result < 0) { 258 request_->Cancel(); 259 result_ = DISKCACHE_ERROR; 260 OnResponseCompleted(); 261 return; 262 } 263 ReadResponseData(); 264 } 265 266 void AppCacheUpdateJob::URLFetcher::ReadResponseData() { 267 InternalUpdateState state = job_->internal_state_; 268 if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED) 269 return; 270 int bytes_read = 0; 271 request_->Read(buffer_.get(), kBufferSize, &bytes_read); 272 OnReadCompleted(request_.get(), bytes_read); 273 } 274 275 // Returns false if response data is processed asynchronously, in which 276 // case ReadResponseData will be invoked when it is safe to continue 277 // reading more response data from the request. 278 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) { 279 DCHECK_GT(bytes_read, 0); 280 switch (fetch_type_) { 281 case MANIFEST_FETCH: 282 case MANIFEST_REFETCH: 283 manifest_data_.append(buffer_->data(), bytes_read); 284 break; 285 case URL_FETCH: 286 case MASTER_ENTRY_FETCH: 287 DCHECK(response_writer_.get()); 288 response_writer_->WriteData( 289 buffer_.get(), 290 bytes_read, 291 base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this))); 292 return false; // wait for async write completion to continue reading 293 default: 294 NOTREACHED(); 295 } 296 return true; 297 } 298 299 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() { 300 if (request_->status().is_success()) 301 job_->MadeProgress(); 302 303 // Retry for 503s where retry-after is 0. 304 if (request_->status().is_success() && 305 request_->GetResponseCode() == 503 && 306 MaybeRetryRequest()) { 307 return; 308 } 309 310 switch (fetch_type_) { 311 case MANIFEST_FETCH: 312 job_->HandleManifestFetchCompleted(this); 313 break; 314 case URL_FETCH: 315 job_->HandleUrlFetchCompleted(this); 316 break; 317 case MASTER_ENTRY_FETCH: 318 job_->HandleMasterEntryFetchCompleted(this); 319 break; 320 case MANIFEST_REFETCH: 321 job_->HandleManifestRefetchCompleted(this); 322 break; 323 default: 324 NOTREACHED(); 325 } 326 327 delete this; 328 } 329 330 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() { 331 if (retry_503_attempts_ >= kMax503Retries || 332 !request_->response_headers()->HasHeaderValue("retry-after", "0")) { 333 return false; 334 } 335 ++retry_503_attempts_; 336 result_ = UPDATE_OK; 337 request_ = job_->service_->request_context()->CreateRequest( 338 url_, net::DEFAULT_PRIORITY, this, NULL); 339 Start(); 340 return true; 341 } 342 343 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheServiceImpl* service, 344 AppCacheGroup* group) 345 : service_(service), 346 manifest_url_(group->manifest_url()), 347 group_(group), 348 update_type_(UNKNOWN_TYPE), 349 internal_state_(FETCH_MANIFEST), 350 master_entries_completed_(0), 351 url_fetches_completed_(0), 352 manifest_fetcher_(NULL), 353 manifest_has_valid_mime_type_(false), 354 stored_state_(UNSTORED), 355 storage_(service->storage()) { 356 service_->AddObserver(this); 357 } 358 359 AppCacheUpdateJob::~AppCacheUpdateJob() { 360 if (service_) 361 service_->RemoveObserver(this); 362 if (internal_state_ != COMPLETED) 363 Cancel(); 364 365 DCHECK(!manifest_fetcher_); 366 DCHECK(pending_url_fetches_.empty()); 367 DCHECK(!inprogress_cache_.get()); 368 DCHECK(pending_master_entries_.empty()); 369 DCHECK(master_entry_fetches_.empty()); 370 371 if (group_) 372 group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE); 373 } 374 375 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host, 376 const GURL& new_master_resource) { 377 DCHECK(group_->update_job() == this); 378 DCHECK(!group_->is_obsolete()); 379 380 bool is_new_pending_master_entry = false; 381 if (!new_master_resource.is_empty()) { 382 DCHECK(new_master_resource == host->pending_master_entry_url()); 383 DCHECK(!new_master_resource.has_ref()); 384 DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin()); 385 386 // Cannot add more to this update if already terminating. 387 if (IsTerminating()) { 388 group_->QueueUpdate(host, new_master_resource); 389 return; 390 } 391 392 std::pair<PendingMasters::iterator, bool> ret = 393 pending_master_entries_.insert( 394 PendingMasters::value_type(new_master_resource, PendingHosts())); 395 is_new_pending_master_entry = ret.second; 396 ret.first->second.push_back(host); 397 host->AddObserver(this); 398 } 399 400 // Notify host (if any) if already checking or downloading. 401 AppCacheGroup::UpdateAppCacheStatus update_status = group_->update_status(); 402 if (update_status == AppCacheGroup::CHECKING || 403 update_status == AppCacheGroup::DOWNLOADING) { 404 if (host) { 405 NotifySingleHost(host, APPCACHE_CHECKING_EVENT); 406 if (update_status == AppCacheGroup::DOWNLOADING) 407 NotifySingleHost(host, APPCACHE_DOWNLOADING_EVENT); 408 409 // Add to fetch list or an existing entry if already fetched. 410 if (!new_master_resource.is_empty()) { 411 AddMasterEntryToFetchList(host, new_master_resource, 412 is_new_pending_master_entry); 413 } 414 } 415 return; 416 } 417 418 // Begin update process for the group. 419 MadeProgress(); 420 group_->SetUpdateAppCacheStatus(AppCacheGroup::CHECKING); 421 if (group_->HasCache()) { 422 update_type_ = UPGRADE_ATTEMPT; 423 NotifyAllAssociatedHosts(APPCACHE_CHECKING_EVENT); 424 } else { 425 update_type_ = CACHE_ATTEMPT; 426 DCHECK(host); 427 NotifySingleHost(host, APPCACHE_CHECKING_EVENT); 428 } 429 430 if (!new_master_resource.is_empty()) { 431 AddMasterEntryToFetchList(host, new_master_resource, 432 is_new_pending_master_entry); 433 } 434 435 FetchManifest(true); 436 } 437 438 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() { 439 AppCacheResponseWriter* writer = 440 storage_->CreateResponseWriter(manifest_url_, 441 group_->group_id()); 442 stored_response_ids_.push_back(writer->response_id()); 443 return writer; 444 } 445 446 void AppCacheUpdateJob::HandleCacheFailure( 447 const AppCacheErrorDetails& error_details, 448 ResultType result, 449 const GURL& failed_resource_url) { 450 // 6.9.4 cache failure steps 2-8. 451 DCHECK(internal_state_ != CACHE_FAILURE); 452 DCHECK(!error_details.message.empty()); 453 DCHECK(result != UPDATE_OK); 454 internal_state_ = CACHE_FAILURE; 455 LogHistogramStats(result, failed_resource_url); 456 CancelAllUrlFetches(); 457 CancelAllMasterEntryFetches(error_details); 458 NotifyAllError(error_details); 459 DiscardInprogressCache(); 460 internal_state_ = COMPLETED; 461 DeleteSoon(); // To unwind the stack prior to deletion. 462 } 463 464 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) { 465 DCHECK(!manifest_fetcher_); 466 manifest_fetcher_ = new URLFetcher( 467 manifest_url_, 468 is_first_fetch ? URLFetcher::MANIFEST_FETCH : 469 URLFetcher::MANIFEST_REFETCH, 470 this); 471 472 // Add any necessary Http headers before sending fetch request. 473 if (is_first_fetch) { 474 AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ? 475 group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL; 476 if (entry) { 477 // Asynchronously load response info for manifest from newest cache. 478 storage_->LoadResponseInfo(manifest_url_, group_->group_id(), 479 entry->response_id(), this); 480 } else { 481 manifest_fetcher_->Start(); 482 } 483 } else { 484 DCHECK(internal_state_ == REFETCH_MANIFEST); 485 DCHECK(manifest_response_info_.get()); 486 manifest_fetcher_->set_existing_response_headers( 487 manifest_response_info_->headers.get()); 488 manifest_fetcher_->Start(); 489 } 490 } 491 492 493 void AppCacheUpdateJob::HandleManifestFetchCompleted( 494 URLFetcher* fetcher) { 495 DCHECK_EQ(internal_state_, FETCH_MANIFEST); 496 DCHECK_EQ(manifest_fetcher_, fetcher); 497 manifest_fetcher_ = NULL; 498 499 net::URLRequest* request = fetcher->request(); 500 int response_code = -1; 501 bool is_valid_response_code = false; 502 if (request->status().is_success()) { 503 response_code = request->GetResponseCode(); 504 is_valid_response_code = (response_code / 100 == 2); 505 506 std::string mime_type; 507 request->GetMimeType(&mime_type); 508 manifest_has_valid_mime_type_ = (mime_type == "text/cache-manifest"); 509 } 510 511 if (is_valid_response_code) { 512 manifest_data_ = fetcher->manifest_data(); 513 manifest_response_info_.reset( 514 new net::HttpResponseInfo(request->response_info())); 515 if (update_type_ == UPGRADE_ATTEMPT) 516 CheckIfManifestChanged(); // continues asynchronously 517 else 518 ContinueHandleManifestFetchCompleted(true); 519 } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) { 520 ContinueHandleManifestFetchCompleted(false); 521 } else if ((response_code == 404 || response_code == 410) && 522 update_type_ == UPGRADE_ATTEMPT) { 523 storage_->MakeGroupObsolete(group_, this, response_code); // async 524 } else { 525 const char* kFormatString = "Manifest fetch failed (%d) %s"; 526 std::string message = FormatUrlErrorMessage( 527 kFormatString, manifest_url_, fetcher->result(), response_code); 528 HandleCacheFailure(AppCacheErrorDetails(message, 529 appcache::APPCACHE_MANIFEST_ERROR, 530 manifest_url_, 531 response_code, 532 false /*is_cross_origin*/), 533 fetcher->result(), 534 GURL()); 535 } 536 } 537 538 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group, 539 bool success, 540 int response_code) { 541 DCHECK(master_entry_fetches_.empty()); 542 CancelAllMasterEntryFetches(AppCacheErrorDetails( 543 "The cache has been made obsolete, " 544 "the manifest file returned 404 or 410", 545 appcache::APPCACHE_MANIFEST_ERROR, 546 GURL(), 547 response_code, 548 false /*is_cross_origin*/)); 549 if (success) { 550 DCHECK(group->is_obsolete()); 551 NotifyAllAssociatedHosts(APPCACHE_OBSOLETE_EVENT); 552 internal_state_ = COMPLETED; 553 MaybeCompleteUpdate(); 554 } else { 555 // Treat failure to mark group obsolete as a cache failure. 556 HandleCacheFailure(AppCacheErrorDetails( 557 "Failed to mark the cache as obsolete", 558 APPCACHE_UNKNOWN_ERROR, 559 GURL(), 560 0, 561 false /*is_cross_origin*/), 562 DB_ERROR, 563 GURL()); 564 } 565 } 566 567 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) { 568 DCHECK(internal_state_ == FETCH_MANIFEST); 569 570 if (!changed) { 571 DCHECK(update_type_ == UPGRADE_ATTEMPT); 572 internal_state_ = NO_UPDATE; 573 574 // Wait for pending master entries to download. 575 FetchMasterEntries(); 576 MaybeCompleteUpdate(); // if not done, run async 6.9.4 step 7 substeps 577 return; 578 } 579 580 Manifest manifest; 581 if (!ParseManifest(manifest_url_, manifest_data_.data(), 582 manifest_data_.length(), 583 manifest_has_valid_mime_type_ ? 584 PARSE_MANIFEST_ALLOWING_INTERCEPTS : 585 PARSE_MANIFEST_PER_STANDARD, 586 manifest)) { 587 const char* kFormatString = "Failed to parse manifest %s"; 588 const std::string message = base::StringPrintf(kFormatString, 589 manifest_url_.spec().c_str()); 590 HandleCacheFailure( 591 AppCacheErrorDetails( 592 message, APPCACHE_SIGNATURE_ERROR, GURL(), 0, 593 false /*is_cross_origin*/), 594 APPCACHE_MANIFEST_ERROR, 595 GURL()); 596 VLOG(1) << message; 597 return; 598 } 599 600 // Proceed with update process. Section 6.9.4 steps 8-20. 601 internal_state_ = DOWNLOADING; 602 inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId()); 603 BuildUrlFileList(manifest); 604 inprogress_cache_->InitializeWithManifest(&manifest); 605 606 // Associate all pending master hosts with the newly created cache. 607 for (PendingMasters::iterator it = pending_master_entries_.begin(); 608 it != pending_master_entries_.end(); ++it) { 609 PendingHosts& hosts = it->second; 610 for (PendingHosts::iterator host_it = hosts.begin(); 611 host_it != hosts.end(); ++host_it) { 612 (*host_it) 613 ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_); 614 } 615 } 616 617 if (manifest.did_ignore_intercept_namespaces) { 618 // Must be done after associating all pending master hosts. 619 std::string message( 620 "Ignoring the INTERCEPT section of the application cache manifest " 621 "because the content type is not text/cache-manifest"); 622 LogConsoleMessageToAll(message); 623 } 624 625 group_->SetUpdateAppCacheStatus(AppCacheGroup::DOWNLOADING); 626 NotifyAllAssociatedHosts(APPCACHE_DOWNLOADING_EVENT); 627 FetchUrls(); 628 FetchMasterEntries(); 629 MaybeCompleteUpdate(); // if not done, continues when async fetches complete 630 } 631 632 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) { 633 DCHECK(internal_state_ == DOWNLOADING); 634 635 net::URLRequest* request = fetcher->request(); 636 const GURL& url = request->original_url(); 637 pending_url_fetches_.erase(url); 638 NotifyAllProgress(url); 639 ++url_fetches_completed_; 640 641 int response_code = request->status().is_success() 642 ? request->GetResponseCode() 643 : fetcher->redirect_response_code(); 644 645 AppCacheEntry& entry = url_file_list_.find(url)->second; 646 647 if (response_code / 100 == 2) { 648 // Associate storage with the new entry. 649 DCHECK(fetcher->response_writer()); 650 entry.set_response_id(fetcher->response_writer()->response_id()); 651 entry.set_response_size(fetcher->response_writer()->amount_written()); 652 if (!inprogress_cache_->AddOrModifyEntry(url, entry)) 653 duplicate_response_ids_.push_back(entry.response_id()); 654 655 // TODO(michaeln): Check for <html manifest=xxx> 656 // See http://code.google.com/p/chromium/issues/detail?id=97930 657 // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept)) 658 // if (!manifestAttribute) skip it 659 660 // Foreign entries will be detected during cache selection. 661 // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML 662 // file whose root element is an html element with a manifest attribute 663 // whose value doesn't match the manifest url of the application cache 664 // being processed, mark the entry as being foreign. 665 } else { 666 VLOG(1) << "Request status: " << request->status().status() 667 << " error: " << request->status().error() 668 << " response code: " << response_code; 669 if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) { 670 if (response_code == 304 && fetcher->existing_entry().has_response_id()) { 671 // Keep the existing response. 672 entry.set_response_id(fetcher->existing_entry().response_id()); 673 entry.set_response_size(fetcher->existing_entry().response_size()); 674 inprogress_cache_->AddOrModifyEntry(url, entry); 675 } else { 676 const char* kFormatString = "Resource fetch failed (%d) %s"; 677 std::string message = FormatUrlErrorMessage( 678 kFormatString, url, fetcher->result(), response_code); 679 ResultType result = fetcher->result(); 680 bool is_cross_origin = url.GetOrigin() != manifest_url_.GetOrigin(); 681 switch (result) { 682 case DISKCACHE_ERROR: 683 HandleCacheFailure( 684 AppCacheErrorDetails( 685 message, APPCACHE_UNKNOWN_ERROR, GURL(), 0, 686 is_cross_origin), 687 result, 688 url); 689 break; 690 case NETWORK_ERROR: 691 HandleCacheFailure( 692 AppCacheErrorDetails(message, APPCACHE_RESOURCE_ERROR, url, 0, 693 is_cross_origin), 694 result, 695 url); 696 break; 697 default: 698 HandleCacheFailure(AppCacheErrorDetails(message, 699 APPCACHE_RESOURCE_ERROR, 700 url, 701 response_code, 702 is_cross_origin), 703 result, 704 url); 705 break; 706 } 707 return; 708 } 709 } else if (response_code == 404 || response_code == 410) { 710 // Entry is skipped. They are dropped from the cache. 711 } else if (update_type_ == UPGRADE_ATTEMPT && 712 fetcher->existing_entry().has_response_id()) { 713 // Keep the existing response. 714 // TODO(michaeln): Not sure this is a good idea. This is spec compliant 715 // but the old resource may or may not be compatible with the new contents 716 // of the cache. Impossible to know one way or the other. 717 entry.set_response_id(fetcher->existing_entry().response_id()); 718 entry.set_response_size(fetcher->existing_entry().response_size()); 719 inprogress_cache_->AddOrModifyEntry(url, entry); 720 } 721 } 722 723 // Fetch another URL now that one request has completed. 724 DCHECK(internal_state_ != CACHE_FAILURE); 725 FetchUrls(); 726 MaybeCompleteUpdate(); 727 } 728 729 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted( 730 URLFetcher* fetcher) { 731 DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING); 732 733 // TODO(jennb): Handle downloads completing during cache failure when update 734 // no longer fetches master entries directly. For now, we cancel all pending 735 // master entry fetches when entering cache failure state so this will never 736 // be called in CACHE_FAILURE state. 737 738 net::URLRequest* request = fetcher->request(); 739 const GURL& url = request->original_url(); 740 master_entry_fetches_.erase(url); 741 ++master_entries_completed_; 742 743 int response_code = request->status().is_success() 744 ? request->GetResponseCode() : -1; 745 746 PendingMasters::iterator found = pending_master_entries_.find(url); 747 DCHECK(found != pending_master_entries_.end()); 748 PendingHosts& hosts = found->second; 749 750 // Section 6.9.4. No update case: step 7.3, else step 22. 751 if (response_code / 100 == 2) { 752 // Add fetched master entry to the appropriate cache. 753 AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get() 754 : group_->newest_complete_cache(); 755 DCHECK(fetcher->response_writer()); 756 AppCacheEntry master_entry(AppCacheEntry::MASTER, 757 fetcher->response_writer()->response_id(), 758 fetcher->response_writer()->amount_written()); 759 if (cache->AddOrModifyEntry(url, master_entry)) 760 added_master_entries_.push_back(url); 761 else 762 duplicate_response_ids_.push_back(master_entry.response_id()); 763 764 // In no-update case, associate host with the newest cache. 765 if (!inprogress_cache_.get()) { 766 // TODO(michaeln): defer until the updated cache has been stored 767 DCHECK(cache == group_->newest_complete_cache()); 768 for (PendingHosts::iterator host_it = hosts.begin(); 769 host_it != hosts.end(); ++host_it) { 770 (*host_it)->AssociateCompleteCache(cache); 771 } 772 } 773 } else { 774 HostNotifier host_notifier; 775 for (PendingHosts::iterator host_it = hosts.begin(); 776 host_it != hosts.end(); ++host_it) { 777 AppCacheHost* host = *host_it; 778 host_notifier.AddHost(host); 779 780 // In downloading case, disassociate host from inprogress cache. 781 if (inprogress_cache_.get()) 782 host->AssociateNoCache(GURL()); 783 784 host->RemoveObserver(this); 785 } 786 hosts.clear(); 787 788 const char* kFormatString = "Manifest fetch failed (%d) %s"; 789 std::string message = FormatUrlErrorMessage( 790 kFormatString, request->url(), fetcher->result(), response_code); 791 host_notifier.SendErrorNotifications( 792 AppCacheErrorDetails(message, 793 appcache::APPCACHE_MANIFEST_ERROR, 794 request->url(), 795 response_code, 796 false /*is_cross_origin*/)); 797 798 // In downloading case, update result is different if all master entries 799 // failed vs. only some failing. 800 if (inprogress_cache_.get()) { 801 // Only count successful downloads to know if all master entries failed. 802 pending_master_entries_.erase(found); 803 --master_entries_completed_; 804 805 // Section 6.9.4, step 22.3. 806 if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) { 807 HandleCacheFailure(AppCacheErrorDetails(message, 808 appcache::APPCACHE_MANIFEST_ERROR, 809 request->url(), 810 response_code, 811 false /*is_cross_origin*/), 812 fetcher->result(), 813 GURL()); 814 return; 815 } 816 } 817 } 818 819 DCHECK(internal_state_ != CACHE_FAILURE); 820 FetchMasterEntries(); 821 MaybeCompleteUpdate(); 822 } 823 824 void AppCacheUpdateJob::HandleManifestRefetchCompleted( 825 URLFetcher* fetcher) { 826 DCHECK(internal_state_ == REFETCH_MANIFEST); 827 DCHECK(manifest_fetcher_ == fetcher); 828 manifest_fetcher_ = NULL; 829 830 net::URLRequest* request = fetcher->request(); 831 int response_code = request->status().is_success() 832 ? request->GetResponseCode() : -1; 833 if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) { 834 // Only need to store response in storage if manifest is not already 835 // an entry in the cache. 836 AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_); 837 if (entry) { 838 entry->add_types(AppCacheEntry::MANIFEST); 839 StoreGroupAndCache(); 840 } else { 841 manifest_response_writer_.reset(CreateResponseWriter()); 842 scoped_refptr<HttpResponseInfoIOBuffer> io_buffer( 843 new HttpResponseInfoIOBuffer(manifest_response_info_.release())); 844 manifest_response_writer_->WriteInfo( 845 io_buffer.get(), 846 base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete, 847 base::Unretained(this))); 848 } 849 } else { 850 VLOG(1) << "Request status: " << request->status().status() 851 << " error: " << request->status().error() 852 << " response code: " << response_code; 853 ScheduleUpdateRetry(kRerunDelayMs); 854 if (response_code == 200) { 855 HandleCacheFailure(AppCacheErrorDetails("Manifest changed during update", 856 APPCACHE_CHANGED_ERROR, 857 GURL(), 858 0, 859 false /*is_cross_origin*/), 860 APPCACHE_MANIFEST_ERROR, 861 GURL()); 862 } else { 863 const char* kFormatString = "Manifest re-fetch failed (%d) %s"; 864 std::string message = FormatUrlErrorMessage( 865 kFormatString, manifest_url_, fetcher->result(), response_code); 866 HandleCacheFailure(AppCacheErrorDetails(message, 867 appcache::APPCACHE_MANIFEST_ERROR, 868 GURL(), 869 response_code, 870 false /*is_cross_origin*/), 871 fetcher->result(), 872 GURL()); 873 } 874 } 875 } 876 877 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) { 878 if (result > 0) { 879 scoped_refptr<net::StringIOBuffer> io_buffer( 880 new net::StringIOBuffer(manifest_data_)); 881 manifest_response_writer_->WriteData( 882 io_buffer.get(), 883 manifest_data_.length(), 884 base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete, 885 base::Unretained(this))); 886 } else { 887 HandleCacheFailure( 888 AppCacheErrorDetails("Failed to write the manifest headers to storage", 889 APPCACHE_UNKNOWN_ERROR, 890 GURL(), 891 0, 892 false /*is_cross_origin*/), 893 DISKCACHE_ERROR, 894 GURL()); 895 } 896 } 897 898 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) { 899 if (result > 0) { 900 AppCacheEntry entry(AppCacheEntry::MANIFEST, 901 manifest_response_writer_->response_id(), 902 manifest_response_writer_->amount_written()); 903 if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry)) 904 duplicate_response_ids_.push_back(entry.response_id()); 905 StoreGroupAndCache(); 906 } else { 907 HandleCacheFailure( 908 AppCacheErrorDetails("Failed to write the manifest data to storage", 909 APPCACHE_UNKNOWN_ERROR, 910 GURL(), 911 0, 912 false /*is_cross_origin*/), 913 DISKCACHE_ERROR, 914 GURL()); 915 } 916 } 917 918 void AppCacheUpdateJob::StoreGroupAndCache() { 919 DCHECK(stored_state_ == UNSTORED); 920 stored_state_ = STORING; 921 scoped_refptr<AppCache> newest_cache; 922 if (inprogress_cache_.get()) 923 newest_cache.swap(inprogress_cache_); 924 else 925 newest_cache = group_->newest_complete_cache(); 926 newest_cache->set_update_time(base::Time::Now()); 927 928 // TODO(michaeln): dcheck is fishing for clues to crbug/95101 929 DCHECK_EQ(manifest_url_, group_->manifest_url()); 930 storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this); 931 } 932 933 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group, 934 AppCache* newest_cache, 935 bool success, 936 bool would_exceed_quota) { 937 DCHECK(stored_state_ == STORING); 938 if (success) { 939 stored_state_ = STORED; 940 MaybeCompleteUpdate(); // will definitely complete 941 } else { 942 stored_state_ = UNSTORED; 943 944 // Restore inprogress_cache_ to get the proper events delivered 945 // and the proper cleanup to occur. 946 if (newest_cache != group->newest_complete_cache()) 947 inprogress_cache_ = newest_cache; 948 949 ResultType result = DB_ERROR; 950 AppCacheErrorReason reason = APPCACHE_UNKNOWN_ERROR; 951 std::string message("Failed to commit new cache to storage"); 952 if (would_exceed_quota) { 953 message.append(", would exceed quota"); 954 result = APPCACHE_QUOTA_ERROR; 955 reason = appcache::APPCACHE_QUOTA_ERROR; 956 } 957 HandleCacheFailure( 958 AppCacheErrorDetails(message, reason, GURL(), 0, 959 false /*is_cross_origin*/), 960 result, 961 GURL()); 962 } 963 } 964 965 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host, 966 AppCacheEventID event_id) { 967 std::vector<int> ids(1, host->host_id()); 968 host->frontend()->OnEventRaised(ids, event_id); 969 } 970 971 void AppCacheUpdateJob::NotifyAllAssociatedHosts(AppCacheEventID event_id) { 972 HostNotifier host_notifier; 973 AddAllAssociatedHostsToNotifier(&host_notifier); 974 host_notifier.SendNotifications(event_id); 975 } 976 977 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) { 978 HostNotifier host_notifier; 979 AddAllAssociatedHostsToNotifier(&host_notifier); 980 host_notifier.SendProgressNotifications( 981 url, url_file_list_.size(), url_fetches_completed_); 982 } 983 984 void AppCacheUpdateJob::NotifyAllFinalProgress() { 985 DCHECK(url_file_list_.size() == url_fetches_completed_); 986 NotifyAllProgress(GURL()); 987 } 988 989 void AppCacheUpdateJob::NotifyAllError(const AppCacheErrorDetails& details) { 990 HostNotifier host_notifier; 991 AddAllAssociatedHostsToNotifier(&host_notifier); 992 host_notifier.SendErrorNotifications(details); 993 } 994 995 void AppCacheUpdateJob::LogConsoleMessageToAll(const std::string& message) { 996 HostNotifier host_notifier; 997 AddAllAssociatedHostsToNotifier(&host_notifier); 998 host_notifier.SendLogMessage(message); 999 } 1000 1001 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier( 1002 HostNotifier* host_notifier) { 1003 // Collect hosts so we only send one notification per frontend. 1004 // A host can only be associated with a single cache so no need to worry 1005 // about duplicate hosts being added to the notifier. 1006 if (inprogress_cache_.get()) { 1007 DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE); 1008 host_notifier->AddHosts(inprogress_cache_->associated_hosts()); 1009 } 1010 1011 AppCacheGroup::Caches old_caches = group_->old_caches(); 1012 for (AppCacheGroup::Caches::const_iterator it = old_caches.begin(); 1013 it != old_caches.end(); ++it) { 1014 host_notifier->AddHosts((*it)->associated_hosts()); 1015 } 1016 1017 AppCache* newest_cache = group_->newest_complete_cache(); 1018 if (newest_cache) 1019 host_notifier->AddHosts(newest_cache->associated_hosts()); 1020 } 1021 1022 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) { 1023 // The host is about to be deleted; remove from our collection. 1024 PendingMasters::iterator found = 1025 pending_master_entries_.find(host->pending_master_entry_url()); 1026 DCHECK(found != pending_master_entries_.end()); 1027 PendingHosts& hosts = found->second; 1028 PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host); 1029 DCHECK(it != hosts.end()); 1030 hosts.erase(it); 1031 } 1032 1033 void AppCacheUpdateJob::OnServiceReinitialized( 1034 AppCacheStorageReference* old_storage_ref) { 1035 // We continue to use the disabled instance, but arrange for its 1036 // deletion when its no longer needed. 1037 if (old_storage_ref->storage() == storage_) 1038 disabled_storage_reference_ = old_storage_ref; 1039 } 1040 1041 void AppCacheUpdateJob::CheckIfManifestChanged() { 1042 DCHECK(update_type_ == UPGRADE_ATTEMPT); 1043 AppCacheEntry* entry = NULL; 1044 if (group_->newest_complete_cache()) 1045 entry = group_->newest_complete_cache()->GetEntry(manifest_url_); 1046 if (!entry) { 1047 // TODO(michaeln): This is just a bandaid to avoid a crash. 1048 // http://code.google.com/p/chromium/issues/detail?id=95101 1049 if (service_->storage() == storage_) { 1050 // Use a local variable because service_ is reset in HandleCacheFailure. 1051 AppCacheServiceImpl* service = service_; 1052 HandleCacheFailure( 1053 AppCacheErrorDetails("Manifest entry not found in existing cache", 1054 APPCACHE_UNKNOWN_ERROR, 1055 GURL(), 1056 0, 1057 false /*is_cross_origin*/), 1058 DB_ERROR, 1059 GURL()); 1060 AppCacheHistograms::AddMissingManifestEntrySample(); 1061 service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback()); 1062 } 1063 return; 1064 } 1065 1066 // Load manifest data from storage to compare against fetched manifest. 1067 manifest_response_reader_.reset( 1068 storage_->CreateResponseReader(manifest_url_, 1069 group_->group_id(), 1070 entry->response_id())); 1071 read_manifest_buffer_ = new net::IOBuffer(kBufferSize); 1072 manifest_response_reader_->ReadData( 1073 read_manifest_buffer_.get(), 1074 kBufferSize, 1075 base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete, 1076 base::Unretained(this))); // async read 1077 } 1078 1079 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) { 1080 if (result > 0) { 1081 loaded_manifest_data_.append(read_manifest_buffer_->data(), result); 1082 manifest_response_reader_->ReadData( 1083 read_manifest_buffer_.get(), 1084 kBufferSize, 1085 base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete, 1086 base::Unretained(this))); // read more 1087 } else { 1088 read_manifest_buffer_ = NULL; 1089 manifest_response_reader_.reset(); 1090 ContinueHandleManifestFetchCompleted( 1091 result < 0 || manifest_data_ != loaded_manifest_data_); 1092 } 1093 } 1094 1095 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) { 1096 for (base::hash_set<std::string>::const_iterator it = 1097 manifest.explicit_urls.begin(); 1098 it != manifest.explicit_urls.end(); ++it) { 1099 AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT); 1100 } 1101 1102 const std::vector<Namespace>& intercepts = 1103 manifest.intercept_namespaces; 1104 for (std::vector<Namespace>::const_iterator it = intercepts.begin(); 1105 it != intercepts.end(); ++it) { 1106 int flags = AppCacheEntry::INTERCEPT; 1107 if (it->is_executable) 1108 flags |= AppCacheEntry::EXECUTABLE; 1109 AddUrlToFileList(it->target_url, flags); 1110 } 1111 1112 const std::vector<Namespace>& fallbacks = 1113 manifest.fallback_namespaces; 1114 for (std::vector<Namespace>::const_iterator it = fallbacks.begin(); 1115 it != fallbacks.end(); ++it) { 1116 AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK); 1117 } 1118 1119 // Add all master entries from newest complete cache. 1120 if (update_type_ == UPGRADE_ATTEMPT) { 1121 const AppCache::EntryMap& entries = 1122 group_->newest_complete_cache()->entries(); 1123 for (AppCache::EntryMap::const_iterator it = entries.begin(); 1124 it != entries.end(); ++it) { 1125 const AppCacheEntry& entry = it->second; 1126 if (entry.IsMaster()) 1127 AddUrlToFileList(it->first, AppCacheEntry::MASTER); 1128 } 1129 } 1130 } 1131 1132 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) { 1133 std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert( 1134 AppCache::EntryMap::value_type(url, AppCacheEntry(type))); 1135 1136 if (ret.second) 1137 urls_to_fetch_.push_back(UrlToFetch(url, false, NULL)); 1138 else 1139 ret.first->second.add_types(type); // URL already exists. Merge types. 1140 } 1141 1142 void AppCacheUpdateJob::FetchUrls() { 1143 DCHECK(internal_state_ == DOWNLOADING); 1144 1145 // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3. 1146 // Fetch up to the concurrent limit. Other fetches will be triggered as each 1147 // each fetch completes. 1148 while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches && 1149 !urls_to_fetch_.empty()) { 1150 UrlToFetch url_to_fetch = urls_to_fetch_.front(); 1151 urls_to_fetch_.pop_front(); 1152 1153 AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url); 1154 DCHECK(it != url_file_list_.end()); 1155 AppCacheEntry& entry = it->second; 1156 if (ShouldSkipUrlFetch(entry)) { 1157 NotifyAllProgress(url_to_fetch.url); 1158 ++url_fetches_completed_; 1159 } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) { 1160 NotifyAllProgress(url_to_fetch.url); 1161 ++url_fetches_completed_; // saved a URL request 1162 } else if (!url_to_fetch.storage_checked && 1163 MaybeLoadFromNewestCache(url_to_fetch.url, entry)) { 1164 // Continues asynchronously after data is loaded from newest cache. 1165 } else { 1166 URLFetcher* fetcher = new URLFetcher( 1167 url_to_fetch.url, URLFetcher::URL_FETCH, this); 1168 if (url_to_fetch.existing_response_info.get()) { 1169 DCHECK(group_->newest_complete_cache()); 1170 AppCacheEntry* existing_entry = 1171 group_->newest_complete_cache()->GetEntry(url_to_fetch.url); 1172 DCHECK(existing_entry); 1173 DCHECK(existing_entry->response_id() == 1174 url_to_fetch.existing_response_info->response_id()); 1175 fetcher->set_existing_response_headers( 1176 url_to_fetch.existing_response_info->http_response_info()->headers 1177 .get()); 1178 fetcher->set_existing_entry(*existing_entry); 1179 } 1180 fetcher->Start(); 1181 pending_url_fetches_.insert( 1182 PendingUrlFetches::value_type(url_to_fetch.url, fetcher)); 1183 } 1184 } 1185 } 1186 1187 void AppCacheUpdateJob::CancelAllUrlFetches() { 1188 // Cancel any pending URL requests. 1189 for (PendingUrlFetches::iterator it = pending_url_fetches_.begin(); 1190 it != pending_url_fetches_.end(); ++it) { 1191 delete it->second; 1192 } 1193 1194 url_fetches_completed_ += 1195 pending_url_fetches_.size() + urls_to_fetch_.size(); 1196 pending_url_fetches_.clear(); 1197 urls_to_fetch_.clear(); 1198 } 1199 1200 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) { 1201 // 6.6.4 Step 17 1202 // If the resource URL being processed was flagged as neither an 1203 // "explicit entry" nor or a "fallback entry", then the user agent 1204 // may skip this URL. 1205 if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) 1206 return false; 1207 1208 // TODO(jennb): decide if entry should be skipped to expire it from cache 1209 return false; 1210 } 1211 1212 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url, 1213 int entry_type) { 1214 DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE); 1215 AppCacheEntry* existing = 1216 inprogress_cache_.get() ? inprogress_cache_->GetEntry(url) 1217 : group_->newest_complete_cache()->GetEntry(url); 1218 if (existing) { 1219 existing->add_types(entry_type); 1220 return true; 1221 } 1222 return false; 1223 } 1224 1225 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host, 1226 const GURL& url, 1227 bool is_new) { 1228 DCHECK(!IsTerminating()); 1229 1230 if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) { 1231 AppCache* cache; 1232 if (inprogress_cache_.get()) { 1233 // always associate 1234 host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_); 1235 cache = inprogress_cache_.get(); 1236 } else { 1237 cache = group_->newest_complete_cache(); 1238 } 1239 1240 // Update existing entry if it has already been fetched. 1241 AppCacheEntry* entry = cache->GetEntry(url); 1242 if (entry) { 1243 entry->add_types(AppCacheEntry::MASTER); 1244 if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) { 1245 // only associate if have entry 1246 host->AssociateCompleteCache(cache); 1247 } 1248 if (is_new) 1249 ++master_entries_completed_; // pretend fetching completed 1250 return; 1251 } 1252 } 1253 1254 // Add to fetch list if not already fetching. 1255 if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) { 1256 master_entries_to_fetch_.insert(url); 1257 if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) 1258 FetchMasterEntries(); 1259 } 1260 } 1261 1262 void AppCacheUpdateJob::FetchMasterEntries() { 1263 DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING); 1264 1265 // Fetch each master entry in the list, up to the concurrent limit. 1266 // Additional fetches will be triggered as each fetch completes. 1267 while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches && 1268 !master_entries_to_fetch_.empty()) { 1269 const GURL& url = *master_entries_to_fetch_.begin(); 1270 1271 if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) { 1272 ++master_entries_completed_; // saved a URL request 1273 1274 // In no update case, associate hosts to newest cache in group 1275 // now that master entry has been "successfully downloaded". 1276 if (internal_state_ == NO_UPDATE) { 1277 // TODO(michaeln): defer until the updated cache has been stored. 1278 DCHECK(!inprogress_cache_.get()); 1279 AppCache* cache = group_->newest_complete_cache(); 1280 PendingMasters::iterator found = pending_master_entries_.find(url); 1281 DCHECK(found != pending_master_entries_.end()); 1282 PendingHosts& hosts = found->second; 1283 for (PendingHosts::iterator host_it = hosts.begin(); 1284 host_it != hosts.end(); ++host_it) { 1285 (*host_it)->AssociateCompleteCache(cache); 1286 } 1287 } 1288 } else { 1289 URLFetcher* fetcher = new URLFetcher( 1290 url, URLFetcher::MASTER_ENTRY_FETCH, this); 1291 fetcher->Start(); 1292 master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher)); 1293 } 1294 1295 master_entries_to_fetch_.erase(master_entries_to_fetch_.begin()); 1296 } 1297 } 1298 1299 void AppCacheUpdateJob::CancelAllMasterEntryFetches( 1300 const AppCacheErrorDetails& error_details) { 1301 // For now, cancel all in-progress fetches for master entries and pretend 1302 // all master entries fetches have completed. 1303 // TODO(jennb): Delete this when update no longer fetches master entries 1304 // directly. 1305 1306 // Cancel all in-progress fetches. 1307 for (PendingUrlFetches::iterator it = master_entry_fetches_.begin(); 1308 it != master_entry_fetches_.end(); ++it) { 1309 delete it->second; 1310 master_entries_to_fetch_.insert(it->first); // back in unfetched list 1311 } 1312 master_entry_fetches_.clear(); 1313 1314 master_entries_completed_ += master_entries_to_fetch_.size(); 1315 1316 // Cache failure steps, step 2. 1317 // Pretend all master entries that have not yet been fetched have completed 1318 // downloading. Unassociate hosts from any appcache and send ERROR event. 1319 HostNotifier host_notifier; 1320 while (!master_entries_to_fetch_.empty()) { 1321 const GURL& url = *master_entries_to_fetch_.begin(); 1322 PendingMasters::iterator found = pending_master_entries_.find(url); 1323 DCHECK(found != pending_master_entries_.end()); 1324 PendingHosts& hosts = found->second; 1325 for (PendingHosts::iterator host_it = hosts.begin(); 1326 host_it != hosts.end(); ++host_it) { 1327 AppCacheHost* host = *host_it; 1328 host->AssociateNoCache(GURL()); 1329 host_notifier.AddHost(host); 1330 host->RemoveObserver(this); 1331 } 1332 hosts.clear(); 1333 1334 master_entries_to_fetch_.erase(master_entries_to_fetch_.begin()); 1335 } 1336 host_notifier.SendErrorNotifications(error_details); 1337 } 1338 1339 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url, 1340 AppCacheEntry& entry) { 1341 if (update_type_ != UPGRADE_ATTEMPT) 1342 return false; 1343 1344 AppCache* newest = group_->newest_complete_cache(); 1345 AppCacheEntry* copy_me = newest->GetEntry(url); 1346 if (!copy_me || !copy_me->has_response_id()) 1347 return false; 1348 1349 // Load HTTP headers for entry from newest cache. 1350 loading_responses_.insert( 1351 LoadingResponses::value_type(copy_me->response_id(), url)); 1352 storage_->LoadResponseInfo(manifest_url_, group_->group_id(), 1353 copy_me->response_id(), 1354 this); 1355 // Async: wait for OnResponseInfoLoaded to complete. 1356 return true; 1357 } 1358 1359 void AppCacheUpdateJob::OnResponseInfoLoaded( 1360 AppCacheResponseInfo* response_info, int64 response_id) { 1361 const net::HttpResponseInfo* http_info = response_info ? 1362 response_info->http_response_info() : NULL; 1363 1364 // Needed response info for a manifest fetch request. 1365 if (internal_state_ == FETCH_MANIFEST) { 1366 if (http_info) 1367 manifest_fetcher_->set_existing_response_headers( 1368 http_info->headers.get()); 1369 manifest_fetcher_->Start(); 1370 return; 1371 } 1372 1373 LoadingResponses::iterator found = loading_responses_.find(response_id); 1374 DCHECK(found != loading_responses_.end()); 1375 const GURL& url = found->second; 1376 1377 if (!http_info) { 1378 LoadFromNewestCacheFailed(url, NULL); // no response found 1379 } else { 1380 // Check if response can be re-used according to HTTP caching semantics. 1381 // Responses with a "vary" header get treated as expired. 1382 const std::string name = "vary"; 1383 std::string value; 1384 void* iter = NULL; 1385 if (!http_info->headers.get() || 1386 http_info->headers->RequiresValidation(http_info->request_time, 1387 http_info->response_time, 1388 base::Time::Now()) || 1389 http_info->headers->EnumerateHeader(&iter, name, &value)) { 1390 LoadFromNewestCacheFailed(url, response_info); 1391 } else { 1392 DCHECK(group_->newest_complete_cache()); 1393 AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url); 1394 DCHECK(copy_me); 1395 DCHECK(copy_me->response_id() == response_id); 1396 1397 AppCache::EntryMap::iterator it = url_file_list_.find(url); 1398 DCHECK(it != url_file_list_.end()); 1399 AppCacheEntry& entry = it->second; 1400 entry.set_response_id(response_id); 1401 entry.set_response_size(copy_me->response_size()); 1402 inprogress_cache_->AddOrModifyEntry(url, entry); 1403 NotifyAllProgress(url); 1404 ++url_fetches_completed_; 1405 } 1406 } 1407 loading_responses_.erase(found); 1408 1409 MaybeCompleteUpdate(); 1410 } 1411 1412 void AppCacheUpdateJob::LoadFromNewestCacheFailed( 1413 const GURL& url, AppCacheResponseInfo* response_info) { 1414 if (internal_state_ == CACHE_FAILURE) 1415 return; 1416 1417 // Re-insert url at front of fetch list. Indicate storage has been checked. 1418 urls_to_fetch_.push_front(UrlToFetch(url, true, response_info)); 1419 FetchUrls(); 1420 } 1421 1422 void AppCacheUpdateJob::MaybeCompleteUpdate() { 1423 DCHECK(internal_state_ != CACHE_FAILURE); 1424 1425 // Must wait for any pending master entries or url fetches to complete. 1426 if (master_entries_completed_ != pending_master_entries_.size() || 1427 url_fetches_completed_ != url_file_list_.size()) { 1428 DCHECK(internal_state_ != COMPLETED); 1429 return; 1430 } 1431 1432 switch (internal_state_) { 1433 case NO_UPDATE: 1434 if (master_entries_completed_ > 0) { 1435 switch (stored_state_) { 1436 case UNSTORED: 1437 StoreGroupAndCache(); 1438 return; 1439 case STORING: 1440 return; 1441 case STORED: 1442 break; 1443 } 1444 } 1445 // 6.9.4 steps 7.3-7.7. 1446 NotifyAllAssociatedHosts(APPCACHE_NO_UPDATE_EVENT); 1447 DiscardDuplicateResponses(); 1448 internal_state_ = COMPLETED; 1449 break; 1450 case DOWNLOADING: 1451 internal_state_ = REFETCH_MANIFEST; 1452 FetchManifest(false); 1453 break; 1454 case REFETCH_MANIFEST: 1455 DCHECK(stored_state_ == STORED); 1456 NotifyAllFinalProgress(); 1457 if (update_type_ == CACHE_ATTEMPT) 1458 NotifyAllAssociatedHosts(APPCACHE_CACHED_EVENT); 1459 else 1460 NotifyAllAssociatedHosts(APPCACHE_UPDATE_READY_EVENT); 1461 DiscardDuplicateResponses(); 1462 internal_state_ = COMPLETED; 1463 LogHistogramStats(UPDATE_OK, GURL()); 1464 break; 1465 case CACHE_FAILURE: 1466 NOTREACHED(); // See HandleCacheFailure 1467 break; 1468 default: 1469 break; 1470 } 1471 1472 // Let the stack unwind before deletion to make it less risky as this 1473 // method is called from multiple places in this file. 1474 if (internal_state_ == COMPLETED) 1475 DeleteSoon(); 1476 } 1477 1478 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) { 1479 // TODO(jennb): post a delayed task with the "same parameters" as this job 1480 // to retry the update at a later time. Need group, URLs of pending master 1481 // entries and their hosts. 1482 } 1483 1484 void AppCacheUpdateJob::Cancel() { 1485 internal_state_ = CANCELLED; 1486 1487 LogHistogramStats(CANCELLED_ERROR, GURL()); 1488 1489 if (manifest_fetcher_) { 1490 delete manifest_fetcher_; 1491 manifest_fetcher_ = NULL; 1492 } 1493 1494 for (PendingUrlFetches::iterator it = pending_url_fetches_.begin(); 1495 it != pending_url_fetches_.end(); ++it) { 1496 delete it->second; 1497 } 1498 pending_url_fetches_.clear(); 1499 1500 for (PendingUrlFetches::iterator it = master_entry_fetches_.begin(); 1501 it != master_entry_fetches_.end(); ++it) { 1502 delete it->second; 1503 } 1504 master_entry_fetches_.clear(); 1505 1506 ClearPendingMasterEntries(); 1507 DiscardInprogressCache(); 1508 1509 // Delete response writer to avoid any callbacks. 1510 if (manifest_response_writer_) 1511 manifest_response_writer_.reset(); 1512 1513 storage_->CancelDelegateCallbacks(this); 1514 } 1515 1516 void AppCacheUpdateJob::ClearPendingMasterEntries() { 1517 for (PendingMasters::iterator it = pending_master_entries_.begin(); 1518 it != pending_master_entries_.end(); ++it) { 1519 PendingHosts& hosts = it->second; 1520 for (PendingHosts::iterator host_it = hosts.begin(); 1521 host_it != hosts.end(); ++host_it) { 1522 (*host_it)->RemoveObserver(this); 1523 } 1524 } 1525 1526 pending_master_entries_.clear(); 1527 } 1528 1529 void AppCacheUpdateJob::DiscardInprogressCache() { 1530 if (stored_state_ == STORING) { 1531 // We can make no assumptions about whether the StoreGroupAndCacheTask 1532 // actually completed or not. This condition should only be reachable 1533 // during shutdown. Free things up and return to do no harm. 1534 inprogress_cache_ = NULL; 1535 added_master_entries_.clear(); 1536 return; 1537 } 1538 1539 storage_->DoomResponses(manifest_url_, stored_response_ids_); 1540 1541 if (!inprogress_cache_.get()) { 1542 // We have to undo the changes we made, if any, to the existing cache. 1543 if (group_ && group_->newest_complete_cache()) { 1544 for (std::vector<GURL>::iterator iter = added_master_entries_.begin(); 1545 iter != added_master_entries_.end(); ++iter) { 1546 group_->newest_complete_cache()->RemoveEntry(*iter); 1547 } 1548 } 1549 added_master_entries_.clear(); 1550 return; 1551 } 1552 1553 AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts(); 1554 while (!hosts.empty()) 1555 (*hosts.begin())->AssociateNoCache(GURL()); 1556 1557 inprogress_cache_ = NULL; 1558 added_master_entries_.clear(); 1559 } 1560 1561 void AppCacheUpdateJob::DiscardDuplicateResponses() { 1562 storage_->DoomResponses(manifest_url_, duplicate_response_ids_); 1563 } 1564 1565 void AppCacheUpdateJob::LogHistogramStats( 1566 ResultType result, const GURL& failed_resource_url) { 1567 AppCacheHistograms::CountUpdateJobResult(result, manifest_url_.GetOrigin()); 1568 if (result == UPDATE_OK) 1569 return; 1570 1571 int percent_complete = 0; 1572 if (url_file_list_.size() > 0) { 1573 size_t actual_fetches_completed = url_fetches_completed_; 1574 if (!failed_resource_url.is_empty() && actual_fetches_completed) 1575 --actual_fetches_completed; 1576 percent_complete = (static_cast<double>(actual_fetches_completed) / 1577 static_cast<double>(url_file_list_.size())) * 100.0; 1578 percent_complete = std::min(percent_complete, 99); 1579 } 1580 1581 bool was_making_progress = 1582 base::Time::Now() - last_progress_time_ < 1583 base::TimeDelta::FromMinutes(5); 1584 1585 bool off_origin_resource_failure = 1586 !failed_resource_url.is_empty() && 1587 (failed_resource_url.GetOrigin() != manifest_url_.GetOrigin()); 1588 1589 AppCacheHistograms::LogUpdateFailureStats( 1590 manifest_url_.GetOrigin(), 1591 percent_complete, 1592 was_making_progress, 1593 off_origin_resource_failure); 1594 } 1595 1596 void AppCacheUpdateJob::DeleteSoon() { 1597 ClearPendingMasterEntries(); 1598 manifest_response_writer_.reset(); 1599 storage_->CancelDelegateCallbacks(this); 1600 service_->RemoveObserver(this); 1601 service_ = NULL; 1602 1603 // Break the connection with the group so the group cannot call delete 1604 // on this object after we've posted a task to delete ourselves. 1605 group_->SetUpdateAppCacheStatus(AppCacheGroup::IDLE); 1606 group_ = NULL; 1607 1608 base::MessageLoop::current()->DeleteSoon(FROM_HERE, this); 1609 } 1610 1611 } // namespace appcache 1612
