Home | History | Annotate | Download | only in appcache 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "webkit/browser/appcache/appcache_update_job.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/bind_helpers.h"
      9 #include "base/compiler_specific.h"
     10 #include "base/message_loop/message_loop.h"
     11 #include "base/strings/string_util.h"
     12 #include "base/strings/stringprintf.h"
     13 #include "net/base/io_buffer.h"
     14 #include "net/base/load_flags.h"
     15 #include "net/base/net_errors.h"
     16 #include "net/base/request_priority.h"
     17 #include "net/http/http_request_headers.h"
     18 #include "net/http/http_response_headers.h"
     19 #include "net/url_request/url_request_context.h"
     20 #include "webkit/browser/appcache/appcache_group.h"
     21 #include "webkit/browser/appcache/appcache_histograms.h"
     22 
     23 namespace appcache {
     24 
     25 static const int kBufferSize = 32768;
     26 static const size_t kMaxConcurrentUrlFetches = 2;
     27 static const int kMax503Retries = 3;
     28 
     29 // Helper class for collecting hosts per frontend when sending notifications
     30 // so that only one notification is sent for all hosts using the same frontend.
     31 class HostNotifier {
     32  public:
     33   typedef std::vector<int> HostIds;
     34   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
     35 
     36   // Caller is responsible for ensuring there will be no duplicate hosts.
     37   void AddHost(AppCacheHost* host) {
     38     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
     39         NotifyHostMap::value_type(host->frontend(), HostIds()));
     40     ret.first->second.push_back(host->host_id());
     41   }
     42 
     43   void AddHosts(const std::set<AppCacheHost*>& hosts) {
     44     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
     45          it != hosts.end(); ++it) {
     46       AddHost(*it);
     47     }
     48   }
     49 
     50   void SendNotifications(EventID event_id) {
     51     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
     52          it != hosts_to_notify.end(); ++it) {
     53       AppCacheFrontend* frontend = it->first;
     54       frontend->OnEventRaised(it->second, event_id);
     55     }
     56   }
     57 
     58   void SendProgressNotifications(
     59       const GURL& url, int num_total, int num_complete) {
     60     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
     61          it != hosts_to_notify.end(); ++it) {
     62       AppCacheFrontend* frontend = it->first;
     63       frontend->OnProgressEventRaised(it->second, url,
     64                                       num_total, num_complete);
     65     }
     66   }
     67 
     68   void SendErrorNotifications(const std::string& error_message) {
     69     DCHECK(!error_message.empty());
     70     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
     71          it != hosts_to_notify.end(); ++it) {
     72       AppCacheFrontend* frontend = it->first;
     73       frontend->OnErrorEventRaised(it->second, error_message);
     74     }
     75   }
     76 
     77  private:
     78   NotifyHostMap hosts_to_notify;
     79 };
     80 
     81 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
     82                                           bool checked,
     83                                           AppCacheResponseInfo* info)
     84     : url(url),
     85       storage_checked(checked),
     86       existing_response_info(info) {
     87 }
     88 
     89 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
     90 }
     91 
     92 // Helper class to fetch resources. Depending on the fetch type,
     93 // can either fetch to an in-memory string or write the response
     94 // data out to the disk cache.
     95 AppCacheUpdateJob::URLFetcher::URLFetcher(const GURL& url,
     96                                           FetchType fetch_type,
     97                                           AppCacheUpdateJob* job)
     98     : url_(url),
     99       job_(job),
    100       fetch_type_(fetch_type),
    101       retry_503_attempts_(0),
    102       buffer_(new net::IOBuffer(kBufferSize)),
    103       request_(job->service_->request_context()
    104                    ->CreateRequest(url, net::DEFAULT_PRIORITY, this)) {}
    105 
    106 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
    107 }
    108 
    109 void AppCacheUpdateJob::URLFetcher::Start() {
    110   request_->set_first_party_for_cookies(job_->manifest_url_);
    111   request_->SetLoadFlags(request_->load_flags() |
    112                          net::LOAD_DISABLE_INTERCEPT);
    113   if (existing_response_headers_.get())
    114     AddConditionalHeaders(existing_response_headers_.get());
    115   request_->Start();
    116 }
    117 
    118 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
    119     net::URLRequest* request, const GURL& new_url,  bool* defer_redirect) {
    120   DCHECK(request_ == request);
    121   // Redirect is not allowed by the update process.
    122   request->Cancel();
    123   OnResponseCompleted();
    124 }
    125 
    126 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
    127     net::URLRequest *request) {
    128   DCHECK(request == request_);
    129   if (request->status().is_success() &&
    130       (request->GetResponseCode() / 100) == 2) {
    131 
    132     // See http://code.google.com/p/chromium/issues/detail?id=69594
    133     // We willfully violate the HTML5 spec at this point in order
    134     // to support the appcaching of cross-origin HTTPS resources.
    135     // We've opted for a milder constraint and allow caching unless
    136     // the resource has a "no-store" header. A spec change has been
    137     // requested on the whatwg list.
    138     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
    139     if (url_.SchemeIsSecure() &&
    140         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
    141       if (request->response_headers()->
    142               HasHeaderValue("cache-control", "no-store")) {
    143         request->Cancel();
    144         OnResponseCompleted();
    145         return;
    146       }
    147     }
    148 
    149     // Write response info to storage for URL fetches. Wait for async write
    150     // completion before reading any response data.
    151     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
    152       response_writer_.reset(job_->CreateResponseWriter());
    153       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
    154           new HttpResponseInfoIOBuffer(
    155               new net::HttpResponseInfo(request->response_info())));
    156       response_writer_->WriteInfo(
    157           io_buffer.get(),
    158           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
    159     } else {
    160       ReadResponseData();
    161     }
    162   } else {
    163     OnResponseCompleted();
    164   }
    165 }
    166 
    167 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
    168     net::URLRequest* request, int bytes_read) {
    169   DCHECK(request_ == request);
    170   bool data_consumed = true;
    171   if (request->status().is_success() && bytes_read > 0) {
    172     data_consumed = ConsumeResponseData(bytes_read);
    173     if (data_consumed) {
    174       bytes_read = 0;
    175       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
    176         if (bytes_read > 0) {
    177           data_consumed = ConsumeResponseData(bytes_read);
    178           if (!data_consumed)
    179             break;  // wait for async data processing, then read more
    180         } else {
    181           break;
    182         }
    183       }
    184     }
    185   }
    186   if (data_consumed && !request->status().is_io_pending())
    187     OnResponseCompleted();
    188 }
    189 
    190 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
    191     const net::HttpResponseHeaders* headers) {
    192   DCHECK(request_.get() && headers);
    193   net::HttpRequestHeaders extra_headers;
    194 
    195   // Add If-Modified-Since header if response info has Last-Modified header.
    196   const std::string last_modified = "Last-Modified";
    197   std::string last_modified_value;
    198   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
    199   if (!last_modified_value.empty()) {
    200     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
    201                             last_modified_value);
    202   }
    203 
    204   // Add If-None-Match header if response info has ETag header.
    205   const std::string etag = "ETag";
    206   std::string etag_value;
    207   headers->EnumerateHeader(NULL, etag, &etag_value);
    208   if (!etag_value.empty()) {
    209     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
    210                             etag_value);
    211   }
    212   if (!extra_headers.IsEmpty())
    213     request_->SetExtraRequestHeaders(extra_headers);
    214 }
    215 
    216 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
    217   if (result < 0) {
    218     request_->Cancel();
    219     OnResponseCompleted();
    220     return;
    221   }
    222   ReadResponseData();
    223 }
    224 
    225 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
    226   InternalUpdateState state = job_->internal_state_;
    227   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
    228     return;
    229   int bytes_read = 0;
    230   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
    231   OnReadCompleted(request_.get(), bytes_read);
    232 }
    233 
    234 // Returns false if response data is processed asynchronously, in which
    235 // case ReadResponseData will be invoked when it is safe to continue
    236 // reading more response data from the request.
    237 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
    238   DCHECK_GT(bytes_read, 0);
    239   switch (fetch_type_) {
    240     case MANIFEST_FETCH:
    241     case MANIFEST_REFETCH:
    242       manifest_data_.append(buffer_->data(), bytes_read);
    243       break;
    244     case URL_FETCH:
    245     case MASTER_ENTRY_FETCH:
    246       DCHECK(response_writer_.get());
    247       response_writer_->WriteData(
    248           buffer_.get(),
    249           bytes_read,
    250           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
    251       return false;  // wait for async write completion to continue reading
    252     default:
    253       NOTREACHED();
    254   }
    255   return true;
    256 }
    257 
    258 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
    259   // Retry for 503s where retry-after is 0.
    260   if (request_->status().is_success() &&
    261       request_->GetResponseCode() == 503 &&
    262       MaybeRetryRequest()) {
    263     return;
    264   }
    265 
    266   switch (fetch_type_) {
    267     case MANIFEST_FETCH:
    268       job_->HandleManifestFetchCompleted(this);
    269       break;
    270     case URL_FETCH:
    271       job_->HandleUrlFetchCompleted(this);
    272       break;
    273     case MASTER_ENTRY_FETCH:
    274       job_->HandleMasterEntryFetchCompleted(this);
    275       break;
    276     case MANIFEST_REFETCH:
    277       job_->HandleManifestRefetchCompleted(this);
    278       break;
    279     default:
    280       NOTREACHED();
    281   }
    282 
    283   delete this;
    284 }
    285 
    286 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
    287   if (retry_503_attempts_ >= kMax503Retries ||
    288       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
    289     return false;
    290   }
    291   ++retry_503_attempts_;
    292   request_ = job_->service_->request_context()->CreateRequest(
    293       url_, net::DEFAULT_PRIORITY, this);
    294   Start();
    295   return true;
    296 }
    297 
    298 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheService* service,
    299                                      AppCacheGroup* group)
    300     : service_(service),
    301       manifest_url_(group->manifest_url()),
    302       group_(group),
    303       update_type_(UNKNOWN_TYPE),
    304       internal_state_(FETCH_MANIFEST),
    305       master_entries_completed_(0),
    306       url_fetches_completed_(0),
    307       manifest_fetcher_(NULL),
    308       stored_state_(UNSTORED),
    309       storage_(service->storage()) {
    310     service_->AddObserver(this);
    311 }
    312 
    313 AppCacheUpdateJob::~AppCacheUpdateJob() {
    314   if (service_)
    315     service_->RemoveObserver(this);
    316   if (internal_state_ != COMPLETED)
    317     Cancel();
    318 
    319   DCHECK(!manifest_fetcher_);
    320   DCHECK(pending_url_fetches_.empty());
    321   DCHECK(!inprogress_cache_.get());
    322   DCHECK(pending_master_entries_.empty());
    323   DCHECK(master_entry_fetches_.empty());
    324 
    325   if (group_)
    326     group_->SetUpdateStatus(AppCacheGroup::IDLE);
    327 }
    328 
    329 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
    330                                     const GURL& new_master_resource) {
    331   DCHECK(group_->update_job() == this);
    332   DCHECK(!group_->is_obsolete());
    333 
    334   bool is_new_pending_master_entry = false;
    335   if (!new_master_resource.is_empty()) {
    336     DCHECK(new_master_resource == host->pending_master_entry_url());
    337     DCHECK(!new_master_resource.has_ref());
    338     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
    339 
    340     // Cannot add more to this update if already terminating.
    341     if (IsTerminating()) {
    342       group_->QueueUpdate(host, new_master_resource);
    343       return;
    344     }
    345 
    346     std::pair<PendingMasters::iterator, bool> ret =
    347         pending_master_entries_.insert(
    348             PendingMasters::value_type(new_master_resource, PendingHosts()));
    349     is_new_pending_master_entry = ret.second;
    350     ret.first->second.push_back(host);
    351     host->AddObserver(this);
    352   }
    353 
    354   // Notify host (if any) if already checking or downloading.
    355   AppCacheGroup::UpdateStatus update_status = group_->update_status();
    356   if (update_status == AppCacheGroup::CHECKING ||
    357       update_status == AppCacheGroup::DOWNLOADING) {
    358     if (host) {
    359       NotifySingleHost(host, CHECKING_EVENT);
    360       if (update_status == AppCacheGroup::DOWNLOADING)
    361         NotifySingleHost(host, DOWNLOADING_EVENT);
    362 
    363       // Add to fetch list or an existing entry if already fetched.
    364       if (!new_master_resource.is_empty()) {
    365         AddMasterEntryToFetchList(host, new_master_resource,
    366                                   is_new_pending_master_entry);
    367       }
    368     }
    369     return;
    370   }
    371 
    372   // Begin update process for the group.
    373   group_->SetUpdateStatus(AppCacheGroup::CHECKING);
    374   if (group_->HasCache()) {
    375     update_type_ = UPGRADE_ATTEMPT;
    376     NotifyAllAssociatedHosts(CHECKING_EVENT);
    377   } else {
    378     update_type_ = CACHE_ATTEMPT;
    379     DCHECK(host);
    380     NotifySingleHost(host, CHECKING_EVENT);
    381   }
    382 
    383   if (!new_master_resource.is_empty()) {
    384     AddMasterEntryToFetchList(host, new_master_resource,
    385                               is_new_pending_master_entry);
    386   }
    387 
    388   FetchManifest(true);
    389 }
    390 
    391 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
    392   AppCacheResponseWriter* writer =
    393       storage_->CreateResponseWriter(manifest_url_,
    394                                                 group_->group_id());
    395   stored_response_ids_.push_back(writer->response_id());
    396   return writer;
    397 }
    398 
    399 void AppCacheUpdateJob::HandleCacheFailure(const std::string& error_message) {
    400   // 6.9.4 cache failure steps 2-8.
    401   DCHECK(internal_state_ != CACHE_FAILURE);
    402   DCHECK(!error_message.empty());
    403   internal_state_ = CACHE_FAILURE;
    404   CancelAllUrlFetches();
    405   CancelAllMasterEntryFetches(error_message);
    406   NotifyAllError(error_message);
    407   DiscardInprogressCache();
    408   internal_state_ = COMPLETED;
    409   DeleteSoon();  // To unwind the stack prior to deletion.
    410 }
    411 
    412 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
    413   DCHECK(!manifest_fetcher_);
    414   manifest_fetcher_ = new URLFetcher(
    415      manifest_url_,
    416      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
    417                       URLFetcher::MANIFEST_REFETCH,
    418      this);
    419 
    420   // Add any necessary Http headers before sending fetch request.
    421   if (is_first_fetch) {
    422     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
    423         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
    424     if (entry) {
    425       // Asynchronously load response info for manifest from newest cache.
    426       storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
    427                                  entry->response_id(), this);
    428     } else {
    429       manifest_fetcher_->Start();
    430     }
    431   } else {
    432     DCHECK(internal_state_ == REFETCH_MANIFEST);
    433     DCHECK(manifest_response_info_.get());
    434     manifest_fetcher_->set_existing_response_headers(
    435         manifest_response_info_->headers.get());
    436     manifest_fetcher_->Start();
    437   }
    438 }
    439 
    440 
    441 void AppCacheUpdateJob::HandleManifestFetchCompleted(
    442     URLFetcher* fetcher) {
    443   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
    444   DCHECK_EQ(manifest_fetcher_, fetcher);
    445   manifest_fetcher_ = NULL;
    446 
    447   net::URLRequest* request = fetcher->request();
    448   int response_code = -1;
    449   bool is_valid_response_code = false;
    450   if (request->status().is_success()) {
    451     response_code = request->GetResponseCode();
    452     is_valid_response_code = (response_code / 100 == 2);
    453   }
    454 
    455   if (is_valid_response_code) {
    456     manifest_data_ = fetcher->manifest_data();
    457     manifest_response_info_.reset(
    458         new net::HttpResponseInfo(request->response_info()));
    459     if (update_type_ == UPGRADE_ATTEMPT)
    460       CheckIfManifestChanged();  // continues asynchronously
    461     else
    462       ContinueHandleManifestFetchCompleted(true);
    463   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
    464     ContinueHandleManifestFetchCompleted(false);
    465   } else if ((response_code == 404 || response_code == 410) &&
    466              update_type_ == UPGRADE_ATTEMPT) {
    467     storage_->MakeGroupObsolete(group_, this);  // async
    468   } else {
    469     const char* kFormatString = "Manifest fetch failed (%d) %s";
    470     std::string message = base::StringPrintf(kFormatString, response_code,
    471                                              manifest_url_.spec().c_str());
    472     HandleCacheFailure(message);
    473   }
    474 }
    475 
    476 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
    477                                             bool success) {
    478   DCHECK(master_entry_fetches_.empty());
    479   CancelAllMasterEntryFetches("The cache has been made obsolete, "
    480                               "the manifest file returned 404 or 410");
    481   if (success) {
    482     DCHECK(group->is_obsolete());
    483     NotifyAllAssociatedHosts(OBSOLETE_EVENT);
    484     internal_state_ = COMPLETED;
    485     MaybeCompleteUpdate();
    486   } else {
    487     // Treat failure to mark group obsolete as a cache failure.
    488     HandleCacheFailure("Failed to mark the cache as obsolete");
    489   }
    490 }
    491 
    492 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
    493   DCHECK(internal_state_ == FETCH_MANIFEST);
    494 
    495   if (!changed) {
    496     DCHECK(update_type_ == UPGRADE_ATTEMPT);
    497     internal_state_ = NO_UPDATE;
    498 
    499     // Wait for pending master entries to download.
    500     FetchMasterEntries();
    501     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
    502     return;
    503   }
    504 
    505   Manifest manifest;
    506   if (!ParseManifest(manifest_url_, manifest_data_.data(),
    507                      manifest_data_.length(), manifest)) {
    508     const char* kFormatString = "Failed to parse manifest %s";
    509     const std::string message = base::StringPrintf(kFormatString,
    510         manifest_url_.spec().c_str());
    511     HandleCacheFailure(message);
    512     VLOG(1) << message;
    513     return;
    514   }
    515 
    516   // Proceed with update process. Section 6.9.4 steps 8-20.
    517   internal_state_ = DOWNLOADING;
    518   inprogress_cache_ = new AppCache(storage_, storage_->NewCacheId());
    519   BuildUrlFileList(manifest);
    520   inprogress_cache_->InitializeWithManifest(&manifest);
    521 
    522   // Associate all pending master hosts with the newly created cache.
    523   for (PendingMasters::iterator it = pending_master_entries_.begin();
    524        it != pending_master_entries_.end(); ++it) {
    525     PendingHosts& hosts = it->second;
    526     for (PendingHosts::iterator host_it = hosts.begin();
    527          host_it != hosts.end(); ++host_it) {
    528       (*host_it)
    529           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
    530     }
    531   }
    532 
    533   group_->SetUpdateStatus(AppCacheGroup::DOWNLOADING);
    534   NotifyAllAssociatedHosts(DOWNLOADING_EVENT);
    535   FetchUrls();
    536   FetchMasterEntries();
    537   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
    538 }
    539 
    540 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
    541   DCHECK(internal_state_ == DOWNLOADING);
    542 
    543   net::URLRequest* request = fetcher->request();
    544   const GURL& url = request->original_url();
    545   pending_url_fetches_.erase(url);
    546   NotifyAllProgress(url);
    547   ++url_fetches_completed_;
    548 
    549   int response_code = request->status().is_success()
    550       ? request->GetResponseCode() : -1;
    551   AppCacheEntry& entry = url_file_list_.find(url)->second;
    552 
    553   if (response_code / 100 == 2) {
    554     // Associate storage with the new entry.
    555     DCHECK(fetcher->response_writer());
    556     entry.set_response_id(fetcher->response_writer()->response_id());
    557     entry.set_response_size(fetcher->response_writer()->amount_written());
    558     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
    559       duplicate_response_ids_.push_back(entry.response_id());
    560 
    561     // TODO(michaeln): Check for <html manifest=xxx>
    562     // See http://code.google.com/p/chromium/issues/detail?id=97930
    563     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
    564     //   if (!manifestAttribute) skip it
    565 
    566     // Foreign entries will be detected during cache selection.
    567     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
    568     // file whose root element is an html element with a manifest attribute
    569     // whose value doesn't match the manifest url of the application cache
    570     // being processed, mark the entry as being foreign.
    571   } else {
    572     VLOG(1) << "Request status: " << request->status().status()
    573             << " error: " << request->status().error()
    574             << " response code: " << response_code;
    575     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
    576       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
    577         // Keep the existing response.
    578         entry.set_response_id(fetcher->existing_entry().response_id());
    579         entry.set_response_size(fetcher->existing_entry().response_size());
    580         inprogress_cache_->AddOrModifyEntry(url, entry);
    581       } else {
    582         const char* kFormatString = "Resource fetch failed (%d) %s";
    583         const std::string message = base::StringPrintf(kFormatString,
    584             response_code, url.spec().c_str());
    585         HandleCacheFailure(message);
    586         return;
    587       }
    588     } else if (response_code == 404 || response_code == 410) {
    589       // Entry is skipped.  They are dropped from the cache.
    590     } else if (update_type_ == UPGRADE_ATTEMPT &&
    591                fetcher->existing_entry().has_response_id()) {
    592       // Keep the existing response.
    593       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
    594       // but the old resource may or may not be compatible with the new contents
    595       // of the cache. Impossible to know one way or the other.
    596       entry.set_response_id(fetcher->existing_entry().response_id());
    597       entry.set_response_size(fetcher->existing_entry().response_size());
    598       inprogress_cache_->AddOrModifyEntry(url, entry);
    599     }
    600   }
    601 
    602   // Fetch another URL now that one request has completed.
    603   DCHECK(internal_state_ != CACHE_FAILURE);
    604   FetchUrls();
    605   MaybeCompleteUpdate();
    606 }
    607 
    608 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
    609     URLFetcher* fetcher) {
    610   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
    611 
    612   // TODO(jennb): Handle downloads completing during cache failure when update
    613   // no longer fetches master entries directly. For now, we cancel all pending
    614   // master entry fetches when entering cache failure state so this will never
    615   // be called in CACHE_FAILURE state.
    616 
    617   net::URLRequest* request = fetcher->request();
    618   const GURL& url = request->original_url();
    619   master_entry_fetches_.erase(url);
    620   ++master_entries_completed_;
    621 
    622   int response_code = request->status().is_success()
    623       ? request->GetResponseCode() : -1;
    624 
    625   PendingMasters::iterator found = pending_master_entries_.find(url);
    626   DCHECK(found != pending_master_entries_.end());
    627   PendingHosts& hosts = found->second;
    628 
    629   // Section 6.9.4. No update case: step 7.3, else step 22.
    630   if (response_code / 100 == 2) {
    631     // Add fetched master entry to the appropriate cache.
    632     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
    633                                               : group_->newest_complete_cache();
    634     DCHECK(fetcher->response_writer());
    635     AppCacheEntry master_entry(AppCacheEntry::MASTER,
    636                                fetcher->response_writer()->response_id(),
    637                                fetcher->response_writer()->amount_written());
    638     if (cache->AddOrModifyEntry(url, master_entry))
    639       added_master_entries_.push_back(url);
    640     else
    641       duplicate_response_ids_.push_back(master_entry.response_id());
    642 
    643     // In no-update case, associate host with the newest cache.
    644     if (!inprogress_cache_.get()) {
    645       // TODO(michaeln): defer until the updated cache has been stored
    646       DCHECK(cache == group_->newest_complete_cache());
    647       for (PendingHosts::iterator host_it = hosts.begin();
    648            host_it != hosts.end(); ++host_it) {
    649         (*host_it)->AssociateCompleteCache(cache);
    650       }
    651     }
    652   } else {
    653     HostNotifier host_notifier;
    654     for (PendingHosts::iterator host_it = hosts.begin();
    655          host_it != hosts.end(); ++host_it) {
    656       AppCacheHost* host = *host_it;
    657       host_notifier.AddHost(host);
    658 
    659       // In downloading case, disassociate host from inprogress cache.
    660       if (inprogress_cache_.get())
    661         host->AssociateNoCache(GURL());
    662 
    663       host->RemoveObserver(this);
    664     }
    665     hosts.clear();
    666 
    667     const char* kFormatString = "Master entry fetch failed (%d) %s";
    668     const std::string message = base::StringPrintf(kFormatString,
    669         response_code, request->url().spec().c_str());
    670     host_notifier.SendErrorNotifications(message);
    671 
    672     // In downloading case, update result is different if all master entries
    673     // failed vs. only some failing.
    674     if (inprogress_cache_.get()) {
    675       // Only count successful downloads to know if all master entries failed.
    676       pending_master_entries_.erase(found);
    677       --master_entries_completed_;
    678 
    679       // Section 6.9.4, step 22.3.
    680       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
    681         HandleCacheFailure(message);
    682         return;
    683       }
    684     }
    685   }
    686 
    687   DCHECK(internal_state_ != CACHE_FAILURE);
    688   FetchMasterEntries();
    689   MaybeCompleteUpdate();
    690 }
    691 
    692 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
    693     URLFetcher* fetcher) {
    694   DCHECK(internal_state_ == REFETCH_MANIFEST);
    695   DCHECK(manifest_fetcher_ == fetcher);
    696   manifest_fetcher_ = NULL;
    697 
    698   net::URLRequest* request = fetcher->request();
    699   int response_code = request->status().is_success()
    700       ? request->GetResponseCode() : -1;
    701   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
    702     // Only need to store response in storage if manifest is not already
    703     // an entry in the cache.
    704     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
    705     if (entry) {
    706       entry->add_types(AppCacheEntry::MANIFEST);
    707       StoreGroupAndCache();
    708     } else {
    709       manifest_response_writer_.reset(CreateResponseWriter());
    710       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
    711           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
    712       manifest_response_writer_->WriteInfo(
    713           io_buffer.get(),
    714           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
    715                      base::Unretained(this)));
    716     }
    717   } else {
    718     VLOG(1) << "Request status: " << request->status().status()
    719             << " error: " << request->status().error()
    720             << " response code: " << response_code;
    721     ScheduleUpdateRetry(kRerunDelayMs);
    722     HandleCacheFailure("Manifest changed during update, scheduling retry");
    723   }
    724 }
    725 
    726 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
    727   if (result > 0) {
    728     scoped_refptr<net::StringIOBuffer> io_buffer(
    729         new net::StringIOBuffer(manifest_data_));
    730     manifest_response_writer_->WriteData(
    731         io_buffer.get(),
    732         manifest_data_.length(),
    733         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
    734                    base::Unretained(this)));
    735   } else {
    736     HandleCacheFailure("Failed to write the manifest headers to storage");
    737   }
    738 }
    739 
    740 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
    741   if (result > 0) {
    742     AppCacheEntry entry(AppCacheEntry::MANIFEST,
    743         manifest_response_writer_->response_id(),
    744         manifest_response_writer_->amount_written());
    745     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
    746       duplicate_response_ids_.push_back(entry.response_id());
    747     StoreGroupAndCache();
    748   } else {
    749     HandleCacheFailure("Failed to write the manifest data to storage");
    750   }
    751 }
    752 
    753 void AppCacheUpdateJob::StoreGroupAndCache() {
    754   DCHECK(stored_state_ == UNSTORED);
    755   stored_state_ = STORING;
    756   scoped_refptr<AppCache> newest_cache;
    757   if (inprogress_cache_.get())
    758     newest_cache.swap(inprogress_cache_);
    759   else
    760     newest_cache = group_->newest_complete_cache();
    761   newest_cache->set_update_time(base::Time::Now());
    762 
    763   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
    764   DCHECK_EQ(manifest_url_, group_->manifest_url());
    765   storage_->StoreGroupAndNewestCache(group_, newest_cache.get(), this);
    766 }
    767 
    768 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
    769                                                     AppCache* newest_cache,
    770                                                     bool success,
    771                                                     bool would_exceed_quota) {
    772   DCHECK(stored_state_ == STORING);
    773   if (success) {
    774     stored_state_ = STORED;
    775     MaybeCompleteUpdate();  // will definitely complete
    776   } else {
    777     // Restore inprogress_cache_ to get the proper events delivered
    778     // and the proper cleanup to occur.
    779     if (newest_cache != group->newest_complete_cache())
    780       inprogress_cache_ = newest_cache;
    781 
    782     std::string message("Failed to commit new cache to storage");
    783     if (would_exceed_quota)
    784       message.append(", would exceed quota");
    785     HandleCacheFailure(message);
    786   }
    787 }
    788 
    789 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
    790                                          EventID event_id) {
    791   std::vector<int> ids(1, host->host_id());
    792   host->frontend()->OnEventRaised(ids, event_id);
    793 }
    794 
    795 void AppCacheUpdateJob::NotifyAllAssociatedHosts(EventID event_id) {
    796   HostNotifier host_notifier;
    797   AddAllAssociatedHostsToNotifier(&host_notifier);
    798   host_notifier.SendNotifications(event_id);
    799 }
    800 
    801 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
    802   HostNotifier host_notifier;
    803   AddAllAssociatedHostsToNotifier(&host_notifier);
    804   host_notifier.SendProgressNotifications(
    805       url, url_file_list_.size(), url_fetches_completed_);
    806 }
    807 
    808 void AppCacheUpdateJob::NotifyAllFinalProgress() {
    809   DCHECK(url_file_list_.size() == url_fetches_completed_);
    810   NotifyAllProgress(GURL());
    811 }
    812 
    813 void AppCacheUpdateJob::NotifyAllError(const std::string& error_message) {
    814   HostNotifier host_notifier;
    815   AddAllAssociatedHostsToNotifier(&host_notifier);
    816   host_notifier.SendErrorNotifications(error_message);
    817 }
    818 
    819 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
    820     HostNotifier* host_notifier) {
    821   // Collect hosts so we only send one notification per frontend.
    822   // A host can only be associated with a single cache so no need to worry
    823   // about duplicate hosts being added to the notifier.
    824   if (inprogress_cache_.get()) {
    825     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
    826     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
    827   }
    828 
    829   AppCacheGroup::Caches old_caches = group_->old_caches();
    830   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
    831        it != old_caches.end(); ++it) {
    832     host_notifier->AddHosts((*it)->associated_hosts());
    833   }
    834 
    835   AppCache* newest_cache = group_->newest_complete_cache();
    836   if (newest_cache)
    837     host_notifier->AddHosts(newest_cache->associated_hosts());
    838 }
    839 
    840 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
    841   // The host is about to be deleted; remove from our collection.
    842   PendingMasters::iterator found =
    843       pending_master_entries_.find(host->pending_master_entry_url());
    844   DCHECK(found != pending_master_entries_.end());
    845   PendingHosts& hosts = found->second;
    846   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
    847   DCHECK(it != hosts.end());
    848   hosts.erase(it);
    849 }
    850 
    851 void AppCacheUpdateJob::OnServiceReinitialized(
    852     AppCacheStorageReference* old_storage_ref) {
    853   // We continue to use the disabled instance, but arrange for its
    854   // deletion when its no longer needed.
    855   if (old_storage_ref->storage() == storage_)
    856     disabled_storage_reference_ = old_storage_ref;
    857 }
    858 
    859 void AppCacheUpdateJob::CheckIfManifestChanged() {
    860   DCHECK(update_type_ == UPGRADE_ATTEMPT);
    861   AppCacheEntry* entry = NULL;
    862   if (group_->newest_complete_cache())
    863     entry = group_->newest_complete_cache()->GetEntry(manifest_url_);
    864   if (!entry) {
    865     // TODO(michaeln): This is just a bandaid to avoid a crash.
    866     // http://code.google.com/p/chromium/issues/detail?id=95101
    867     if (service_->storage() == storage_) {
    868       // Use a local variable because service_ is reset in HandleCacheFailure.
    869       AppCacheService* service = service_;
    870       HandleCacheFailure("Manifest entry not found in existing cache");
    871       AppCacheHistograms::AddMissingManifestEntrySample();
    872       service->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
    873     }
    874     return;
    875   }
    876 
    877   // Load manifest data from storage to compare against fetched manifest.
    878   manifest_response_reader_.reset(
    879       storage_->CreateResponseReader(manifest_url_,
    880                                      group_->group_id(),
    881                                      entry->response_id()));
    882   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
    883   manifest_response_reader_->ReadData(
    884       read_manifest_buffer_.get(),
    885       kBufferSize,
    886       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
    887                  base::Unretained(this)));  // async read
    888 }
    889 
    890 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
    891   if (result > 0) {
    892     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
    893     manifest_response_reader_->ReadData(
    894         read_manifest_buffer_.get(),
    895         kBufferSize,
    896         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
    897                    base::Unretained(this)));  // read more
    898   } else {
    899     read_manifest_buffer_ = NULL;
    900     manifest_response_reader_.reset();
    901     ContinueHandleManifestFetchCompleted(
    902         result < 0 || manifest_data_ != loaded_manifest_data_);
    903   }
    904 }
    905 
    906 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) {
    907   for (base::hash_set<std::string>::const_iterator it =
    908            manifest.explicit_urls.begin();
    909        it != manifest.explicit_urls.end(); ++it) {
    910     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
    911   }
    912 
    913   const std::vector<Namespace>& intercepts =
    914       manifest.intercept_namespaces;
    915   for (std::vector<Namespace>::const_iterator it = intercepts.begin();
    916        it != intercepts.end(); ++it) {
    917     int flags = AppCacheEntry::INTERCEPT;
    918     if (it->is_executable)
    919       flags |= AppCacheEntry::EXECUTABLE;
    920     AddUrlToFileList(it->target_url, flags);
    921   }
    922 
    923   const std::vector<Namespace>& fallbacks =
    924       manifest.fallback_namespaces;
    925   for (std::vector<Namespace>::const_iterator it = fallbacks.begin();
    926        it != fallbacks.end(); ++it) {
    927      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
    928   }
    929 
    930   // Add all master entries from newest complete cache.
    931   if (update_type_ == UPGRADE_ATTEMPT) {
    932     const AppCache::EntryMap& entries =
    933         group_->newest_complete_cache()->entries();
    934     for (AppCache::EntryMap::const_iterator it = entries.begin();
    935          it != entries.end(); ++it) {
    936       const AppCacheEntry& entry = it->second;
    937       if (entry.IsMaster())
    938         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
    939     }
    940   }
    941 }
    942 
    943 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
    944   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
    945       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
    946 
    947   if (ret.second)
    948     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
    949   else
    950     ret.first->second.add_types(type);  // URL already exists. Merge types.
    951 }
    952 
    953 void AppCacheUpdateJob::FetchUrls() {
    954   DCHECK(internal_state_ == DOWNLOADING);
    955 
    956   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
    957   // Fetch up to the concurrent limit. Other fetches will be triggered as each
    958   // each fetch completes.
    959   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
    960          !urls_to_fetch_.empty()) {
    961     UrlToFetch url_to_fetch = urls_to_fetch_.front();
    962     urls_to_fetch_.pop_front();
    963 
    964     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
    965     DCHECK(it != url_file_list_.end());
    966     AppCacheEntry& entry = it->second;
    967     if (ShouldSkipUrlFetch(entry)) {
    968       NotifyAllProgress(url_to_fetch.url);
    969       ++url_fetches_completed_;
    970     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
    971       NotifyAllProgress(url_to_fetch.url);
    972       ++url_fetches_completed_;  // saved a URL request
    973     } else if (!url_to_fetch.storage_checked &&
    974                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
    975       // Continues asynchronously after data is loaded from newest cache.
    976     } else {
    977       URLFetcher* fetcher = new URLFetcher(
    978           url_to_fetch.url, URLFetcher::URL_FETCH, this);
    979       if (url_to_fetch.existing_response_info.get()) {
    980         DCHECK(group_->newest_complete_cache());
    981         AppCacheEntry* existing_entry =
    982             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
    983         DCHECK(existing_entry);
    984         DCHECK(existing_entry->response_id() ==
    985                url_to_fetch.existing_response_info->response_id());
    986         fetcher->set_existing_response_headers(
    987             url_to_fetch.existing_response_info->http_response_info()->headers
    988                 .get());
    989         fetcher->set_existing_entry(*existing_entry);
    990       }
    991       fetcher->Start();
    992       pending_url_fetches_.insert(
    993           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
    994     }
    995   }
    996 }
    997 
    998 void AppCacheUpdateJob::CancelAllUrlFetches() {
    999   // Cancel any pending URL requests.
   1000   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
   1001        it != pending_url_fetches_.end(); ++it) {
   1002     delete it->second;
   1003   }
   1004 
   1005   url_fetches_completed_ +=
   1006       pending_url_fetches_.size() + urls_to_fetch_.size();
   1007   pending_url_fetches_.clear();
   1008   urls_to_fetch_.clear();
   1009 }
   1010 
   1011 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
   1012   // 6.6.4 Step 17
   1013   // If the resource URL being processed was flagged as neither an
   1014   // "explicit entry" nor or a "fallback entry", then the user agent
   1015   // may skip this URL.
   1016   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
   1017     return false;
   1018 
   1019   // TODO(jennb): decide if entry should be skipped to expire it from cache
   1020   return false;
   1021 }
   1022 
   1023 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
   1024                                             int entry_type) {
   1025   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
   1026   AppCacheEntry* existing =
   1027       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
   1028                               : group_->newest_complete_cache()->GetEntry(url);
   1029   if (existing) {
   1030     existing->add_types(entry_type);
   1031     return true;
   1032   }
   1033   return false;
   1034 }
   1035 
   1036 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
   1037                                                   const GURL& url,
   1038                                                   bool is_new) {
   1039   DCHECK(!IsTerminating());
   1040 
   1041   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
   1042     AppCache* cache;
   1043     if (inprogress_cache_.get()) {
   1044       // always associate
   1045       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
   1046       cache = inprogress_cache_.get();
   1047     } else {
   1048       cache = group_->newest_complete_cache();
   1049     }
   1050 
   1051     // Update existing entry if it has already been fetched.
   1052     AppCacheEntry* entry = cache->GetEntry(url);
   1053     if (entry) {
   1054       entry->add_types(AppCacheEntry::MASTER);
   1055       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
   1056         // only associate if have entry
   1057         host->AssociateCompleteCache(cache);
   1058       }
   1059       if (is_new)
   1060         ++master_entries_completed_;  // pretend fetching completed
   1061       return;
   1062     }
   1063   }
   1064 
   1065   // Add to fetch list if not already fetching.
   1066   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
   1067     master_entries_to_fetch_.insert(url);
   1068     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
   1069       FetchMasterEntries();
   1070   }
   1071 }
   1072 
   1073 void AppCacheUpdateJob::FetchMasterEntries() {
   1074   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
   1075 
   1076   // Fetch each master entry in the list, up to the concurrent limit.
   1077   // Additional fetches will be triggered as each fetch completes.
   1078   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
   1079          !master_entries_to_fetch_.empty()) {
   1080     const GURL& url = *master_entries_to_fetch_.begin();
   1081 
   1082     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
   1083       ++master_entries_completed_;  // saved a URL request
   1084 
   1085       // In no update case, associate hosts to newest cache in group
   1086       // now that master entry has been "successfully downloaded".
   1087       if (internal_state_ == NO_UPDATE) {
   1088         // TODO(michaeln): defer until the updated cache has been stored.
   1089         DCHECK(!inprogress_cache_.get());
   1090         AppCache* cache = group_->newest_complete_cache();
   1091         PendingMasters::iterator found = pending_master_entries_.find(url);
   1092         DCHECK(found != pending_master_entries_.end());
   1093         PendingHosts& hosts = found->second;
   1094         for (PendingHosts::iterator host_it = hosts.begin();
   1095              host_it != hosts.end(); ++host_it) {
   1096           (*host_it)->AssociateCompleteCache(cache);
   1097         }
   1098       }
   1099     } else {
   1100       URLFetcher* fetcher = new URLFetcher(
   1101           url, URLFetcher::MASTER_ENTRY_FETCH, this);
   1102       fetcher->Start();
   1103       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
   1104     }
   1105 
   1106     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
   1107   }
   1108 }
   1109 
   1110 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
   1111     const std::string& error_message) {
   1112   // For now, cancel all in-progress fetches for master entries and pretend
   1113   // all master entries fetches have completed.
   1114   // TODO(jennb): Delete this when update no longer fetches master entries
   1115   // directly.
   1116 
   1117   // Cancel all in-progress fetches.
   1118   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
   1119        it != master_entry_fetches_.end(); ++it) {
   1120     delete it->second;
   1121     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
   1122   }
   1123   master_entry_fetches_.clear();
   1124 
   1125   master_entries_completed_ += master_entries_to_fetch_.size();
   1126 
   1127   // Cache failure steps, step 2.
   1128   // Pretend all master entries that have not yet been fetched have completed
   1129   // downloading. Unassociate hosts from any appcache and send ERROR event.
   1130   HostNotifier host_notifier;
   1131   while (!master_entries_to_fetch_.empty()) {
   1132     const GURL& url = *master_entries_to_fetch_.begin();
   1133     PendingMasters::iterator found = pending_master_entries_.find(url);
   1134     DCHECK(found != pending_master_entries_.end());
   1135     PendingHosts& hosts = found->second;
   1136     for (PendingHosts::iterator host_it = hosts.begin();
   1137          host_it != hosts.end(); ++host_it) {
   1138       AppCacheHost* host = *host_it;
   1139       host->AssociateNoCache(GURL());
   1140       host_notifier.AddHost(host);
   1141       host->RemoveObserver(this);
   1142     }
   1143     hosts.clear();
   1144 
   1145     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
   1146   }
   1147   host_notifier.SendErrorNotifications(error_message);
   1148 }
   1149 
   1150 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
   1151                                                  AppCacheEntry& entry) {
   1152   if (update_type_ != UPGRADE_ATTEMPT)
   1153     return false;
   1154 
   1155   AppCache* newest = group_->newest_complete_cache();
   1156   AppCacheEntry* copy_me = newest->GetEntry(url);
   1157   if (!copy_me || !copy_me->has_response_id())
   1158     return false;
   1159 
   1160   // Load HTTP headers for entry from newest cache.
   1161   loading_responses_.insert(
   1162       LoadingResponses::value_type(copy_me->response_id(), url));
   1163   storage_->LoadResponseInfo(manifest_url_, group_->group_id(),
   1164                              copy_me->response_id(),
   1165                              this);
   1166   // Async: wait for OnResponseInfoLoaded to complete.
   1167   return true;
   1168 }
   1169 
   1170 void AppCacheUpdateJob::OnResponseInfoLoaded(
   1171     AppCacheResponseInfo* response_info, int64 response_id) {
   1172   const net::HttpResponseInfo* http_info = response_info ?
   1173       response_info->http_response_info() : NULL;
   1174 
   1175   // Needed response info for a manifest fetch request.
   1176   if (internal_state_ == FETCH_MANIFEST) {
   1177     if (http_info)
   1178       manifest_fetcher_->set_existing_response_headers(
   1179           http_info->headers.get());
   1180     manifest_fetcher_->Start();
   1181     return;
   1182   }
   1183 
   1184   LoadingResponses::iterator found = loading_responses_.find(response_id);
   1185   DCHECK(found != loading_responses_.end());
   1186   const GURL& url = found->second;
   1187 
   1188   if (!http_info) {
   1189     LoadFromNewestCacheFailed(url, NULL);  // no response found
   1190   } else {
   1191     // Check if response can be re-used according to HTTP caching semantics.
   1192     // Responses with a "vary" header get treated as expired.
   1193     const std::string name = "vary";
   1194     std::string value;
   1195     void* iter = NULL;
   1196     if (!http_info->headers.get() ||
   1197         http_info->headers->RequiresValidation(http_info->request_time,
   1198                                                http_info->response_time,
   1199                                                base::Time::Now()) ||
   1200         http_info->headers->EnumerateHeader(&iter, name, &value)) {
   1201       LoadFromNewestCacheFailed(url, response_info);
   1202     } else {
   1203       DCHECK(group_->newest_complete_cache());
   1204       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
   1205       DCHECK(copy_me);
   1206       DCHECK(copy_me->response_id() == response_id);
   1207 
   1208       AppCache::EntryMap::iterator it = url_file_list_.find(url);
   1209       DCHECK(it != url_file_list_.end());
   1210       AppCacheEntry& entry = it->second;
   1211       entry.set_response_id(response_id);
   1212       entry.set_response_size(copy_me->response_size());
   1213       inprogress_cache_->AddOrModifyEntry(url, entry);
   1214       NotifyAllProgress(url);
   1215       ++url_fetches_completed_;
   1216     }
   1217   }
   1218   loading_responses_.erase(found);
   1219 
   1220   MaybeCompleteUpdate();
   1221 }
   1222 
   1223 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
   1224     const GURL& url, AppCacheResponseInfo* response_info) {
   1225   if (internal_state_ == CACHE_FAILURE)
   1226     return;
   1227 
   1228   // Re-insert url at front of fetch list. Indicate storage has been checked.
   1229   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
   1230   FetchUrls();
   1231 }
   1232 
   1233 void AppCacheUpdateJob::MaybeCompleteUpdate() {
   1234   DCHECK(internal_state_ != CACHE_FAILURE);
   1235 
   1236   // Must wait for any pending master entries or url fetches to complete.
   1237   if (master_entries_completed_ != pending_master_entries_.size() ||
   1238       url_fetches_completed_ != url_file_list_.size()) {
   1239     DCHECK(internal_state_ != COMPLETED);
   1240     return;
   1241   }
   1242 
   1243   switch (internal_state_) {
   1244     case NO_UPDATE:
   1245       if (master_entries_completed_ > 0) {
   1246         switch (stored_state_) {
   1247           case UNSTORED:
   1248             StoreGroupAndCache();
   1249             return;
   1250           case STORING:
   1251             return;
   1252           case STORED:
   1253             break;
   1254         }
   1255       }
   1256       // 6.9.4 steps 7.3-7.7.
   1257       NotifyAllAssociatedHosts(NO_UPDATE_EVENT);
   1258       DiscardDuplicateResponses();
   1259       internal_state_ = COMPLETED;
   1260       break;
   1261     case DOWNLOADING:
   1262       internal_state_ = REFETCH_MANIFEST;
   1263       FetchManifest(false);
   1264       break;
   1265     case REFETCH_MANIFEST:
   1266       DCHECK(stored_state_ == STORED);
   1267       NotifyAllFinalProgress();
   1268       if (update_type_ == CACHE_ATTEMPT)
   1269         NotifyAllAssociatedHosts(CACHED_EVENT);
   1270       else
   1271         NotifyAllAssociatedHosts(UPDATE_READY_EVENT);
   1272       DiscardDuplicateResponses();
   1273       internal_state_ = COMPLETED;
   1274       break;
   1275     case CACHE_FAILURE:
   1276       NOTREACHED();  // See HandleCacheFailure
   1277       break;
   1278     default:
   1279       break;
   1280   }
   1281 
   1282   // Let the stack unwind before deletion to make it less risky as this
   1283   // method is called from multiple places in this file.
   1284   if (internal_state_ == COMPLETED)
   1285     DeleteSoon();
   1286 }
   1287 
   1288 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
   1289   // TODO(jennb): post a delayed task with the "same parameters" as this job
   1290   // to retry the update at a later time. Need group, URLs of pending master
   1291   // entries and their hosts.
   1292 }
   1293 
   1294 void AppCacheUpdateJob::Cancel() {
   1295   internal_state_ = CANCELLED;
   1296 
   1297   if (manifest_fetcher_) {
   1298     delete manifest_fetcher_;
   1299     manifest_fetcher_ = NULL;
   1300   }
   1301 
   1302   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
   1303        it != pending_url_fetches_.end(); ++it) {
   1304     delete it->second;
   1305   }
   1306   pending_url_fetches_.clear();
   1307 
   1308   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
   1309        it != master_entry_fetches_.end(); ++it) {
   1310     delete it->second;
   1311   }
   1312   master_entry_fetches_.clear();
   1313 
   1314   ClearPendingMasterEntries();
   1315   DiscardInprogressCache();
   1316 
   1317   // Delete response writer to avoid any callbacks.
   1318   if (manifest_response_writer_)
   1319     manifest_response_writer_.reset();
   1320 
   1321   storage_->CancelDelegateCallbacks(this);
   1322 }
   1323 
   1324 void AppCacheUpdateJob::ClearPendingMasterEntries() {
   1325   for (PendingMasters::iterator it = pending_master_entries_.begin();
   1326        it != pending_master_entries_.end(); ++it) {
   1327     PendingHosts& hosts = it->second;
   1328     for (PendingHosts::iterator host_it = hosts.begin();
   1329          host_it != hosts.end(); ++host_it) {
   1330       (*host_it)->RemoveObserver(this);
   1331     }
   1332   }
   1333 
   1334   pending_master_entries_.clear();
   1335 }
   1336 
   1337 void AppCacheUpdateJob::DiscardInprogressCache() {
   1338   storage_->DoomResponses(manifest_url_, stored_response_ids_);
   1339 
   1340   if (!inprogress_cache_.get()) {
   1341     // We have to undo the changes we made, if any, to the existing cache.
   1342     for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
   1343          iter != added_master_entries_.end(); ++iter) {
   1344       DCHECK(group_->newest_complete_cache());
   1345       group_->newest_complete_cache()->RemoveEntry(*iter);
   1346     }
   1347     return;
   1348   }
   1349 
   1350   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
   1351   while (!hosts.empty())
   1352     (*hosts.begin())->AssociateNoCache(GURL());
   1353 
   1354   inprogress_cache_ = NULL;
   1355 }
   1356 
   1357 void AppCacheUpdateJob::DiscardDuplicateResponses() {
   1358   storage_->DoomResponses(manifest_url_, duplicate_response_ids_);
   1359 }
   1360 
   1361 void AppCacheUpdateJob::DeleteSoon() {
   1362   ClearPendingMasterEntries();
   1363   manifest_response_writer_.reset();
   1364   storage_->CancelDelegateCallbacks(this);
   1365   service_->RemoveObserver(this);
   1366   service_ = NULL;
   1367 
   1368   // Break the connection with the group so the group cannot call delete
   1369   // on this object after we've posted a task to delete ourselves.
   1370   group_->SetUpdateStatus(AppCacheGroup::IDLE);
   1371   group_ = NULL;
   1372 
   1373   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
   1374 }
   1375 
   1376 }  // namespace appcache
   1377