Home | History | Annotate | Download | only in appcache 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "webkit/browser/appcache/appcache_update_job.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/bind_helpers.h"
      9 #include "base/compiler_specific.h"
     10 #include "base/message_loop/message_loop.h"
     11 #include "base/strings/string_util.h"
     12 #include "base/strings/stringprintf.h"
     13 #include "net/base/io_buffer.h"
     14 #include "net/base/load_flags.h"
     15 #include "net/base/net_errors.h"
     16 #include "net/http/http_request_headers.h"
     17 #include "net/http/http_response_headers.h"
     18 #include "net/url_request/url_request_context.h"
     19 #include "webkit/browser/appcache/appcache_group.h"
     20 #include "webkit/browser/appcache/appcache_histograms.h"
     21 
     22 namespace appcache {
     23 
     24 static const int kBufferSize = 32768;
     25 static const size_t kMaxConcurrentUrlFetches = 2;
     26 static const int kMax503Retries = 3;
     27 
     28 // Helper class for collecting hosts per frontend when sending notifications
     29 // so that only one notification is sent for all hosts using the same frontend.
     30 class HostNotifier {
     31  public:
     32   typedef std::vector<int> HostIds;
     33   typedef std::map<AppCacheFrontend*, HostIds> NotifyHostMap;
     34 
     35   // Caller is responsible for ensuring there will be no duplicate hosts.
     36   void AddHost(AppCacheHost* host) {
     37     std::pair<NotifyHostMap::iterator , bool> ret = hosts_to_notify.insert(
     38         NotifyHostMap::value_type(host->frontend(), HostIds()));
     39     ret.first->second.push_back(host->host_id());
     40   }
     41 
     42   void AddHosts(const std::set<AppCacheHost*>& hosts) {
     43     for (std::set<AppCacheHost*>::const_iterator it = hosts.begin();
     44          it != hosts.end(); ++it) {
     45       AddHost(*it);
     46     }
     47   }
     48 
     49   void SendNotifications(EventID event_id) {
     50     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
     51          it != hosts_to_notify.end(); ++it) {
     52       AppCacheFrontend* frontend = it->first;
     53       frontend->OnEventRaised(it->second, event_id);
     54     }
     55   }
     56 
     57   void SendProgressNotifications(
     58       const GURL& url, int num_total, int num_complete) {
     59     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
     60          it != hosts_to_notify.end(); ++it) {
     61       AppCacheFrontend* frontend = it->first;
     62       frontend->OnProgressEventRaised(it->second, url,
     63                                       num_total, num_complete);
     64     }
     65   }
     66 
     67   void SendErrorNotifications(const std::string& error_message) {
     68     DCHECK(!error_message.empty());
     69     for (NotifyHostMap::iterator it = hosts_to_notify.begin();
     70          it != hosts_to_notify.end(); ++it) {
     71       AppCacheFrontend* frontend = it->first;
     72       frontend->OnErrorEventRaised(it->second, error_message);
     73     }
     74   }
     75 
     76  private:
     77   NotifyHostMap hosts_to_notify;
     78 };
     79 
     80 AppCacheUpdateJob::UrlToFetch::UrlToFetch(const GURL& url,
     81                                           bool checked,
     82                                           AppCacheResponseInfo* info)
     83     : url(url),
     84       storage_checked(checked),
     85       existing_response_info(info) {
     86 }
     87 
     88 AppCacheUpdateJob::UrlToFetch::~UrlToFetch() {
     89 }
     90 
     91 // Helper class to fetch resources. Depending on the fetch type,
     92 // can either fetch to an in-memory string or write the response
     93 // data out to the disk cache.
     94 AppCacheUpdateJob::URLFetcher::URLFetcher(
     95     const GURL& url, FetchType fetch_type, AppCacheUpdateJob* job)
     96     : url_(url),
     97       job_(job),
     98       fetch_type_(fetch_type),
     99       retry_503_attempts_(0),
    100       buffer_(new net::IOBuffer(kBufferSize)),
    101       request_(job->service_->request_context()->CreateRequest(url, this)) {
    102 }
    103 
    104 AppCacheUpdateJob::URLFetcher::~URLFetcher() {
    105 }
    106 
    107 void AppCacheUpdateJob::URLFetcher::Start() {
    108   request_->set_first_party_for_cookies(job_->manifest_url_);
    109   request_->set_load_flags(request_->load_flags() |
    110                            net::LOAD_DISABLE_INTERCEPT);
    111   if (existing_response_headers_.get())
    112     AddConditionalHeaders(existing_response_headers_.get());
    113   request_->Start();
    114 }
    115 
    116 void AppCacheUpdateJob::URLFetcher::OnReceivedRedirect(
    117     net::URLRequest* request, const GURL& new_url,  bool* defer_redirect) {
    118   DCHECK(request_ == request);
    119   // Redirect is not allowed by the update process.
    120   request->Cancel();
    121   OnResponseCompleted();
    122 }
    123 
    124 void AppCacheUpdateJob::URLFetcher::OnResponseStarted(
    125     net::URLRequest *request) {
    126   DCHECK(request == request_);
    127   if (request->status().is_success() &&
    128       (request->GetResponseCode() / 100) == 2) {
    129 
    130     // See http://code.google.com/p/chromium/issues/detail?id=69594
    131     // We willfully violate the HTML5 spec at this point in order
    132     // to support the appcaching of cross-origin HTTPS resources.
    133     // We've opted for a milder constraint and allow caching unless
    134     // the resource has a "no-store" header. A spec change has been
    135     // requested on the whatwg list.
    136     // TODO(michaeln): Consider doing this for cross-origin HTTP resources too.
    137     if (url_.SchemeIsSecure() &&
    138         url_.GetOrigin() != job_->manifest_url_.GetOrigin()) {
    139       if (request->response_headers()->
    140               HasHeaderValue("cache-control", "no-store")) {
    141         request->Cancel();
    142         OnResponseCompleted();
    143         return;
    144       }
    145     }
    146 
    147     // Write response info to storage for URL fetches. Wait for async write
    148     // completion before reading any response data.
    149     if (fetch_type_ == URL_FETCH || fetch_type_ == MASTER_ENTRY_FETCH) {
    150       response_writer_.reset(job_->CreateResponseWriter());
    151       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
    152           new HttpResponseInfoIOBuffer(
    153               new net::HttpResponseInfo(request->response_info())));
    154       response_writer_->WriteInfo(
    155           io_buffer.get(),
    156           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
    157     } else {
    158       ReadResponseData();
    159     }
    160   } else {
    161     OnResponseCompleted();
    162   }
    163 }
    164 
    165 void AppCacheUpdateJob::URLFetcher::OnReadCompleted(
    166     net::URLRequest* request, int bytes_read) {
    167   DCHECK(request_ == request);
    168   bool data_consumed = true;
    169   if (request->status().is_success() && bytes_read > 0) {
    170     data_consumed = ConsumeResponseData(bytes_read);
    171     if (data_consumed) {
    172       bytes_read = 0;
    173       while (request->Read(buffer_.get(), kBufferSize, &bytes_read)) {
    174         if (bytes_read > 0) {
    175           data_consumed = ConsumeResponseData(bytes_read);
    176           if (!data_consumed)
    177             break;  // wait for async data processing, then read more
    178         } else {
    179           break;
    180         }
    181       }
    182     }
    183   }
    184   if (data_consumed && !request->status().is_io_pending())
    185     OnResponseCompleted();
    186 }
    187 
    188 void AppCacheUpdateJob::URLFetcher::AddConditionalHeaders(
    189     const net::HttpResponseHeaders* headers) {
    190   DCHECK(request_.get() && headers);
    191   net::HttpRequestHeaders extra_headers;
    192 
    193   // Add If-Modified-Since header if response info has Last-Modified header.
    194   const std::string last_modified = "Last-Modified";
    195   std::string last_modified_value;
    196   headers->EnumerateHeader(NULL, last_modified, &last_modified_value);
    197   if (!last_modified_value.empty()) {
    198     extra_headers.SetHeader(net::HttpRequestHeaders::kIfModifiedSince,
    199                             last_modified_value);
    200   }
    201 
    202   // Add If-None-Match header if response info has ETag header.
    203   const std::string etag = "ETag";
    204   std::string etag_value;
    205   headers->EnumerateHeader(NULL, etag, &etag_value);
    206   if (!etag_value.empty()) {
    207     extra_headers.SetHeader(net::HttpRequestHeaders::kIfNoneMatch,
    208                             etag_value);
    209   }
    210   if (!extra_headers.IsEmpty())
    211     request_->SetExtraRequestHeaders(extra_headers);
    212 }
    213 
    214 void  AppCacheUpdateJob::URLFetcher::OnWriteComplete(int result) {
    215   if (result < 0) {
    216     request_->Cancel();
    217     OnResponseCompleted();
    218     return;
    219   }
    220   ReadResponseData();
    221 }
    222 
    223 void AppCacheUpdateJob::URLFetcher::ReadResponseData() {
    224   InternalUpdateState state = job_->internal_state_;
    225   if (state == CACHE_FAILURE || state == CANCELLED || state == COMPLETED)
    226     return;
    227   int bytes_read = 0;
    228   request_->Read(buffer_.get(), kBufferSize, &bytes_read);
    229   OnReadCompleted(request_.get(), bytes_read);
    230 }
    231 
    232 // Returns false if response data is processed asynchronously, in which
    233 // case ReadResponseData will be invoked when it is safe to continue
    234 // reading more response data from the request.
    235 bool AppCacheUpdateJob::URLFetcher::ConsumeResponseData(int bytes_read) {
    236   DCHECK_GT(bytes_read, 0);
    237   switch (fetch_type_) {
    238     case MANIFEST_FETCH:
    239     case MANIFEST_REFETCH:
    240       manifest_data_.append(buffer_->data(), bytes_read);
    241       break;
    242     case URL_FETCH:
    243     case MASTER_ENTRY_FETCH:
    244       DCHECK(response_writer_.get());
    245       response_writer_->WriteData(
    246           buffer_.get(),
    247           bytes_read,
    248           base::Bind(&URLFetcher::OnWriteComplete, base::Unretained(this)));
    249       return false;  // wait for async write completion to continue reading
    250     default:
    251       NOTREACHED();
    252   }
    253   return true;
    254 }
    255 
    256 void AppCacheUpdateJob::URLFetcher::OnResponseCompleted() {
    257   // Retry for 503s where retry-after is 0.
    258   if (request_->status().is_success() &&
    259       request_->GetResponseCode() == 503 &&
    260       MaybeRetryRequest()) {
    261     return;
    262   }
    263 
    264   switch (fetch_type_) {
    265     case MANIFEST_FETCH:
    266       job_->HandleManifestFetchCompleted(this);
    267       break;
    268     case URL_FETCH:
    269       job_->HandleUrlFetchCompleted(this);
    270       break;
    271     case MASTER_ENTRY_FETCH:
    272       job_->HandleMasterEntryFetchCompleted(this);
    273       break;
    274     case MANIFEST_REFETCH:
    275       job_->HandleManifestRefetchCompleted(this);
    276       break;
    277     default:
    278       NOTREACHED();
    279   }
    280 
    281   delete this;
    282 }
    283 
    284 bool AppCacheUpdateJob::URLFetcher::MaybeRetryRequest() {
    285   if (retry_503_attempts_ >= kMax503Retries ||
    286       !request_->response_headers()->HasHeaderValue("retry-after", "0")) {
    287     return false;
    288   }
    289   ++retry_503_attempts_;
    290   request_.reset(job_->service_->request_context()->CreateRequest(url_, this));
    291   Start();
    292   return true;
    293 }
    294 
    295 AppCacheUpdateJob::AppCacheUpdateJob(AppCacheService* service,
    296                                      AppCacheGroup* group)
    297     : service_(service),
    298       manifest_url_(group->manifest_url()),
    299       group_(group),
    300       update_type_(UNKNOWN_TYPE),
    301       internal_state_(FETCH_MANIFEST),
    302       master_entries_completed_(0),
    303       url_fetches_completed_(0),
    304       manifest_fetcher_(NULL),
    305       stored_state_(UNSTORED) {
    306 }
    307 
    308 AppCacheUpdateJob::~AppCacheUpdateJob() {
    309   if (internal_state_ != COMPLETED)
    310     Cancel();
    311 
    312   DCHECK(!manifest_fetcher_);
    313   DCHECK(pending_url_fetches_.empty());
    314   DCHECK(!inprogress_cache_.get());
    315   DCHECK(pending_master_entries_.empty());
    316   DCHECK(master_entry_fetches_.empty());
    317 
    318   if (group_)
    319     group_->SetUpdateStatus(AppCacheGroup::IDLE);
    320 }
    321 
    322 void AppCacheUpdateJob::StartUpdate(AppCacheHost* host,
    323                                     const GURL& new_master_resource) {
    324   DCHECK(group_->update_job() == this);
    325   DCHECK(!group_->is_obsolete());
    326 
    327   bool is_new_pending_master_entry = false;
    328   if (!new_master_resource.is_empty()) {
    329     DCHECK(new_master_resource == host->pending_master_entry_url());
    330     DCHECK(!new_master_resource.has_ref());
    331     DCHECK(new_master_resource.GetOrigin() == manifest_url_.GetOrigin());
    332 
    333     // Cannot add more to this update if already terminating.
    334     if (IsTerminating()) {
    335       group_->QueueUpdate(host, new_master_resource);
    336       return;
    337     }
    338 
    339     std::pair<PendingMasters::iterator, bool> ret =
    340         pending_master_entries_.insert(
    341             PendingMasters::value_type(new_master_resource, PendingHosts()));
    342     is_new_pending_master_entry = ret.second;
    343     ret.first->second.push_back(host);
    344     host->AddObserver(this);
    345   }
    346 
    347   // Notify host (if any) if already checking or downloading.
    348   AppCacheGroup::UpdateStatus update_status = group_->update_status();
    349   if (update_status == AppCacheGroup::CHECKING ||
    350       update_status == AppCacheGroup::DOWNLOADING) {
    351     if (host) {
    352       NotifySingleHost(host, CHECKING_EVENT);
    353       if (update_status == AppCacheGroup::DOWNLOADING)
    354         NotifySingleHost(host, DOWNLOADING_EVENT);
    355 
    356       // Add to fetch list or an existing entry if already fetched.
    357       if (!new_master_resource.is_empty()) {
    358         AddMasterEntryToFetchList(host, new_master_resource,
    359                                   is_new_pending_master_entry);
    360       }
    361     }
    362     return;
    363   }
    364 
    365   // Begin update process for the group.
    366   group_->SetUpdateStatus(AppCacheGroup::CHECKING);
    367   if (group_->HasCache()) {
    368     update_type_ = UPGRADE_ATTEMPT;
    369     NotifyAllAssociatedHosts(CHECKING_EVENT);
    370   } else {
    371     update_type_ = CACHE_ATTEMPT;
    372     DCHECK(host);
    373     NotifySingleHost(host, CHECKING_EVENT);
    374   }
    375 
    376   if (!new_master_resource.is_empty()) {
    377     AddMasterEntryToFetchList(host, new_master_resource,
    378                               is_new_pending_master_entry);
    379   }
    380 
    381   FetchManifest(true);
    382 }
    383 
    384 AppCacheResponseWriter* AppCacheUpdateJob::CreateResponseWriter() {
    385   AppCacheResponseWriter* writer =
    386       service_->storage()->CreateResponseWriter(manifest_url_,
    387                                                 group_->group_id());
    388   stored_response_ids_.push_back(writer->response_id());
    389   return writer;
    390 }
    391 
    392 void AppCacheUpdateJob::HandleCacheFailure(const std::string& error_message) {
    393   // 6.9.4 cache failure steps 2-8.
    394   DCHECK(internal_state_ != CACHE_FAILURE);
    395   DCHECK(!error_message.empty());
    396   internal_state_ = CACHE_FAILURE;
    397   CancelAllUrlFetches();
    398   CancelAllMasterEntryFetches(error_message);
    399   NotifyAllError(error_message);
    400   DiscardInprogressCache();
    401   internal_state_ = COMPLETED;
    402   DeleteSoon();  // To unwind the stack prior to deletion.
    403 }
    404 
    405 void AppCacheUpdateJob::FetchManifest(bool is_first_fetch) {
    406   DCHECK(!manifest_fetcher_);
    407   manifest_fetcher_ = new URLFetcher(
    408      manifest_url_,
    409      is_first_fetch ? URLFetcher::MANIFEST_FETCH :
    410                       URLFetcher::MANIFEST_REFETCH,
    411      this);
    412 
    413   // Add any necessary Http headers before sending fetch request.
    414   if (is_first_fetch) {
    415     AppCacheEntry* entry = (update_type_ == UPGRADE_ATTEMPT) ?
    416         group_->newest_complete_cache()->GetEntry(manifest_url_) : NULL;
    417     if (entry) {
    418       // Asynchronously load response info for manifest from newest cache.
    419       service_->storage()->LoadResponseInfo(manifest_url_, group_->group_id(),
    420                                             entry->response_id(), this);
    421     } else {
    422       manifest_fetcher_->Start();
    423     }
    424   } else {
    425     DCHECK(internal_state_ == REFETCH_MANIFEST);
    426     DCHECK(manifest_response_info_.get());
    427     manifest_fetcher_->set_existing_response_headers(
    428         manifest_response_info_->headers.get());
    429     manifest_fetcher_->Start();
    430   }
    431 }
    432 
    433 
    434 void AppCacheUpdateJob::HandleManifestFetchCompleted(
    435     URLFetcher* fetcher) {
    436   DCHECK_EQ(internal_state_, FETCH_MANIFEST);
    437   DCHECK_EQ(manifest_fetcher_, fetcher);
    438   manifest_fetcher_ = NULL;
    439 
    440   net::URLRequest* request = fetcher->request();
    441   int response_code = -1;
    442   bool is_valid_response_code = false;
    443   if (request->status().is_success()) {
    444     response_code = request->GetResponseCode();
    445     is_valid_response_code = (response_code / 100 == 2);
    446   }
    447 
    448   if (is_valid_response_code) {
    449     manifest_data_ = fetcher->manifest_data();
    450     manifest_response_info_.reset(
    451         new net::HttpResponseInfo(request->response_info()));
    452     if (update_type_ == UPGRADE_ATTEMPT)
    453       CheckIfManifestChanged();  // continues asynchronously
    454     else
    455       ContinueHandleManifestFetchCompleted(true);
    456   } else if (response_code == 304 && update_type_ == UPGRADE_ATTEMPT) {
    457     ContinueHandleManifestFetchCompleted(false);
    458   } else if ((response_code == 404 || response_code == 410) &&
    459              update_type_ == UPGRADE_ATTEMPT) {
    460     service_->storage()->MakeGroupObsolete(group_, this);  // async
    461   } else {
    462     const char* kFormatString = "Manifest fetch failed (%d) %s";
    463     std::string message = base::StringPrintf(kFormatString, response_code,
    464                                              manifest_url_.spec().c_str());
    465     HandleCacheFailure(message);
    466   }
    467 }
    468 
    469 void AppCacheUpdateJob::OnGroupMadeObsolete(AppCacheGroup* group,
    470                                             bool success) {
    471   DCHECK(master_entry_fetches_.empty());
    472   CancelAllMasterEntryFetches("The cache has been made obsolete, "
    473                               "the manifest file returned 404 or 410");
    474   if (success) {
    475     DCHECK(group->is_obsolete());
    476     NotifyAllAssociatedHosts(OBSOLETE_EVENT);
    477     internal_state_ = COMPLETED;
    478     MaybeCompleteUpdate();
    479   } else {
    480     // Treat failure to mark group obsolete as a cache failure.
    481     HandleCacheFailure("Failed to mark the cache as obsolete");
    482   }
    483 }
    484 
    485 void AppCacheUpdateJob::ContinueHandleManifestFetchCompleted(bool changed) {
    486   DCHECK(internal_state_ == FETCH_MANIFEST);
    487 
    488   if (!changed) {
    489     DCHECK(update_type_ == UPGRADE_ATTEMPT);
    490     internal_state_ = NO_UPDATE;
    491 
    492     // Wait for pending master entries to download.
    493     FetchMasterEntries();
    494     MaybeCompleteUpdate();  // if not done, run async 6.9.4 step 7 substeps
    495     return;
    496   }
    497 
    498   Manifest manifest;
    499   if (!ParseManifest(manifest_url_, manifest_data_.data(),
    500                      manifest_data_.length(), manifest)) {
    501     const char* kFormatString = "Failed to parse manifest %s";
    502     const std::string message = base::StringPrintf(kFormatString,
    503         manifest_url_.spec().c_str());
    504     HandleCacheFailure(message);
    505     VLOG(1) << message;
    506     return;
    507   }
    508 
    509   // Proceed with update process. Section 6.9.4 steps 8-20.
    510   internal_state_ = DOWNLOADING;
    511   inprogress_cache_ = new AppCache(service_->storage(),
    512                                    service_->storage()->NewCacheId());
    513   BuildUrlFileList(manifest);
    514   inprogress_cache_->InitializeWithManifest(&manifest);
    515 
    516   // Associate all pending master hosts with the newly created cache.
    517   for (PendingMasters::iterator it = pending_master_entries_.begin();
    518        it != pending_master_entries_.end(); ++it) {
    519     PendingHosts& hosts = it->second;
    520     for (PendingHosts::iterator host_it = hosts.begin();
    521          host_it != hosts.end(); ++host_it) {
    522       (*host_it)
    523           ->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
    524     }
    525   }
    526 
    527   group_->SetUpdateStatus(AppCacheGroup::DOWNLOADING);
    528   NotifyAllAssociatedHosts(DOWNLOADING_EVENT);
    529   FetchUrls();
    530   FetchMasterEntries();
    531   MaybeCompleteUpdate();  // if not done, continues when async fetches complete
    532 }
    533 
    534 void AppCacheUpdateJob::HandleUrlFetchCompleted(URLFetcher* fetcher) {
    535   DCHECK(internal_state_ == DOWNLOADING);
    536 
    537   net::URLRequest* request = fetcher->request();
    538   const GURL& url = request->original_url();
    539   pending_url_fetches_.erase(url);
    540   NotifyAllProgress(url);
    541   ++url_fetches_completed_;
    542 
    543   int response_code = request->status().is_success()
    544       ? request->GetResponseCode() : -1;
    545   AppCacheEntry& entry = url_file_list_.find(url)->second;
    546 
    547   if (response_code / 100 == 2) {
    548     // Associate storage with the new entry.
    549     DCHECK(fetcher->response_writer());
    550     entry.set_response_id(fetcher->response_writer()->response_id());
    551     entry.set_response_size(fetcher->response_writer()->amount_written());
    552     if (!inprogress_cache_->AddOrModifyEntry(url, entry))
    553       duplicate_response_ids_.push_back(entry.response_id());
    554 
    555     // TODO(michaeln): Check for <html manifest=xxx>
    556     // See http://code.google.com/p/chromium/issues/detail?id=97930
    557     // if (entry.IsMaster() && !(entry.IsExplicit() || fallback || intercept))
    558     //   if (!manifestAttribute) skip it
    559 
    560     // Foreign entries will be detected during cache selection.
    561     // Note: 6.9.4, step 17.9 possible optimization: if resource is HTML or XML
    562     // file whose root element is an html element with a manifest attribute
    563     // whose value doesn't match the manifest url of the application cache
    564     // being processed, mark the entry as being foreign.
    565   } else {
    566     VLOG(1) << "Request status: " << request->status().status()
    567             << " error: " << request->status().error()
    568             << " response code: " << response_code;
    569     if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept()) {
    570       if (response_code == 304 && fetcher->existing_entry().has_response_id()) {
    571         // Keep the existing response.
    572         entry.set_response_id(fetcher->existing_entry().response_id());
    573         entry.set_response_size(fetcher->existing_entry().response_size());
    574         inprogress_cache_->AddOrModifyEntry(url, entry);
    575       } else {
    576         const char* kFormatString = "Resource fetch failed (%d) %s";
    577         const std::string message = base::StringPrintf(kFormatString,
    578             response_code, url.spec().c_str());
    579         HandleCacheFailure(message);
    580         return;
    581       }
    582     } else if (response_code == 404 || response_code == 410) {
    583       // Entry is skipped.  They are dropped from the cache.
    584     } else if (update_type_ == UPGRADE_ATTEMPT &&
    585                fetcher->existing_entry().has_response_id()) {
    586       // Keep the existing response.
    587       // TODO(michaeln): Not sure this is a good idea. This is spec compliant
    588       // but the old resource may or may not be compatible with the new contents
    589       // of the cache. Impossible to know one way or the other.
    590       entry.set_response_id(fetcher->existing_entry().response_id());
    591       entry.set_response_size(fetcher->existing_entry().response_size());
    592       inprogress_cache_->AddOrModifyEntry(url, entry);
    593     }
    594   }
    595 
    596   // Fetch another URL now that one request has completed.
    597   DCHECK(internal_state_ != CACHE_FAILURE);
    598   FetchUrls();
    599   MaybeCompleteUpdate();
    600 }
    601 
    602 void AppCacheUpdateJob::HandleMasterEntryFetchCompleted(
    603     URLFetcher* fetcher) {
    604   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
    605 
    606   // TODO(jennb): Handle downloads completing during cache failure when update
    607   // no longer fetches master entries directly. For now, we cancel all pending
    608   // master entry fetches when entering cache failure state so this will never
    609   // be called in CACHE_FAILURE state.
    610 
    611   net::URLRequest* request = fetcher->request();
    612   const GURL& url = request->original_url();
    613   master_entry_fetches_.erase(url);
    614   ++master_entries_completed_;
    615 
    616   int response_code = request->status().is_success()
    617       ? request->GetResponseCode() : -1;
    618 
    619   PendingMasters::iterator found = pending_master_entries_.find(url);
    620   DCHECK(found != pending_master_entries_.end());
    621   PendingHosts& hosts = found->second;
    622 
    623   // Section 6.9.4. No update case: step 7.3, else step 22.
    624   if (response_code / 100 == 2) {
    625     // Add fetched master entry to the appropriate cache.
    626     AppCache* cache = inprogress_cache_.get() ? inprogress_cache_.get()
    627                                               : group_->newest_complete_cache();
    628     DCHECK(fetcher->response_writer());
    629     AppCacheEntry master_entry(AppCacheEntry::MASTER,
    630                                fetcher->response_writer()->response_id(),
    631                                fetcher->response_writer()->amount_written());
    632     if (cache->AddOrModifyEntry(url, master_entry))
    633       added_master_entries_.push_back(url);
    634     else
    635       duplicate_response_ids_.push_back(master_entry.response_id());
    636 
    637     // In no-update case, associate host with the newest cache.
    638     if (!inprogress_cache_.get()) {
    639       // TODO(michaeln): defer until the updated cache has been stored
    640       DCHECK(cache == group_->newest_complete_cache());
    641       for (PendingHosts::iterator host_it = hosts.begin();
    642            host_it != hosts.end(); ++host_it) {
    643         (*host_it)->AssociateCompleteCache(cache);
    644       }
    645     }
    646   } else {
    647     HostNotifier host_notifier;
    648     for (PendingHosts::iterator host_it = hosts.begin();
    649          host_it != hosts.end(); ++host_it) {
    650       AppCacheHost* host = *host_it;
    651       host_notifier.AddHost(host);
    652 
    653       // In downloading case, disassociate host from inprogress cache.
    654       if (inprogress_cache_.get())
    655         host->AssociateNoCache(GURL());
    656 
    657       host->RemoveObserver(this);
    658     }
    659     hosts.clear();
    660 
    661     const char* kFormatString = "Master entry fetch failed (%d) %s";
    662     const std::string message = base::StringPrintf(kFormatString,
    663         response_code, request->url().spec().c_str());
    664     host_notifier.SendErrorNotifications(message);
    665 
    666     // In downloading case, update result is different if all master entries
    667     // failed vs. only some failing.
    668     if (inprogress_cache_.get()) {
    669       // Only count successful downloads to know if all master entries failed.
    670       pending_master_entries_.erase(found);
    671       --master_entries_completed_;
    672 
    673       // Section 6.9.4, step 22.3.
    674       if (update_type_ == CACHE_ATTEMPT && pending_master_entries_.empty()) {
    675         HandleCacheFailure(message);
    676         return;
    677       }
    678     }
    679   }
    680 
    681   DCHECK(internal_state_ != CACHE_FAILURE);
    682   FetchMasterEntries();
    683   MaybeCompleteUpdate();
    684 }
    685 
    686 void AppCacheUpdateJob::HandleManifestRefetchCompleted(
    687     URLFetcher* fetcher) {
    688   DCHECK(internal_state_ == REFETCH_MANIFEST);
    689   DCHECK(manifest_fetcher_ == fetcher);
    690   manifest_fetcher_ = NULL;
    691 
    692   net::URLRequest* request = fetcher->request();
    693   int response_code = request->status().is_success()
    694       ? request->GetResponseCode() : -1;
    695   if (response_code == 304 || manifest_data_ == fetcher->manifest_data()) {
    696     // Only need to store response in storage if manifest is not already
    697     // an entry in the cache.
    698     AppCacheEntry* entry = inprogress_cache_->GetEntry(manifest_url_);
    699     if (entry) {
    700       entry->add_types(AppCacheEntry::MANIFEST);
    701       StoreGroupAndCache();
    702     } else {
    703       manifest_response_writer_.reset(CreateResponseWriter());
    704       scoped_refptr<HttpResponseInfoIOBuffer> io_buffer(
    705           new HttpResponseInfoIOBuffer(manifest_response_info_.release()));
    706       manifest_response_writer_->WriteInfo(
    707           io_buffer.get(),
    708           base::Bind(&AppCacheUpdateJob::OnManifestInfoWriteComplete,
    709                      base::Unretained(this)));
    710     }
    711   } else {
    712     VLOG(1) << "Request status: " << request->status().status()
    713             << " error: " << request->status().error()
    714             << " response code: " << response_code;
    715     ScheduleUpdateRetry(kRerunDelayMs);
    716     HandleCacheFailure("Manifest changed during update, scheduling retry");
    717   }
    718 }
    719 
    720 void AppCacheUpdateJob::OnManifestInfoWriteComplete(int result) {
    721   if (result > 0) {
    722     scoped_refptr<net::StringIOBuffer> io_buffer(
    723         new net::StringIOBuffer(manifest_data_));
    724     manifest_response_writer_->WriteData(
    725         io_buffer.get(),
    726         manifest_data_.length(),
    727         base::Bind(&AppCacheUpdateJob::OnManifestDataWriteComplete,
    728                    base::Unretained(this)));
    729   } else {
    730     HandleCacheFailure("Failed to write the manifest headers to storage");
    731   }
    732 }
    733 
    734 void AppCacheUpdateJob::OnManifestDataWriteComplete(int result) {
    735   if (result > 0) {
    736     AppCacheEntry entry(AppCacheEntry::MANIFEST,
    737         manifest_response_writer_->response_id(),
    738         manifest_response_writer_->amount_written());
    739     if (!inprogress_cache_->AddOrModifyEntry(manifest_url_, entry))
    740       duplicate_response_ids_.push_back(entry.response_id());
    741     StoreGroupAndCache();
    742   } else {
    743     HandleCacheFailure("Failed to write the manifest data to storage");
    744   }
    745 }
    746 
    747 void AppCacheUpdateJob::StoreGroupAndCache() {
    748   DCHECK(stored_state_ == UNSTORED);
    749   stored_state_ = STORING;
    750   scoped_refptr<AppCache> newest_cache;
    751   if (inprogress_cache_.get())
    752     newest_cache.swap(inprogress_cache_);
    753   else
    754     newest_cache = group_->newest_complete_cache();
    755   newest_cache->set_update_time(base::Time::Now());
    756 
    757   // TODO(michaeln): dcheck is fishing for clues to crbug/95101
    758   DCHECK_EQ(manifest_url_, group_->manifest_url());
    759   service_->storage()
    760       ->StoreGroupAndNewestCache(group_, newest_cache.get(), this);  // async
    761 }
    762 
    763 void AppCacheUpdateJob::OnGroupAndNewestCacheStored(AppCacheGroup* group,
    764                                                     AppCache* newest_cache,
    765                                                     bool success,
    766                                                     bool would_exceed_quota) {
    767   DCHECK(stored_state_ == STORING);
    768   if (success) {
    769     stored_state_ = STORED;
    770     MaybeCompleteUpdate();  // will definitely complete
    771   } else {
    772     // Restore inprogress_cache_ to get the proper events delivered
    773     // and the proper cleanup to occur.
    774     if (newest_cache != group->newest_complete_cache())
    775       inprogress_cache_ = newest_cache;
    776 
    777     std::string message("Failed to commit new cache to storage");
    778     if (would_exceed_quota)
    779       message.append(", would exceed quota");
    780     HandleCacheFailure(message);
    781   }
    782 }
    783 
    784 void AppCacheUpdateJob::NotifySingleHost(AppCacheHost* host,
    785                                          EventID event_id) {
    786   std::vector<int> ids(1, host->host_id());
    787   host->frontend()->OnEventRaised(ids, event_id);
    788 }
    789 
    790 void AppCacheUpdateJob::NotifyAllAssociatedHosts(EventID event_id) {
    791   HostNotifier host_notifier;
    792   AddAllAssociatedHostsToNotifier(&host_notifier);
    793   host_notifier.SendNotifications(event_id);
    794 }
    795 
    796 void AppCacheUpdateJob::NotifyAllProgress(const GURL& url) {
    797   HostNotifier host_notifier;
    798   AddAllAssociatedHostsToNotifier(&host_notifier);
    799   host_notifier.SendProgressNotifications(
    800       url, url_file_list_.size(), url_fetches_completed_);
    801 }
    802 
    803 void AppCacheUpdateJob::NotifyAllFinalProgress() {
    804   DCHECK(url_file_list_.size() == url_fetches_completed_);
    805   NotifyAllProgress(GURL());
    806 }
    807 
    808 void AppCacheUpdateJob::NotifyAllError(const std::string& error_message) {
    809   HostNotifier host_notifier;
    810   AddAllAssociatedHostsToNotifier(&host_notifier);
    811   host_notifier.SendErrorNotifications(error_message);
    812 }
    813 
    814 void AppCacheUpdateJob::AddAllAssociatedHostsToNotifier(
    815     HostNotifier* host_notifier) {
    816   // Collect hosts so we only send one notification per frontend.
    817   // A host can only be associated with a single cache so no need to worry
    818   // about duplicate hosts being added to the notifier.
    819   if (inprogress_cache_.get()) {
    820     DCHECK(internal_state_ == DOWNLOADING || internal_state_ == CACHE_FAILURE);
    821     host_notifier->AddHosts(inprogress_cache_->associated_hosts());
    822   }
    823 
    824   AppCacheGroup::Caches old_caches = group_->old_caches();
    825   for (AppCacheGroup::Caches::const_iterator it = old_caches.begin();
    826        it != old_caches.end(); ++it) {
    827     host_notifier->AddHosts((*it)->associated_hosts());
    828   }
    829 
    830   AppCache* newest_cache = group_->newest_complete_cache();
    831   if (newest_cache)
    832     host_notifier->AddHosts(newest_cache->associated_hosts());
    833 }
    834 
    835 void AppCacheUpdateJob::OnDestructionImminent(AppCacheHost* host) {
    836   // The host is about to be deleted; remove from our collection.
    837   PendingMasters::iterator found =
    838       pending_master_entries_.find(host->pending_master_entry_url());
    839   DCHECK(found != pending_master_entries_.end());
    840   PendingHosts& hosts = found->second;
    841   PendingHosts::iterator it = std::find(hosts.begin(), hosts.end(), host);
    842   DCHECK(it != hosts.end());
    843   hosts.erase(it);
    844 }
    845 
    846 void AppCacheUpdateJob::CheckIfManifestChanged() {
    847   DCHECK(update_type_ == UPGRADE_ATTEMPT);
    848   AppCacheEntry* entry =
    849       group_->newest_complete_cache()->GetEntry(manifest_url_);
    850   if (!entry) {
    851     // TODO(michaeln): This is just a bandaid to avoid a crash.
    852     // http://code.google.com/p/chromium/issues/detail?id=95101
    853     HandleCacheFailure("Manifest entry not found in existing cache");
    854     AppCacheHistograms::AddMissingManifestEntrySample();
    855     service_->DeleteAppCacheGroup(manifest_url_, net::CompletionCallback());
    856     return;
    857   }
    858 
    859   // Load manifest data from storage to compare against fetched manifest.
    860   manifest_response_reader_.reset(
    861       service_->storage()->CreateResponseReader(manifest_url_,
    862                                                 group_->group_id(),
    863                                                 entry->response_id()));
    864   read_manifest_buffer_ = new net::IOBuffer(kBufferSize);
    865   manifest_response_reader_->ReadData(
    866       read_manifest_buffer_.get(),
    867       kBufferSize,
    868       base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
    869                  base::Unretained(this)));  // async read
    870 }
    871 
    872 void AppCacheUpdateJob::OnManifestDataReadComplete(int result) {
    873   if (result > 0) {
    874     loaded_manifest_data_.append(read_manifest_buffer_->data(), result);
    875     manifest_response_reader_->ReadData(
    876         read_manifest_buffer_.get(),
    877         kBufferSize,
    878         base::Bind(&AppCacheUpdateJob::OnManifestDataReadComplete,
    879                    base::Unretained(this)));  // read more
    880   } else {
    881     read_manifest_buffer_ = NULL;
    882     manifest_response_reader_.reset();
    883     ContinueHandleManifestFetchCompleted(
    884         result < 0 || manifest_data_ != loaded_manifest_data_);
    885   }
    886 }
    887 
    888 void AppCacheUpdateJob::BuildUrlFileList(const Manifest& manifest) {
    889   for (base::hash_set<std::string>::const_iterator it =
    890            manifest.explicit_urls.begin();
    891        it != manifest.explicit_urls.end(); ++it) {
    892     AddUrlToFileList(GURL(*it), AppCacheEntry::EXPLICIT);
    893   }
    894 
    895   const std::vector<Namespace>& intercepts =
    896       manifest.intercept_namespaces;
    897   for (std::vector<Namespace>::const_iterator it = intercepts.begin();
    898        it != intercepts.end(); ++it) {
    899     int flags = AppCacheEntry::INTERCEPT;
    900     if (it->is_executable)
    901       flags |= AppCacheEntry::EXECUTABLE;
    902     AddUrlToFileList(it->target_url, flags);
    903   }
    904 
    905   const std::vector<Namespace>& fallbacks =
    906       manifest.fallback_namespaces;
    907   for (std::vector<Namespace>::const_iterator it = fallbacks.begin();
    908        it != fallbacks.end(); ++it) {
    909      AddUrlToFileList(it->target_url, AppCacheEntry::FALLBACK);
    910   }
    911 
    912   // Add all master entries from newest complete cache.
    913   if (update_type_ == UPGRADE_ATTEMPT) {
    914     const AppCache::EntryMap& entries =
    915         group_->newest_complete_cache()->entries();
    916     for (AppCache::EntryMap::const_iterator it = entries.begin();
    917          it != entries.end(); ++it) {
    918       const AppCacheEntry& entry = it->second;
    919       if (entry.IsMaster())
    920         AddUrlToFileList(it->first, AppCacheEntry::MASTER);
    921     }
    922   }
    923 }
    924 
    925 void AppCacheUpdateJob::AddUrlToFileList(const GURL& url, int type) {
    926   std::pair<AppCache::EntryMap::iterator, bool> ret = url_file_list_.insert(
    927       AppCache::EntryMap::value_type(url, AppCacheEntry(type)));
    928 
    929   if (ret.second)
    930     urls_to_fetch_.push_back(UrlToFetch(url, false, NULL));
    931   else
    932     ret.first->second.add_types(type);  // URL already exists. Merge types.
    933 }
    934 
    935 void AppCacheUpdateJob::FetchUrls() {
    936   DCHECK(internal_state_ == DOWNLOADING);
    937 
    938   // Fetch each URL in the list according to section 6.9.4 step 17.1-17.3.
    939   // Fetch up to the concurrent limit. Other fetches will be triggered as each
    940   // each fetch completes.
    941   while (pending_url_fetches_.size() < kMaxConcurrentUrlFetches &&
    942          !urls_to_fetch_.empty()) {
    943     UrlToFetch url_to_fetch = urls_to_fetch_.front();
    944     urls_to_fetch_.pop_front();
    945 
    946     AppCache::EntryMap::iterator it = url_file_list_.find(url_to_fetch.url);
    947     DCHECK(it != url_file_list_.end());
    948     AppCacheEntry& entry = it->second;
    949     if (ShouldSkipUrlFetch(entry)) {
    950       NotifyAllProgress(url_to_fetch.url);
    951       ++url_fetches_completed_;
    952     } else if (AlreadyFetchedEntry(url_to_fetch.url, entry.types())) {
    953       NotifyAllProgress(url_to_fetch.url);
    954       ++url_fetches_completed_;  // saved a URL request
    955     } else if (!url_to_fetch.storage_checked &&
    956                MaybeLoadFromNewestCache(url_to_fetch.url, entry)) {
    957       // Continues asynchronously after data is loaded from newest cache.
    958     } else {
    959       URLFetcher* fetcher = new URLFetcher(
    960           url_to_fetch.url, URLFetcher::URL_FETCH, this);
    961       if (url_to_fetch.existing_response_info.get()) {
    962         DCHECK(group_->newest_complete_cache());
    963         AppCacheEntry* existing_entry =
    964             group_->newest_complete_cache()->GetEntry(url_to_fetch.url);
    965         DCHECK(existing_entry);
    966         DCHECK(existing_entry->response_id() ==
    967                url_to_fetch.existing_response_info->response_id());
    968         fetcher->set_existing_response_headers(
    969             url_to_fetch.existing_response_info->http_response_info()->headers
    970                 .get());
    971         fetcher->set_existing_entry(*existing_entry);
    972       }
    973       fetcher->Start();
    974       pending_url_fetches_.insert(
    975           PendingUrlFetches::value_type(url_to_fetch.url, fetcher));
    976     }
    977   }
    978 }
    979 
    980 void AppCacheUpdateJob::CancelAllUrlFetches() {
    981   // Cancel any pending URL requests.
    982   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
    983        it != pending_url_fetches_.end(); ++it) {
    984     delete it->second;
    985   }
    986 
    987   url_fetches_completed_ +=
    988       pending_url_fetches_.size() + urls_to_fetch_.size();
    989   pending_url_fetches_.clear();
    990   urls_to_fetch_.clear();
    991 }
    992 
    993 bool AppCacheUpdateJob::ShouldSkipUrlFetch(const AppCacheEntry& entry) {
    994   // 6.6.4 Step 17
    995   // If the resource URL being processed was flagged as neither an
    996   // "explicit entry" nor or a "fallback entry", then the user agent
    997   // may skip this URL.
    998   if (entry.IsExplicit() || entry.IsFallback() || entry.IsIntercept())
    999     return false;
   1000 
   1001   // TODO(jennb): decide if entry should be skipped to expire it from cache
   1002   return false;
   1003 }
   1004 
   1005 bool AppCacheUpdateJob::AlreadyFetchedEntry(const GURL& url,
   1006                                             int entry_type) {
   1007   DCHECK(internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE);
   1008   AppCacheEntry* existing =
   1009       inprogress_cache_.get() ? inprogress_cache_->GetEntry(url)
   1010                               : group_->newest_complete_cache()->GetEntry(url);
   1011   if (existing) {
   1012     existing->add_types(entry_type);
   1013     return true;
   1014   }
   1015   return false;
   1016 }
   1017 
   1018 void AppCacheUpdateJob::AddMasterEntryToFetchList(AppCacheHost* host,
   1019                                                   const GURL& url,
   1020                                                   bool is_new) {
   1021   DCHECK(!IsTerminating());
   1022 
   1023   if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE) {
   1024     AppCache* cache;
   1025     if (inprogress_cache_.get()) {
   1026       // always associate
   1027       host->AssociateIncompleteCache(inprogress_cache_.get(), manifest_url_);
   1028       cache = inprogress_cache_.get();
   1029     } else {
   1030       cache = group_->newest_complete_cache();
   1031     }
   1032 
   1033     // Update existing entry if it has already been fetched.
   1034     AppCacheEntry* entry = cache->GetEntry(url);
   1035     if (entry) {
   1036       entry->add_types(AppCacheEntry::MASTER);
   1037       if (internal_state_ == NO_UPDATE && !inprogress_cache_.get()) {
   1038         // only associate if have entry
   1039         host->AssociateCompleteCache(cache);
   1040       }
   1041       if (is_new)
   1042         ++master_entries_completed_;  // pretend fetching completed
   1043       return;
   1044     }
   1045   }
   1046 
   1047   // Add to fetch list if not already fetching.
   1048   if (master_entry_fetches_.find(url) == master_entry_fetches_.end()) {
   1049     master_entries_to_fetch_.insert(url);
   1050     if (internal_state_ == DOWNLOADING || internal_state_ == NO_UPDATE)
   1051       FetchMasterEntries();
   1052   }
   1053 }
   1054 
   1055 void AppCacheUpdateJob::FetchMasterEntries() {
   1056   DCHECK(internal_state_ == NO_UPDATE || internal_state_ == DOWNLOADING);
   1057 
   1058   // Fetch each master entry in the list, up to the concurrent limit.
   1059   // Additional fetches will be triggered as each fetch completes.
   1060   while (master_entry_fetches_.size() < kMaxConcurrentUrlFetches &&
   1061          !master_entries_to_fetch_.empty()) {
   1062     const GURL& url = *master_entries_to_fetch_.begin();
   1063 
   1064     if (AlreadyFetchedEntry(url, AppCacheEntry::MASTER)) {
   1065       ++master_entries_completed_;  // saved a URL request
   1066 
   1067       // In no update case, associate hosts to newest cache in group
   1068       // now that master entry has been "successfully downloaded".
   1069       if (internal_state_ == NO_UPDATE) {
   1070         // TODO(michaeln): defer until the updated cache has been stored.
   1071         DCHECK(!inprogress_cache_.get());
   1072         AppCache* cache = group_->newest_complete_cache();
   1073         PendingMasters::iterator found = pending_master_entries_.find(url);
   1074         DCHECK(found != pending_master_entries_.end());
   1075         PendingHosts& hosts = found->second;
   1076         for (PendingHosts::iterator host_it = hosts.begin();
   1077              host_it != hosts.end(); ++host_it) {
   1078           (*host_it)->AssociateCompleteCache(cache);
   1079         }
   1080       }
   1081     } else {
   1082       URLFetcher* fetcher = new URLFetcher(
   1083           url, URLFetcher::MASTER_ENTRY_FETCH, this);
   1084       fetcher->Start();
   1085       master_entry_fetches_.insert(PendingUrlFetches::value_type(url, fetcher));
   1086     }
   1087 
   1088     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
   1089   }
   1090 }
   1091 
   1092 void AppCacheUpdateJob::CancelAllMasterEntryFetches(
   1093     const std::string& error_message) {
   1094   // For now, cancel all in-progress fetches for master entries and pretend
   1095   // all master entries fetches have completed.
   1096   // TODO(jennb): Delete this when update no longer fetches master entries
   1097   // directly.
   1098 
   1099   // Cancel all in-progress fetches.
   1100   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
   1101        it != master_entry_fetches_.end(); ++it) {
   1102     delete it->second;
   1103     master_entries_to_fetch_.insert(it->first);  // back in unfetched list
   1104   }
   1105   master_entry_fetches_.clear();
   1106 
   1107   master_entries_completed_ += master_entries_to_fetch_.size();
   1108 
   1109   // Cache failure steps, step 2.
   1110   // Pretend all master entries that have not yet been fetched have completed
   1111   // downloading. Unassociate hosts from any appcache and send ERROR event.
   1112   HostNotifier host_notifier;
   1113   while (!master_entries_to_fetch_.empty()) {
   1114     const GURL& url = *master_entries_to_fetch_.begin();
   1115     PendingMasters::iterator found = pending_master_entries_.find(url);
   1116     DCHECK(found != pending_master_entries_.end());
   1117     PendingHosts& hosts = found->second;
   1118     for (PendingHosts::iterator host_it = hosts.begin();
   1119          host_it != hosts.end(); ++host_it) {
   1120       AppCacheHost* host = *host_it;
   1121       host->AssociateNoCache(GURL());
   1122       host_notifier.AddHost(host);
   1123       host->RemoveObserver(this);
   1124     }
   1125     hosts.clear();
   1126 
   1127     master_entries_to_fetch_.erase(master_entries_to_fetch_.begin());
   1128   }
   1129   host_notifier.SendErrorNotifications(error_message);
   1130 }
   1131 
   1132 bool AppCacheUpdateJob::MaybeLoadFromNewestCache(const GURL& url,
   1133                                                  AppCacheEntry& entry) {
   1134   if (update_type_ != UPGRADE_ATTEMPT)
   1135     return false;
   1136 
   1137   AppCache* newest = group_->newest_complete_cache();
   1138   AppCacheEntry* copy_me = newest->GetEntry(url);
   1139   if (!copy_me || !copy_me->has_response_id())
   1140     return false;
   1141 
   1142   // Load HTTP headers for entry from newest cache.
   1143   loading_responses_.insert(
   1144       LoadingResponses::value_type(copy_me->response_id(), url));
   1145   service_->storage()->LoadResponseInfo(manifest_url_, group_->group_id(),
   1146                                         copy_me->response_id(),
   1147                                         this);
   1148   // Async: wait for OnResponseInfoLoaded to complete.
   1149   return true;
   1150 }
   1151 
   1152 void AppCacheUpdateJob::OnResponseInfoLoaded(
   1153     AppCacheResponseInfo* response_info, int64 response_id) {
   1154   const net::HttpResponseInfo* http_info = response_info ?
   1155       response_info->http_response_info() : NULL;
   1156 
   1157   // Needed response info for a manifest fetch request.
   1158   if (internal_state_ == FETCH_MANIFEST) {
   1159     if (http_info)
   1160       manifest_fetcher_->set_existing_response_headers(
   1161           http_info->headers.get());
   1162     manifest_fetcher_->Start();
   1163     return;
   1164   }
   1165 
   1166   LoadingResponses::iterator found = loading_responses_.find(response_id);
   1167   DCHECK(found != loading_responses_.end());
   1168   const GURL& url = found->second;
   1169 
   1170   if (!http_info) {
   1171     LoadFromNewestCacheFailed(url, NULL);  // no response found
   1172   } else {
   1173     // Check if response can be re-used according to HTTP caching semantics.
   1174     // Responses with a "vary" header get treated as expired.
   1175     const std::string name = "vary";
   1176     std::string value;
   1177     void* iter = NULL;
   1178     if (!http_info->headers.get() ||
   1179         http_info->headers->RequiresValidation(http_info->request_time,
   1180                                                http_info->response_time,
   1181                                                base::Time::Now()) ||
   1182         http_info->headers->EnumerateHeader(&iter, name, &value)) {
   1183       LoadFromNewestCacheFailed(url, response_info);
   1184     } else {
   1185       DCHECK(group_->newest_complete_cache());
   1186       AppCacheEntry* copy_me = group_->newest_complete_cache()->GetEntry(url);
   1187       DCHECK(copy_me);
   1188       DCHECK(copy_me->response_id() == response_id);
   1189 
   1190       AppCache::EntryMap::iterator it = url_file_list_.find(url);
   1191       DCHECK(it != url_file_list_.end());
   1192       AppCacheEntry& entry = it->second;
   1193       entry.set_response_id(response_id);
   1194       entry.set_response_size(copy_me->response_size());
   1195       inprogress_cache_->AddOrModifyEntry(url, entry);
   1196       NotifyAllProgress(url);
   1197       ++url_fetches_completed_;
   1198     }
   1199   }
   1200   loading_responses_.erase(found);
   1201 
   1202   MaybeCompleteUpdate();
   1203 }
   1204 
   1205 void AppCacheUpdateJob::LoadFromNewestCacheFailed(
   1206     const GURL& url, AppCacheResponseInfo* response_info) {
   1207   if (internal_state_ == CACHE_FAILURE)
   1208     return;
   1209 
   1210   // Re-insert url at front of fetch list. Indicate storage has been checked.
   1211   urls_to_fetch_.push_front(UrlToFetch(url, true, response_info));
   1212   FetchUrls();
   1213 }
   1214 
   1215 void AppCacheUpdateJob::MaybeCompleteUpdate() {
   1216   DCHECK(internal_state_ != CACHE_FAILURE);
   1217 
   1218   // Must wait for any pending master entries or url fetches to complete.
   1219   if (master_entries_completed_ != pending_master_entries_.size() ||
   1220       url_fetches_completed_ != url_file_list_.size()) {
   1221     DCHECK(internal_state_ != COMPLETED);
   1222     return;
   1223   }
   1224 
   1225   switch (internal_state_) {
   1226     case NO_UPDATE:
   1227       if (master_entries_completed_ > 0) {
   1228         switch (stored_state_) {
   1229           case UNSTORED:
   1230             StoreGroupAndCache();
   1231             return;
   1232           case STORING:
   1233             return;
   1234           case STORED:
   1235             break;
   1236         }
   1237       }
   1238       // 6.9.4 steps 7.3-7.7.
   1239       NotifyAllAssociatedHosts(NO_UPDATE_EVENT);
   1240       DiscardDuplicateResponses();
   1241       internal_state_ = COMPLETED;
   1242       break;
   1243     case DOWNLOADING:
   1244       internal_state_ = REFETCH_MANIFEST;
   1245       FetchManifest(false);
   1246       break;
   1247     case REFETCH_MANIFEST:
   1248       DCHECK(stored_state_ == STORED);
   1249       NotifyAllFinalProgress();
   1250       if (update_type_ == CACHE_ATTEMPT)
   1251         NotifyAllAssociatedHosts(CACHED_EVENT);
   1252       else
   1253         NotifyAllAssociatedHosts(UPDATE_READY_EVENT);
   1254       DiscardDuplicateResponses();
   1255       internal_state_ = COMPLETED;
   1256       break;
   1257     case CACHE_FAILURE:
   1258       NOTREACHED();  // See HandleCacheFailure
   1259       break;
   1260     default:
   1261       break;
   1262   }
   1263 
   1264   // Let the stack unwind before deletion to make it less risky as this
   1265   // method is called from multiple places in this file.
   1266   if (internal_state_ == COMPLETED)
   1267     DeleteSoon();
   1268 }
   1269 
   1270 void AppCacheUpdateJob::ScheduleUpdateRetry(int delay_ms) {
   1271   // TODO(jennb): post a delayed task with the "same parameters" as this job
   1272   // to retry the update at a later time. Need group, URLs of pending master
   1273   // entries and their hosts.
   1274 }
   1275 
   1276 void AppCacheUpdateJob::Cancel() {
   1277   internal_state_ = CANCELLED;
   1278 
   1279   if (manifest_fetcher_) {
   1280     delete manifest_fetcher_;
   1281     manifest_fetcher_ = NULL;
   1282   }
   1283 
   1284   for (PendingUrlFetches::iterator it = pending_url_fetches_.begin();
   1285        it != pending_url_fetches_.end(); ++it) {
   1286     delete it->second;
   1287   }
   1288   pending_url_fetches_.clear();
   1289 
   1290   for (PendingUrlFetches::iterator it = master_entry_fetches_.begin();
   1291        it != master_entry_fetches_.end(); ++it) {
   1292     delete it->second;
   1293   }
   1294   master_entry_fetches_.clear();
   1295 
   1296   ClearPendingMasterEntries();
   1297   DiscardInprogressCache();
   1298 
   1299   // Delete response writer to avoid any callbacks.
   1300   if (manifest_response_writer_)
   1301     manifest_response_writer_.reset();
   1302 
   1303   service_->storage()->CancelDelegateCallbacks(this);
   1304 }
   1305 
   1306 void AppCacheUpdateJob::ClearPendingMasterEntries() {
   1307   for (PendingMasters::iterator it = pending_master_entries_.begin();
   1308        it != pending_master_entries_.end(); ++it) {
   1309     PendingHosts& hosts = it->second;
   1310     for (PendingHosts::iterator host_it = hosts.begin();
   1311          host_it != hosts.end(); ++host_it) {
   1312       (*host_it)->RemoveObserver(this);
   1313     }
   1314   }
   1315 
   1316   pending_master_entries_.clear();
   1317 }
   1318 
   1319 void AppCacheUpdateJob::DiscardInprogressCache() {
   1320   service_->storage()->DoomResponses(manifest_url_, stored_response_ids_);
   1321 
   1322   if (!inprogress_cache_.get()) {
   1323     // We have to undo the changes we made, if any, to the existing cache.
   1324     for (std::vector<GURL>::iterator iter = added_master_entries_.begin();
   1325          iter != added_master_entries_.end(); ++iter) {
   1326       DCHECK(group_->newest_complete_cache());
   1327       group_->newest_complete_cache()->RemoveEntry(*iter);
   1328     }
   1329     return;
   1330   }
   1331 
   1332   AppCache::AppCacheHosts& hosts = inprogress_cache_->associated_hosts();
   1333   while (!hosts.empty())
   1334     (*hosts.begin())->AssociateNoCache(GURL());
   1335 
   1336   inprogress_cache_ = NULL;
   1337 }
   1338 
   1339 void AppCacheUpdateJob::DiscardDuplicateResponses() {
   1340   service_->storage()->DoomResponses(manifest_url_, duplicate_response_ids_);
   1341 }
   1342 
   1343 void AppCacheUpdateJob::DeleteSoon() {
   1344   ClearPendingMasterEntries();
   1345   manifest_response_writer_.reset();
   1346   service_->storage()->CancelDelegateCallbacks(this);
   1347 
   1348   // Break the connection with the group so the group cannot call delete
   1349   // on this object after we've posted a task to delete ourselves.
   1350   group_->SetUpdateStatus(AppCacheGroup::IDLE);
   1351   group_ = NULL;
   1352 
   1353   base::MessageLoop::current()->DeleteSoon(FROM_HERE, this);
   1354 }
   1355 
   1356 }  // namespace appcache
   1357