Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/url_request/url_fetcher_core.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/logging.h"
      9 #include "base/metrics/histogram.h"
     10 #include "base/sequenced_task_runner.h"
     11 #include "base/single_thread_task_runner.h"
     12 #include "base/stl_util.h"
     13 #include "base/thread_task_runner_handle.h"
     14 #include "base/tracked_objects.h"
     15 #include "net/base/io_buffer.h"
     16 #include "net/base/load_flags.h"
     17 #include "net/base/net_errors.h"
     18 #include "net/base/request_priority.h"
     19 #include "net/base/upload_bytes_element_reader.h"
     20 #include "net/base/upload_data_stream.h"
     21 #include "net/base/upload_file_element_reader.h"
     22 #include "net/http/http_response_headers.h"
     23 #include "net/url_request/url_fetcher_delegate.h"
     24 #include "net/url_request/url_fetcher_response_writer.h"
     25 #include "net/url_request/url_request_context.h"
     26 #include "net/url_request/url_request_context_getter.h"
     27 #include "net/url_request/url_request_throttler_manager.h"
     28 
     29 namespace {
     30 
     31 const int kBufferSize = 4096;
     32 const int kUploadProgressTimerInterval = 100;
     33 bool g_interception_enabled = false;
     34 bool g_ignore_certificate_requests = false;
     35 
     36 void EmptyCompletionCallback(int result) {}
     37 
     38 }  // namespace
     39 
     40 namespace net {
     41 
     42 // URLFetcherCore::Registry ---------------------------------------------------
     43 
     44 URLFetcherCore::Registry::Registry() {}
     45 URLFetcherCore::Registry::~Registry() {}
     46 
     47 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
     48   DCHECK(!ContainsKey(fetchers_, core));
     49   fetchers_.insert(core);
     50 }
     51 
     52 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
     53   DCHECK(ContainsKey(fetchers_, core));
     54   fetchers_.erase(core);
     55 }
     56 
     57 void URLFetcherCore::Registry::CancelAll() {
     58   while (!fetchers_.empty())
     59     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
     60 }
     61 
     62 // URLFetcherCore -------------------------------------------------------------
     63 
     64 // static
     65 base::LazyInstance<URLFetcherCore::Registry>
     66     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
     67 
     68 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
     69                                const GURL& original_url,
     70                                URLFetcher::RequestType request_type,
     71                                URLFetcherDelegate* d)
     72     : fetcher_(fetcher),
     73       original_url_(original_url),
     74       request_type_(request_type),
     75       delegate_(d),
     76       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
     77       load_flags_(LOAD_NORMAL),
     78       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
     79       buffer_(new IOBuffer(kBufferSize)),
     80       url_request_data_key_(NULL),
     81       was_fetched_via_proxy_(false),
     82       upload_content_set_(false),
     83       upload_range_offset_(0),
     84       upload_range_length_(0),
     85       is_chunked_upload_(false),
     86       was_cancelled_(false),
     87       stop_on_redirect_(false),
     88       stopped_on_redirect_(false),
     89       automatically_retry_on_5xx_(true),
     90       num_retries_on_5xx_(0),
     91       max_retries_on_5xx_(0),
     92       num_retries_on_network_changes_(0),
     93       max_retries_on_network_changes_(0),
     94       current_upload_bytes_(-1),
     95       current_response_bytes_(0),
     96       total_response_bytes_(-1) {
     97   CHECK(original_url_.is_valid());
     98 }
     99 
    100 void URLFetcherCore::Start() {
    101   DCHECK(delegate_task_runner_.get());
    102   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
    103   if (network_task_runner_.get()) {
    104     DCHECK_EQ(network_task_runner_,
    105               request_context_getter_->GetNetworkTaskRunner());
    106   } else {
    107     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
    108   }
    109   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
    110 
    111   network_task_runner_->PostTask(
    112       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
    113 }
    114 
    115 void URLFetcherCore::Stop() {
    116   if (delegate_task_runner_.get())  // May be NULL in tests.
    117     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    118 
    119   delegate_ = NULL;
    120   fetcher_ = NULL;
    121   if (!network_task_runner_.get())
    122     return;
    123   if (network_task_runner_->RunsTasksOnCurrentThread()) {
    124     CancelURLRequest(ERR_ABORTED);
    125   } else {
    126     network_task_runner_->PostTask(
    127         FROM_HERE,
    128         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
    129   }
    130 }
    131 
    132 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
    133                                    const std::string& upload_content) {
    134   DCHECK(!is_chunked_upload_);
    135   DCHECK(!upload_content_set_);
    136   DCHECK(upload_content_.empty());
    137   DCHECK(upload_file_path_.empty());
    138   DCHECK(upload_content_type_.empty());
    139 
    140   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
    141   DCHECK(upload_content.empty() || !upload_content_type.empty());
    142 
    143   upload_content_type_ = upload_content_type;
    144   upload_content_ = upload_content;
    145   upload_content_set_ = true;
    146 }
    147 
    148 void URLFetcherCore::SetUploadFilePath(
    149     const std::string& upload_content_type,
    150     const base::FilePath& file_path,
    151     uint64 range_offset,
    152     uint64 range_length,
    153     scoped_refptr<base::TaskRunner> file_task_runner) {
    154   DCHECK(!is_chunked_upload_);
    155   DCHECK(!upload_content_set_);
    156   DCHECK(upload_content_.empty());
    157   DCHECK(upload_file_path_.empty());
    158   DCHECK_EQ(upload_range_offset_, 0ULL);
    159   DCHECK_EQ(upload_range_length_, 0ULL);
    160   DCHECK(upload_content_type_.empty());
    161   DCHECK(!upload_content_type.empty());
    162 
    163   upload_content_type_ = upload_content_type;
    164   upload_file_path_ = file_path;
    165   upload_range_offset_ = range_offset;
    166   upload_range_length_ = range_length;
    167   upload_file_task_runner_ = file_task_runner;
    168   upload_content_set_ = true;
    169 }
    170 
    171 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
    172   DCHECK(is_chunked_upload_ ||
    173          (upload_content_type_.empty() &&
    174           upload_content_.empty()));
    175 
    176   // Empty |content_type| is not allowed here, because it is impossible
    177   // to ensure non-empty upload content as it is not yet supplied.
    178   DCHECK(!content_type.empty());
    179 
    180   upload_content_type_ = content_type;
    181   upload_content_.clear();
    182   is_chunked_upload_ = true;
    183 }
    184 
    185 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
    186                                          bool is_last_chunk) {
    187   DCHECK(delegate_task_runner_.get());
    188   DCHECK(network_task_runner_.get());
    189   network_task_runner_->PostTask(
    190       FROM_HERE,
    191       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
    192                  is_last_chunk));
    193 }
    194 
    195 void URLFetcherCore::SetLoadFlags(int load_flags) {
    196   load_flags_ = load_flags;
    197 }
    198 
    199 int URLFetcherCore::GetLoadFlags() const {
    200   return load_flags_;
    201 }
    202 
    203 void URLFetcherCore::SetReferrer(const std::string& referrer) {
    204   referrer_ = referrer;
    205 }
    206 
    207 void URLFetcherCore::SetExtraRequestHeaders(
    208     const std::string& extra_request_headers) {
    209   extra_request_headers_.Clear();
    210   extra_request_headers_.AddHeadersFromString(extra_request_headers);
    211 }
    212 
    213 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
    214   extra_request_headers_.AddHeaderFromString(header_line);
    215 }
    216 
    217 void URLFetcherCore::GetExtraRequestHeaders(
    218     HttpRequestHeaders* headers) const {
    219   headers->CopyFrom(extra_request_headers_);
    220 }
    221 
    222 void URLFetcherCore::SetRequestContext(
    223     URLRequestContextGetter* request_context_getter) {
    224   DCHECK(!request_context_getter_.get());
    225   DCHECK(request_context_getter);
    226   request_context_getter_ = request_context_getter;
    227 }
    228 
    229 void URLFetcherCore::SetFirstPartyForCookies(
    230     const GURL& first_party_for_cookies) {
    231   DCHECK(first_party_for_cookies_.is_empty());
    232   first_party_for_cookies_ = first_party_for_cookies;
    233 }
    234 
    235 void URLFetcherCore::SetURLRequestUserData(
    236     const void* key,
    237     const URLFetcher::CreateDataCallback& create_data_callback) {
    238   DCHECK(key);
    239   DCHECK(!create_data_callback.is_null());
    240   url_request_data_key_ = key;
    241   url_request_create_data_callback_ = create_data_callback;
    242 }
    243 
    244 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
    245   stop_on_redirect_ = stop_on_redirect;
    246 }
    247 
    248 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
    249   automatically_retry_on_5xx_ = retry;
    250 }
    251 
    252 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
    253   max_retries_on_5xx_ = max_retries;
    254 }
    255 
    256 int URLFetcherCore::GetMaxRetriesOn5xx() const {
    257   return max_retries_on_5xx_;
    258 }
    259 
    260 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
    261   return backoff_delay_;
    262 }
    263 
    264 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
    265   max_retries_on_network_changes_ = max_retries;
    266 }
    267 
    268 void URLFetcherCore::SaveResponseToFileAtPath(
    269     const base::FilePath& file_path,
    270     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
    271   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    272   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
    273       new URLFetcherFileWriter(file_task_runner, file_path)));
    274 }
    275 
    276 void URLFetcherCore::SaveResponseToTemporaryFile(
    277     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
    278   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    279   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
    280       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
    281 }
    282 
    283 void URLFetcherCore::SaveResponseWithWriter(
    284     scoped_ptr<URLFetcherResponseWriter> response_writer) {
    285   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    286   response_writer_ = response_writer.Pass();
    287 }
    288 
    289 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
    290   return response_headers_.get();
    291 }
    292 
    293 // TODO(panayiotis): socket_address_ is written in the IO thread,
    294 // if this is accessed in the UI thread, this could result in a race.
    295 // Same for response_headers_ above and was_fetched_via_proxy_ below.
    296 HostPortPair URLFetcherCore::GetSocketAddress() const {
    297   return socket_address_;
    298 }
    299 
    300 bool URLFetcherCore::WasFetchedViaProxy() const {
    301   return was_fetched_via_proxy_;
    302 }
    303 
    304 const GURL& URLFetcherCore::GetOriginalURL() const {
    305   return original_url_;
    306 }
    307 
    308 const GURL& URLFetcherCore::GetURL() const {
    309   return url_;
    310 }
    311 
    312 const URLRequestStatus& URLFetcherCore::GetStatus() const {
    313   return status_;
    314 }
    315 
    316 int URLFetcherCore::GetResponseCode() const {
    317   return response_code_;
    318 }
    319 
    320 const ResponseCookies& URLFetcherCore::GetCookies() const {
    321   return cookies_;
    322 }
    323 
    324 void URLFetcherCore::ReceivedContentWasMalformed() {
    325   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    326   if (network_task_runner_.get()) {
    327     network_task_runner_->PostTask(
    328         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
    329   }
    330 }
    331 
    332 bool URLFetcherCore::GetResponseAsString(
    333     std::string* out_response_string) const {
    334   URLFetcherStringWriter* string_writer =
    335       response_writer_ ? response_writer_->AsStringWriter() : NULL;
    336   if (!string_writer)
    337     return false;
    338 
    339   *out_response_string = string_writer->data();
    340   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
    341                           (string_writer->data().length() / 1024));
    342   return true;
    343 }
    344 
    345 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
    346                                            base::FilePath* out_response_path) {
    347   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    348 
    349   URLFetcherFileWriter* file_writer =
    350       response_writer_ ? response_writer_->AsFileWriter() : NULL;
    351   if (!file_writer)
    352     return false;
    353 
    354   *out_response_path = file_writer->file_path();
    355 
    356   if (take_ownership) {
    357     // Intentionally calling a file_writer_ method directly without posting
    358     // the task to network_task_runner_.
    359     //
    360     // This is for correctly handling the case when file_writer_->DisownFile()
    361     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
    362     // DisownFile takes effect before Stop deletes file_writer_.
    363     //
    364     // This direct call should be thread-safe, since DisownFile itself does no
    365     // file operation. It just flips the state to be referred in destruction.
    366     file_writer->DisownFile();
    367   }
    368   return true;
    369 }
    370 
    371 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
    372                                         const GURL& new_url,
    373                                         bool* defer_redirect) {
    374   DCHECK_EQ(request, request_.get());
    375   DCHECK(network_task_runner_->BelongsToCurrentThread());
    376   if (stop_on_redirect_) {
    377     stopped_on_redirect_ = true;
    378     url_ = new_url;
    379     response_code_ = request_->GetResponseCode();
    380     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
    381     request->Cancel();
    382     OnReadCompleted(request, 0);
    383   }
    384 }
    385 
    386 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
    387   DCHECK_EQ(request, request_.get());
    388   DCHECK(network_task_runner_->BelongsToCurrentThread());
    389   if (request_->status().is_success()) {
    390     response_code_ = request_->GetResponseCode();
    391     response_headers_ = request_->response_headers();
    392     socket_address_ = request_->GetSocketAddress();
    393     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
    394     total_response_bytes_ = request_->GetExpectedContentSize();
    395   }
    396 
    397   ReadResponse();
    398 }
    399 
    400 void URLFetcherCore::OnCertificateRequested(
    401     URLRequest* request,
    402     SSLCertRequestInfo* cert_request_info) {
    403   DCHECK_EQ(request, request_.get());
    404   DCHECK(network_task_runner_->BelongsToCurrentThread());
    405 
    406   if (g_ignore_certificate_requests) {
    407     request->ContinueWithCertificate(NULL);
    408   } else {
    409     request->Cancel();
    410   }
    411 }
    412 
    413 void URLFetcherCore::OnReadCompleted(URLRequest* request,
    414                                      int bytes_read) {
    415   DCHECK(request == request_);
    416   DCHECK(network_task_runner_->BelongsToCurrentThread());
    417 
    418   if (!stopped_on_redirect_)
    419     url_ = request->url();
    420   URLRequestThrottlerManager* throttler_manager =
    421       request->context()->throttler_manager();
    422   if (throttler_manager) {
    423     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
    424   }
    425 
    426   do {
    427     if (!request_->status().is_success() || bytes_read <= 0)
    428       break;
    429 
    430     current_response_bytes_ += bytes_read;
    431     InformDelegateDownloadProgress();
    432 
    433     const int result =
    434         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
    435     if (result < 0) {
    436       // Write failed or waiting for write completion.
    437       return;
    438     }
    439   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
    440 
    441   const URLRequestStatus status = request_->status();
    442 
    443   if (status.is_success())
    444     request_->GetResponseCookies(&cookies_);
    445 
    446   // See comments re: HEAD requests in ReadResponse().
    447   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
    448     status_ = status;
    449     ReleaseRequest();
    450 
    451     // No more data to write.
    452     const int result = response_writer_->Finish(
    453         base::Bind(&URLFetcherCore::DidFinishWriting, this));
    454     if (result != ERR_IO_PENDING)
    455       DidFinishWriting(result);
    456   }
    457 }
    458 
    459 void URLFetcherCore::CancelAll() {
    460   g_registry.Get().CancelAll();
    461 }
    462 
    463 int URLFetcherCore::GetNumFetcherCores() {
    464   return g_registry.Get().size();
    465 }
    466 
    467 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
    468   g_interception_enabled = enabled;
    469 }
    470 
    471 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
    472   g_ignore_certificate_requests = ignored;
    473 }
    474 
    475 URLFetcherCore::~URLFetcherCore() {
    476   // |request_| should be NULL.  If not, it's unsafe to delete it here since we
    477   // may not be on the IO thread.
    478   DCHECK(!request_.get());
    479 }
    480 
    481 void URLFetcherCore::StartOnIOThread() {
    482   DCHECK(network_task_runner_->BelongsToCurrentThread());
    483 
    484   if (!response_writer_)
    485     response_writer_.reset(new URLFetcherStringWriter);
    486 
    487   const int result = response_writer_->Initialize(
    488       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
    489   if (result != ERR_IO_PENDING)
    490     DidInitializeWriter(result);
    491 }
    492 
    493 void URLFetcherCore::StartURLRequest() {
    494   DCHECK(network_task_runner_->BelongsToCurrentThread());
    495 
    496   if (was_cancelled_) {
    497     // Since StartURLRequest() is posted as a *delayed* task, it may
    498     // run after the URLFetcher was already stopped.
    499     return;
    500   }
    501 
    502   DCHECK(request_context_getter_.get());
    503   DCHECK(!request_.get());
    504 
    505   g_registry.Get().AddURLFetcherCore(this);
    506   current_response_bytes_ = 0;
    507   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
    508       original_url_, DEFAULT_PRIORITY, this);
    509   request_->set_stack_trace(stack_trace_);
    510   int flags = request_->load_flags() | load_flags_;
    511   if (!g_interception_enabled)
    512     flags = flags | LOAD_DISABLE_INTERCEPT;
    513 
    514   if (is_chunked_upload_)
    515     request_->EnableChunkedUpload();
    516   request_->SetLoadFlags(flags);
    517   request_->SetReferrer(referrer_);
    518   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
    519       original_url_ : first_party_for_cookies_);
    520   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
    521     request_->SetUserData(url_request_data_key_,
    522                           url_request_create_data_callback_.Run());
    523   }
    524 
    525   switch (request_type_) {
    526     case URLFetcher::GET:
    527       break;
    528 
    529     case URLFetcher::POST:
    530     case URLFetcher::PUT:
    531     case URLFetcher::PATCH:
    532       // Upload content must be set.
    533       DCHECK(is_chunked_upload_ || upload_content_set_);
    534 
    535       request_->set_method(
    536           request_type_ == URLFetcher::POST ? "POST" :
    537           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
    538       if (!upload_content_type_.empty()) {
    539         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
    540                                          upload_content_type_);
    541       }
    542       if (!upload_content_.empty()) {
    543         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
    544             upload_content_.data(), upload_content_.size()));
    545         request_->set_upload(make_scoped_ptr(
    546             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
    547       } else if (!upload_file_path_.empty()) {
    548         scoped_ptr<UploadElementReader> reader(
    549             new UploadFileElementReader(upload_file_task_runner_.get(),
    550                                         upload_file_path_,
    551                                         upload_range_offset_,
    552                                         upload_range_length_,
    553                                         base::Time()));
    554         request_->set_upload(make_scoped_ptr(
    555             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
    556       }
    557 
    558       current_upload_bytes_ = -1;
    559       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
    560       //  layer and avoid using timer here.
    561       upload_progress_checker_timer_.reset(
    562           new base::RepeatingTimer<URLFetcherCore>());
    563       upload_progress_checker_timer_->Start(
    564           FROM_HERE,
    565           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
    566           this,
    567           &URLFetcherCore::InformDelegateUploadProgress);
    568       break;
    569 
    570     case URLFetcher::HEAD:
    571       request_->set_method("HEAD");
    572       break;
    573 
    574     case URLFetcher::DELETE_REQUEST:
    575       request_->set_method("DELETE");
    576       break;
    577 
    578     default:
    579       NOTREACHED();
    580   }
    581 
    582   if (!extra_request_headers_.IsEmpty())
    583     request_->SetExtraRequestHeaders(extra_request_headers_);
    584 
    585   request_->Start();
    586 }
    587 
    588 void URLFetcherCore::DidInitializeWriter(int result) {
    589   if (result != OK) {
    590     CancelURLRequest(result);
    591     delegate_task_runner_->PostTask(
    592         FROM_HERE,
    593         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    594     return;
    595   }
    596   StartURLRequestWhenAppropriate();
    597 }
    598 
    599 void URLFetcherCore::StartURLRequestWhenAppropriate() {
    600   DCHECK(network_task_runner_->BelongsToCurrentThread());
    601 
    602   if (was_cancelled_)
    603     return;
    604 
    605   DCHECK(request_context_getter_.get());
    606 
    607   int64 delay = 0LL;
    608   if (original_url_throttler_entry_.get() == NULL) {
    609     URLRequestThrottlerManager* manager =
    610         request_context_getter_->GetURLRequestContext()->throttler_manager();
    611     if (manager) {
    612       original_url_throttler_entry_ =
    613           manager->RegisterRequestUrl(original_url_);
    614     }
    615   }
    616   if (original_url_throttler_entry_.get() != NULL) {
    617     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
    618         GetBackoffReleaseTime());
    619   }
    620 
    621   if (delay == 0) {
    622     StartURLRequest();
    623   } else {
    624     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
    625         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
    626         base::TimeDelta::FromMilliseconds(delay));
    627   }
    628 }
    629 
    630 void URLFetcherCore::CancelURLRequest(int error) {
    631   DCHECK(network_task_runner_->BelongsToCurrentThread());
    632 
    633   if (request_.get()) {
    634     request_->CancelWithError(error);
    635     ReleaseRequest();
    636   }
    637 
    638   // Set the error manually.
    639   // Normally, calling URLRequest::CancelWithError() results in calling
    640   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
    641   // URLRequestJob::NotifyDone(). But, because the request was released
    642   // immediately after being canceled, the request could not call
    643   // OnReadCompleted() which overwrites |status_| with the error status.
    644   status_.set_status(URLRequestStatus::CANCELED);
    645   status_.set_error(error);
    646 
    647   // Release the reference to the request context. There could be multiple
    648   // references to URLFetcher::Core at this point so it may take a while to
    649   // delete the object, but we cannot delay the destruction of the request
    650   // context.
    651   request_context_getter_ = NULL;
    652   first_party_for_cookies_ = GURL();
    653   url_request_data_key_ = NULL;
    654   url_request_create_data_callback_.Reset();
    655   was_cancelled_ = true;
    656 }
    657 
    658 void URLFetcherCore::OnCompletedURLRequest(
    659     base::TimeDelta backoff_delay) {
    660   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    661 
    662   // Save the status and backoff_delay so that delegates can read it.
    663   if (delegate_) {
    664     backoff_delay_ = backoff_delay;
    665     InformDelegateFetchIsComplete();
    666   }
    667 }
    668 
    669 void URLFetcherCore::InformDelegateFetchIsComplete() {
    670   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    671   if (delegate_)
    672     delegate_->OnURLFetchComplete(fetcher_);
    673 }
    674 
    675 void URLFetcherCore::NotifyMalformedContent() {
    676   DCHECK(network_task_runner_->BelongsToCurrentThread());
    677   if (url_throttler_entry_.get() != NULL) {
    678     int status_code = response_code_;
    679     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
    680       // The status code will generally be known by the time clients
    681       // call the |ReceivedContentWasMalformed()| function (which ends up
    682       // calling the current function) but if it's not, we need to assume
    683       // the response was successful so that the total failure count
    684       // used to calculate exponential back-off goes up.
    685       status_code = 200;
    686     }
    687     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
    688   }
    689 }
    690 
    691 void URLFetcherCore::DidFinishWriting(int result) {
    692   if (result != OK) {
    693     CancelURLRequest(result);
    694     delegate_task_runner_->PostTask(
    695         FROM_HERE,
    696         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    697     return;
    698   }
    699   // If the file was successfully closed, then the URL request is complete.
    700   RetryOrCompleteUrlFetch();
    701 }
    702 
    703 void URLFetcherCore::RetryOrCompleteUrlFetch() {
    704   DCHECK(network_task_runner_->BelongsToCurrentThread());
    705   base::TimeDelta backoff_delay;
    706 
    707   // Checks the response from server.
    708   if (response_code_ >= 500 ||
    709       status_.error() == ERR_TEMPORARILY_THROTTLED) {
    710     // When encountering a server error, we will send the request again
    711     // after backoff time.
    712     ++num_retries_on_5xx_;
    713 
    714     // Note that backoff_delay may be 0 because (a) the
    715     // URLRequestThrottlerManager and related code does not
    716     // necessarily back off on the first error, (b) it only backs off
    717     // on some of the 5xx status codes, (c) not all URLRequestContexts
    718     // have a throttler manager.
    719     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
    720     backoff_delay = backoff_release_time - base::TimeTicks::Now();
    721     if (backoff_delay < base::TimeDelta())
    722       backoff_delay = base::TimeDelta();
    723 
    724     if (automatically_retry_on_5xx_ &&
    725         num_retries_on_5xx_ <= max_retries_on_5xx_) {
    726       StartOnIOThread();
    727       return;
    728     }
    729   } else {
    730     backoff_delay = base::TimeDelta();
    731   }
    732 
    733   // Retry if the request failed due to network changes.
    734   if (status_.error() == ERR_NETWORK_CHANGED &&
    735       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
    736     ++num_retries_on_network_changes_;
    737 
    738     // Retry soon, after flushing all the current tasks which may include
    739     // further network change observers.
    740     network_task_runner_->PostTask(
    741         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
    742     return;
    743   }
    744 
    745   request_context_getter_ = NULL;
    746   first_party_for_cookies_ = GURL();
    747   url_request_data_key_ = NULL;
    748   url_request_create_data_callback_.Reset();
    749   bool posted = delegate_task_runner_->PostTask(
    750       FROM_HERE,
    751       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
    752 
    753   // If the delegate message loop does not exist any more, then the delegate
    754   // should be gone too.
    755   DCHECK(posted || !delegate_);
    756 }
    757 
    758 void URLFetcherCore::ReleaseRequest() {
    759   upload_progress_checker_timer_.reset();
    760   request_.reset();
    761   g_registry.Get().RemoveURLFetcherCore(this);
    762 }
    763 
    764 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
    765   DCHECK(network_task_runner_->BelongsToCurrentThread());
    766 
    767   if (original_url_throttler_entry_.get()) {
    768     base::TimeTicks original_url_backoff =
    769         original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
    770     base::TimeTicks destination_url_backoff;
    771     if (url_throttler_entry_.get() != NULL &&
    772         original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
    773       destination_url_backoff =
    774           url_throttler_entry_->GetExponentialBackoffReleaseTime();
    775     }
    776 
    777     return original_url_backoff > destination_url_backoff ?
    778         original_url_backoff : destination_url_backoff;
    779   } else {
    780     return base::TimeTicks();
    781   }
    782 }
    783 
    784 void URLFetcherCore::CompleteAddingUploadDataChunk(
    785     const std::string& content, bool is_last_chunk) {
    786   if (was_cancelled_) {
    787     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
    788     // may run after the URLFetcher was already stopped.
    789     return;
    790   }
    791   DCHECK(is_chunked_upload_);
    792   DCHECK(request_.get());
    793   DCHECK(!content.empty());
    794   request_->AppendChunkToUpload(content.data(),
    795                                 static_cast<int>(content.length()),
    796                                 is_last_chunk);
    797 }
    798 
    799 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
    800   while (data->BytesRemaining() > 0) {
    801     const int result = response_writer_->Write(
    802         data.get(),
    803         data->BytesRemaining(),
    804         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
    805     if (result < 0) {
    806       if (result != ERR_IO_PENDING)
    807         DidWriteBuffer(data, result);
    808       return result;
    809     }
    810     data->DidConsume(result);
    811   }
    812   return OK;
    813 }
    814 
    815 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
    816                                     int result) {
    817   if (result < 0) {  // Handle errors.
    818     CancelURLRequest(result);
    819     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
    820     delegate_task_runner_->PostTask(
    821         FROM_HERE,
    822         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    823     return;
    824   }
    825 
    826   // Continue writing.
    827   data->DidConsume(result);
    828   if (WriteBuffer(data) < 0)
    829     return;
    830 
    831   // Finished writing buffer_. Read some more, unless the request has been
    832   // cancelled and deleted.
    833   DCHECK_EQ(0, data->BytesRemaining());
    834   if (request_.get())
    835     ReadResponse();
    836 }
    837 
    838 void URLFetcherCore::ReadResponse() {
    839   // Some servers may treat HEAD requests as GET requests.  To free up the
    840   // network connection as soon as possible, signal that the request has
    841   // completed immediately, without trying to read any data back (all we care
    842   // about is the response code and headers, which we already have).
    843   int bytes_read = 0;
    844   if (request_->status().is_success() &&
    845       (request_type_ != URLFetcher::HEAD))
    846     request_->Read(buffer_.get(), kBufferSize, &bytes_read);
    847   OnReadCompleted(request_.get(), bytes_read);
    848 }
    849 
    850 void URLFetcherCore::InformDelegateUploadProgress() {
    851   DCHECK(network_task_runner_->BelongsToCurrentThread());
    852   if (request_.get()) {
    853     int64 current = request_->GetUploadProgress().position();
    854     if (current_upload_bytes_ != current) {
    855       current_upload_bytes_ = current;
    856       int64 total = -1;
    857       if (!is_chunked_upload_) {
    858         total = static_cast<int64>(request_->GetUploadProgress().size());
    859         // Total may be zero if the UploadDataStream::Init has not been called
    860         // yet.  Don't send the upload progress until the size is initialized.
    861         if (!total)
    862           return;
    863       }
    864       delegate_task_runner_->PostTask(
    865           FROM_HERE,
    866           base::Bind(
    867               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
    868               this, current, total));
    869     }
    870   }
    871 }
    872 
    873 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
    874     int64 current, int64 total) {
    875   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    876   if (delegate_)
    877     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
    878 }
    879 
    880 void URLFetcherCore::InformDelegateDownloadProgress() {
    881   DCHECK(network_task_runner_->BelongsToCurrentThread());
    882   delegate_task_runner_->PostTask(
    883       FROM_HERE,
    884       base::Bind(
    885           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
    886           this, current_response_bytes_, total_response_bytes_));
    887 }
    888 
    889 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
    890     int64 current, int64 total) {
    891   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    892   if (delegate_)
    893     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
    894 }
    895 
    896 }  // namespace net
    897