Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/url_request/url_fetcher_core.h"
      6 
      7 #include <stdint.h>
      8 
      9 #include "base/bind.h"
     10 #include "base/logging.h"
     11 #include "base/metrics/histogram.h"
     12 #include "base/sequenced_task_runner.h"
     13 #include "base/single_thread_task_runner.h"
     14 #include "base/stl_util.h"
     15 #include "base/thread_task_runner_handle.h"
     16 #include "base/tracked_objects.h"
     17 #include "net/base/io_buffer.h"
     18 #include "net/base/load_flags.h"
     19 #include "net/base/net_errors.h"
     20 #include "net/base/request_priority.h"
     21 #include "net/base/upload_bytes_element_reader.h"
     22 #include "net/base/upload_data_stream.h"
     23 #include "net/base/upload_file_element_reader.h"
     24 #include "net/http/http_response_headers.h"
     25 #include "net/url_request/redirect_info.h"
     26 #include "net/url_request/url_fetcher_delegate.h"
     27 #include "net/url_request/url_fetcher_response_writer.h"
     28 #include "net/url_request/url_request_context.h"
     29 #include "net/url_request/url_request_context_getter.h"
     30 #include "net/url_request/url_request_throttler_manager.h"
     31 
     32 namespace {
     33 
     34 const int kBufferSize = 4096;
     35 const int kUploadProgressTimerInterval = 100;
     36 bool g_interception_enabled = false;
     37 bool g_ignore_certificate_requests = false;
     38 
     39 void EmptyCompletionCallback(int result) {}
     40 
     41 }  // namespace
     42 
     43 namespace net {
     44 
     45 // URLFetcherCore::Registry ---------------------------------------------------
     46 
     47 URLFetcherCore::Registry::Registry() {}
     48 URLFetcherCore::Registry::~Registry() {}
     49 
     50 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
     51   DCHECK(!ContainsKey(fetchers_, core));
     52   fetchers_.insert(core);
     53 }
     54 
     55 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
     56   DCHECK(ContainsKey(fetchers_, core));
     57   fetchers_.erase(core);
     58 }
     59 
     60 void URLFetcherCore::Registry::CancelAll() {
     61   while (!fetchers_.empty())
     62     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
     63 }
     64 
     65 // URLFetcherCore -------------------------------------------------------------
     66 
     67 // static
     68 base::LazyInstance<URLFetcherCore::Registry>
     69     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
     70 
     71 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
     72                                const GURL& original_url,
     73                                URLFetcher::RequestType request_type,
     74                                URLFetcherDelegate* d)
     75     : fetcher_(fetcher),
     76       original_url_(original_url),
     77       request_type_(request_type),
     78       delegate_(d),
     79       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
     80       load_flags_(LOAD_NORMAL),
     81       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
     82       buffer_(new IOBuffer(kBufferSize)),
     83       url_request_data_key_(NULL),
     84       was_fetched_via_proxy_(false),
     85       upload_content_set_(false),
     86       upload_range_offset_(0),
     87       upload_range_length_(0),
     88       referrer_policy_(
     89           URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
     90       is_chunked_upload_(false),
     91       was_cancelled_(false),
     92       stop_on_redirect_(false),
     93       stopped_on_redirect_(false),
     94       automatically_retry_on_5xx_(true),
     95       num_retries_on_5xx_(0),
     96       max_retries_on_5xx_(0),
     97       num_retries_on_network_changes_(0),
     98       max_retries_on_network_changes_(0),
     99       current_upload_bytes_(-1),
    100       current_response_bytes_(0),
    101       total_response_bytes_(-1) {
    102   CHECK(original_url_.is_valid());
    103 }
    104 
    105 void URLFetcherCore::Start() {
    106   DCHECK(delegate_task_runner_.get());
    107   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
    108   if (network_task_runner_.get()) {
    109     DCHECK_EQ(network_task_runner_,
    110               request_context_getter_->GetNetworkTaskRunner());
    111   } else {
    112     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
    113   }
    114   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
    115 
    116   network_task_runner_->PostTask(
    117       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
    118 }
    119 
    120 void URLFetcherCore::Stop() {
    121   if (delegate_task_runner_.get())  // May be NULL in tests.
    122     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    123 
    124   delegate_ = NULL;
    125   fetcher_ = NULL;
    126   if (!network_task_runner_.get())
    127     return;
    128   if (network_task_runner_->RunsTasksOnCurrentThread()) {
    129     CancelURLRequest(ERR_ABORTED);
    130   } else {
    131     network_task_runner_->PostTask(
    132         FROM_HERE,
    133         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
    134   }
    135 }
    136 
    137 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
    138                                    const std::string& upload_content) {
    139   DCHECK(!is_chunked_upload_);
    140   DCHECK(!upload_content_set_);
    141   DCHECK(upload_content_.empty());
    142   DCHECK(upload_file_path_.empty());
    143   DCHECK(upload_content_type_.empty());
    144 
    145   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
    146   DCHECK(upload_content.empty() || !upload_content_type.empty());
    147 
    148   upload_content_type_ = upload_content_type;
    149   upload_content_ = upload_content;
    150   upload_content_set_ = true;
    151 }
    152 
    153 void URLFetcherCore::SetUploadFilePath(
    154     const std::string& upload_content_type,
    155     const base::FilePath& file_path,
    156     uint64 range_offset,
    157     uint64 range_length,
    158     scoped_refptr<base::TaskRunner> file_task_runner) {
    159   DCHECK(!is_chunked_upload_);
    160   DCHECK(!upload_content_set_);
    161   DCHECK(upload_content_.empty());
    162   DCHECK(upload_file_path_.empty());
    163   DCHECK_EQ(upload_range_offset_, 0ULL);
    164   DCHECK_EQ(upload_range_length_, 0ULL);
    165   DCHECK(upload_content_type_.empty());
    166   DCHECK(!upload_content_type.empty());
    167 
    168   upload_content_type_ = upload_content_type;
    169   upload_file_path_ = file_path;
    170   upload_range_offset_ = range_offset;
    171   upload_range_length_ = range_length;
    172   upload_file_task_runner_ = file_task_runner;
    173   upload_content_set_ = true;
    174 }
    175 
    176 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
    177   DCHECK(is_chunked_upload_ ||
    178          (upload_content_type_.empty() &&
    179           upload_content_.empty()));
    180 
    181   // Empty |content_type| is not allowed here, because it is impossible
    182   // to ensure non-empty upload content as it is not yet supplied.
    183   DCHECK(!content_type.empty());
    184 
    185   upload_content_type_ = content_type;
    186   upload_content_.clear();
    187   is_chunked_upload_ = true;
    188 }
    189 
    190 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
    191                                          bool is_last_chunk) {
    192   DCHECK(delegate_task_runner_.get());
    193   DCHECK(network_task_runner_.get());
    194   network_task_runner_->PostTask(
    195       FROM_HERE,
    196       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
    197                  is_last_chunk));
    198 }
    199 
    200 void URLFetcherCore::SetLoadFlags(int load_flags) {
    201   load_flags_ = load_flags;
    202 }
    203 
    204 int URLFetcherCore::GetLoadFlags() const {
    205   return load_flags_;
    206 }
    207 
    208 void URLFetcherCore::SetReferrer(const std::string& referrer) {
    209   referrer_ = referrer;
    210 }
    211 
    212 void URLFetcherCore::SetReferrerPolicy(
    213     URLRequest::ReferrerPolicy referrer_policy) {
    214   referrer_policy_ = referrer_policy;
    215 }
    216 
    217 void URLFetcherCore::SetExtraRequestHeaders(
    218     const std::string& extra_request_headers) {
    219   extra_request_headers_.Clear();
    220   extra_request_headers_.AddHeadersFromString(extra_request_headers);
    221 }
    222 
    223 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
    224   extra_request_headers_.AddHeaderFromString(header_line);
    225 }
    226 
    227 void URLFetcherCore::SetRequestContext(
    228     URLRequestContextGetter* request_context_getter) {
    229   DCHECK(!request_context_getter_.get());
    230   DCHECK(request_context_getter);
    231   request_context_getter_ = request_context_getter;
    232 }
    233 
    234 void URLFetcherCore::SetFirstPartyForCookies(
    235     const GURL& first_party_for_cookies) {
    236   DCHECK(first_party_for_cookies_.is_empty());
    237   first_party_for_cookies_ = first_party_for_cookies;
    238 }
    239 
    240 void URLFetcherCore::SetURLRequestUserData(
    241     const void* key,
    242     const URLFetcher::CreateDataCallback& create_data_callback) {
    243   DCHECK(key);
    244   DCHECK(!create_data_callback.is_null());
    245   url_request_data_key_ = key;
    246   url_request_create_data_callback_ = create_data_callback;
    247 }
    248 
    249 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
    250   stop_on_redirect_ = stop_on_redirect;
    251 }
    252 
    253 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
    254   automatically_retry_on_5xx_ = retry;
    255 }
    256 
    257 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
    258   max_retries_on_5xx_ = max_retries;
    259 }
    260 
    261 int URLFetcherCore::GetMaxRetriesOn5xx() const {
    262   return max_retries_on_5xx_;
    263 }
    264 
    265 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
    266   return backoff_delay_;
    267 }
    268 
    269 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
    270   max_retries_on_network_changes_ = max_retries;
    271 }
    272 
    273 void URLFetcherCore::SaveResponseToFileAtPath(
    274     const base::FilePath& file_path,
    275     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
    276   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    277   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
    278       new URLFetcherFileWriter(file_task_runner, file_path)));
    279 }
    280 
    281 void URLFetcherCore::SaveResponseToTemporaryFile(
    282     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
    283   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    284   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
    285       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
    286 }
    287 
    288 void URLFetcherCore::SaveResponseWithWriter(
    289     scoped_ptr<URLFetcherResponseWriter> response_writer) {
    290   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    291   response_writer_ = response_writer.Pass();
    292 }
    293 
    294 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
    295   return response_headers_.get();
    296 }
    297 
    298 // TODO(panayiotis): socket_address_ is written in the IO thread,
    299 // if this is accessed in the UI thread, this could result in a race.
    300 // Same for response_headers_ above and was_fetched_via_proxy_ below.
    301 HostPortPair URLFetcherCore::GetSocketAddress() const {
    302   return socket_address_;
    303 }
    304 
    305 bool URLFetcherCore::WasFetchedViaProxy() const {
    306   return was_fetched_via_proxy_;
    307 }
    308 
    309 const GURL& URLFetcherCore::GetOriginalURL() const {
    310   return original_url_;
    311 }
    312 
    313 const GURL& URLFetcherCore::GetURL() const {
    314   return url_;
    315 }
    316 
    317 const URLRequestStatus& URLFetcherCore::GetStatus() const {
    318   return status_;
    319 }
    320 
    321 int URLFetcherCore::GetResponseCode() const {
    322   return response_code_;
    323 }
    324 
    325 const ResponseCookies& URLFetcherCore::GetCookies() const {
    326   return cookies_;
    327 }
    328 
    329 void URLFetcherCore::ReceivedContentWasMalformed() {
    330   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    331   if (network_task_runner_.get()) {
    332     network_task_runner_->PostTask(
    333         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
    334   }
    335 }
    336 
    337 bool URLFetcherCore::GetResponseAsString(
    338     std::string* out_response_string) const {
    339   URLFetcherStringWriter* string_writer =
    340       response_writer_ ? response_writer_->AsStringWriter() : NULL;
    341   if (!string_writer)
    342     return false;
    343 
    344   *out_response_string = string_writer->data();
    345   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
    346                           (string_writer->data().length() / 1024));
    347   return true;
    348 }
    349 
    350 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
    351                                            base::FilePath* out_response_path) {
    352   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    353 
    354   URLFetcherFileWriter* file_writer =
    355       response_writer_ ? response_writer_->AsFileWriter() : NULL;
    356   if (!file_writer)
    357     return false;
    358 
    359   *out_response_path = file_writer->file_path();
    360 
    361   if (take_ownership) {
    362     // Intentionally calling a file_writer_ method directly without posting
    363     // the task to network_task_runner_.
    364     //
    365     // This is for correctly handling the case when file_writer_->DisownFile()
    366     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
    367     // DisownFile takes effect before Stop deletes file_writer_.
    368     //
    369     // This direct call should be thread-safe, since DisownFile itself does no
    370     // file operation. It just flips the state to be referred in destruction.
    371     file_writer->DisownFile();
    372   }
    373   return true;
    374 }
    375 
    376 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
    377                                         const RedirectInfo& redirect_info,
    378                                         bool* defer_redirect) {
    379   DCHECK_EQ(request, request_.get());
    380   DCHECK(network_task_runner_->BelongsToCurrentThread());
    381   if (stop_on_redirect_) {
    382     stopped_on_redirect_ = true;
    383     url_ = redirect_info.new_url;
    384     response_code_ = request_->GetResponseCode();
    385     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
    386     request->Cancel();
    387     OnReadCompleted(request, 0);
    388   }
    389 }
    390 
    391 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
    392   DCHECK_EQ(request, request_.get());
    393   DCHECK(network_task_runner_->BelongsToCurrentThread());
    394   if (request_->status().is_success()) {
    395     response_code_ = request_->GetResponseCode();
    396     response_headers_ = request_->response_headers();
    397     socket_address_ = request_->GetSocketAddress();
    398     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
    399     total_response_bytes_ = request_->GetExpectedContentSize();
    400   }
    401 
    402   ReadResponse();
    403 }
    404 
    405 void URLFetcherCore::OnCertificateRequested(
    406     URLRequest* request,
    407     SSLCertRequestInfo* cert_request_info) {
    408   DCHECK_EQ(request, request_.get());
    409   DCHECK(network_task_runner_->BelongsToCurrentThread());
    410 
    411   if (g_ignore_certificate_requests) {
    412     request->ContinueWithCertificate(NULL);
    413   } else {
    414     request->Cancel();
    415   }
    416 }
    417 
    418 void URLFetcherCore::OnReadCompleted(URLRequest* request,
    419                                      int bytes_read) {
    420   DCHECK(request == request_);
    421   DCHECK(network_task_runner_->BelongsToCurrentThread());
    422 
    423   if (!stopped_on_redirect_)
    424     url_ = request->url();
    425   URLRequestThrottlerManager* throttler_manager =
    426       request->context()->throttler_manager();
    427   if (throttler_manager) {
    428     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
    429   }
    430 
    431   do {
    432     if (!request_->status().is_success() || bytes_read <= 0)
    433       break;
    434 
    435     current_response_bytes_ += bytes_read;
    436     InformDelegateDownloadProgress();
    437 
    438     const int result =
    439         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
    440     if (result < 0) {
    441       // Write failed or waiting for write completion.
    442       return;
    443     }
    444   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
    445 
    446   const URLRequestStatus status = request_->status();
    447 
    448   if (status.is_success())
    449     request_->GetResponseCookies(&cookies_);
    450 
    451   // See comments re: HEAD requests in ReadResponse().
    452   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
    453     status_ = status;
    454     ReleaseRequest();
    455 
    456     // No more data to write.
    457     const int result = response_writer_->Finish(
    458         base::Bind(&URLFetcherCore::DidFinishWriting, this));
    459     if (result != ERR_IO_PENDING)
    460       DidFinishWriting(result);
    461   }
    462 }
    463 
    464 void URLFetcherCore::CancelAll() {
    465   g_registry.Get().CancelAll();
    466 }
    467 
    468 int URLFetcherCore::GetNumFetcherCores() {
    469   return g_registry.Get().size();
    470 }
    471 
    472 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
    473   g_interception_enabled = enabled;
    474 }
    475 
    476 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
    477   g_ignore_certificate_requests = ignored;
    478 }
    479 
    480 URLFetcherCore::~URLFetcherCore() {
    481   // |request_| should be NULL. If not, it's unsafe to delete it here since we
    482   // may not be on the IO thread.
    483   DCHECK(!request_.get());
    484 }
    485 
    486 void URLFetcherCore::StartOnIOThread() {
    487   DCHECK(network_task_runner_->BelongsToCurrentThread());
    488 
    489   if (!response_writer_)
    490     response_writer_.reset(new URLFetcherStringWriter);
    491 
    492   const int result = response_writer_->Initialize(
    493       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
    494   if (result != ERR_IO_PENDING)
    495     DidInitializeWriter(result);
    496 }
    497 
    498 void URLFetcherCore::StartURLRequest() {
    499   DCHECK(network_task_runner_->BelongsToCurrentThread());
    500 
    501   if (was_cancelled_) {
    502     // Since StartURLRequest() is posted as a *delayed* task, it may
    503     // run after the URLFetcher was already stopped.
    504     return;
    505   }
    506 
    507   DCHECK(request_context_getter_.get());
    508   DCHECK(!request_.get());
    509 
    510   g_registry.Get().AddURLFetcherCore(this);
    511   current_response_bytes_ = 0;
    512   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
    513       original_url_, DEFAULT_PRIORITY, this, NULL);
    514   request_->set_stack_trace(stack_trace_);
    515   int flags = request_->load_flags() | load_flags_;
    516   if (!g_interception_enabled)
    517     flags = flags | LOAD_DISABLE_INTERCEPT;
    518 
    519   if (is_chunked_upload_)
    520     request_->EnableChunkedUpload();
    521   request_->SetLoadFlags(flags);
    522   request_->SetReferrer(referrer_);
    523   request_->set_referrer_policy(referrer_policy_);
    524   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
    525       original_url_ : first_party_for_cookies_);
    526   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
    527     request_->SetUserData(url_request_data_key_,
    528                           url_request_create_data_callback_.Run());
    529   }
    530 
    531   switch (request_type_) {
    532     case URLFetcher::GET:
    533       break;
    534 
    535     case URLFetcher::POST:
    536     case URLFetcher::PUT:
    537     case URLFetcher::PATCH:
    538       // Upload content must be set.
    539       DCHECK(is_chunked_upload_ || upload_content_set_);
    540 
    541       request_->set_method(
    542           request_type_ == URLFetcher::POST ? "POST" :
    543           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
    544       if (!upload_content_type_.empty()) {
    545         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
    546                                          upload_content_type_);
    547       }
    548       if (!upload_content_.empty()) {
    549         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
    550             upload_content_.data(), upload_content_.size()));
    551         request_->set_upload(make_scoped_ptr(
    552             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
    553       } else if (!upload_file_path_.empty()) {
    554         scoped_ptr<UploadElementReader> reader(
    555             new UploadFileElementReader(upload_file_task_runner_.get(),
    556                                         upload_file_path_,
    557                                         upload_range_offset_,
    558                                         upload_range_length_,
    559                                         base::Time()));
    560         request_->set_upload(make_scoped_ptr(
    561             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
    562       }
    563 
    564       current_upload_bytes_ = -1;
    565       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
    566       //  layer and avoid using timer here.
    567       upload_progress_checker_timer_.reset(
    568           new base::RepeatingTimer<URLFetcherCore>());
    569       upload_progress_checker_timer_->Start(
    570           FROM_HERE,
    571           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
    572           this,
    573           &URLFetcherCore::InformDelegateUploadProgress);
    574       break;
    575 
    576     case URLFetcher::HEAD:
    577       request_->set_method("HEAD");
    578       break;
    579 
    580     case URLFetcher::DELETE_REQUEST:
    581       request_->set_method("DELETE");
    582       break;
    583 
    584     default:
    585       NOTREACHED();
    586   }
    587 
    588   if (!extra_request_headers_.IsEmpty())
    589     request_->SetExtraRequestHeaders(extra_request_headers_);
    590 
    591   request_->Start();
    592 }
    593 
    594 void URLFetcherCore::DidInitializeWriter(int result) {
    595   if (result != OK) {
    596     CancelURLRequest(result);
    597     delegate_task_runner_->PostTask(
    598         FROM_HERE,
    599         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    600     return;
    601   }
    602   StartURLRequestWhenAppropriate();
    603 }
    604 
    605 void URLFetcherCore::StartURLRequestWhenAppropriate() {
    606   DCHECK(network_task_runner_->BelongsToCurrentThread());
    607 
    608   if (was_cancelled_)
    609     return;
    610 
    611   DCHECK(request_context_getter_.get());
    612 
    613   int64 delay = INT64_C(0);
    614   if (!original_url_throttler_entry_.get()) {
    615     URLRequestThrottlerManager* manager =
    616         request_context_getter_->GetURLRequestContext()->throttler_manager();
    617     if (manager) {
    618       original_url_throttler_entry_ =
    619           manager->RegisterRequestUrl(original_url_);
    620     }
    621   }
    622   if (original_url_throttler_entry_.get()) {
    623     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
    624         GetBackoffReleaseTime());
    625   }
    626 
    627   if (delay == INT64_C(0)) {
    628     StartURLRequest();
    629   } else {
    630     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
    631         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
    632         base::TimeDelta::FromMilliseconds(delay));
    633   }
    634 }
    635 
    636 void URLFetcherCore::CancelURLRequest(int error) {
    637   DCHECK(network_task_runner_->BelongsToCurrentThread());
    638 
    639   if (request_.get()) {
    640     request_->CancelWithError(error);
    641     ReleaseRequest();
    642   }
    643 
    644   // Set the error manually.
    645   // Normally, calling URLRequest::CancelWithError() results in calling
    646   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
    647   // URLRequestJob::NotifyDone(). But, because the request was released
    648   // immediately after being canceled, the request could not call
    649   // OnReadCompleted() which overwrites |status_| with the error status.
    650   status_.set_status(URLRequestStatus::CANCELED);
    651   status_.set_error(error);
    652 
    653   // Release the reference to the request context. There could be multiple
    654   // references to URLFetcher::Core at this point so it may take a while to
    655   // delete the object, but we cannot delay the destruction of the request
    656   // context.
    657   request_context_getter_ = NULL;
    658   first_party_for_cookies_ = GURL();
    659   url_request_data_key_ = NULL;
    660   url_request_create_data_callback_.Reset();
    661   was_cancelled_ = true;
    662 }
    663 
    664 void URLFetcherCore::OnCompletedURLRequest(
    665     base::TimeDelta backoff_delay) {
    666   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    667 
    668   // Save the status and backoff_delay so that delegates can read it.
    669   if (delegate_) {
    670     backoff_delay_ = backoff_delay;
    671     InformDelegateFetchIsComplete();
    672   }
    673 }
    674 
    675 void URLFetcherCore::InformDelegateFetchIsComplete() {
    676   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    677   if (delegate_)
    678     delegate_->OnURLFetchComplete(fetcher_);
    679 }
    680 
    681 void URLFetcherCore::NotifyMalformedContent() {
    682   DCHECK(network_task_runner_->BelongsToCurrentThread());
    683   if (url_throttler_entry_.get()) {
    684     int status_code = response_code_;
    685     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
    686       // The status code will generally be known by the time clients
    687       // call the |ReceivedContentWasMalformed()| function (which ends up
    688       // calling the current function) but if it's not, we need to assume
    689       // the response was successful so that the total failure count
    690       // used to calculate exponential back-off goes up.
    691       status_code = 200;
    692     }
    693     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
    694   }
    695 }
    696 
    697 void URLFetcherCore::DidFinishWriting(int result) {
    698   if (result != OK) {
    699     CancelURLRequest(result);
    700     delegate_task_runner_->PostTask(
    701         FROM_HERE,
    702         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    703     return;
    704   }
    705   // If the file was successfully closed, then the URL request is complete.
    706   RetryOrCompleteUrlFetch();
    707 }
    708 
    709 void URLFetcherCore::RetryOrCompleteUrlFetch() {
    710   DCHECK(network_task_runner_->BelongsToCurrentThread());
    711   base::TimeDelta backoff_delay;
    712 
    713   // Checks the response from server.
    714   if (response_code_ >= 500 ||
    715       status_.error() == ERR_TEMPORARILY_THROTTLED) {
    716     // When encountering a server error, we will send the request again
    717     // after backoff time.
    718     ++num_retries_on_5xx_;
    719 
    720     // Note that backoff_delay may be 0 because (a) the
    721     // URLRequestThrottlerManager and related code does not
    722     // necessarily back off on the first error, (b) it only backs off
    723     // on some of the 5xx status codes, (c) not all URLRequestContexts
    724     // have a throttler manager.
    725     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
    726     backoff_delay = backoff_release_time - base::TimeTicks::Now();
    727     if (backoff_delay < base::TimeDelta())
    728       backoff_delay = base::TimeDelta();
    729 
    730     if (automatically_retry_on_5xx_ &&
    731         num_retries_on_5xx_ <= max_retries_on_5xx_) {
    732       StartOnIOThread();
    733       return;
    734     }
    735   } else {
    736     backoff_delay = base::TimeDelta();
    737   }
    738 
    739   // Retry if the request failed due to network changes.
    740   if (status_.error() == ERR_NETWORK_CHANGED &&
    741       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
    742     ++num_retries_on_network_changes_;
    743 
    744     // Retry soon, after flushing all the current tasks which may include
    745     // further network change observers.
    746     network_task_runner_->PostTask(
    747         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
    748     return;
    749   }
    750 
    751   request_context_getter_ = NULL;
    752   first_party_for_cookies_ = GURL();
    753   url_request_data_key_ = NULL;
    754   url_request_create_data_callback_.Reset();
    755   bool posted = delegate_task_runner_->PostTask(
    756       FROM_HERE,
    757       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
    758 
    759   // If the delegate message loop does not exist any more, then the delegate
    760   // should be gone too.
    761   DCHECK(posted || !delegate_);
    762 }
    763 
    764 void URLFetcherCore::ReleaseRequest() {
    765   upload_progress_checker_timer_.reset();
    766   request_.reset();
    767   g_registry.Get().RemoveURLFetcherCore(this);
    768 }
    769 
    770 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
    771   DCHECK(network_task_runner_->BelongsToCurrentThread());
    772 
    773   if (!original_url_throttler_entry_.get())
    774     return base::TimeTicks();
    775 
    776   base::TimeTicks original_url_backoff =
    777       original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
    778   base::TimeTicks destination_url_backoff;
    779   if (url_throttler_entry_.get() &&
    780       original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
    781     destination_url_backoff =
    782         url_throttler_entry_->GetExponentialBackoffReleaseTime();
    783   }
    784 
    785   return original_url_backoff > destination_url_backoff ?
    786       original_url_backoff : destination_url_backoff;
    787 }
    788 
    789 void URLFetcherCore::CompleteAddingUploadDataChunk(
    790     const std::string& content, bool is_last_chunk) {
    791   if (was_cancelled_) {
    792     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
    793     // may run after the URLFetcher was already stopped.
    794     return;
    795   }
    796   DCHECK(is_chunked_upload_);
    797   DCHECK(request_.get());
    798   DCHECK(!content.empty());
    799   request_->AppendChunkToUpload(content.data(),
    800                                 static_cast<int>(content.length()),
    801                                 is_last_chunk);
    802 }
    803 
    804 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
    805   while (data->BytesRemaining() > 0) {
    806     const int result = response_writer_->Write(
    807         data.get(),
    808         data->BytesRemaining(),
    809         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
    810     if (result < 0) {
    811       if (result != ERR_IO_PENDING)
    812         DidWriteBuffer(data, result);
    813       return result;
    814     }
    815     data->DidConsume(result);
    816   }
    817   return OK;
    818 }
    819 
    820 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
    821                                     int result) {
    822   if (result < 0) {  // Handle errors.
    823     CancelURLRequest(result);
    824     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
    825     delegate_task_runner_->PostTask(
    826         FROM_HERE,
    827         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    828     return;
    829   }
    830 
    831   // Continue writing.
    832   data->DidConsume(result);
    833   if (WriteBuffer(data) < 0)
    834     return;
    835 
    836   // Finished writing buffer_. Read some more, unless the request has been
    837   // cancelled and deleted.
    838   DCHECK_EQ(0, data->BytesRemaining());
    839   if (request_.get())
    840     ReadResponse();
    841 }
    842 
    843 void URLFetcherCore::ReadResponse() {
    844   // Some servers may treat HEAD requests as GET requests. To free up the
    845   // network connection as soon as possible, signal that the request has
    846   // completed immediately, without trying to read any data back (all we care
    847   // about is the response code and headers, which we already have).
    848   int bytes_read = 0;
    849   if (request_->status().is_success() &&
    850       (request_type_ != URLFetcher::HEAD)) {
    851     if (!request_->Read(buffer_.get(), kBufferSize, &bytes_read))
    852       bytes_read = -1;  // Match OnReadCompleted() interface contract.
    853   }
    854   OnReadCompleted(request_.get(), bytes_read);
    855 }
    856 
    857 void URLFetcherCore::InformDelegateUploadProgress() {
    858   DCHECK(network_task_runner_->BelongsToCurrentThread());
    859   if (request_.get()) {
    860     int64 current = request_->GetUploadProgress().position();
    861     if (current_upload_bytes_ != current) {
    862       current_upload_bytes_ = current;
    863       int64 total = -1;
    864       if (!is_chunked_upload_) {
    865         total = static_cast<int64>(request_->GetUploadProgress().size());
    866         // Total may be zero if the UploadDataStream::Init has not been called
    867         // yet. Don't send the upload progress until the size is initialized.
    868         if (!total)
    869           return;
    870       }
    871       delegate_task_runner_->PostTask(
    872           FROM_HERE,
    873           base::Bind(
    874               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
    875               this, current, total));
    876     }
    877   }
    878 }
    879 
    880 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
    881     int64 current, int64 total) {
    882   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    883   if (delegate_)
    884     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
    885 }
    886 
    887 void URLFetcherCore::InformDelegateDownloadProgress() {
    888   DCHECK(network_task_runner_->BelongsToCurrentThread());
    889   delegate_task_runner_->PostTask(
    890       FROM_HERE,
    891       base::Bind(
    892           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
    893           this, current_response_bytes_, total_response_bytes_));
    894 }
    895 
    896 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
    897     int64 current, int64 total) {
    898   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    899   if (delegate_)
    900     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
    901 }
    902 
    903 }  // namespace net
    904