Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/url_request/url_fetcher_core.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/logging.h"
      9 #include "base/metrics/histogram.h"
     10 #include "base/sequenced_task_runner.h"
     11 #include "base/single_thread_task_runner.h"
     12 #include "base/stl_util.h"
     13 #include "base/thread_task_runner_handle.h"
     14 #include "base/tracked_objects.h"
     15 #include "net/base/io_buffer.h"
     16 #include "net/base/load_flags.h"
     17 #include "net/base/net_errors.h"
     18 #include "net/base/request_priority.h"
     19 #include "net/base/upload_bytes_element_reader.h"
     20 #include "net/base/upload_data_stream.h"
     21 #include "net/base/upload_file_element_reader.h"
     22 #include "net/http/http_response_headers.h"
     23 #include "net/url_request/url_fetcher_delegate.h"
     24 #include "net/url_request/url_fetcher_response_writer.h"
     25 #include "net/url_request/url_request_context.h"
     26 #include "net/url_request/url_request_context_getter.h"
     27 #include "net/url_request/url_request_throttler_manager.h"
     28 
     29 namespace {
     30 
     31 const int kBufferSize = 4096;
     32 const int kUploadProgressTimerInterval = 100;
     33 bool g_interception_enabled = false;
     34 bool g_ignore_certificate_requests = false;
     35 
     36 void EmptyCompletionCallback(int result) {}
     37 
     38 }  // namespace
     39 
     40 namespace net {
     41 
     42 // URLFetcherCore::Registry ---------------------------------------------------
     43 
     44 URLFetcherCore::Registry::Registry() {}
     45 URLFetcherCore::Registry::~Registry() {}
     46 
     47 void URLFetcherCore::Registry::AddURLFetcherCore(URLFetcherCore* core) {
     48   DCHECK(!ContainsKey(fetchers_, core));
     49   fetchers_.insert(core);
     50 }
     51 
     52 void URLFetcherCore::Registry::RemoveURLFetcherCore(URLFetcherCore* core) {
     53   DCHECK(ContainsKey(fetchers_, core));
     54   fetchers_.erase(core);
     55 }
     56 
     57 void URLFetcherCore::Registry::CancelAll() {
     58   while (!fetchers_.empty())
     59     (*fetchers_.begin())->CancelURLRequest(ERR_ABORTED);
     60 }
     61 
     62 // URLFetcherCore -------------------------------------------------------------
     63 
     64 // static
     65 base::LazyInstance<URLFetcherCore::Registry>
     66     URLFetcherCore::g_registry = LAZY_INSTANCE_INITIALIZER;
     67 
     68 URLFetcherCore::URLFetcherCore(URLFetcher* fetcher,
     69                                const GURL& original_url,
     70                                URLFetcher::RequestType request_type,
     71                                URLFetcherDelegate* d)
     72     : fetcher_(fetcher),
     73       original_url_(original_url),
     74       request_type_(request_type),
     75       delegate_(d),
     76       delegate_task_runner_(base::ThreadTaskRunnerHandle::Get()),
     77       load_flags_(LOAD_NORMAL),
     78       response_code_(URLFetcher::RESPONSE_CODE_INVALID),
     79       buffer_(new IOBuffer(kBufferSize)),
     80       url_request_data_key_(NULL),
     81       was_fetched_via_proxy_(false),
     82       upload_content_set_(false),
     83       upload_range_offset_(0),
     84       upload_range_length_(0),
     85       referrer_policy_(
     86           URLRequest::CLEAR_REFERRER_ON_TRANSITION_FROM_SECURE_TO_INSECURE),
     87       is_chunked_upload_(false),
     88       was_cancelled_(false),
     89       stop_on_redirect_(false),
     90       stopped_on_redirect_(false),
     91       automatically_retry_on_5xx_(true),
     92       num_retries_on_5xx_(0),
     93       max_retries_on_5xx_(0),
     94       num_retries_on_network_changes_(0),
     95       max_retries_on_network_changes_(0),
     96       current_upload_bytes_(-1),
     97       current_response_bytes_(0),
     98       total_response_bytes_(-1) {
     99   CHECK(original_url_.is_valid());
    100 }
    101 
    102 void URLFetcherCore::Start() {
    103   DCHECK(delegate_task_runner_.get());
    104   DCHECK(request_context_getter_.get()) << "We need an URLRequestContext!";
    105   if (network_task_runner_.get()) {
    106     DCHECK_EQ(network_task_runner_,
    107               request_context_getter_->GetNetworkTaskRunner());
    108   } else {
    109     network_task_runner_ = request_context_getter_->GetNetworkTaskRunner();
    110   }
    111   DCHECK(network_task_runner_.get()) << "We need an IO task runner";
    112 
    113   network_task_runner_->PostTask(
    114       FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
    115 }
    116 
    117 void URLFetcherCore::Stop() {
    118   if (delegate_task_runner_.get())  // May be NULL in tests.
    119     DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    120 
    121   delegate_ = NULL;
    122   fetcher_ = NULL;
    123   if (!network_task_runner_.get())
    124     return;
    125   if (network_task_runner_->RunsTasksOnCurrentThread()) {
    126     CancelURLRequest(ERR_ABORTED);
    127   } else {
    128     network_task_runner_->PostTask(
    129         FROM_HERE,
    130         base::Bind(&URLFetcherCore::CancelURLRequest, this, ERR_ABORTED));
    131   }
    132 }
    133 
    134 void URLFetcherCore::SetUploadData(const std::string& upload_content_type,
    135                                    const std::string& upload_content) {
    136   DCHECK(!is_chunked_upload_);
    137   DCHECK(!upload_content_set_);
    138   DCHECK(upload_content_.empty());
    139   DCHECK(upload_file_path_.empty());
    140   DCHECK(upload_content_type_.empty());
    141 
    142   // Empty |upload_content_type| is allowed iff the |upload_content| is empty.
    143   DCHECK(upload_content.empty() || !upload_content_type.empty());
    144 
    145   upload_content_type_ = upload_content_type;
    146   upload_content_ = upload_content;
    147   upload_content_set_ = true;
    148 }
    149 
    150 void URLFetcherCore::SetUploadFilePath(
    151     const std::string& upload_content_type,
    152     const base::FilePath& file_path,
    153     uint64 range_offset,
    154     uint64 range_length,
    155     scoped_refptr<base::TaskRunner> file_task_runner) {
    156   DCHECK(!is_chunked_upload_);
    157   DCHECK(!upload_content_set_);
    158   DCHECK(upload_content_.empty());
    159   DCHECK(upload_file_path_.empty());
    160   DCHECK_EQ(upload_range_offset_, 0ULL);
    161   DCHECK_EQ(upload_range_length_, 0ULL);
    162   DCHECK(upload_content_type_.empty());
    163   DCHECK(!upload_content_type.empty());
    164 
    165   upload_content_type_ = upload_content_type;
    166   upload_file_path_ = file_path;
    167   upload_range_offset_ = range_offset;
    168   upload_range_length_ = range_length;
    169   upload_file_task_runner_ = file_task_runner;
    170   upload_content_set_ = true;
    171 }
    172 
    173 void URLFetcherCore::SetChunkedUpload(const std::string& content_type) {
    174   DCHECK(is_chunked_upload_ ||
    175          (upload_content_type_.empty() &&
    176           upload_content_.empty()));
    177 
    178   // Empty |content_type| is not allowed here, because it is impossible
    179   // to ensure non-empty upload content as it is not yet supplied.
    180   DCHECK(!content_type.empty());
    181 
    182   upload_content_type_ = content_type;
    183   upload_content_.clear();
    184   is_chunked_upload_ = true;
    185 }
    186 
    187 void URLFetcherCore::AppendChunkToUpload(const std::string& content,
    188                                          bool is_last_chunk) {
    189   DCHECK(delegate_task_runner_.get());
    190   DCHECK(network_task_runner_.get());
    191   network_task_runner_->PostTask(
    192       FROM_HERE,
    193       base::Bind(&URLFetcherCore::CompleteAddingUploadDataChunk, this, content,
    194                  is_last_chunk));
    195 }
    196 
    197 void URLFetcherCore::SetLoadFlags(int load_flags) {
    198   load_flags_ = load_flags;
    199 }
    200 
    201 int URLFetcherCore::GetLoadFlags() const {
    202   return load_flags_;
    203 }
    204 
    205 void URLFetcherCore::SetReferrer(const std::string& referrer) {
    206   referrer_ = referrer;
    207 }
    208 
    209 void URLFetcherCore::SetReferrerPolicy(
    210     URLRequest::ReferrerPolicy referrer_policy) {
    211   referrer_policy_ = referrer_policy;
    212 }
    213 
    214 void URLFetcherCore::SetExtraRequestHeaders(
    215     const std::string& extra_request_headers) {
    216   extra_request_headers_.Clear();
    217   extra_request_headers_.AddHeadersFromString(extra_request_headers);
    218 }
    219 
    220 void URLFetcherCore::AddExtraRequestHeader(const std::string& header_line) {
    221   extra_request_headers_.AddHeaderFromString(header_line);
    222 }
    223 
    224 void URLFetcherCore::SetRequestContext(
    225     URLRequestContextGetter* request_context_getter) {
    226   DCHECK(!request_context_getter_.get());
    227   DCHECK(request_context_getter);
    228   request_context_getter_ = request_context_getter;
    229 }
    230 
    231 void URLFetcherCore::SetFirstPartyForCookies(
    232     const GURL& first_party_for_cookies) {
    233   DCHECK(first_party_for_cookies_.is_empty());
    234   first_party_for_cookies_ = first_party_for_cookies;
    235 }
    236 
    237 void URLFetcherCore::SetURLRequestUserData(
    238     const void* key,
    239     const URLFetcher::CreateDataCallback& create_data_callback) {
    240   DCHECK(key);
    241   DCHECK(!create_data_callback.is_null());
    242   url_request_data_key_ = key;
    243   url_request_create_data_callback_ = create_data_callback;
    244 }
    245 
    246 void URLFetcherCore::SetStopOnRedirect(bool stop_on_redirect) {
    247   stop_on_redirect_ = stop_on_redirect;
    248 }
    249 
    250 void URLFetcherCore::SetAutomaticallyRetryOn5xx(bool retry) {
    251   automatically_retry_on_5xx_ = retry;
    252 }
    253 
    254 void URLFetcherCore::SetMaxRetriesOn5xx(int max_retries) {
    255   max_retries_on_5xx_ = max_retries;
    256 }
    257 
    258 int URLFetcherCore::GetMaxRetriesOn5xx() const {
    259   return max_retries_on_5xx_;
    260 }
    261 
    262 base::TimeDelta URLFetcherCore::GetBackoffDelay() const {
    263   return backoff_delay_;
    264 }
    265 
    266 void URLFetcherCore::SetAutomaticallyRetryOnNetworkChanges(int max_retries) {
    267   max_retries_on_network_changes_ = max_retries;
    268 }
    269 
    270 void URLFetcherCore::SaveResponseToFileAtPath(
    271     const base::FilePath& file_path,
    272     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
    273   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    274   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
    275       new URLFetcherFileWriter(file_task_runner, file_path)));
    276 }
    277 
    278 void URLFetcherCore::SaveResponseToTemporaryFile(
    279     scoped_refptr<base::SequencedTaskRunner> file_task_runner) {
    280   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    281   SaveResponseWithWriter(scoped_ptr<URLFetcherResponseWriter>(
    282       new URLFetcherFileWriter(file_task_runner, base::FilePath())));
    283 }
    284 
    285 void URLFetcherCore::SaveResponseWithWriter(
    286     scoped_ptr<URLFetcherResponseWriter> response_writer) {
    287   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    288   response_writer_ = response_writer.Pass();
    289 }
    290 
    291 HttpResponseHeaders* URLFetcherCore::GetResponseHeaders() const {
    292   return response_headers_.get();
    293 }
    294 
    295 // TODO(panayiotis): socket_address_ is written in the IO thread,
    296 // if this is accessed in the UI thread, this could result in a race.
    297 // Same for response_headers_ above and was_fetched_via_proxy_ below.
    298 HostPortPair URLFetcherCore::GetSocketAddress() const {
    299   return socket_address_;
    300 }
    301 
    302 bool URLFetcherCore::WasFetchedViaProxy() const {
    303   return was_fetched_via_proxy_;
    304 }
    305 
    306 const GURL& URLFetcherCore::GetOriginalURL() const {
    307   return original_url_;
    308 }
    309 
    310 const GURL& URLFetcherCore::GetURL() const {
    311   return url_;
    312 }
    313 
    314 const URLRequestStatus& URLFetcherCore::GetStatus() const {
    315   return status_;
    316 }
    317 
    318 int URLFetcherCore::GetResponseCode() const {
    319   return response_code_;
    320 }
    321 
    322 const ResponseCookies& URLFetcherCore::GetCookies() const {
    323   return cookies_;
    324 }
    325 
    326 void URLFetcherCore::ReceivedContentWasMalformed() {
    327   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    328   if (network_task_runner_.get()) {
    329     network_task_runner_->PostTask(
    330         FROM_HERE, base::Bind(&URLFetcherCore::NotifyMalformedContent, this));
    331   }
    332 }
    333 
    334 bool URLFetcherCore::GetResponseAsString(
    335     std::string* out_response_string) const {
    336   URLFetcherStringWriter* string_writer =
    337       response_writer_ ? response_writer_->AsStringWriter() : NULL;
    338   if (!string_writer)
    339     return false;
    340 
    341   *out_response_string = string_writer->data();
    342   UMA_HISTOGRAM_MEMORY_KB("UrlFetcher.StringResponseSize",
    343                           (string_writer->data().length() / 1024));
    344   return true;
    345 }
    346 
    347 bool URLFetcherCore::GetResponseAsFilePath(bool take_ownership,
    348                                            base::FilePath* out_response_path) {
    349   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    350 
    351   URLFetcherFileWriter* file_writer =
    352       response_writer_ ? response_writer_->AsFileWriter() : NULL;
    353   if (!file_writer)
    354     return false;
    355 
    356   *out_response_path = file_writer->file_path();
    357 
    358   if (take_ownership) {
    359     // Intentionally calling a file_writer_ method directly without posting
    360     // the task to network_task_runner_.
    361     //
    362     // This is for correctly handling the case when file_writer_->DisownFile()
    363     // is soon followed by URLFetcherCore::Stop(). We have to make sure that
    364     // DisownFile takes effect before Stop deletes file_writer_.
    365     //
    366     // This direct call should be thread-safe, since DisownFile itself does no
    367     // file operation. It just flips the state to be referred in destruction.
    368     file_writer->DisownFile();
    369   }
    370   return true;
    371 }
    372 
    373 void URLFetcherCore::OnReceivedRedirect(URLRequest* request,
    374                                         const GURL& new_url,
    375                                         bool* defer_redirect) {
    376   DCHECK_EQ(request, request_.get());
    377   DCHECK(network_task_runner_->BelongsToCurrentThread());
    378   if (stop_on_redirect_) {
    379     stopped_on_redirect_ = true;
    380     url_ = new_url;
    381     response_code_ = request_->GetResponseCode();
    382     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
    383     request->Cancel();
    384     OnReadCompleted(request, 0);
    385   }
    386 }
    387 
    388 void URLFetcherCore::OnResponseStarted(URLRequest* request) {
    389   DCHECK_EQ(request, request_.get());
    390   DCHECK(network_task_runner_->BelongsToCurrentThread());
    391   if (request_->status().is_success()) {
    392     response_code_ = request_->GetResponseCode();
    393     response_headers_ = request_->response_headers();
    394     socket_address_ = request_->GetSocketAddress();
    395     was_fetched_via_proxy_ = request_->was_fetched_via_proxy();
    396     total_response_bytes_ = request_->GetExpectedContentSize();
    397   }
    398 
    399   ReadResponse();
    400 }
    401 
    402 void URLFetcherCore::OnCertificateRequested(
    403     URLRequest* request,
    404     SSLCertRequestInfo* cert_request_info) {
    405   DCHECK_EQ(request, request_.get());
    406   DCHECK(network_task_runner_->BelongsToCurrentThread());
    407 
    408   if (g_ignore_certificate_requests) {
    409     request->ContinueWithCertificate(NULL);
    410   } else {
    411     request->Cancel();
    412   }
    413 }
    414 
    415 void URLFetcherCore::OnReadCompleted(URLRequest* request,
    416                                      int bytes_read) {
    417   DCHECK(request == request_);
    418   DCHECK(network_task_runner_->BelongsToCurrentThread());
    419 
    420   if (!stopped_on_redirect_)
    421     url_ = request->url();
    422   URLRequestThrottlerManager* throttler_manager =
    423       request->context()->throttler_manager();
    424   if (throttler_manager) {
    425     url_throttler_entry_ = throttler_manager->RegisterRequestUrl(url_);
    426   }
    427 
    428   do {
    429     if (!request_->status().is_success() || bytes_read <= 0)
    430       break;
    431 
    432     current_response_bytes_ += bytes_read;
    433     InformDelegateDownloadProgress();
    434 
    435     const int result =
    436         WriteBuffer(new DrainableIOBuffer(buffer_.get(), bytes_read));
    437     if (result < 0) {
    438       // Write failed or waiting for write completion.
    439       return;
    440     }
    441   } while (request_->Read(buffer_.get(), kBufferSize, &bytes_read));
    442 
    443   const URLRequestStatus status = request_->status();
    444 
    445   if (status.is_success())
    446     request_->GetResponseCookies(&cookies_);
    447 
    448   // See comments re: HEAD requests in ReadResponse().
    449   if (!status.is_io_pending() || request_type_ == URLFetcher::HEAD) {
    450     status_ = status;
    451     ReleaseRequest();
    452 
    453     // No more data to write.
    454     const int result = response_writer_->Finish(
    455         base::Bind(&URLFetcherCore::DidFinishWriting, this));
    456     if (result != ERR_IO_PENDING)
    457       DidFinishWriting(result);
    458   }
    459 }
    460 
    461 void URLFetcherCore::CancelAll() {
    462   g_registry.Get().CancelAll();
    463 }
    464 
    465 int URLFetcherCore::GetNumFetcherCores() {
    466   return g_registry.Get().size();
    467 }
    468 
    469 void URLFetcherCore::SetEnableInterceptionForTests(bool enabled) {
    470   g_interception_enabled = enabled;
    471 }
    472 
    473 void URLFetcherCore::SetIgnoreCertificateRequests(bool ignored) {
    474   g_ignore_certificate_requests = ignored;
    475 }
    476 
    477 URLFetcherCore::~URLFetcherCore() {
    478   // |request_| should be NULL.  If not, it's unsafe to delete it here since we
    479   // may not be on the IO thread.
    480   DCHECK(!request_.get());
    481 }
    482 
    483 void URLFetcherCore::StartOnIOThread() {
    484   DCHECK(network_task_runner_->BelongsToCurrentThread());
    485 
    486   if (!response_writer_)
    487     response_writer_.reset(new URLFetcherStringWriter);
    488 
    489   const int result = response_writer_->Initialize(
    490       base::Bind(&URLFetcherCore::DidInitializeWriter, this));
    491   if (result != ERR_IO_PENDING)
    492     DidInitializeWriter(result);
    493 }
    494 
    495 void URLFetcherCore::StartURLRequest() {
    496   DCHECK(network_task_runner_->BelongsToCurrentThread());
    497 
    498   if (was_cancelled_) {
    499     // Since StartURLRequest() is posted as a *delayed* task, it may
    500     // run after the URLFetcher was already stopped.
    501     return;
    502   }
    503 
    504   DCHECK(request_context_getter_.get());
    505   DCHECK(!request_.get());
    506 
    507   g_registry.Get().AddURLFetcherCore(this);
    508   current_response_bytes_ = 0;
    509   request_ = request_context_getter_->GetURLRequestContext()->CreateRequest(
    510       original_url_, DEFAULT_PRIORITY, this, NULL);
    511   request_->set_stack_trace(stack_trace_);
    512   int flags = request_->load_flags() | load_flags_;
    513   if (!g_interception_enabled)
    514     flags = flags | LOAD_DISABLE_INTERCEPT;
    515 
    516   if (is_chunked_upload_)
    517     request_->EnableChunkedUpload();
    518   request_->SetLoadFlags(flags);
    519   request_->SetReferrer(referrer_);
    520   request_->set_referrer_policy(referrer_policy_);
    521   request_->set_first_party_for_cookies(first_party_for_cookies_.is_empty() ?
    522       original_url_ : first_party_for_cookies_);
    523   if (url_request_data_key_ && !url_request_create_data_callback_.is_null()) {
    524     request_->SetUserData(url_request_data_key_,
    525                           url_request_create_data_callback_.Run());
    526   }
    527 
    528   switch (request_type_) {
    529     case URLFetcher::GET:
    530       break;
    531 
    532     case URLFetcher::POST:
    533     case URLFetcher::PUT:
    534     case URLFetcher::PATCH:
    535       // Upload content must be set.
    536       DCHECK(is_chunked_upload_ || upload_content_set_);
    537 
    538       request_->set_method(
    539           request_type_ == URLFetcher::POST ? "POST" :
    540           request_type_ == URLFetcher::PUT ? "PUT" : "PATCH");
    541       if (!upload_content_type_.empty()) {
    542         extra_request_headers_.SetHeader(HttpRequestHeaders::kContentType,
    543                                          upload_content_type_);
    544       }
    545       if (!upload_content_.empty()) {
    546         scoped_ptr<UploadElementReader> reader(new UploadBytesElementReader(
    547             upload_content_.data(), upload_content_.size()));
    548         request_->set_upload(make_scoped_ptr(
    549             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
    550       } else if (!upload_file_path_.empty()) {
    551         scoped_ptr<UploadElementReader> reader(
    552             new UploadFileElementReader(upload_file_task_runner_.get(),
    553                                         upload_file_path_,
    554                                         upload_range_offset_,
    555                                         upload_range_length_,
    556                                         base::Time()));
    557         request_->set_upload(make_scoped_ptr(
    558             UploadDataStream::CreateWithReader(reader.Pass(), 0)));
    559       }
    560 
    561       current_upload_bytes_ = -1;
    562       // TODO(kinaba): http://crbug.com/118103. Implement upload callback in the
    563       //  layer and avoid using timer here.
    564       upload_progress_checker_timer_.reset(
    565           new base::RepeatingTimer<URLFetcherCore>());
    566       upload_progress_checker_timer_->Start(
    567           FROM_HERE,
    568           base::TimeDelta::FromMilliseconds(kUploadProgressTimerInterval),
    569           this,
    570           &URLFetcherCore::InformDelegateUploadProgress);
    571       break;
    572 
    573     case URLFetcher::HEAD:
    574       request_->set_method("HEAD");
    575       break;
    576 
    577     case URLFetcher::DELETE_REQUEST:
    578       request_->set_method("DELETE");
    579       break;
    580 
    581     default:
    582       NOTREACHED();
    583   }
    584 
    585   if (!extra_request_headers_.IsEmpty())
    586     request_->SetExtraRequestHeaders(extra_request_headers_);
    587 
    588   request_->Start();
    589 }
    590 
    591 void URLFetcherCore::DidInitializeWriter(int result) {
    592   if (result != OK) {
    593     CancelURLRequest(result);
    594     delegate_task_runner_->PostTask(
    595         FROM_HERE,
    596         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    597     return;
    598   }
    599   StartURLRequestWhenAppropriate();
    600 }
    601 
    602 void URLFetcherCore::StartURLRequestWhenAppropriate() {
    603   DCHECK(network_task_runner_->BelongsToCurrentThread());
    604 
    605   if (was_cancelled_)
    606     return;
    607 
    608   DCHECK(request_context_getter_.get());
    609 
    610   int64 delay = 0LL;
    611   if (original_url_throttler_entry_.get() == NULL) {
    612     URLRequestThrottlerManager* manager =
    613         request_context_getter_->GetURLRequestContext()->throttler_manager();
    614     if (manager) {
    615       original_url_throttler_entry_ =
    616           manager->RegisterRequestUrl(original_url_);
    617     }
    618   }
    619   if (original_url_throttler_entry_.get() != NULL) {
    620     delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
    621         GetBackoffReleaseTime());
    622   }
    623 
    624   if (delay == 0) {
    625     StartURLRequest();
    626   } else {
    627     base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
    628         FROM_HERE, base::Bind(&URLFetcherCore::StartURLRequest, this),
    629         base::TimeDelta::FromMilliseconds(delay));
    630   }
    631 }
    632 
    633 void URLFetcherCore::CancelURLRequest(int error) {
    634   DCHECK(network_task_runner_->BelongsToCurrentThread());
    635 
    636   if (request_.get()) {
    637     request_->CancelWithError(error);
    638     ReleaseRequest();
    639   }
    640 
    641   // Set the error manually.
    642   // Normally, calling URLRequest::CancelWithError() results in calling
    643   // OnReadCompleted() with bytes_read = -1 via an asynchronous task posted by
    644   // URLRequestJob::NotifyDone(). But, because the request was released
    645   // immediately after being canceled, the request could not call
    646   // OnReadCompleted() which overwrites |status_| with the error status.
    647   status_.set_status(URLRequestStatus::CANCELED);
    648   status_.set_error(error);
    649 
    650   // Release the reference to the request context. There could be multiple
    651   // references to URLFetcher::Core at this point so it may take a while to
    652   // delete the object, but we cannot delay the destruction of the request
    653   // context.
    654   request_context_getter_ = NULL;
    655   first_party_for_cookies_ = GURL();
    656   url_request_data_key_ = NULL;
    657   url_request_create_data_callback_.Reset();
    658   was_cancelled_ = true;
    659 }
    660 
    661 void URLFetcherCore::OnCompletedURLRequest(
    662     base::TimeDelta backoff_delay) {
    663   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    664 
    665   // Save the status and backoff_delay so that delegates can read it.
    666   if (delegate_) {
    667     backoff_delay_ = backoff_delay;
    668     InformDelegateFetchIsComplete();
    669   }
    670 }
    671 
    672 void URLFetcherCore::InformDelegateFetchIsComplete() {
    673   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    674   if (delegate_)
    675     delegate_->OnURLFetchComplete(fetcher_);
    676 }
    677 
    678 void URLFetcherCore::NotifyMalformedContent() {
    679   DCHECK(network_task_runner_->BelongsToCurrentThread());
    680   if (url_throttler_entry_.get() != NULL) {
    681     int status_code = response_code_;
    682     if (status_code == URLFetcher::RESPONSE_CODE_INVALID) {
    683       // The status code will generally be known by the time clients
    684       // call the |ReceivedContentWasMalformed()| function (which ends up
    685       // calling the current function) but if it's not, we need to assume
    686       // the response was successful so that the total failure count
    687       // used to calculate exponential back-off goes up.
    688       status_code = 200;
    689     }
    690     url_throttler_entry_->ReceivedContentWasMalformed(status_code);
    691   }
    692 }
    693 
    694 void URLFetcherCore::DidFinishWriting(int result) {
    695   if (result != OK) {
    696     CancelURLRequest(result);
    697     delegate_task_runner_->PostTask(
    698         FROM_HERE,
    699         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    700     return;
    701   }
    702   // If the file was successfully closed, then the URL request is complete.
    703   RetryOrCompleteUrlFetch();
    704 }
    705 
    706 void URLFetcherCore::RetryOrCompleteUrlFetch() {
    707   DCHECK(network_task_runner_->BelongsToCurrentThread());
    708   base::TimeDelta backoff_delay;
    709 
    710   // Checks the response from server.
    711   if (response_code_ >= 500 ||
    712       status_.error() == ERR_TEMPORARILY_THROTTLED) {
    713     // When encountering a server error, we will send the request again
    714     // after backoff time.
    715     ++num_retries_on_5xx_;
    716 
    717     // Note that backoff_delay may be 0 because (a) the
    718     // URLRequestThrottlerManager and related code does not
    719     // necessarily back off on the first error, (b) it only backs off
    720     // on some of the 5xx status codes, (c) not all URLRequestContexts
    721     // have a throttler manager.
    722     base::TimeTicks backoff_release_time = GetBackoffReleaseTime();
    723     backoff_delay = backoff_release_time - base::TimeTicks::Now();
    724     if (backoff_delay < base::TimeDelta())
    725       backoff_delay = base::TimeDelta();
    726 
    727     if (automatically_retry_on_5xx_ &&
    728         num_retries_on_5xx_ <= max_retries_on_5xx_) {
    729       StartOnIOThread();
    730       return;
    731     }
    732   } else {
    733     backoff_delay = base::TimeDelta();
    734   }
    735 
    736   // Retry if the request failed due to network changes.
    737   if (status_.error() == ERR_NETWORK_CHANGED &&
    738       num_retries_on_network_changes_ < max_retries_on_network_changes_) {
    739     ++num_retries_on_network_changes_;
    740 
    741     // Retry soon, after flushing all the current tasks which may include
    742     // further network change observers.
    743     network_task_runner_->PostTask(
    744         FROM_HERE, base::Bind(&URLFetcherCore::StartOnIOThread, this));
    745     return;
    746   }
    747 
    748   request_context_getter_ = NULL;
    749   first_party_for_cookies_ = GURL();
    750   url_request_data_key_ = NULL;
    751   url_request_create_data_callback_.Reset();
    752   bool posted = delegate_task_runner_->PostTask(
    753       FROM_HERE,
    754       base::Bind(&URLFetcherCore::OnCompletedURLRequest, this, backoff_delay));
    755 
    756   // If the delegate message loop does not exist any more, then the delegate
    757   // should be gone too.
    758   DCHECK(posted || !delegate_);
    759 }
    760 
    761 void URLFetcherCore::ReleaseRequest() {
    762   upload_progress_checker_timer_.reset();
    763   request_.reset();
    764   g_registry.Get().RemoveURLFetcherCore(this);
    765 }
    766 
    767 base::TimeTicks URLFetcherCore::GetBackoffReleaseTime() {
    768   DCHECK(network_task_runner_->BelongsToCurrentThread());
    769 
    770   if (original_url_throttler_entry_.get()) {
    771     base::TimeTicks original_url_backoff =
    772         original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
    773     base::TimeTicks destination_url_backoff;
    774     if (url_throttler_entry_.get() != NULL &&
    775         original_url_throttler_entry_.get() != url_throttler_entry_.get()) {
    776       destination_url_backoff =
    777           url_throttler_entry_->GetExponentialBackoffReleaseTime();
    778     }
    779 
    780     return original_url_backoff > destination_url_backoff ?
    781         original_url_backoff : destination_url_backoff;
    782   } else {
    783     return base::TimeTicks();
    784   }
    785 }
    786 
    787 void URLFetcherCore::CompleteAddingUploadDataChunk(
    788     const std::string& content, bool is_last_chunk) {
    789   if (was_cancelled_) {
    790     // Since CompleteAddingUploadDataChunk() is posted as a *delayed* task, it
    791     // may run after the URLFetcher was already stopped.
    792     return;
    793   }
    794   DCHECK(is_chunked_upload_);
    795   DCHECK(request_.get());
    796   DCHECK(!content.empty());
    797   request_->AppendChunkToUpload(content.data(),
    798                                 static_cast<int>(content.length()),
    799                                 is_last_chunk);
    800 }
    801 
    802 int URLFetcherCore::WriteBuffer(scoped_refptr<DrainableIOBuffer> data) {
    803   while (data->BytesRemaining() > 0) {
    804     const int result = response_writer_->Write(
    805         data.get(),
    806         data->BytesRemaining(),
    807         base::Bind(&URLFetcherCore::DidWriteBuffer, this, data));
    808     if (result < 0) {
    809       if (result != ERR_IO_PENDING)
    810         DidWriteBuffer(data, result);
    811       return result;
    812     }
    813     data->DidConsume(result);
    814   }
    815   return OK;
    816 }
    817 
    818 void URLFetcherCore::DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data,
    819                                     int result) {
    820   if (result < 0) {  // Handle errors.
    821     CancelURLRequest(result);
    822     response_writer_->Finish(base::Bind(&EmptyCompletionCallback));
    823     delegate_task_runner_->PostTask(
    824         FROM_HERE,
    825         base::Bind(&URLFetcherCore::InformDelegateFetchIsComplete, this));
    826     return;
    827   }
    828 
    829   // Continue writing.
    830   data->DidConsume(result);
    831   if (WriteBuffer(data) < 0)
    832     return;
    833 
    834   // Finished writing buffer_. Read some more, unless the request has been
    835   // cancelled and deleted.
    836   DCHECK_EQ(0, data->BytesRemaining());
    837   if (request_.get())
    838     ReadResponse();
    839 }
    840 
    841 void URLFetcherCore::ReadResponse() {
    842   // Some servers may treat HEAD requests as GET requests.  To free up the
    843   // network connection as soon as possible, signal that the request has
    844   // completed immediately, without trying to read any data back (all we care
    845   // about is the response code and headers, which we already have).
    846   int bytes_read = 0;
    847   if (request_->status().is_success() &&
    848       (request_type_ != URLFetcher::HEAD))
    849     request_->Read(buffer_.get(), kBufferSize, &bytes_read);
    850   OnReadCompleted(request_.get(), bytes_read);
    851 }
    852 
    853 void URLFetcherCore::InformDelegateUploadProgress() {
    854   DCHECK(network_task_runner_->BelongsToCurrentThread());
    855   if (request_.get()) {
    856     int64 current = request_->GetUploadProgress().position();
    857     if (current_upload_bytes_ != current) {
    858       current_upload_bytes_ = current;
    859       int64 total = -1;
    860       if (!is_chunked_upload_) {
    861         total = static_cast<int64>(request_->GetUploadProgress().size());
    862         // Total may be zero if the UploadDataStream::Init has not been called
    863         // yet.  Don't send the upload progress until the size is initialized.
    864         if (!total)
    865           return;
    866       }
    867       delegate_task_runner_->PostTask(
    868           FROM_HERE,
    869           base::Bind(
    870               &URLFetcherCore::InformDelegateUploadProgressInDelegateThread,
    871               this, current, total));
    872     }
    873   }
    874 }
    875 
    876 void URLFetcherCore::InformDelegateUploadProgressInDelegateThread(
    877     int64 current, int64 total) {
    878   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    879   if (delegate_)
    880     delegate_->OnURLFetchUploadProgress(fetcher_, current, total);
    881 }
    882 
    883 void URLFetcherCore::InformDelegateDownloadProgress() {
    884   DCHECK(network_task_runner_->BelongsToCurrentThread());
    885   delegate_task_runner_->PostTask(
    886       FROM_HERE,
    887       base::Bind(
    888           &URLFetcherCore::InformDelegateDownloadProgressInDelegateThread,
    889           this, current_response_bytes_, total_response_bytes_));
    890 }
    891 
    892 void URLFetcherCore::InformDelegateDownloadProgressInDelegateThread(
    893     int64 current, int64 total) {
    894   DCHECK(delegate_task_runner_->BelongsToCurrentThread());
    895   if (delegate_)
    896     delegate_->OnURLFetchDownloadProgress(fetcher_, current, total);
    897 }
    898 
    899 }  // namespace net
    900