Home | History | Annotate | Download | only in net 
      1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/common/net/url_fetcher.h"
      6 
      7 #include <set>
      8 
      9 #include "base/compiler_specific.h"
     10 #include "base/lazy_instance.h"
     11 #include "base/memory/scoped_ptr.h"
     12 #include "base/message_loop_proxy.h"
     13 #include "base/stl_util-inl.h"
     14 #include "base/string_util.h"
     15 #include "base/threading/thread.h"
     16 #include "googleurl/src/gurl.h"
     17 #include "net/base/load_flags.h"
     18 #include "net/base/io_buffer.h"
     19 #include "net/base/net_errors.h"
     20 #include "net/http/http_request_headers.h"
     21 #include "net/http/http_response_headers.h"
     22 #include "net/url_request/url_request.h"
     23 #include "net/url_request/url_request_context.h"
     24 #include "net/url_request/url_request_context_getter.h"
     25 #include "net/url_request/url_request_throttler_manager.h"
     26 
     27 #ifdef ANDROID
     28 #include "android/autofill/url_fetcher_proxy.h"
     29 #endif
     30 
     31 static const int kBufferSize = 4096;
     32 
     33 class URLFetcher::Core
     34     : public base::RefCountedThreadSafe<URLFetcher::Core>,
     35       public net::URLRequest::Delegate {
     36  public:
     37   // For POST requests, set |content_type| to the MIME type of the content
     38   // and set |content| to the data to upload.  |flags| are flags to apply to
     39   // the load operation--these should be one or more of the LOAD_* flags
     40   // defined in net/base/load_flags.h.
     41   Core(URLFetcher* fetcher,
     42        const GURL& original_url,
     43        RequestType request_type,
     44        URLFetcher::Delegate* d);
     45 
     46   // Starts the load.  It's important that this not happen in the constructor
     47   // because it causes the IO thread to begin AddRef()ing and Release()ing
     48   // us.  If our caller hasn't had time to fully construct us and take a
     49   // reference, the IO thread could interrupt things, run a task, Release()
     50   // us, and destroy us, leaving the caller with an already-destroyed object
     51   // when construction finishes.
     52   void Start();
     53 
     54   // Stops any in-progress load and ensures no callback will happen.  It is
     55   // safe to call this multiple times.
     56   void Stop();
     57 
     58   // Reports that the received content was malformed.
     59   void ReceivedContentWasMalformed();
     60 
     61   // Overridden from net::URLRequest::Delegate:
     62   virtual void OnResponseStarted(net::URLRequest* request);
     63   virtual void OnReadCompleted(net::URLRequest* request, int bytes_read);
     64 
     65   URLFetcher::Delegate* delegate() const { return delegate_; }
     66 
     67   static void CancelAll();
     68 
     69  private:
     70   friend class base::RefCountedThreadSafe<URLFetcher::Core>;
     71 
     72   class Registry {
     73    public:
     74     Registry();
     75     ~Registry();
     76 
     77     void AddURLFetcherCore(Core* core);
     78     void RemoveURLFetcherCore(Core* core);
     79 
     80     void CancelAll();
     81 
     82     int size() const {
     83       return fetchers_.size();
     84     }
     85 
     86    private:
     87     std::set<Core*> fetchers_;
     88 
     89     DISALLOW_COPY_AND_ASSIGN(Registry);
     90   };
     91 
     92   virtual ~Core();
     93 
     94   // Wrapper functions that allow us to ensure actions happen on the right
     95   // thread.
     96   void StartURLRequest();
     97   void StartURLRequestWhenAppropriate();
     98   void CancelURLRequest();
     99   void OnCompletedURLRequest(const net::URLRequestStatus& status);
    100   void NotifyMalformedContent();
    101 
    102   // Deletes the request, removes it from the registry, and removes the
    103   // destruction observer.
    104   void ReleaseRequest();
    105 
    106   // Returns the max value of exponential back-off release time for
    107   // |original_url_| and |url_|.
    108   base::TimeTicks GetBackoffReleaseTime();
    109 
    110   void CompleteAddingUploadDataChunk(const std::string& data,
    111                                      bool is_last_chunk);
    112 
    113   // Adds a block of data to be uploaded in a POST body. This can only be called
    114   // after Start().
    115   void AppendChunkToUpload(const std::string& data, bool is_last_chunk);
    116 
    117   URLFetcher* fetcher_;              // Corresponding fetcher object
    118   GURL original_url_;                // The URL we were asked to fetch
    119   GURL url_;                         // The URL we eventually wound up at
    120   RequestType request_type_;         // What type of request is this?
    121   URLFetcher::Delegate* delegate_;   // Object to notify on completion
    122   scoped_refptr<base::MessageLoopProxy> delegate_loop_proxy_;
    123                                      // Message loop proxy of the creating
    124                                      // thread.
    125   scoped_refptr<base::MessageLoopProxy> io_message_loop_proxy_;
    126                                      // The message loop proxy for the thread
    127                                      // on which the request IO happens.
    128   scoped_ptr<net::URLRequest> request_;   // The actual request this wraps
    129   int load_flags_;                   // Flags for the load operation
    130   int response_code_;                // HTTP status code for the request
    131   std::string data_;                 // Results of the request
    132   scoped_refptr<net::IOBuffer> buffer_;
    133                                      // Read buffer
    134   scoped_refptr<net::URLRequestContextGetter> request_context_getter_;
    135                                      // Cookie/cache info for the request
    136   ResponseCookies cookies_;          // Response cookies
    137   net::HttpRequestHeaders extra_request_headers_;
    138   scoped_refptr<net::HttpResponseHeaders> response_headers_;
    139 
    140   std::string upload_content_;       // HTTP POST payload
    141   std::string upload_content_type_;  // MIME type of POST payload
    142   std::string referrer_;             // HTTP Referer header value
    143   bool is_chunked_upload_;           // True if using chunked transfer encoding
    144 
    145   // Used to determine how long to wait before making a request or doing a
    146   // retry.
    147   // Both of them can only be accessed on the IO thread.
    148   // We need not only the throttler entry for |original_URL|, but also the one
    149   // for |url|. For example, consider the case that URL A redirects to URL B,
    150   // for which the server returns a 500 response. In this case, the exponential
    151   // back-off release time of URL A won't increase. If we retry without
    152   // considering the back-off constraint of URL B, we may send out too many
    153   // requests for URL A in a short period of time.
    154   scoped_refptr<net::URLRequestThrottlerEntryInterface>
    155       original_url_throttler_entry_;
    156   scoped_refptr<net::URLRequestThrottlerEntryInterface> url_throttler_entry_;
    157 
    158   // |num_retries_| indicates how many times we've failed to successfully
    159   // fetch this URL.  Once this value exceeds the maximum number of retries
    160   // specified by the owner URLFetcher instance, we'll give up.
    161   int num_retries_;
    162 
    163   // True if the URLFetcher has been cancelled.
    164   bool was_cancelled_;
    165 
    166   // Since GetBackoffReleaseTime() can only be called on the IO thread, we cache
    167   // its value to be used by OnCompletedURLRequest on the creating thread.
    168   base::TimeTicks backoff_release_time_;
    169 
    170   static base::LazyInstance<Registry> g_registry;
    171 
    172   friend class URLFetcher;
    173   DISALLOW_COPY_AND_ASSIGN(Core);
    174 };
    175 
    176 URLFetcher::Core::Registry::Registry() {}
    177 URLFetcher::Core::Registry::~Registry() {}
    178 
    179 void URLFetcher::Core::Registry::AddURLFetcherCore(Core* core) {
    180   DCHECK(!ContainsKey(fetchers_, core));
    181   fetchers_.insert(core);
    182 }
    183 
    184 void URLFetcher::Core::Registry::RemoveURLFetcherCore(Core* core) {
    185   DCHECK(ContainsKey(fetchers_, core));
    186   fetchers_.erase(core);
    187 }
    188 
    189 void URLFetcher::Core::Registry::CancelAll() {
    190   while (!fetchers_.empty())
    191     (*fetchers_.begin())->CancelURLRequest();
    192 }
    193 
    194 // static
    195 base::LazyInstance<URLFetcher::Core::Registry>
    196     URLFetcher::Core::g_registry(base::LINKER_INITIALIZED);
    197 
    198 // static
    199 URLFetcher::Factory* URLFetcher::factory_ = NULL;
    200 
    201 // static
    202 bool URLFetcher::g_interception_enabled = false;
    203 
    204 URLFetcher::URLFetcher(const GURL& url,
    205                        RequestType request_type,
    206                        Delegate* d)
    207     : ALLOW_THIS_IN_INITIALIZER_LIST(
    208       core_(new Core(this, url, request_type, d))),
    209       automatically_retry_on_5xx_(true),
    210       max_retries_(0) {
    211 }
    212 
    213 URLFetcher::~URLFetcher() {
    214   core_->Stop();
    215 }
    216 
    217 // static
    218 URLFetcher* URLFetcher::Create(int id, const GURL& url,
    219                                RequestType request_type, Delegate* d) {
    220 #ifdef ANDROID
    221   // TODO: Upstream.
    222   return new URLFetcherProxy(url, request_type, d);
    223 #else
    224   return factory_ ? factory_->CreateURLFetcher(id, url, request_type, d) :
    225                     new URLFetcher(url, request_type, d);
    226 #endif
    227 }
    228 
    229 URLFetcher::Core::Core(URLFetcher* fetcher,
    230                        const GURL& original_url,
    231                        RequestType request_type,
    232                        URLFetcher::Delegate* d)
    233     : fetcher_(fetcher),
    234       original_url_(original_url),
    235       request_type_(request_type),
    236       delegate_(d),
    237       delegate_loop_proxy_(base::MessageLoopProxy::CreateForCurrentThread()),
    238       request_(NULL),
    239       load_flags_(net::LOAD_NORMAL),
    240       response_code_(-1),
    241       buffer_(new net::IOBuffer(kBufferSize)),
    242       is_chunked_upload_(false),
    243       num_retries_(0),
    244       was_cancelled_(false) {
    245 }
    246 
    247 URLFetcher::Core::~Core() {
    248   // |request_| should be NULL.  If not, it's unsafe to delete it here since we
    249   // may not be on the IO thread.
    250   DCHECK(!request_.get());
    251 }
    252 
    253 void URLFetcher::Core::Start() {
    254   DCHECK(delegate_loop_proxy_);
    255   CHECK(request_context_getter_) << "We need an URLRequestContext!";
    256   io_message_loop_proxy_ = request_context_getter_->GetIOMessageLoopProxy();
    257   CHECK(io_message_loop_proxy_.get()) << "We need an IO message loop proxy";
    258 
    259   io_message_loop_proxy_->PostTask(
    260       FROM_HERE,
    261       NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate));
    262 }
    263 
    264 void URLFetcher::Core::Stop() {
    265   DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
    266   delegate_ = NULL;
    267   fetcher_ = NULL;
    268   if (io_message_loop_proxy_.get()) {
    269     io_message_loop_proxy_->PostTask(
    270         FROM_HERE, NewRunnableMethod(this, &Core::CancelURLRequest));
    271   }
    272 }
    273 
    274 void URLFetcher::Core::ReceivedContentWasMalformed() {
    275   DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
    276   if (io_message_loop_proxy_.get()) {
    277     io_message_loop_proxy_->PostTask(
    278         FROM_HERE, NewRunnableMethod(this, &Core::NotifyMalformedContent));
    279   }
    280 }
    281 
    282 void URLFetcher::Core::CancelAll() {
    283   g_registry.Get().CancelAll();
    284 }
    285 
    286 void URLFetcher::Core::OnResponseStarted(net::URLRequest* request) {
    287   DCHECK_EQ(request, request_.get());
    288   DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
    289   if (request_->status().is_success()) {
    290     response_code_ = request_->GetResponseCode();
    291     response_headers_ = request_->response_headers();
    292   }
    293 
    294   int bytes_read = 0;
    295   // Some servers may treat HEAD requests as GET requests.  To free up the
    296   // network connection as soon as possible, signal that the request has
    297   // completed immediately, without trying to read any data back (all we care
    298   // about is the response code and headers, which we already have).
    299   if (request_->status().is_success() && (request_type_ != HEAD))
    300     request_->Read(buffer_, kBufferSize, &bytes_read);
    301   OnReadCompleted(request_.get(), bytes_read);
    302 }
    303 
    304 void URLFetcher::Core::CompleteAddingUploadDataChunk(
    305     const std::string& content, bool is_last_chunk) {
    306   DCHECK(is_chunked_upload_);
    307   DCHECK(request_.get());
    308   DCHECK(!content.empty());
    309   request_->AppendChunkToUpload(content.data(),
    310                                 static_cast<int>(content.length()),
    311                                 is_last_chunk);
    312 }
    313 
    314 void URLFetcher::Core::AppendChunkToUpload(const std::string& content,
    315                                            bool is_last_chunk) {
    316   DCHECK(delegate_loop_proxy_);
    317   CHECK(io_message_loop_proxy_.get());
    318   io_message_loop_proxy_->PostTask(
    319       FROM_HERE,
    320       NewRunnableMethod(this, &Core::CompleteAddingUploadDataChunk, content,
    321                         is_last_chunk));
    322 }
    323 
    324 void URLFetcher::Core::OnReadCompleted(net::URLRequest* request,
    325                                        int bytes_read) {
    326   DCHECK(request == request_);
    327   DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
    328 
    329   url_ = request->url();
    330   url_throttler_entry_ =
    331       net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(url_);
    332 
    333   do {
    334     if (!request_->status().is_success() || bytes_read <= 0)
    335       break;
    336     data_.append(buffer_->data(), bytes_read);
    337   } while (request_->Read(buffer_, kBufferSize, &bytes_read));
    338 
    339   if (request_->status().is_success())
    340     request_->GetResponseCookies(&cookies_);
    341 
    342   // See comments re: HEAD requests in OnResponseStarted().
    343   if (!request_->status().is_io_pending() || (request_type_ == HEAD)) {
    344     backoff_release_time_ = GetBackoffReleaseTime();
    345 
    346     bool posted = delegate_loop_proxy_->PostTask(
    347         FROM_HERE,
    348         NewRunnableMethod(this,
    349                           &Core::OnCompletedURLRequest,
    350                           request_->status()));
    351     // If the delegate message loop does not exist any more, then the delegate
    352     // should be gone too.
    353     DCHECK(posted || !delegate_);
    354     ReleaseRequest();
    355   }
    356 }
    357 
    358 void URLFetcher::Core::StartURLRequest() {
    359   DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
    360 
    361   if (was_cancelled_) {
    362     // Since StartURLRequest() is posted as a *delayed* task, it may
    363     // run after the URLFetcher was already stopped.
    364     return;
    365   }
    366 
    367   CHECK(request_context_getter_);
    368   DCHECK(!request_.get());
    369 
    370   g_registry.Get().AddURLFetcherCore(this);
    371   request_.reset(new net::URLRequest(original_url_, this));
    372   int flags = request_->load_flags() | load_flags_;
    373   if (!g_interception_enabled) {
    374     flags = flags | net::LOAD_DISABLE_INTERCEPT;
    375   }
    376   if (is_chunked_upload_)
    377     request_->EnableChunkedUpload();
    378   request_->set_load_flags(flags);
    379   request_->set_context(request_context_getter_->GetURLRequestContext());
    380   request_->set_referrer(referrer_);
    381 
    382   switch (request_type_) {
    383     case GET:
    384       break;
    385 
    386     case POST:
    387       DCHECK(!upload_content_.empty() || is_chunked_upload_);
    388       DCHECK(!upload_content_type_.empty());
    389 
    390       request_->set_method("POST");
    391       extra_request_headers_.SetHeader(net::HttpRequestHeaders::kContentType,
    392                                        upload_content_type_);
    393       if (!upload_content_.empty()) {
    394         request_->AppendBytesToUpload(
    395             upload_content_.data(), static_cast<int>(upload_content_.length()));
    396       }
    397       break;
    398 
    399     case HEAD:
    400       request_->set_method("HEAD");
    401       break;
    402 
    403     default:
    404       NOTREACHED();
    405   }
    406 
    407   if (!extra_request_headers_.IsEmpty())
    408     request_->SetExtraRequestHeaders(extra_request_headers_);
    409 
    410   // There might be data left over from a previous request attempt.
    411   data_.clear();
    412 
    413   request_->Start();
    414 }
    415 
    416 void URLFetcher::Core::StartURLRequestWhenAppropriate() {
    417   DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
    418 
    419   if (was_cancelled_)
    420     return;
    421 
    422   if (original_url_throttler_entry_ == NULL) {
    423     original_url_throttler_entry_ =
    424         net::URLRequestThrottlerManager::GetInstance()->RegisterRequestUrl(
    425             original_url_);
    426   }
    427 
    428   int64 delay = original_url_throttler_entry_->ReserveSendingTimeForNextRequest(
    429       GetBackoffReleaseTime());
    430   if (delay == 0) {
    431     StartURLRequest();
    432   } else {
    433     MessageLoop::current()->PostDelayedTask(
    434         FROM_HERE,
    435         NewRunnableMethod(this, &Core::StartURLRequest),
    436         delay);
    437   }
    438 }
    439 
    440 void URLFetcher::Core::CancelURLRequest() {
    441   DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
    442 
    443   if (request_.get()) {
    444     request_->Cancel();
    445     ReleaseRequest();
    446   }
    447   // Release the reference to the request context. There could be multiple
    448   // references to URLFetcher::Core at this point so it may take a while to
    449   // delete the object, but we cannot delay the destruction of the request
    450   // context.
    451   request_context_getter_ = NULL;
    452   was_cancelled_ = true;
    453 }
    454 
    455 void URLFetcher::Core::OnCompletedURLRequest(
    456     const net::URLRequestStatus& status) {
    457   DCHECK(delegate_loop_proxy_->BelongsToCurrentThread());
    458 
    459   // Checks the response from server.
    460   if (response_code_ >= 500 ||
    461       status.os_error() == net::ERR_TEMPORARILY_THROTTLED) {
    462     // When encountering a server error, we will send the request again
    463     // after backoff time.
    464     ++num_retries_;
    465     // Restarts the request if we still need to notify the delegate.
    466     if (delegate_) {
    467       fetcher_->backoff_delay_ = backoff_release_time_ - base::TimeTicks::Now();
    468       if (fetcher_->backoff_delay_ < base::TimeDelta())
    469         fetcher_->backoff_delay_ = base::TimeDelta();
    470 
    471       if (fetcher_->automatically_retry_on_5xx_ &&
    472           num_retries_ <= fetcher_->max_retries()) {
    473         io_message_loop_proxy_->PostTask(
    474             FROM_HERE,
    475             NewRunnableMethod(this, &Core::StartURLRequestWhenAppropriate));
    476       } else {
    477         delegate_->OnURLFetchComplete(fetcher_, url_, status, response_code_,
    478                                       cookies_, data_);
    479       }
    480     }
    481   } else {
    482     if (delegate_) {
    483       fetcher_->backoff_delay_ = base::TimeDelta();
    484       delegate_->OnURLFetchComplete(fetcher_, url_, status, response_code_,
    485                                     cookies_, data_);
    486     }
    487   }
    488 }
    489 
    490 void URLFetcher::Core::NotifyMalformedContent() {
    491   DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
    492   if (url_throttler_entry_ != NULL)
    493     url_throttler_entry_->ReceivedContentWasMalformed();
    494 }
    495 
    496 void URLFetcher::Core::ReleaseRequest() {
    497   request_.reset();
    498   g_registry.Get().RemoveURLFetcherCore(this);
    499 }
    500 
    501 base::TimeTicks URLFetcher::Core::GetBackoffReleaseTime() {
    502   DCHECK(io_message_loop_proxy_->BelongsToCurrentThread());
    503   DCHECK(original_url_throttler_entry_ != NULL);
    504 
    505   base::TimeTicks original_url_backoff =
    506       original_url_throttler_entry_->GetExponentialBackoffReleaseTime();
    507   base::TimeTicks destination_url_backoff;
    508   if (url_throttler_entry_ != NULL &&
    509       original_url_throttler_entry_ != url_throttler_entry_) {
    510     destination_url_backoff =
    511         url_throttler_entry_->GetExponentialBackoffReleaseTime();
    512   }
    513 
    514   return original_url_backoff > destination_url_backoff ?
    515       original_url_backoff : destination_url_backoff;
    516 }
    517 
    518 void URLFetcher::set_upload_data(const std::string& upload_content_type,
    519                                  const std::string& upload_content) {
    520   DCHECK(!core_->is_chunked_upload_);
    521   core_->upload_content_type_ = upload_content_type;
    522   core_->upload_content_ = upload_content;
    523 }
    524 
    525 void URLFetcher::set_chunked_upload(const std::string& content_type) {
    526   DCHECK(core_->is_chunked_upload_ ||
    527          (core_->upload_content_type_.empty() &&
    528           core_->upload_content_.empty()));
    529   core_->upload_content_type_ = content_type;
    530   core_->upload_content_.clear();
    531   core_->is_chunked_upload_ = true;
    532 }
    533 
    534 void URLFetcher::AppendChunkToUpload(const std::string& data,
    535                                      bool is_last_chunk) {
    536   DCHECK(data.length());
    537   core_->AppendChunkToUpload(data, is_last_chunk);
    538 }
    539 
    540 const std::string& URLFetcher::upload_data() const {
    541   return core_->upload_content_;
    542 }
    543 
    544 void URLFetcher::set_referrer(const std::string& referrer) {
    545   core_->referrer_ = referrer;
    546 }
    547 
    548 void URLFetcher::set_load_flags(int load_flags) {
    549   core_->load_flags_ = load_flags;
    550 }
    551 
    552 int URLFetcher::load_flags() const {
    553   return core_->load_flags_;
    554 }
    555 
    556 void URLFetcher::set_extra_request_headers(
    557     const std::string& extra_request_headers) {
    558   core_->extra_request_headers_.Clear();
    559   core_->extra_request_headers_.AddHeadersFromString(extra_request_headers);
    560 }
    561 
    562 void URLFetcher::set_request_context(
    563     net::URLRequestContextGetter* request_context_getter) {
    564   core_->request_context_getter_ = request_context_getter;
    565 }
    566 
    567 #ifdef ANDROID
    568 net::URLRequestContextGetter* URLFetcher::request_context() {
    569     return core_->request_context_getter_;
    570 }
    571 #endif
    572 
    573 void URLFetcher::set_automatically_retry_on_5xx(bool retry) {
    574   automatically_retry_on_5xx_ = retry;
    575 }
    576 
    577 net::HttpResponseHeaders* URLFetcher::response_headers() const {
    578   return core_->response_headers_;
    579 }
    580 
    581 void URLFetcher::Start() {
    582   core_->Start();
    583 }
    584 
    585 const GURL& URLFetcher::url() const {
    586   return core_->url_;
    587 }
    588 
    589 void URLFetcher::ReceivedContentWasMalformed() {
    590   core_->ReceivedContentWasMalformed();
    591 }
    592 
    593 // static
    594 void URLFetcher::CancelAll() {
    595   Core::CancelAll();
    596 }
    597 
    598 // static
    599 int URLFetcher::GetNumFetcherCores() {
    600   return Core::g_registry.Get().size();
    601 }
    602 
    603 URLFetcher::Delegate* URLFetcher::delegate() const {
    604   return core_->delegate();
    605 }
    606