Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #ifndef NET_URL_REQUEST_URL_FETCHER_CORE_H_
      6 #define NET_URL_REQUEST_URL_FETCHER_CORE_H_
      7 
      8 #include <set>
      9 #include <string>
     10 
     11 #include "base/basictypes.h"
     12 #include "base/compiler_specific.h"
     13 #include "base/debug/stack_trace.h"
     14 #include "base/files/file_path.h"
     15 #include "base/lazy_instance.h"
     16 #include "base/memory/ref_counted.h"
     17 #include "base/memory/scoped_ptr.h"
     18 #include "base/timer/timer.h"
     19 #include "net/base/host_port_pair.h"
     20 #include "net/http/http_request_headers.h"
     21 #include "net/url_request/url_fetcher.h"
     22 #include "net/url_request/url_request.h"
     23 #include "net/url_request/url_request_status.h"
     24 #include "url/gurl.h"
     25 
     26 namespace base {
     27 class SingleThreadTaskRunner;
     28 }  // namespace base
     29 
     30 namespace net {
     31 class DrainableIOBuffer;
     32 class HttpResponseHeaders;
     33 class IOBuffer;
     34 class URLFetcherDelegate;
     35 class URLFetcherResponseWriter;
     36 class URLRequestContextGetter;
     37 class URLRequestThrottlerEntryInterface;
     38 
     39 class URLFetcherCore
     40     : public base::RefCountedThreadSafe<URLFetcherCore>,
     41       public URLRequest::Delegate {
     42  public:
     43   URLFetcherCore(URLFetcher* fetcher,
     44                  const GURL& original_url,
     45                  URLFetcher::RequestType request_type,
     46                  URLFetcherDelegate* d);
     47 
     48   // Starts the load. It's important that this not happen in the constructor
     49   // because it causes the IO thread to begin AddRef()ing and Release()ing
     50   // us. If our caller hasn't had time to fully construct us and take a
     51   // reference, the IO thread could interrupt things, run a task, Release()
     52   // us, and destroy us, leaving the caller with an already-destroyed object
     53   // when construction finishes.
     54   void Start();
     55 
     56   // Stops any in-progress load and ensures no callback will happen. It is
     57   // safe to call this multiple times.
     58   void Stop();
     59 
     60   // URLFetcher-like functions.
     61 
     62   // For POST requests, set |content_type| to the MIME type of the
     63   // content and set |content| to the data to upload.
     64   void SetUploadData(const std::string& upload_content_type,
     65                      const std::string& upload_content);
     66   void SetUploadFilePath(const std::string& upload_content_type,
     67                          const base::FilePath& file_path,
     68                          uint64 range_offset,
     69                          uint64 range_length,
     70                          scoped_refptr<base::TaskRunner> file_task_runner);
     71   void SetChunkedUpload(const std::string& upload_content_type);
     72   // Adds a block of data to be uploaded in a POST body. This can only be
     73   // called after Start().
     74   void AppendChunkToUpload(const std::string& data, bool is_last_chunk);
     75   // |flags| are flags to apply to the load operation--these should be
     76   // one or more of the LOAD_* flags defined in net/base/load_flags.h.
     77   void SetLoadFlags(int load_flags);
     78   int GetLoadFlags() const;
     79   void SetReferrer(const std::string& referrer);
     80   void SetReferrerPolicy(URLRequest::ReferrerPolicy referrer_policy);
     81   void SetExtraRequestHeaders(const std::string& extra_request_headers);
     82   void AddExtraRequestHeader(const std::string& header_line);
     83   void SetRequestContext(URLRequestContextGetter* request_context_getter);
     84   // Set the URL that should be consulted for the third-party cookie
     85   // blocking policy.
     86   void SetFirstPartyForCookies(const GURL& first_party_for_cookies);
     87   // Set the key and data callback that is used when setting the user
     88   // data on any URLRequest objects this object creates.
     89   void SetURLRequestUserData(
     90       const void* key,
     91       const URLFetcher::CreateDataCallback& create_data_callback);
     92   void SetStopOnRedirect(bool stop_on_redirect);
     93   void SetAutomaticallyRetryOn5xx(bool retry);
     94   void SetMaxRetriesOn5xx(int max_retries);
     95   int GetMaxRetriesOn5xx() const;
     96   base::TimeDelta GetBackoffDelay() const;
     97   void SetAutomaticallyRetryOnNetworkChanges(int max_retries);
     98   void SaveResponseToFileAtPath(
     99       const base::FilePath& file_path,
    100       scoped_refptr<base::SequencedTaskRunner> file_task_runner);
    101   void SaveResponseToTemporaryFile(
    102       scoped_refptr<base::SequencedTaskRunner> file_task_runner);
    103   void SaveResponseWithWriter(
    104       scoped_ptr<URLFetcherResponseWriter> response_writer);
    105   HttpResponseHeaders* GetResponseHeaders() const;
    106   HostPortPair GetSocketAddress() const;
    107   bool WasFetchedViaProxy() const;
    108   const GURL& GetOriginalURL() const;
    109   const GURL& GetURL() const;
    110   const URLRequestStatus& GetStatus() const;
    111   int GetResponseCode() const;
    112   const ResponseCookies& GetCookies() const;
    113   // Reports that the received content was malformed (i.e. failed parsing
    114   // or validation). This makes the throttling logic that does exponential
    115   // back-off when servers are having problems treat the current request as
    116   // a failure. Your call to this method will be ignored if your request is
    117   // already considered a failure based on the HTTP response code or response
    118   // headers.
    119   void ReceivedContentWasMalformed();
    120   bool GetResponseAsString(std::string* out_response_string) const;
    121   bool GetResponseAsFilePath(bool take_ownership,
    122                              base::FilePath* out_response_path);
    123 
    124   // Overridden from URLRequest::Delegate:
    125   virtual void OnReceivedRedirect(URLRequest* request,
    126                                   const RedirectInfo& redirect_info,
    127                                   bool* defer_redirect) OVERRIDE;
    128   virtual void OnResponseStarted(URLRequest* request) OVERRIDE;
    129   virtual void OnReadCompleted(URLRequest* request,
    130                                int bytes_read) OVERRIDE;
    131   virtual void OnCertificateRequested(
    132       URLRequest* request,
    133       SSLCertRequestInfo* cert_request_info) OVERRIDE;
    134 
    135   URLFetcherDelegate* delegate() const { return delegate_; }
    136   static void CancelAll();
    137   static int GetNumFetcherCores();
    138   static void SetEnableInterceptionForTests(bool enabled);
    139   static void SetIgnoreCertificateRequests(bool ignored);
    140 
    141  private:
    142   friend class base::RefCountedThreadSafe<URLFetcherCore>;
    143 
    144   class Registry {
    145    public:
    146     Registry();
    147     ~Registry();
    148 
    149     void AddURLFetcherCore(URLFetcherCore* core);
    150     void RemoveURLFetcherCore(URLFetcherCore* core);
    151 
    152     void CancelAll();
    153 
    154     int size() const {
    155       return fetchers_.size();
    156     }
    157 
    158    private:
    159     std::set<URLFetcherCore*> fetchers_;
    160 
    161     DISALLOW_COPY_AND_ASSIGN(Registry);
    162   };
    163 
    164   virtual ~URLFetcherCore();
    165 
    166   // Wrapper functions that allow us to ensure actions happen on the right
    167   // thread.
    168   void StartOnIOThread();
    169   void StartURLRequest();
    170   void DidInitializeWriter(int result);
    171   void StartURLRequestWhenAppropriate();
    172   void CancelURLRequest(int error);
    173   void OnCompletedURLRequest(base::TimeDelta backoff_delay);
    174   void InformDelegateFetchIsComplete();
    175   void NotifyMalformedContent();
    176   void DidFinishWriting(int result);
    177   void RetryOrCompleteUrlFetch();
    178 
    179   // Deletes the request, removes it from the registry, and removes the
    180   // destruction observer.
    181   void ReleaseRequest();
    182 
    183   // Returns the max value of exponential back-off release time for
    184   // |original_url_| and |url_|.
    185   base::TimeTicks GetBackoffReleaseTime();
    186 
    187   void CompleteAddingUploadDataChunk(const std::string& data,
    188                                      bool is_last_chunk);
    189 
    190   // Writes all bytes stored in |data| with |response_writer_|.
    191   // Returns OK if all bytes in |data| get written synchronously. Otherwise,
    192   // returns ERR_IO_PENDING or a network error code.
    193   int WriteBuffer(scoped_refptr<DrainableIOBuffer> data);
    194 
    195   // Used to implement WriteBuffer().
    196   void DidWriteBuffer(scoped_refptr<DrainableIOBuffer> data, int result);
    197 
    198   // Read response bytes from the request.
    199   void ReadResponse();
    200 
    201   // Notify Delegate about the progress of upload/download.
    202   void InformDelegateUploadProgress();
    203   void InformDelegateUploadProgressInDelegateThread(int64 current, int64 total);
    204   void InformDelegateDownloadProgress();
    205   void InformDelegateDownloadProgressInDelegateThread(int64 current,
    206                                                       int64 total);
    207 
    208   URLFetcher* fetcher_;              // Corresponding fetcher object
    209   GURL original_url_;                // The URL we were asked to fetch
    210   GURL url_;                         // The URL we eventually wound up at
    211   URLFetcher::RequestType request_type_;  // What type of request is this?
    212   URLRequestStatus status_;          // Status of the request
    213   URLFetcherDelegate* delegate_;     // Object to notify on completion
    214   // Task runner for the creating thread. Used to interact with the delegate.
    215   scoped_refptr<base::SingleThreadTaskRunner> delegate_task_runner_;
    216   // Task runner for network operations.
    217   scoped_refptr<base::SingleThreadTaskRunner> network_task_runner_;
    218   // Task runner for upload file access.
    219   scoped_refptr<base::TaskRunner> upload_file_task_runner_;
    220   scoped_ptr<URLRequest> request_;   // The actual request this wraps
    221   int load_flags_;                   // Flags for the load operation
    222   int response_code_;                // HTTP status code for the request
    223   scoped_refptr<IOBuffer> buffer_;
    224                                      // Read buffer
    225   scoped_refptr<URLRequestContextGetter> request_context_getter_;
    226                                      // Cookie/cache info for the request
    227   GURL first_party_for_cookies_;     // The first party URL for the request
    228   // The user data to add to each newly-created URLRequest.
    229   const void* url_request_data_key_;
    230   URLFetcher::CreateDataCallback url_request_create_data_callback_;
    231   ResponseCookies cookies_;          // Response cookies
    232   HttpRequestHeaders extra_request_headers_;
    233   scoped_refptr<HttpResponseHeaders> response_headers_;
    234   bool was_fetched_via_proxy_;
    235   HostPortPair socket_address_;
    236 
    237   bool upload_content_set_;          // SetUploadData has been called
    238   std::string upload_content_;       // HTTP POST payload
    239   base::FilePath upload_file_path_;  // Path to file containing POST payload
    240   uint64 upload_range_offset_;       // Offset from the beginning of the file
    241                                      // to be uploaded.
    242   uint64 upload_range_length_;       // The length of the part of file to be
    243                                      // uploaded.
    244   std::string upload_content_type_;  // MIME type of POST payload
    245   std::string referrer_;             // HTTP Referer header value and policy
    246   URLRequest::ReferrerPolicy referrer_policy_;
    247   bool is_chunked_upload_;           // True if using chunked transfer encoding
    248 
    249   // Used to determine how long to wait before making a request or doing a
    250   // retry.
    251   //
    252   // Both of them can only be accessed on the IO thread.
    253   //
    254   // To determine the proper backoff timing, throttler entries for
    255   // both |original_URL| and |url| are needed. For example, consider
    256   // the case that URL A redirects to URL B, for which the server
    257   // returns a 500 response. In this case, the exponential back-off
    258   // release time of URL A won't increase. If only the backoff
    259   // constraints for URL A are considered, too many requests for URL A
    260   // may be sent in a short period of time.
    261   //
    262   // Both of these will be NULL if
    263   // URLRequestContext::throttler_manager() is NULL.
    264   scoped_refptr<URLRequestThrottlerEntryInterface>
    265       original_url_throttler_entry_;
    266   scoped_refptr<URLRequestThrottlerEntryInterface> url_throttler_entry_;
    267 
    268   // True if the URLFetcher has been cancelled.
    269   bool was_cancelled_;
    270 
    271   // Writer object to write response to the destination like file and string.
    272   scoped_ptr<URLFetcherResponseWriter> response_writer_;
    273 
    274   // By default any server-initiated redirects are automatically followed. If
    275   // this flag is set to true, however, a redirect will halt the fetch and call
    276   // back to to the delegate immediately.
    277   bool stop_on_redirect_;
    278   // True when we're actually stopped due to a redirect halted by the above. We
    279   // use this to ensure that |url_| is set to the redirect destination rather
    280   // than the originally-fetched URL.
    281   bool stopped_on_redirect_;
    282 
    283   // If |automatically_retry_on_5xx_| is false, 5xx responses will be
    284   // propagated to the observer, if it is true URLFetcher will automatically
    285   // re-execute the request, after the back-off delay has expired.
    286   // true by default.
    287   bool automatically_retry_on_5xx_;
    288   // |num_retries_on_5xx_| indicates how many times we've failed to successfully
    289   // fetch this URL due to 5xx responses. Once this value exceeds the maximum
    290   // number of retries specified by the owner URLFetcher instance,
    291   // we'll give up.
    292   int num_retries_on_5xx_;
    293   // Maximum retries allowed when 5xx responses are received.
    294   int max_retries_on_5xx_;
    295   // Back-off time delay. 0 by default.
    296   base::TimeDelta backoff_delay_;
    297 
    298   // The number of retries that have been attempted due to ERR_NETWORK_CHANGED.
    299   int num_retries_on_network_changes_;
    300   // Maximum retries allowed when the request fails with ERR_NETWORK_CHANGED.
    301   // 0 by default.
    302   int max_retries_on_network_changes_;
    303 
    304   // Timer to poll the progress of uploading for POST and PUT requests.
    305   // When crbug.com/119629 is fixed, scoped_ptr is not necessary here.
    306   scoped_ptr<base::RepeatingTimer<URLFetcherCore> >
    307       upload_progress_checker_timer_;
    308   // Number of bytes sent so far.
    309   int64 current_upload_bytes_;
    310   // Number of bytes received so far.
    311   int64 current_response_bytes_;
    312   // Total expected bytes to receive (-1 if it cannot be determined).
    313   int64 total_response_bytes_;
    314 
    315   // TODO(willchan): Get rid of this after debugging crbug.com/90971.
    316   base::debug::StackTrace stack_trace_;
    317 
    318   static base::LazyInstance<Registry> g_registry;
    319 
    320   DISALLOW_COPY_AND_ASSIGN(URLFetcherCore);
    321 };
    322 
    323 }  // namespace net
    324 
    325 #endif  // NET_URL_REQUEST_URL_FETCHER_CORE_H_
    326