Home | History | Annotate | Download | only in url_request 
      1 // Copyright (c) 2010 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "net/url_request/url_request_http_job.h"
      6 
      7 #include "base/base_switches.h"
      8 #include "base/command_line.h"
      9 #include "base/compiler_specific.h"
     10 #include "base/file_util.h"
     11 #include "base/file_version_info.h"
     12 #include "base/message_loop.h"
     13 #include "base/rand_util.h"
     14 #include "base/string_util.h"
     15 #include "net/base/cert_status_flags.h"
     16 #include "net/base/cookie_policy.h"
     17 #include "net/base/filter.h"
     18 #include "net/base/https_prober.h"
     19 #include "net/base/transport_security_state.h"
     20 #include "net/base/load_flags.h"
     21 #include "net/base/net_errors.h"
     22 #include "net/base/net_util.h"
     23 #include "net/base/sdch_manager.h"
     24 #include "net/base/ssl_cert_request_info.h"
     25 #include "net/http/http_response_headers.h"
     26 #include "net/http/http_response_info.h"
     27 #include "net/http/http_transaction.h"
     28 #include "net/http/http_transaction_factory.h"
     29 #include "net/http/http_util.h"
     30 #include "net/url_request/url_request.h"
     31 #include "net/url_request/url_request_context.h"
     32 #include "net/url_request/url_request_error_job.h"
     33 #include "net/url_request/url_request_redirect_job.h"
     34 
     35 // TODO(darin): make sure the port blocking code is not lost
     36 // static
     37 URLRequestJob* URLRequestHttpJob::Factory(URLRequest* request,
     38                                           const std::string& scheme) {
     39   DCHECK(scheme == "http" || scheme == "https");
     40 
     41   int port = request->url().IntPort();
     42   if (!net::IsPortAllowedByDefault(port) && !net::IsPortAllowedByOverride(port))
     43     return new URLRequestErrorJob(request, net::ERR_UNSAFE_PORT);
     44 
     45   if (!request->context() ||
     46       !request->context()->http_transaction_factory()) {
     47     NOTREACHED() << "requires a valid context";
     48     return new URLRequestErrorJob(request, net::ERR_INVALID_ARGUMENT);
     49   }
     50 
     51   net::TransportSecurityState::DomainState domain_state;
     52   if (scheme == "http" &&
     53       (request->url().port().empty() || port == 80) &&
     54       request->context()->transport_security_state() &&
     55       request->context()->transport_security_state()->IsEnabledForHost(
     56           &domain_state, request->url().host())) {
     57     if (domain_state.mode ==
     58          net::TransportSecurityState::DomainState::MODE_STRICT) {
     59       DCHECK_EQ(request->url().scheme(), "http");
     60       url_canon::Replacements<char> replacements;
     61       static const char kNewScheme[] = "https";
     62       replacements.SetScheme(kNewScheme,
     63                              url_parse::Component(0, strlen(kNewScheme)));
     64       GURL new_location = request->url().ReplaceComponents(replacements);
     65       return new URLRequestRedirectJob(request, new_location);
     66     } else {
     67       // TODO(agl): implement opportunistic HTTPS upgrade.
     68     }
     69   }
     70 
     71   return new URLRequestHttpJob(request);
     72 }
     73 
     74 URLRequestHttpJob::URLRequestHttpJob(URLRequest* request)
     75     : URLRequestJob(request),
     76       context_(request->context()),
     77       response_info_(NULL),
     78       response_cookies_save_index_(0),
     79       proxy_auth_state_(net::AUTH_STATE_DONT_NEED_AUTH),
     80       server_auth_state_(net::AUTH_STATE_DONT_NEED_AUTH),
     81       ALLOW_THIS_IN_INITIALIZER_LIST(can_get_cookies_callback_(
     82           this, &URLRequestHttpJob::OnCanGetCookiesCompleted)),
     83       ALLOW_THIS_IN_INITIALIZER_LIST(can_set_cookie_callback_(
     84           this, &URLRequestHttpJob::OnCanSetCookieCompleted)),
     85       ALLOW_THIS_IN_INITIALIZER_LIST(start_callback_(
     86           this, &URLRequestHttpJob::OnStartCompleted)),
     87       ALLOW_THIS_IN_INITIALIZER_LIST(read_callback_(
     88           this, &URLRequestHttpJob::OnReadCompleted)),
     89       read_in_progress_(false),
     90       transaction_(NULL),
     91       sdch_dictionary_advertised_(false),
     92       sdch_test_activated_(false),
     93       sdch_test_control_(false),
     94       is_cached_content_(false) {
     95 }
     96 
     97 URLRequestHttpJob::~URLRequestHttpJob() {
     98   DCHECK(!sdch_test_control_ || !sdch_test_activated_);
     99   if (!IsCachedContent()) {
    100     if (sdch_test_control_)
    101       RecordPacketStats(SDCH_EXPERIMENT_HOLDBACK);
    102     if (sdch_test_activated_)
    103       RecordPacketStats(SDCH_EXPERIMENT_DECODE);
    104   }
    105   // Make sure SDCH filters are told to emit histogram data while this class
    106   // can still service the IsCachedContent() call.
    107   DestroyFilters();
    108 
    109   if (sdch_dictionary_url_.is_valid()) {
    110     // Prior to reaching the destructor, request_ has been set to a NULL
    111     // pointer, so request_->url() is no longer valid in the destructor, and we
    112     // use an alternate copy |request_info_.url|.
    113     SdchManager* manager = SdchManager::Global();
    114     // To be extra safe, since this is a "different time" from when we decided
    115     // to get the dictionary, we'll validate that an SdchManager is available.
    116     // At shutdown time, care is taken to be sure that we don't delete this
    117     // globally useful instance "too soon," so this check is just defensive
    118     // coding to assure that IF the system is shutting down, we don't have any
    119     // problem if the manager was deleted ahead of time.
    120     if (manager)  // Defensive programming.
    121       manager->FetchDictionary(request_info_.url, sdch_dictionary_url_);
    122   }
    123 }
    124 
    125 void URLRequestHttpJob::SetUpload(net::UploadData* upload) {
    126   DCHECK(!transaction_.get()) << "cannot change once started";
    127   request_info_.upload_data = upload;
    128 }
    129 
    130 void URLRequestHttpJob::SetExtraRequestHeaders(
    131     const std::string& headers) {
    132   DCHECK(!transaction_.get()) << "cannot change once started";
    133   request_info_.extra_headers = headers;
    134 }
    135 
    136 void URLRequestHttpJob::Start() {
    137   DCHECK(!transaction_.get());
    138 
    139   // Ensure that we do not send username and password fields in the referrer.
    140   GURL referrer(request_->GetSanitizedReferrer());
    141 
    142   request_info_.url = request_->url();
    143   request_info_.referrer = referrer;
    144   request_info_.method = request_->method();
    145   request_info_.load_flags = request_->load_flags();
    146   request_info_.priority = request_->priority();
    147 
    148   if (request_->context()) {
    149     request_info_.user_agent =
    150         request_->context()->GetUserAgent(request_->url());
    151   }
    152 
    153   AddExtraHeaders();
    154   AddCookieHeaderAndStart();
    155 }
    156 
    157 void URLRequestHttpJob::Kill() {
    158   if (!transaction_.get())
    159     return;
    160 
    161   DestroyTransaction();
    162   URLRequestJob::Kill();
    163 }
    164 
    165 net::LoadState URLRequestHttpJob::GetLoadState() const {
    166   return transaction_.get() ?
    167       transaction_->GetLoadState() : net::LOAD_STATE_IDLE;
    168 }
    169 
    170 uint64 URLRequestHttpJob::GetUploadProgress() const {
    171   return transaction_.get() ? transaction_->GetUploadProgress() : 0;
    172 }
    173 
    174 bool URLRequestHttpJob::GetMimeType(std::string* mime_type) const {
    175   DCHECK(transaction_.get());
    176 
    177   if (!response_info_)
    178     return false;
    179 
    180   return response_info_->headers->GetMimeType(mime_type);
    181 }
    182 
    183 bool URLRequestHttpJob::GetCharset(std::string* charset) {
    184   DCHECK(transaction_.get());
    185 
    186   if (!response_info_)
    187     return false;
    188 
    189   return response_info_->headers->GetCharset(charset);
    190 }
    191 
    192 void URLRequestHttpJob::GetResponseInfo(net::HttpResponseInfo* info) {
    193   DCHECK(request_);
    194   DCHECK(transaction_.get());
    195 
    196   if (response_info_)
    197     *info = *response_info_;
    198 }
    199 
    200 bool URLRequestHttpJob::GetResponseCookies(
    201     std::vector<std::string>* cookies) {
    202   DCHECK(transaction_.get());
    203 
    204   if (!response_info_)
    205     return false;
    206 
    207   // TODO(darin): Why are we extracting response cookies again?  Perhaps we
    208   // should just leverage response_cookies_.
    209 
    210   cookies->clear();
    211   FetchResponseCookies(response_info_, cookies);
    212   return true;
    213 }
    214 
    215 int URLRequestHttpJob::GetResponseCode() const {
    216   DCHECK(transaction_.get());
    217 
    218   if (!response_info_)
    219     return -1;
    220 
    221   return response_info_->headers->response_code();
    222 }
    223 
    224 bool URLRequestHttpJob::GetContentEncodings(
    225     std::vector<Filter::FilterType>* encoding_types) {
    226   DCHECK(transaction_.get());
    227   if (!response_info_)
    228     return false;
    229   DCHECK(encoding_types->empty());
    230 
    231   std::string encoding_type;
    232   void* iter = NULL;
    233   while (response_info_->headers->EnumerateHeader(&iter, "Content-Encoding",
    234                                                   &encoding_type)) {
    235     encoding_types->push_back(Filter::ConvertEncodingToType(encoding_type));
    236   }
    237 
    238   // Even if encoding types are empty, there is a chance that we need to add
    239   // some decoding, as some proxies strip encoding completely. In such cases,
    240   // we may need to add (for example) SDCH filtering (when the context suggests
    241   // it is appropriate).
    242   Filter::FixupEncodingTypes(*this, encoding_types);
    243 
    244   return !encoding_types->empty();
    245 }
    246 
    247 bool URLRequestHttpJob::IsSdchResponse() const {
    248   return sdch_dictionary_advertised_;
    249 }
    250 
    251 bool URLRequestHttpJob::IsSafeRedirect(const GURL& location) {
    252   // We only allow redirects to certain "safe" protocols.  This does not
    253   // restrict redirects to externally handled protocols.  Our consumer would
    254   // need to take care of those.
    255 
    256   if (!URLRequest::IsHandledURL(location))
    257     return true;
    258 
    259   static const char* kSafeSchemes[] = {
    260     "http",
    261     "https",
    262     "ftp"
    263   };
    264 
    265   for (size_t i = 0; i < arraysize(kSafeSchemes); ++i) {
    266     if (location.SchemeIs(kSafeSchemes[i]))
    267       return true;
    268   }
    269 
    270   return false;
    271 }
    272 
    273 bool URLRequestHttpJob::NeedsAuth() {
    274   int code = GetResponseCode();
    275   if (code == -1)
    276     return false;
    277 
    278   // Check if we need either Proxy or WWW Authentication.  This could happen
    279   // because we either provided no auth info, or provided incorrect info.
    280   switch (code) {
    281     case 407:
    282       if (proxy_auth_state_ == net::AUTH_STATE_CANCELED)
    283         return false;
    284       proxy_auth_state_ = net::AUTH_STATE_NEED_AUTH;
    285       return true;
    286     case 401:
    287       if (server_auth_state_ == net::AUTH_STATE_CANCELED)
    288         return false;
    289       server_auth_state_ = net::AUTH_STATE_NEED_AUTH;
    290       return true;
    291   }
    292   return false;
    293 }
    294 
    295 void URLRequestHttpJob::GetAuthChallengeInfo(
    296     scoped_refptr<net::AuthChallengeInfo>* result) {
    297   DCHECK(transaction_.get());
    298   DCHECK(response_info_);
    299 
    300   // sanity checks:
    301   DCHECK(proxy_auth_state_ == net::AUTH_STATE_NEED_AUTH ||
    302          server_auth_state_ == net::AUTH_STATE_NEED_AUTH);
    303   DCHECK(response_info_->headers->response_code() == 401 ||
    304          response_info_->headers->response_code() == 407);
    305 
    306   *result = response_info_->auth_challenge;
    307 }
    308 
    309 void URLRequestHttpJob::SetAuth(const std::wstring& username,
    310                                 const std::wstring& password) {
    311   DCHECK(transaction_.get());
    312 
    313   // Proxy gets set first, then WWW.
    314   if (proxy_auth_state_ == net::AUTH_STATE_NEED_AUTH) {
    315     proxy_auth_state_ = net::AUTH_STATE_HAVE_AUTH;
    316   } else {
    317     DCHECK(server_auth_state_ == net::AUTH_STATE_NEED_AUTH);
    318     server_auth_state_ = net::AUTH_STATE_HAVE_AUTH;
    319   }
    320 
    321   RestartTransactionWithAuth(username, password);
    322 }
    323 
    324 void URLRequestHttpJob::RestartTransactionWithAuth(
    325     const std::wstring& username,
    326     const std::wstring& password) {
    327   username_ = username;
    328   password_ = password;
    329 
    330   // These will be reset in OnStartCompleted.
    331   response_info_ = NULL;
    332   response_cookies_.clear();
    333 
    334   // Update the cookies, since the cookie store may have been updated from the
    335   // headers in the 401/407. Since cookies were already appended to
    336   // extra_headers, we need to strip them out before adding them again.
    337   static const char* const cookie_name[] = { "cookie" };
    338   request_info_.extra_headers = net::HttpUtil::StripHeaders(
    339       request_info_.extra_headers, cookie_name, arraysize(cookie_name));
    340 
    341   AddCookieHeaderAndStart();
    342 }
    343 
    344 void URLRequestHttpJob::CancelAuth() {
    345   // Proxy gets set first, then WWW.
    346   if (proxy_auth_state_ == net::AUTH_STATE_NEED_AUTH) {
    347     proxy_auth_state_ = net::AUTH_STATE_CANCELED;
    348   } else {
    349     DCHECK(server_auth_state_ == net::AUTH_STATE_NEED_AUTH);
    350     server_auth_state_ = net::AUTH_STATE_CANCELED;
    351   }
    352 
    353   // These will be reset in OnStartCompleted.
    354   response_info_ = NULL;
    355   response_cookies_.clear();
    356 
    357   // OK, let the consumer read the error page...
    358   //
    359   // Because we set the AUTH_STATE_CANCELED flag, NeedsAuth will return false,
    360   // which will cause the consumer to receive OnResponseStarted instead of
    361   // OnAuthRequired.
    362   //
    363   // We have to do this via InvokeLater to avoid "recursing" the consumer.
    364   //
    365   MessageLoop::current()->PostTask(FROM_HERE, NewRunnableMethod(
    366       this, &URLRequestHttpJob::OnStartCompleted, net::OK));
    367 }
    368 
    369 void URLRequestHttpJob::ContinueWithCertificate(
    370     net::X509Certificate* client_cert) {
    371   DCHECK(transaction_.get());
    372 
    373   DCHECK(!response_info_) << "should not have a response yet";
    374 
    375   // No matter what, we want to report our status as IO pending since we will
    376   // be notifying our consumer asynchronously via OnStartCompleted.
    377   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
    378 
    379   int rv = transaction_->RestartWithCertificate(client_cert, &start_callback_);
    380   if (rv == net::ERR_IO_PENDING)
    381     return;
    382 
    383   // The transaction started synchronously, but we need to notify the
    384   // URLRequest delegate via the message loop.
    385   MessageLoop::current()->PostTask(FROM_HERE, NewRunnableMethod(
    386       this, &URLRequestHttpJob::OnStartCompleted, rv));
    387 }
    388 
    389 void URLRequestHttpJob::ContinueDespiteLastError() {
    390   // If the transaction was destroyed, then the job was cancelled.
    391   if (!transaction_.get())
    392     return;
    393 
    394   DCHECK(!response_info_) << "should not have a response yet";
    395 
    396   // No matter what, we want to report our status as IO pending since we will
    397   // be notifying our consumer asynchronously via OnStartCompleted.
    398   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
    399 
    400   int rv = transaction_->RestartIgnoringLastError(&start_callback_);
    401   if (rv == net::ERR_IO_PENDING)
    402     return;
    403 
    404   // The transaction started synchronously, but we need to notify the
    405   // URLRequest delegate via the message loop.
    406   MessageLoop::current()->PostTask(FROM_HERE, NewRunnableMethod(
    407       this, &URLRequestHttpJob::OnStartCompleted, rv));
    408 }
    409 
    410 bool URLRequestHttpJob::ReadRawData(net::IOBuffer* buf, int buf_size,
    411                                     int *bytes_read) {
    412   DCHECK_NE(buf_size, 0);
    413   DCHECK(bytes_read);
    414   DCHECK(!read_in_progress_);
    415 
    416   int rv = transaction_->Read(buf, buf_size, &read_callback_);
    417   if (rv >= 0) {
    418     *bytes_read = rv;
    419     return true;
    420   }
    421 
    422   if (rv == net::ERR_IO_PENDING) {
    423     read_in_progress_ = true;
    424     SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
    425   } else {
    426     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, rv));
    427   }
    428 
    429   return false;
    430 }
    431 
    432 void URLRequestHttpJob::OnCanGetCookiesCompleted(int policy) {
    433   // If the request was destroyed, then there is no more work to do.
    434   if (request_ && request_->delegate()) {
    435     if (policy == net::OK && request_->context()->cookie_store()) {
    436       net::CookieOptions options;
    437       options.set_include_httponly();
    438       std::string cookies =
    439           request_->context()->cookie_store()->GetCookiesWithOptions(
    440               request_->url(), options);
    441       if (request_->context()->InterceptRequestCookies(request_, cookies) &&
    442           !cookies.empty())
    443         request_info_.extra_headers += "Cookie: " + cookies + "\r\n";
    444     }
    445     StartTransaction();
    446   }
    447   Release();  // Balance AddRef taken in AddCookieHeaderAndStart
    448 }
    449 
    450 void URLRequestHttpJob::OnCanSetCookieCompleted(int policy) {
    451   // If the request was destroyed, then there is no more work to do.
    452   if (request_ && request_->delegate()) {
    453     if (policy == net::OK && request_->context()->cookie_store()) {
    454       // OK to save the current response cookie now.
    455       net::CookieOptions options;
    456       options.set_include_httponly();
    457       request_->context()->cookie_store()->SetCookieWithOptions(
    458           request_->url(), response_cookies_[response_cookies_save_index_],
    459           options);
    460     }
    461     response_cookies_save_index_++;
    462     SaveNextCookie();
    463   }
    464   Release();  // Balance AddRef taken in SaveNextCookie
    465 }
    466 
    467 void URLRequestHttpJob::OnStartCompleted(int result) {
    468   // If the request was destroyed, then there is no more work to do.
    469   if (!request_ || !request_->delegate())
    470     return;
    471 
    472   // If the transaction was destroyed, then the job was cancelled, and
    473   // we can just ignore this notification.
    474   if (!transaction_.get())
    475     return;
    476 
    477   // Clear the IO_PENDING status
    478   SetStatus(URLRequestStatus());
    479 
    480   if (result == net::OK) {
    481     SaveCookiesAndNotifyHeadersComplete();
    482   } else if (ShouldTreatAsCertificateError(result)) {
    483     // We encountered an SSL certificate error.  Ask our delegate to decide
    484     // what we should do.
    485     // TODO(wtc): also pass ssl_info.cert_status, or just pass the whole
    486     // ssl_info.
    487     request_->delegate()->OnSSLCertificateError(
    488         request_, result, transaction_->GetResponseInfo()->ssl_info.cert);
    489   } else if (result == net::ERR_SSL_CLIENT_AUTH_CERT_NEEDED) {
    490     request_->delegate()->OnCertificateRequested(
    491         request_, transaction_->GetResponseInfo()->cert_request_info);
    492   } else {
    493     NotifyStartError(URLRequestStatus(URLRequestStatus::FAILED, result));
    494   }
    495 }
    496 
    497 void URLRequestHttpJob::OnReadCompleted(int result) {
    498   read_in_progress_ = false;
    499 
    500   if (result == 0) {
    501     NotifyDone(URLRequestStatus());
    502   } else if (result < 0) {
    503     NotifyDone(URLRequestStatus(URLRequestStatus::FAILED, result));
    504   } else {
    505     // Clear the IO_PENDING status
    506     SetStatus(URLRequestStatus());
    507   }
    508 
    509   NotifyReadComplete(result);
    510 }
    511 
    512 bool URLRequestHttpJob::ShouldTreatAsCertificateError(int result) {
    513   if (!net::IsCertificateError(result))
    514     return false;
    515 
    516   // Check whether our context is using Strict-Transport-Security.
    517   if (!context_->transport_security_state())
    518     return true;
    519 
    520   net::TransportSecurityState::DomainState domain_state;
    521   // TODO(agl): don't ignore opportunistic mode.
    522   const bool r = context_->transport_security_state()->IsEnabledForHost(
    523       &domain_state, request_info_.url.host());
    524 
    525   return !r || domain_state.mode ==
    526                net::TransportSecurityState::DomainState::MODE_OPPORTUNISTIC;
    527 }
    528 
    529 void URLRequestHttpJob::NotifyHeadersComplete() {
    530   DCHECK(!response_info_);
    531 
    532   response_info_ = transaction_->GetResponseInfo();
    533 
    534   // Save boolean, as we'll need this info at destruction time, and filters may
    535   // also need this info.
    536   is_cached_content_ = response_info_->was_cached;
    537 
    538   ProcessStrictTransportSecurityHeader();
    539 
    540   if (SdchManager::Global() &&
    541       SdchManager::Global()->IsInSupportedDomain(request_->url())) {
    542     static const std::string name = "Get-Dictionary";
    543     std::string url_text;
    544     void* iter = NULL;
    545     // TODO(jar): We need to not fetch dictionaries the first time they are
    546     // seen, but rather wait until we can justify their usefulness.
    547     // For now, we will only fetch the first dictionary, which will at least
    548     // require multiple suggestions before we get additional ones for this site.
    549     // Eventually we should wait until a dictionary is requested several times
    550     // before we even download it (so that we don't waste memory or bandwidth).
    551     if (response_info_->headers->EnumerateHeader(&iter, name, &url_text)) {
    552       // request_->url() won't be valid in the destructor, so we use an
    553       // alternate copy.
    554       DCHECK(request_->url() == request_info_.url);
    555       // Resolve suggested URL relative to request url.
    556       sdch_dictionary_url_ = request_info_.url.Resolve(url_text);
    557     }
    558   }
    559 
    560   // The HTTP transaction may be restarted several times for the purposes
    561   // of sending authorization information. Each time it restarts, we get
    562   // notified of the headers completion so that we can update the cookie store.
    563   if (transaction_->IsReadyToRestartForAuth()) {
    564     DCHECK(!response_info_->auth_challenge.get());
    565     RestartTransactionWithAuth(std::wstring(), std::wstring());
    566     return;
    567   }
    568 
    569   URLRequestJob::NotifyHeadersComplete();
    570 }
    571 
    572 #if defined(OS_WIN)
    573 #pragma optimize("", off)
    574 #pragma warning(disable:4748)
    575 #endif
    576 void URLRequestHttpJob::DestroyTransaction() {
    577   CHECK(transaction_.get());
    578   // TODO(rvargas): remove this after finding the cause for bug 31723.
    579   char local_obj[sizeof(*this)];
    580   memcpy(local_obj, this, sizeof(local_obj));
    581 
    582   transaction_.reset();
    583   response_info_ = NULL;
    584 }
    585 #if defined(OS_WIN)
    586 #pragma warning(default:4748)
    587 #pragma optimize("", on)
    588 #endif
    589 
    590 void URLRequestHttpJob::StartTransaction() {
    591   // NOTE: This method assumes that request_info_ is already setup properly.
    592 
    593   // If we already have a transaction, then we should restart the transaction
    594   // with auth provided by username_ and password_.
    595 
    596   int rv;
    597   if (transaction_.get()) {
    598     rv = transaction_->RestartWithAuth(username_, password_, &start_callback_);
    599     username_.clear();
    600     password_.clear();
    601   } else {
    602     DCHECK(request_->context());
    603     DCHECK(request_->context()->http_transaction_factory());
    604 
    605     rv = request_->context()->http_transaction_factory()->CreateTransaction(
    606         &transaction_);
    607     if (rv == net::OK) {
    608       rv = transaction_->Start(
    609           &request_info_, &start_callback_, request_->load_log());
    610     }
    611   }
    612 
    613   if (rv == net::ERR_IO_PENDING)
    614     return;
    615 
    616   // The transaction started synchronously, but we need to notify the
    617   // URLRequest delegate via the message loop.
    618   MessageLoop::current()->PostTask(FROM_HERE, NewRunnableMethod(
    619       this, &URLRequestHttpJob::OnStartCompleted, rv));
    620 }
    621 
    622 void URLRequestHttpJob::AddExtraHeaders() {
    623   // TODO(jar): Consider optimizing away SDCH advertising bytes when the URL is
    624   // probably an img or such (and SDCH encoding is not likely).
    625   bool advertise_sdch = SdchManager::Global() &&
    626       SdchManager::Global()->IsInSupportedDomain(request_->url());
    627   std::string avail_dictionaries;
    628   if (advertise_sdch) {
    629     SdchManager::Global()->GetAvailDictionaryList(request_->url(),
    630                                                   &avail_dictionaries);
    631 
    632     // The AllowLatencyExperiment() is only true if we've successfully done a
    633     // full SDCH compression recently in this browser session for this host.
    634     // Note that for this path, there might be no applicable dictionaries, and
    635     // hence we can't participate in the experiment.
    636     if (!avail_dictionaries.empty() &&
    637         SdchManager::Global()->AllowLatencyExperiment(request_->url())) {
    638       // We are participating in the test (or control), and hence we'll
    639       // eventually record statistics via either SDCH_EXPERIMENT_DECODE or
    640       // SDCH_EXPERIMENT_HOLDBACK, and we'll need some packet timing data.
    641       EnablePacketCounting(kSdchPacketHistogramCount);
    642       if (base::RandDouble() < .01) {
    643         sdch_test_control_ = true;  // 1% probability.
    644         advertise_sdch = false;
    645       } else {
    646         sdch_test_activated_ = true;
    647       }
    648     }
    649   }
    650 
    651   // Supply Accept-Encoding headers first so that it is more likely that they
    652   // will be in the first transmitted packet.  This can sometimes make it easier
    653   // to filter and analyze the streams to assure that a proxy has not damaged
    654   // these headers.  Some proxies deliberately corrupt Accept-Encoding headers.
    655   if (!advertise_sdch) {
    656     // Tell the server what compression formats we support (other than SDCH).
    657     request_info_.extra_headers += "Accept-Encoding: gzip,deflate\r\n";
    658   } else {
    659     // Include SDCH in acceptable list.
    660     request_info_.extra_headers += "Accept-Encoding: "
    661         "gzip,deflate,sdch\r\n";
    662     if (!avail_dictionaries.empty()) {
    663       request_info_.extra_headers += "Avail-Dictionary: "
    664           + avail_dictionaries + "\r\n";
    665       sdch_dictionary_advertised_ = true;
    666       // Since we're tagging this transaction as advertising a dictionary, we'll
    667       // definately employ an SDCH filter (or tentative sdch filter) when we get
    668       // a response.  When done, we'll record histograms via SDCH_DECODE or
    669       // SDCH_PASSTHROUGH.  Hence we need to record packet arrival times.
    670       EnablePacketCounting(kSdchPacketHistogramCount);
    671     }
    672   }
    673 
    674   URLRequestContext* context = request_->context();
    675   if (context) {
    676     // Only add default Accept-Language and Accept-Charset if the request
    677     // didn't have them specified.
    678     net::HttpUtil::AppendHeaderIfMissing("Accept-Language",
    679                                          context->accept_language(),
    680                                          &request_info_.extra_headers);
    681     net::HttpUtil::AppendHeaderIfMissing("Accept-Charset",
    682                                          context->accept_charset(),
    683                                          &request_info_.extra_headers);
    684   }
    685 }
    686 
    687 void URLRequestHttpJob::AddCookieHeaderAndStart() {
    688   // No matter what, we want to report our status as IO pending since we will
    689   // be notifying our consumer asynchronously via OnStartCompleted.
    690   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
    691 
    692   AddRef();  // Balanced in OnCanGetCookiesCompleted
    693 
    694   int policy = net::OK;
    695 
    696   if (request_info_.load_flags & net::LOAD_DO_NOT_SEND_COOKIES) {
    697     policy = net::ERR_ACCESS_DENIED;
    698   } else if (request_->context()->cookie_policy()) {
    699     policy = request_->context()->cookie_policy()->CanGetCookies(
    700         request_->url(),
    701         request_->first_party_for_cookies(),
    702         &can_get_cookies_callback_);
    703     if (policy == net::ERR_IO_PENDING)
    704       return;  // Wait for completion callback
    705   }
    706 
    707   OnCanGetCookiesCompleted(policy);
    708 }
    709 
    710 void URLRequestHttpJob::SaveCookiesAndNotifyHeadersComplete() {
    711   DCHECK(transaction_.get());
    712 
    713   const net::HttpResponseInfo* response_info = transaction_->GetResponseInfo();
    714   DCHECK(response_info);
    715 
    716   response_cookies_.clear();
    717   response_cookies_save_index_ = 0;
    718 
    719   FetchResponseCookies(response_info, &response_cookies_);
    720 
    721   // Now, loop over the response cookies, and attempt to persist each.
    722   SaveNextCookie();
    723 }
    724 
    725 void URLRequestHttpJob::SaveNextCookie() {
    726   if (response_cookies_save_index_ == response_cookies_.size()) {
    727     response_cookies_.clear();
    728     response_cookies_save_index_ = 0;
    729     SetStatus(URLRequestStatus());  // Clear the IO_PENDING status
    730     NotifyHeadersComplete();
    731     return;
    732   }
    733 
    734   // No matter what, we want to report our status as IO pending since we will
    735   // be notifying our consumer asynchronously via OnStartCompleted.
    736   SetStatus(URLRequestStatus(URLRequestStatus::IO_PENDING, 0));
    737 
    738   AddRef();  // Balanced in OnCanSetCookieCompleted
    739 
    740   int policy = net::OK;
    741 
    742   if (request_info_.load_flags & net::LOAD_DO_NOT_SAVE_COOKIES) {
    743     policy = net::ERR_ACCESS_DENIED;
    744   } else if (request_->context()->cookie_policy()) {
    745     policy = request_->context()->cookie_policy()->CanSetCookie(
    746         request_->url(),
    747         request_->first_party_for_cookies(),
    748         response_cookies_[response_cookies_save_index_],
    749         &can_set_cookie_callback_);
    750     if (policy == net::ERR_IO_PENDING)
    751       return;  // Wait for completion callback
    752   }
    753 
    754   OnCanSetCookieCompleted(policy);
    755 }
    756 
    757 void URLRequestHttpJob::FetchResponseCookies(
    758     const net::HttpResponseInfo* response_info,
    759     std::vector<std::string>* cookies) {
    760   std::string name = "Set-Cookie";
    761   std::string value;
    762 
    763   void* iter = NULL;
    764   while (response_info->headers->EnumerateHeader(&iter, name, &value)) {
    765     if (request_->context()->InterceptResponseCookie(request_, value))
    766       cookies->push_back(value);
    767   }
    768 }
    769 
    770 class HTTPSProberDelegate : public net::HTTPSProberDelegate {
    771  public:
    772   HTTPSProberDelegate(const std::string& host, int max_age,
    773                       bool include_subdomains,
    774                       net::TransportSecurityState* sts)
    775       : host_(host),
    776         max_age_(max_age),
    777         include_subdomains_(include_subdomains),
    778         sts_(sts) { }
    779 
    780   virtual void ProbeComplete(bool result) {
    781     if (result) {
    782       base::Time current_time(base::Time::Now());
    783       base::TimeDelta max_age_delta = base::TimeDelta::FromSeconds(max_age_);
    784 
    785       net::TransportSecurityState::DomainState domain_state;
    786       domain_state.expiry = current_time + max_age_delta;
    787       domain_state.mode =
    788           net::TransportSecurityState::DomainState::MODE_OPPORTUNISTIC;
    789       domain_state.include_subdomains = include_subdomains_;
    790 
    791       sts_->EnableHost(host_, domain_state);
    792     }
    793 
    794     delete this;
    795   }
    796 
    797  private:
    798   const std::string host_;
    799   const int max_age_;
    800   const bool include_subdomains_;
    801   scoped_refptr<net::TransportSecurityState> sts_;
    802 };
    803 
    804 void URLRequestHttpJob::ProcessStrictTransportSecurityHeader() {
    805   DCHECK(response_info_);
    806 
    807   URLRequestContext* ctx = request_->context();
    808   if (!ctx || !ctx->transport_security_state())
    809     return;
    810 
    811   const bool https = response_info_->ssl_info.is_valid();
    812   const bool valid_https =
    813       https &&
    814       !(response_info_->ssl_info.cert_status & net::CERT_STATUS_ALL_ERRORS);
    815 
    816   std::string name = "Strict-Transport-Security";
    817   std::string value;
    818 
    819   int max_age;
    820   bool include_subdomains;
    821 
    822   void* iter = NULL;
    823   while (response_info_->headers->EnumerateHeader(&iter, name, &value)) {
    824     const bool ok = net::TransportSecurityState::ParseHeader(
    825         value, &max_age, &include_subdomains);
    826     if (!ok)
    827       continue;
    828     // We will only accept strict mode if we saw the header from an HTTPS
    829     // connection with no certificate problems.
    830     if (!valid_https)
    831       continue;
    832     base::Time current_time(base::Time::Now());
    833     base::TimeDelta max_age_delta = base::TimeDelta::FromSeconds(max_age);
    834 
    835     net::TransportSecurityState::DomainState domain_state;
    836     domain_state.expiry = current_time + max_age_delta;
    837     domain_state.mode = net::TransportSecurityState::DomainState::MODE_STRICT;
    838     domain_state.include_subdomains = include_subdomains;
    839 
    840     ctx->transport_security_state()->EnableHost(request_info_.url.host(),
    841                                                 domain_state);
    842   }
    843 
    844   // TODO(agl): change this over when we have fixed things at the server end.
    845   // The string should be "Opportunistic-Transport-Security";
    846   name = "X-Bodge-Transport-Security";
    847 
    848   while (response_info_->headers->EnumerateHeader(&iter, name, &value)) {
    849     const bool ok = net::TransportSecurityState::ParseHeader(
    850         value, &max_age, &include_subdomains);
    851     if (!ok)
    852       continue;
    853     // If we saw an opportunistic request over HTTPS, then clearly we can make
    854     // HTTPS connections to the host so we should remember this.
    855     if (https) {
    856       base::Time current_time(base::Time::Now());
    857       base::TimeDelta max_age_delta = base::TimeDelta::FromSeconds(max_age);
    858 
    859       net::TransportSecurityState::DomainState domain_state;
    860       domain_state.expiry = current_time + max_age_delta;
    861       domain_state.mode =
    862           net::TransportSecurityState::DomainState::MODE_SPDY_ONLY;
    863       domain_state.include_subdomains = include_subdomains;
    864 
    865       ctx->transport_security_state()->EnableHost(request_info_.url.host(),
    866                                                   domain_state);
    867       continue;
    868     }
    869 
    870     if (!request())
    871       break;
    872 
    873     // At this point, we have a request for opportunistic encryption over HTTP.
    874     // In this case we need to probe to check that we can make HTTPS
    875     // connections to that host.
    876     net::HTTPSProber* const prober = Singleton<net::HTTPSProber>::get();
    877     if (prober->HaveProbed(request_info_.url.host()) ||
    878         prober->InFlight(request_info_.url.host())) {
    879       continue;
    880     }
    881 
    882     net::HTTPSProberDelegate* delegate =
    883         new HTTPSProberDelegate(request_info_.url.host(), max_age,
    884                                 include_subdomains,
    885                                 ctx->transport_security_state());
    886     if (!prober->ProbeHost(request_info_.url.host(), request()->context(),
    887                            delegate)) {
    888       delete delegate;
    889     }
    890   }
    891 }
    892