Home | History | Annotate | Download | only in automation 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/automation/url_request_automation_job.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/compiler_specific.h"
      9 #include "base/message_loop/message_loop.h"
     10 #include "base/time/time.h"
     11 #include "chrome/browser/automation/automation_resource_message_filter.h"
     12 #include "chrome/common/automation_messages.h"
     13 #include "content/public/browser/browser_thread.h"
     14 #include "content/public/browser/render_view_host.h"
     15 #include "content/public/browser/resource_request_info.h"
     16 #include "net/base/host_port_pair.h"
     17 #include "net/base/io_buffer.h"
     18 #include "net/base/net_errors.h"
     19 #include "net/base/upload_bytes_element_reader.h"
     20 #include "net/base/upload_data_stream.h"
     21 #include "net/base/upload_file_element_reader.h"
     22 #include "net/cookies/cookie_monster.h"
     23 #include "net/http/http_request_headers.h"
     24 #include "net/http/http_response_headers.h"
     25 #include "net/http/http_util.h"
     26 #include "net/url_request/http_user_agent_settings.h"
     27 #include "net/url_request/url_request.h"
     28 #include "net/url_request/url_request_context.h"
     29 
     30 using base::Time;
     31 using base::TimeDelta;
     32 using content::BrowserThread;
     33 using content::ResourceRequestInfo;
     34 
     35 namespace {
     36 
     37 // The list of filtered headers that are removed from requests sent via
     38 // StartAsync(). These must be lower case.
     39 const char* const kFilteredHeaderStrings[] = {
     40   "connection",
     41   "cookie",
     42   "expect",
     43   "max-forwards",
     44   "proxy-authorization",
     45   "referer",
     46   "te",
     47   "upgrade",
     48   "via"
     49 };
     50 
     51 // Creates UploadData from UploadDataStream.
     52 net::UploadData* CreateUploadData(
     53     const net::UploadDataStream* upload_data_stream) {
     54   net::UploadData* upload_data = new net::UploadData();
     55   const ScopedVector<net::UploadElementReader>& element_readers =
     56       upload_data_stream->element_readers();
     57   for (size_t i = 0; i < element_readers.size(); ++i) {
     58     const net::UploadElementReader* reader = element_readers[i];
     59     if (reader->AsBytesReader()) {
     60       const net::UploadBytesElementReader* bytes_reader =
     61           reader->AsBytesReader();
     62       upload_data->AppendBytes(bytes_reader->bytes(), bytes_reader->length());
     63     } else if (reader->AsFileReader()) {
     64       const net::UploadFileElementReader* file_reader =
     65           reader->AsFileReader();
     66       upload_data->AppendFileRange(file_reader->path(),
     67                                    file_reader->range_offset(),
     68                                    file_reader->range_length(),
     69                                    file_reader->expected_modification_time());
     70     } else {
     71       NOTIMPLEMENTED();
     72     }
     73   }
     74   upload_data->set_identifier(upload_data_stream->identifier());
     75   upload_data->set_is_chunked(upload_data_stream->is_chunked());
     76   upload_data->set_last_chunk_appended(
     77       upload_data_stream->last_chunk_appended());
     78   return upload_data;
     79 }
     80 
     81 }  // namespace
     82 
     83 int URLRequestAutomationJob::instance_count_ = 0;
     84 bool URLRequestAutomationJob::is_protocol_factory_registered_ = false;
     85 
     86 net::URLRequest::ProtocolFactory* URLRequestAutomationJob::old_http_factory_
     87     = NULL;
     88 net::URLRequest::ProtocolFactory* URLRequestAutomationJob::old_https_factory_
     89     = NULL;
     90 
     91 URLRequestAutomationJob::URLRequestAutomationJob(
     92     net::URLRequest* request,
     93     net::NetworkDelegate* network_delegate,
     94     const net::HttpUserAgentSettings* http_user_agent_settings,
     95     int tab,
     96     int request_id,
     97     AutomationResourceMessageFilter* filter,
     98     bool is_pending)
     99     : net::URLRequestJob(request, network_delegate),
    100       http_user_agent_settings_(http_user_agent_settings),
    101       id_(0),
    102       tab_(tab),
    103       message_filter_(filter),
    104       pending_buf_size_(0),
    105       redirect_status_(0),
    106       request_id_(request_id),
    107       is_pending_(is_pending),
    108       upload_size_(0),
    109       weak_factory_(this) {
    110   DVLOG(1) << "URLRequestAutomationJob create. Count: " << ++instance_count_;
    111   DCHECK(message_filter_.get() != NULL);
    112 
    113   if (message_filter_.get()) {
    114     id_ = message_filter_->NewAutomationRequestId();
    115     DCHECK_NE(id_, 0);
    116   }
    117 }
    118 
    119 URLRequestAutomationJob::~URLRequestAutomationJob() {
    120   DVLOG(1) << "URLRequestAutomationJob delete. Count: " << --instance_count_;
    121   Cleanup();
    122 }
    123 
    124 void URLRequestAutomationJob::EnsureProtocolFactoryRegistered() {
    125   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
    126 
    127   if (!is_protocol_factory_registered_) {
    128     old_http_factory_ =
    129         net::URLRequest::Deprecated::RegisterProtocolFactory(
    130             "http", &URLRequestAutomationJob::Factory);
    131     old_https_factory_ =
    132         net::URLRequest::Deprecated::RegisterProtocolFactory(
    133             "https", &URLRequestAutomationJob::Factory);
    134     is_protocol_factory_registered_ = true;
    135   }
    136 }
    137 
    138 net::URLRequestJob* URLRequestAutomationJob::Factory(
    139     net::URLRequest* request,
    140     net::NetworkDelegate* network_delegate,
    141     const std::string& scheme) {
    142   bool scheme_is_http = request->url().SchemeIs("http");
    143   bool scheme_is_https = request->url().SchemeIs("https");
    144 
    145   // Returning null here just means that the built-in handler will be used.
    146   if (scheme_is_http || scheme_is_https) {
    147     const ResourceRequestInfo* info = ResourceRequestInfo::ForRequest(request);
    148     if (info) {
    149       int child_id = info->GetChildID();
    150       int route_id = info->GetRouteID();
    151       AutomationResourceMessageFilter::AutomationDetails details;
    152       if (AutomationResourceMessageFilter::LookupRegisteredRenderView(
    153               child_id, route_id, &details)) {
    154         URLRequestAutomationJob* job = new URLRequestAutomationJob(
    155             request,
    156             network_delegate,
    157             request->context()->http_user_agent_settings(),
    158             details.tab_handle,
    159             info->GetRequestID(),
    160             details.filter.get(),
    161             details.is_pending_render_view);
    162         return job;
    163       }
    164     }
    165 
    166     if (scheme_is_http && old_http_factory_)
    167       return old_http_factory_(request, network_delegate, scheme);
    168     else if (scheme_is_https && old_https_factory_)
    169       return old_https_factory_(request, network_delegate, scheme);
    170   }
    171   return NULL;
    172 }
    173 
    174 // net::URLRequestJob Implementation.
    175 void URLRequestAutomationJob::Start() {
    176   if (!is_pending()) {
    177     // Start reading asynchronously so that all error reporting and data
    178     // callbacks happen as they would for network requests.
    179     base::MessageLoop::current()->PostTask(
    180         FROM_HERE,
    181         base::Bind(&URLRequestAutomationJob::StartAsync,
    182                    weak_factory_.GetWeakPtr()));
    183   } else {
    184     // If this is a pending job, then register it immediately with the message
    185     // filter so it can be serviced later when we receive a request from the
    186     // external host to connect to the corresponding external tab.
    187     message_filter_->RegisterRequest(this);
    188   }
    189 }
    190 
    191 void URLRequestAutomationJob::Kill() {
    192   if (message_filter_.get()) {
    193     if (!is_pending()) {
    194       message_filter_->Send(new AutomationMsg_RequestEnd(tab_, id_,
    195           net::URLRequestStatus(net::URLRequestStatus::CANCELED,
    196                                 net::ERR_ABORTED)));
    197     }
    198   }
    199   DisconnectFromMessageFilter();
    200   receive_headers_end_ = base::TimeTicks();
    201   net::URLRequestJob::Kill();
    202 }
    203 
    204 bool URLRequestAutomationJob::ReadRawData(
    205     net::IOBuffer* buf, int buf_size, int* bytes_read) {
    206   DVLOG(1) << "URLRequestAutomationJob: " << request_->url().spec()
    207            << " - read pending: " << buf_size;
    208 
    209   // We should not receive a read request for a pending job.
    210   DCHECK(!is_pending());
    211 
    212   pending_buf_ = buf;
    213   pending_buf_size_ = buf_size;
    214 
    215   if (message_filter_.get()) {
    216     message_filter_->Send(new AutomationMsg_RequestRead(tab_, id_, buf_size));
    217     SetStatus(net::URLRequestStatus(net::URLRequestStatus::IO_PENDING, 0));
    218   } else {
    219     base::MessageLoop::current()->PostTask(
    220         FROM_HERE,
    221         base::Bind(&URLRequestAutomationJob::NotifyJobCompletionTask,
    222                    weak_factory_.GetWeakPtr()));
    223   }
    224   return false;
    225 }
    226 
    227 bool URLRequestAutomationJob::GetMimeType(std::string* mime_type) const {
    228   if (!mime_type_.empty()) {
    229     *mime_type = mime_type_;
    230   } else if (headers_.get()) {
    231     headers_->GetMimeType(mime_type);
    232   }
    233 
    234   return (!mime_type->empty());
    235 }
    236 
    237 bool URLRequestAutomationJob::GetCharset(std::string* charset) {
    238   if (headers_.get())
    239     return headers_->GetCharset(charset);
    240   return false;
    241 }
    242 
    243 void URLRequestAutomationJob::GetResponseInfo(net::HttpResponseInfo* info) {
    244   if (headers_.get())
    245     info->headers = headers_;
    246   if (request_->url().SchemeIsSecure()) {
    247     // Make up a fake certificate for this response since we don't have
    248     // access to the real SSL info.
    249     const char* kCertIssuer = "Chrome Internal";
    250     const int kLifetimeDays = 100;
    251 
    252     info->ssl_info.cert =
    253         new net::X509Certificate(request_->url().GetWithEmptyPath().spec(),
    254                                  kCertIssuer,
    255                                  Time::Now(),
    256                                  Time::Now() +
    257                                      TimeDelta::FromDays(kLifetimeDays));
    258     info->ssl_info.cert_status = 0;
    259     info->ssl_info.security_bits = -1;
    260   }
    261 }
    262 
    263 void URLRequestAutomationJob::GetLoadTimingInfo(
    264     net::LoadTimingInfo* load_timing_info) const {
    265   if (!receive_headers_end_.is_null()) {
    266     load_timing_info->send_start = request_start_;
    267     // The send ended some time ago, but that information is not available on
    268     // this side of the automation channel. Consider the send to have ended at
    269     // the same time we received the response headers.
    270     load_timing_info->send_end = receive_headers_end_;
    271     load_timing_info->receive_headers_end = receive_headers_end_;
    272   }
    273 }
    274 
    275 int URLRequestAutomationJob::GetResponseCode() const {
    276   if (headers_.get())
    277     return headers_->response_code();
    278 
    279   static const int kDefaultResponseCode = 200;
    280   return kDefaultResponseCode;
    281 }
    282 
    283 bool URLRequestAutomationJob::IsRedirectResponse(
    284     GURL* location, int* http_status_code) {
    285   if (!net::HttpResponseHeaders::IsRedirectResponseCode(redirect_status_))
    286     return false;
    287 
    288   *http_status_code = redirect_status_;
    289   *location = GURL(redirect_url_);
    290   return true;
    291 }
    292 
    293 net::UploadProgress URLRequestAutomationJob::GetUploadProgress() const {
    294   uint64 progress = 0;
    295   if (request_ && request_->status().is_success()) {
    296     // We don't support incremental progress notifications in ChromeFrame. When
    297     // we receive a response for the POST request from Chromeframe, it means
    298     // that the upload is fully complete.
    299     progress = upload_size_;
    300   }
    301   return net::UploadProgress(progress, upload_size_);
    302 }
    303 
    304 net::HostPortPair URLRequestAutomationJob::GetSocketAddress() const {
    305   return socket_address_;
    306 }
    307 
    308 bool URLRequestAutomationJob::MayFilterMessage(const IPC::Message& message,
    309                                                int* request_id) {
    310   switch (message.type()) {
    311     case AutomationMsg_RequestStarted::ID:
    312     case AutomationMsg_RequestData::ID:
    313     case AutomationMsg_RequestEnd::ID: {
    314       PickleIterator iter(message);
    315       if (message.ReadInt(&iter, request_id))
    316         return true;
    317       break;
    318     }
    319   }
    320 
    321   return false;
    322 }
    323 
    324 void URLRequestAutomationJob::OnMessage(const IPC::Message& message) {
    325   if (!request_) {
    326     NOTREACHED() << __FUNCTION__
    327                  << ": Unexpected request received for job:"
    328                  << id();
    329     return;
    330   }
    331 
    332   bool deserialize_success = false;
    333   IPC_BEGIN_MESSAGE_MAP_EX(URLRequestAutomationJob,
    334                            message,
    335                            deserialize_success)
    336     IPC_MESSAGE_HANDLER(AutomationMsg_RequestStarted, OnRequestStarted)
    337     IPC_MESSAGE_HANDLER(AutomationMsg_RequestData, OnDataAvailable)
    338     IPC_MESSAGE_HANDLER(AutomationMsg_RequestEnd, OnRequestEnd)
    339   IPC_END_MESSAGE_MAP_EX()
    340 
    341   if (!deserialize_success) {
    342     LOG(ERROR) << "Failed to deserialize IPC message.";
    343   }
    344 }
    345 
    346 void URLRequestAutomationJob::OnRequestStarted(
    347     int id, const AutomationURLResponse& response) {
    348   DVLOG(1) << "URLRequestAutomationJob: " << request_->url().spec()
    349            << " - response started.";
    350   set_expected_content_size(response.content_length);
    351   mime_type_ = response.mime_type;
    352 
    353   receive_headers_end_ = base::TimeTicks::Now();
    354 
    355   redirect_url_ = response.redirect_url;
    356   redirect_status_ = response.redirect_status;
    357   DCHECK(redirect_status_ == 0 || redirect_status_ == 200 ||
    358          (redirect_status_ >= 300 && redirect_status_ < 400));
    359 
    360   if (!response.headers.empty()) {
    361     headers_ = new net::HttpResponseHeaders(
    362         net::HttpUtil::AssembleRawHeaders(response.headers.data(),
    363                                           response.headers.size()));
    364   }
    365   socket_address_ = response.socket_address;
    366   upload_size_ = response.upload_size;
    367   NotifyHeadersComplete();
    368 }
    369 
    370 void URLRequestAutomationJob::OnDataAvailable(
    371     int id, const std::string& bytes) {
    372   DVLOG(1) << "URLRequestAutomationJob: " << request_->url().spec()
    373            << " - data available, Size: " << bytes.size();
    374   DCHECK(!bytes.empty());
    375 
    376   // The request completed, and we have all the data.
    377   // Clear any IO pending status.
    378   SetStatus(net::URLRequestStatus());
    379 
    380   if (pending_buf_.get() && pending_buf_->data()) {
    381     DCHECK_GE(pending_buf_size_, bytes.size());
    382     const int bytes_to_copy = std::min(bytes.size(), pending_buf_size_);
    383     memcpy(pending_buf_->data(), &bytes[0], bytes_to_copy);
    384 
    385     pending_buf_ = NULL;
    386     pending_buf_size_ = 0;
    387 
    388     NotifyReadComplete(bytes_to_copy);
    389   } else {
    390     NOTREACHED() << "Received unexpected data of length:" << bytes.size();
    391   }
    392 }
    393 
    394 void URLRequestAutomationJob::OnRequestEnd(
    395     int id, const net::URLRequestStatus& status) {
    396 #ifndef NDEBUG
    397   std::string url;
    398   if (request_)
    399     url = request_->url().spec();
    400   DVLOG(1) << "URLRequestAutomationJob: " << url << " - request end. Status: "
    401            << status.status();
    402 #endif
    403 
    404   // TODO(tommi): When we hit certificate errors, notify the delegate via
    405   // OnSSLCertificateError().  Right now we don't have the certificate
    406   // so we don't.  We could possibly call OnSSLCertificateError with a NULL
    407   // certificate, but I'm not sure if all implementations expect it.
    408   // if (status.status() == net::URLRequestStatus::FAILED &&
    409   //    net::IsCertificateError(status.error()) && request_->delegate()) {
    410   //  request_->delegate()->OnSSLCertificateError(request_, status.error());
    411   // }
    412 
    413   DisconnectFromMessageFilter();
    414   // NotifyDone may have been called on the job if the original request was
    415   // redirected.
    416   if (!is_done()) {
    417     // We can complete the job if we have a valid response or a pending read.
    418     // An end request can be received in the following cases
    419     // 1. We failed to connect to the server, in which case we did not receive
    420     //    a valid response.
    421     // 2. In response to a read request.
    422     if (!has_response_started()) {
    423       NotifyStartError(status);
    424     } else if (pending_buf_.get()) {
    425       pending_buf_ = NULL;
    426       pending_buf_size_ = 0;
    427       NotifyDone(status);
    428       NotifyReadComplete(0);
    429     } else {
    430       // Wait for the http stack to issue a Read request where we will notify
    431       // that the job has completed.
    432       request_status_ = status;
    433     }
    434   }
    435   // Note
    436   // The job could have been destroyed above. Please don't attempt to access
    437   // member variables here.
    438 }
    439 
    440 void URLRequestAutomationJob::Cleanup() {
    441   headers_ = NULL;
    442   mime_type_.erase();
    443 
    444   id_ = 0;
    445   tab_ = 0;
    446 
    447   DCHECK(!message_filter_.get());
    448   DisconnectFromMessageFilter();
    449 
    450   pending_buf_ = NULL;
    451   pending_buf_size_ = 0;
    452 }
    453 
    454 void URLRequestAutomationJob::StartAsync() {
    455   DVLOG(1) << "URLRequestAutomationJob: start request: "
    456            << (request_ ? request_->url().spec() : "NULL request");
    457 
    458   // If the job is cancelled before we got a chance to start it
    459   // we have nothing much to do here.
    460   if (is_done())
    461     return;
    462 
    463   // We should not receive a Start request for a pending job.
    464   DCHECK(!is_pending());
    465 
    466   if (!request_) {
    467     NotifyStartError(net::URLRequestStatus(net::URLRequestStatus::FAILED,
    468                                            net::ERR_FAILED));
    469     return;
    470   }
    471 
    472   // Register this request with automation message filter.
    473   message_filter_->RegisterRequest(this);
    474 
    475   // Strip unwanted headers.
    476   net::HttpRequestHeaders new_request_headers;
    477   new_request_headers.MergeFrom(request_->extra_request_headers());
    478   for (size_t i = 0; i < arraysize(kFilteredHeaderStrings); ++i)
    479     new_request_headers.RemoveHeader(kFilteredHeaderStrings[i]);
    480 
    481   // Only add default Accept-Language if the request didn't have it specified.
    482   if (!new_request_headers.HasHeader(
    483       net::HttpRequestHeaders::kAcceptLanguage) &&
    484       http_user_agent_settings_) {
    485     std::string accept_language =
    486         http_user_agent_settings_->GetAcceptLanguage();
    487     if (!accept_language.empty()) {
    488       new_request_headers.SetHeader(net::HttpRequestHeaders::kAcceptLanguage,
    489                                     accept_language);
    490     }
    491   }
    492 
    493   // URLRequest::SetReferrer() ensures that we do not send username and
    494   // password fields in the referrer.
    495   GURL referrer(request_->referrer());
    496 
    497   // The referrer header must be suppressed if the preceding URL was
    498   // a secure one and the new one is not.
    499   if (referrer.SchemeIsSecure() && !request_->url().SchemeIsSecure()) {
    500     DVLOG(1) << "Suppressing referrer header since going from secure to "
    501                 "non-secure";
    502     referrer = GURL();
    503   }
    504 
    505   // Get the resource type (main_frame/script/image/stylesheet etc.
    506   const ResourceRequestInfo* info = ResourceRequestInfo::ForRequest(request_);
    507   ResourceType::Type resource_type = ResourceType::MAIN_FRAME;
    508   if (info) {
    509     resource_type = info->GetResourceType();
    510   }
    511 
    512   // Construct UploadData from UploadDataStream.
    513   scoped_refptr<net::UploadData> upload_data;
    514   if (request_->get_upload())
    515     upload_data = CreateUploadData(request_->get_upload());
    516 
    517   request_start_ = base::TimeTicks::Now();
    518 
    519   // Ask automation to start this request.
    520   AutomationURLRequest automation_request;
    521   automation_request.url = request_->url().spec();
    522   automation_request.method = request_->method();
    523   automation_request.referrer = referrer.spec();
    524   automation_request.extra_request_headers = new_request_headers.ToString();
    525   automation_request.upload_data = upload_data;
    526   automation_request.resource_type = resource_type;
    527   automation_request.load_flags = request_->load_flags();
    528 
    529   DCHECK(message_filter_.get());
    530   message_filter_->Send(
    531       new AutomationMsg_RequestStart(tab_, id_, automation_request));
    532 }
    533 
    534 void URLRequestAutomationJob::DisconnectFromMessageFilter() {
    535   if (message_filter_.get()) {
    536     message_filter_->UnRegisterRequest(this);
    537     message_filter_ = NULL;
    538   }
    539 }
    540 
    541 void URLRequestAutomationJob::StartPendingJob(
    542     int new_tab_handle,
    543     AutomationResourceMessageFilter* new_filter) {
    544   DCHECK(new_filter != NULL);
    545   tab_ = new_tab_handle;
    546   message_filter_ = new_filter;
    547   is_pending_ = false;
    548   Start();
    549 }
    550 
    551 void URLRequestAutomationJob::NotifyJobCompletionTask() {
    552   if (!is_done()) {
    553     NotifyDone(request_status_);
    554   }
    555   // Reset any pending reads.
    556   if (pending_buf_.get()) {
    557     pending_buf_ = NULL;
    558     pending_buf_size_ = 0;
    559     NotifyReadComplete(0);
    560   }
    561 }
    562