1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "net/url_request/url_request.h" 6 7 #include "base/compiler_specific.h" 8 #include "base/memory/singleton.h" 9 #include "base/message_loop.h" 10 #include "base/metrics/stats_counters.h" 11 #include "base/synchronization/lock.h" 12 #include "net/base/host_port_pair.h" 13 #include "net/base/load_flags.h" 14 #include "net/base/net_errors.h" 15 #include "net/base/net_log.h" 16 #include "net/base/network_delegate.h" 17 #include "net/base/ssl_cert_request_info.h" 18 #include "net/base/upload_data.h" 19 #include "net/http/http_response_headers.h" 20 #include "net/http/http_util.h" 21 #include "net/url_request/url_request_context.h" 22 #include "net/url_request/url_request_error_job.h" 23 #include "net/url_request/url_request_job.h" 24 #include "net/url_request/url_request_job_manager.h" 25 #include "net/url_request/url_request_netlog_params.h" 26 #include "net/url_request/url_request_redirect_job.h" 27 28 using base::Time; 29 using std::string; 30 31 namespace net { 32 33 namespace { 34 35 // Max number of http redirects to follow. Same number as gecko. 36 const int kMaxRedirects = 20; 37 38 // Discard headers which have meaning in POST (Content-Length, Content-Type, 39 // Origin). 40 void StripPostSpecificHeaders(HttpRequestHeaders* headers) { 41 // These are headers that may be attached to a POST. 42 headers->RemoveHeader(HttpRequestHeaders::kContentLength); 43 headers->RemoveHeader(HttpRequestHeaders::kContentType); 44 headers->RemoveHeader(HttpRequestHeaders::kOrigin); 45 } 46 47 // This counter keeps track of the identifiers used for URL requests so far. 48 // 0 is reserved to represent an invalid ID. 49 uint64 g_next_url_request_identifier = 1; 50 51 // This lock protects g_next_url_request_identifier. 52 base::Lock g_next_url_request_identifier_lock; 53 54 // Returns an prior unused identifier for URL requests. 55 uint64 GenerateURLRequestIdentifier() { 56 base::AutoLock lock(g_next_url_request_identifier_lock); 57 return g_next_url_request_identifier++; 58 } 59 60 } // namespace 61 62 /////////////////////////////////////////////////////////////////////////////// 63 // URLRequest::Interceptor 64 65 URLRequestJob* URLRequest::Interceptor::MaybeInterceptRedirect( 66 URLRequest* request, 67 const GURL& location) { 68 return NULL; 69 } 70 71 URLRequestJob* URLRequest::Interceptor::MaybeInterceptResponse( 72 URLRequest* request) { 73 return NULL; 74 } 75 76 /////////////////////////////////////////////////////////////////////////////// 77 // URLRequest::Delegate 78 79 void URLRequest::Delegate::OnReceivedRedirect(URLRequest* request, 80 const GURL& new_url, 81 bool* defer_redirect) { 82 } 83 84 void URLRequest::Delegate::OnAuthRequired(URLRequest* request, 85 AuthChallengeInfo* auth_info) { 86 request->CancelAuth(); 87 } 88 89 void URLRequest::Delegate::OnCertificateRequested( 90 URLRequest* request, 91 SSLCertRequestInfo* cert_request_info) { 92 request->ContinueWithCertificate(NULL); 93 } 94 95 void URLRequest::Delegate::OnSSLCertificateError(URLRequest* request, 96 int cert_error, 97 X509Certificate* cert) { 98 request->Cancel(); 99 } 100 101 void URLRequest::Delegate::OnGetCookies(URLRequest* request, 102 bool blocked_by_policy) { 103 } 104 105 void URLRequest::Delegate::OnSetCookie(URLRequest* request, 106 const std::string& cookie_line, 107 const CookieOptions& options, 108 bool blocked_by_policy) { 109 } 110 111 /////////////////////////////////////////////////////////////////////////////// 112 // URLRequest 113 114 URLRequest::URLRequest(const GURL& url, Delegate* delegate) 115 : url_chain_(1, url), 116 method_("GET"), 117 load_flags_(LOAD_NORMAL), 118 delegate_(delegate), 119 is_pending_(false), 120 redirect_limit_(kMaxRedirects), 121 final_upload_progress_(0), 122 priority_(LOWEST), 123 identifier_(GenerateURLRequestIdentifier()), 124 ALLOW_THIS_IN_INITIALIZER_LIST( 125 before_request_callback_(this, &URLRequest::BeforeRequestComplete)) { 126 SIMPLE_STATS_COUNTER("URLRequestCount"); 127 128 // Sanity check out environment. 129 DCHECK(MessageLoop::current()) << 130 "The current MessageLoop must exist"; 131 DCHECK_EQ(MessageLoop::TYPE_IO, MessageLoop::current()->type()) << 132 "The current MessageLoop must be TYPE_IO"; 133 } 134 135 URLRequest::~URLRequest() { 136 if (context_ && context_->network_delegate()) 137 context_->network_delegate()->NotifyURLRequestDestroyed(this); 138 139 Cancel(); 140 141 if (job_) 142 OrphanJob(); 143 144 set_context(NULL); 145 } 146 147 // static 148 URLRequest::ProtocolFactory* URLRequest::RegisterProtocolFactory( 149 const string& scheme, ProtocolFactory* factory) { 150 return URLRequestJobManager::GetInstance()->RegisterProtocolFactory(scheme, 151 factory); 152 } 153 154 // static 155 void URLRequest::RegisterRequestInterceptor(Interceptor* interceptor) { 156 URLRequestJobManager::GetInstance()->RegisterRequestInterceptor(interceptor); 157 } 158 159 // static 160 void URLRequest::UnregisterRequestInterceptor(Interceptor* interceptor) { 161 URLRequestJobManager::GetInstance()->UnregisterRequestInterceptor( 162 interceptor); 163 } 164 165 void URLRequest::AppendBytesToUpload(const char* bytes, int bytes_len) { 166 DCHECK(bytes_len > 0 && bytes); 167 if (!upload_) 168 upload_ = new UploadData(); 169 upload_->AppendBytes(bytes, bytes_len); 170 } 171 172 void URLRequest::AppendFileRangeToUpload( 173 const FilePath& file_path, 174 uint64 offset, 175 uint64 length, 176 const base::Time& expected_modification_time) { 177 DCHECK(file_path.value().length() > 0 && length > 0); 178 if (!upload_) 179 upload_ = new UploadData(); 180 upload_->AppendFileRange(file_path, offset, length, 181 expected_modification_time); 182 } 183 184 void URLRequest::EnableChunkedUpload() { 185 DCHECK(!upload_ || upload_->is_chunked()); 186 if (!upload_) { 187 upload_ = new UploadData(); 188 upload_->set_is_chunked(true); 189 } 190 } 191 192 void URLRequest::AppendChunkToUpload(const char* bytes, 193 int bytes_len, 194 bool is_last_chunk) { 195 DCHECK(upload_); 196 DCHECK(upload_->is_chunked()); 197 DCHECK_GT(bytes_len, 0); 198 upload_->AppendChunk(bytes, bytes_len, is_last_chunk); 199 } 200 201 void URLRequest::set_upload(UploadData* upload) { 202 upload_ = upload; 203 } 204 205 // Get the upload data directly. 206 UploadData* URLRequest::get_upload() { 207 return upload_.get(); 208 } 209 210 bool URLRequest::has_upload() const { 211 return upload_ != NULL; 212 } 213 214 void URLRequest::SetExtraRequestHeaderById(int id, const string& value, 215 bool overwrite) { 216 DCHECK(!is_pending_); 217 NOTREACHED() << "implement me!"; 218 } 219 220 void URLRequest::SetExtraRequestHeaderByName(const string& name, 221 const string& value, 222 bool overwrite) { 223 DCHECK(!is_pending_); 224 if (overwrite) { 225 extra_request_headers_.SetHeader(name, value); 226 } else { 227 extra_request_headers_.SetHeaderIfMissing(name, value); 228 } 229 } 230 231 void URLRequest::SetExtraRequestHeaders( 232 const HttpRequestHeaders& headers) { 233 DCHECK(!is_pending_); 234 extra_request_headers_ = headers; 235 236 // NOTE: This method will likely become non-trivial once the other setters 237 // for request headers are implemented. 238 } 239 240 LoadState URLRequest::GetLoadState() const { 241 return job_ ? job_->GetLoadState() : LOAD_STATE_IDLE; 242 } 243 244 uint64 URLRequest::GetUploadProgress() const { 245 if (!job_) { 246 // We haven't started or the request was cancelled 247 return 0; 248 } 249 if (final_upload_progress_) { 250 // The first job completed and none of the subsequent series of 251 // GETs when following redirects will upload anything, so we return the 252 // cached results from the initial job, the POST. 253 return final_upload_progress_; 254 } 255 return job_->GetUploadProgress(); 256 } 257 258 void URLRequest::GetResponseHeaderById(int id, string* value) { 259 DCHECK(job_); 260 NOTREACHED() << "implement me!"; 261 } 262 263 void URLRequest::GetResponseHeaderByName(const string& name, string* value) { 264 DCHECK(value); 265 if (response_info_.headers) { 266 response_info_.headers->GetNormalizedHeader(name, value); 267 } else { 268 value->clear(); 269 } 270 } 271 272 void URLRequest::GetAllResponseHeaders(string* headers) { 273 DCHECK(headers); 274 if (response_info_.headers) { 275 response_info_.headers->GetNormalizedHeaders(headers); 276 } else { 277 headers->clear(); 278 } 279 } 280 281 HostPortPair URLRequest::GetSocketAddress() const { 282 DCHECK(job_); 283 return job_->GetSocketAddress(); 284 } 285 286 HttpResponseHeaders* URLRequest::response_headers() const { 287 return response_info_.headers.get(); 288 } 289 290 bool URLRequest::GetResponseCookies(ResponseCookies* cookies) { 291 DCHECK(job_); 292 return job_->GetResponseCookies(cookies); 293 } 294 295 void URLRequest::GetMimeType(string* mime_type) { 296 DCHECK(job_); 297 job_->GetMimeType(mime_type); 298 } 299 300 void URLRequest::GetCharset(string* charset) { 301 DCHECK(job_); 302 job_->GetCharset(charset); 303 } 304 305 int URLRequest::GetResponseCode() { 306 DCHECK(job_); 307 return job_->GetResponseCode(); 308 } 309 310 // static 311 bool URLRequest::IsHandledProtocol(const std::string& scheme) { 312 return URLRequestJobManager::GetInstance()->SupportsScheme(scheme); 313 } 314 315 // static 316 bool URLRequest::IsHandledURL(const GURL& url) { 317 if (!url.is_valid()) { 318 // We handle error cases. 319 return true; 320 } 321 322 return IsHandledProtocol(url.scheme()); 323 } 324 325 // static 326 void URLRequest::AllowFileAccess() { 327 URLRequestJobManager::GetInstance()->set_enable_file_access(true); 328 } 329 330 // static 331 bool URLRequest::IsFileAccessAllowed() { 332 return URLRequestJobManager::GetInstance()->enable_file_access(); 333 } 334 335 336 void URLRequest::set_first_party_for_cookies( 337 const GURL& first_party_for_cookies) { 338 first_party_for_cookies_ = first_party_for_cookies; 339 } 340 341 void URLRequest::set_method(const std::string& method) { 342 DCHECK(!is_pending_); 343 method_ = method; 344 } 345 346 void URLRequest::set_referrer(const std::string& referrer) { 347 DCHECK(!is_pending_); 348 referrer_ = referrer; 349 } 350 351 GURL URLRequest::GetSanitizedReferrer() const { 352 GURL ret(referrer()); 353 354 // Ensure that we do not send username and password fields in the referrer. 355 if (ret.has_username() || ret.has_password()) { 356 GURL::Replacements referrer_mods; 357 referrer_mods.ClearUsername(); 358 referrer_mods.ClearPassword(); 359 ret = ret.ReplaceComponents(referrer_mods); 360 } 361 362 return ret; 363 } 364 365 void URLRequest::Start() { 366 response_info_.request_time = Time::Now(); 367 368 // Only notify the delegate for the initial request. 369 if (context_ && context_->network_delegate()) { 370 if (context_->network_delegate()->NotifyBeforeURLRequest( 371 this, &before_request_callback_, &delegate_redirect_url_) == 372 net::ERR_IO_PENDING) { 373 net_log_.BeginEvent(NetLog::TYPE_URL_REQUEST_BLOCKED_ON_EXTENSION, NULL); 374 return; // paused 375 } 376 } 377 378 StartInternal(); 379 } 380 381 /////////////////////////////////////////////////////////////////////////////// 382 383 void URLRequest::BeforeRequestComplete(int error) { 384 DCHECK(!job_); 385 DCHECK_NE(ERR_IO_PENDING, error); 386 387 net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_BLOCKED_ON_EXTENSION, NULL); 388 if (error != OK) { 389 StartJob(new URLRequestErrorJob(this, error)); 390 } else if (!delegate_redirect_url_.is_empty()) { 391 GURL new_url; 392 new_url.Swap(&delegate_redirect_url_); 393 StartJob(new URLRequestRedirectJob(this, new_url)); 394 } else { 395 StartInternal(); 396 } 397 } 398 399 void URLRequest::StartInternal() { 400 StartJob(URLRequestJobManager::GetInstance()->CreateJob(this)); 401 } 402 403 void URLRequest::StartJob(URLRequestJob* job) { 404 DCHECK(!is_pending_); 405 DCHECK(!job_); 406 407 net_log_.BeginEvent( 408 NetLog::TYPE_URL_REQUEST_START_JOB, 409 make_scoped_refptr(new URLRequestStartEventParameters( 410 url(), method_, load_flags_, priority_))); 411 412 job_ = job; 413 job_->SetExtraRequestHeaders(extra_request_headers_); 414 415 if (upload_.get()) 416 job_->SetUpload(upload_.get()); 417 418 is_pending_ = true; 419 420 response_info_.was_cached = false; 421 422 // Don't allow errors to be sent from within Start(). 423 // TODO(brettw) this may cause NotifyDone to be sent synchronously, 424 // we probably don't want this: they should be sent asynchronously so 425 // the caller does not get reentered. 426 job_->Start(); 427 } 428 429 void URLRequest::Restart() { 430 // Should only be called if the original job didn't make any progress. 431 DCHECK(job_ && !job_->has_response_started()); 432 RestartWithJob(URLRequestJobManager::GetInstance()->CreateJob(this)); 433 } 434 435 void URLRequest::RestartWithJob(URLRequestJob *job) { 436 DCHECK(job->request() == this); 437 PrepareToRestart(); 438 StartJob(job); 439 } 440 441 void URLRequest::Cancel() { 442 DoCancel(ERR_ABORTED, SSLInfo()); 443 } 444 445 void URLRequest::SimulateError(int os_error) { 446 DoCancel(os_error, SSLInfo()); 447 } 448 449 void URLRequest::SimulateSSLError(int os_error, const SSLInfo& ssl_info) { 450 // This should only be called on a started request. 451 if (!is_pending_ || !job_ || job_->has_response_started()) { 452 NOTREACHED(); 453 return; 454 } 455 DoCancel(os_error, ssl_info); 456 } 457 458 void URLRequest::DoCancel(int os_error, const SSLInfo& ssl_info) { 459 DCHECK(os_error < 0); 460 461 // If the URL request already has an error status, then canceling is a no-op. 462 // Plus, we don't want to change the error status once it has been set. 463 if (status_.is_success()) { 464 status_.set_status(URLRequestStatus::CANCELED); 465 status_.set_os_error(os_error); 466 response_info_.ssl_info = ssl_info; 467 } 468 469 // There's nothing to do if we are not waiting on a Job. 470 if (!is_pending_ || !job_) 471 return; 472 473 job_->Kill(); 474 475 // The Job will call our NotifyDone method asynchronously. This is done so 476 // that the Delegate implementation can call Cancel without having to worry 477 // about being called recursively. 478 } 479 480 bool URLRequest::Read(IOBuffer* dest, int dest_size, int* bytes_read) { 481 DCHECK(job_); 482 DCHECK(bytes_read); 483 DCHECK(!job_->is_done()); 484 *bytes_read = 0; 485 486 if (dest_size == 0) { 487 // Caller is not too bright. I guess we've done what they asked. 488 return true; 489 } 490 491 // Once the request fails or is cancelled, read will just return 0 bytes 492 // to indicate end of stream. 493 if (!status_.is_success()) { 494 return true; 495 } 496 497 return job_->Read(dest, dest_size, bytes_read); 498 } 499 500 void URLRequest::StopCaching() { 501 DCHECK(job_); 502 job_->StopCaching(); 503 } 504 505 void URLRequest::ReceivedRedirect(const GURL& location, bool* defer_redirect) { 506 URLRequestJob* job = 507 URLRequestJobManager::GetInstance()->MaybeInterceptRedirect(this, 508 location); 509 if (job) { 510 RestartWithJob(job); 511 } else if (delegate_) { 512 delegate_->OnReceivedRedirect(this, location, defer_redirect); 513 } 514 } 515 516 void URLRequest::ResponseStarted() { 517 scoped_refptr<NetLog::EventParameters> params; 518 if (!status_.is_success()) 519 params = new NetLogIntegerParameter("net_error", status_.os_error()); 520 net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB, params); 521 522 URLRequestJob* job = 523 URLRequestJobManager::GetInstance()->MaybeInterceptResponse(this); 524 if (job) { 525 RestartWithJob(job); 526 } else { 527 if (context_ && context_->network_delegate()) 528 context_->network_delegate()->NotifyResponseStarted(this); 529 if (delegate_) 530 delegate_->OnResponseStarted(this); 531 } 532 } 533 534 void URLRequest::FollowDeferredRedirect() { 535 CHECK(job_); 536 CHECK(status_.is_success()); 537 538 job_->FollowDeferredRedirect(); 539 } 540 541 void URLRequest::SetAuth(const string16& username, const string16& password) { 542 DCHECK(job_); 543 DCHECK(job_->NeedsAuth()); 544 545 job_->SetAuth(username, password); 546 } 547 548 void URLRequest::CancelAuth() { 549 DCHECK(job_); 550 DCHECK(job_->NeedsAuth()); 551 552 job_->CancelAuth(); 553 } 554 555 void URLRequest::ContinueWithCertificate(X509Certificate* client_cert) { 556 DCHECK(job_); 557 558 job_->ContinueWithCertificate(client_cert); 559 } 560 561 void URLRequest::ContinueDespiteLastError() { 562 DCHECK(job_); 563 564 job_->ContinueDespiteLastError(); 565 } 566 567 void URLRequest::PrepareToRestart() { 568 DCHECK(job_); 569 570 // Close the current URL_REQUEST_START_JOB, since we will be starting a new 571 // one. 572 net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB, NULL); 573 574 OrphanJob(); 575 576 response_info_ = HttpResponseInfo(); 577 response_info_.request_time = Time::Now(); 578 status_ = URLRequestStatus(); 579 is_pending_ = false; 580 } 581 582 void URLRequest::OrphanJob() { 583 job_->Kill(); 584 job_->DetachRequest(); // ensures that the job will not call us again 585 job_ = NULL; 586 } 587 588 int URLRequest::Redirect(const GURL& location, int http_status_code) { 589 if (net_log_.IsLoggingAllEvents()) { 590 net_log_.AddEvent( 591 NetLog::TYPE_URL_REQUEST_REDIRECTED, 592 make_scoped_refptr(new NetLogStringParameter( 593 "location", location.possibly_invalid_spec()))); 594 } 595 if (redirect_limit_ <= 0) { 596 DVLOG(1) << "disallowing redirect: exceeds limit"; 597 return ERR_TOO_MANY_REDIRECTS; 598 } 599 600 if (!location.is_valid()) 601 return ERR_INVALID_URL; 602 603 if (!job_->IsSafeRedirect(location)) { 604 DVLOG(1) << "disallowing redirect: unsafe protocol"; 605 return ERR_UNSAFE_REDIRECT; 606 } 607 608 bool strip_post_specific_headers = false; 609 if (http_status_code != 307) { 610 // NOTE: Even though RFC 2616 says to preserve the request method when 611 // following a 302 redirect, normal browsers don't do that. Instead, they 612 // all convert a POST into a GET in response to a 302 and so shall we. For 613 // 307 redirects, browsers preserve the method. The RFC says to prompt the 614 // user to confirm the generation of a new POST request, but IE omits this 615 // prompt and so shall we. 616 strip_post_specific_headers = method_ == "POST"; 617 method_ = "GET"; 618 upload_ = NULL; 619 } 620 621 // Suppress the referrer if we're redirecting out of https. 622 if (GURL(referrer_).SchemeIsSecure() && !location.SchemeIsSecure()) 623 referrer_.clear(); 624 625 url_chain_.push_back(location); 626 --redirect_limit_; 627 628 if (strip_post_specific_headers) { 629 // If being switched from POST to GET, must remove headers that were 630 // specific to the POST and don't have meaning in GET. For example 631 // the inclusion of a multipart Content-Type header in GET can cause 632 // problems with some servers: 633 // http://code.google.com/p/chromium/issues/detail?id=843 634 StripPostSpecificHeaders(&extra_request_headers_); 635 } 636 637 if (!final_upload_progress_) 638 final_upload_progress_ = job_->GetUploadProgress(); 639 640 PrepareToRestart(); 641 StartInternal(); 642 return OK; 643 } 644 645 URLRequestContext* URLRequest::context() const { 646 return context_.get(); 647 } 648 649 void URLRequest::set_context(URLRequestContext* context) { 650 scoped_refptr<URLRequestContext> prev_context = context_; 651 652 context_ = context; 653 654 // If the context this request belongs to has changed, update the tracker. 655 if (prev_context != context) { 656 net_log_.EndEvent(NetLog::TYPE_REQUEST_ALIVE, NULL); 657 net_log_ = BoundNetLog(); 658 659 if (context) { 660 net_log_ = BoundNetLog::Make(context->net_log(), 661 NetLog::SOURCE_URL_REQUEST); 662 net_log_.BeginEvent(NetLog::TYPE_REQUEST_ALIVE, NULL); 663 } 664 } 665 } 666 667 int64 URLRequest::GetExpectedContentSize() const { 668 int64 expected_content_size = -1; 669 if (job_) 670 expected_content_size = job_->expected_content_size(); 671 672 return expected_content_size; 673 } 674 675 URLRequest::UserData* URLRequest::GetUserData(const void* key) const { 676 UserDataMap::const_iterator found = user_data_.find(key); 677 if (found != user_data_.end()) 678 return found->second.get(); 679 return NULL; 680 } 681 682 void URLRequest::SetUserData(const void* key, UserData* data) { 683 user_data_[key] = linked_ptr<UserData>(data); 684 } 685 686 } // namespace net 687