1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "net/url_request/url_request.h" 6 7 #include "base/compiler_specific.h" 8 #include "base/memory/singleton.h" 9 #include "base/message_loop.h" 10 #include "base/metrics/stats_counters.h" 11 #include "base/synchronization/lock.h" 12 #include "net/base/host_port_pair.h" 13 #include "net/base/load_flags.h" 14 #include "net/base/net_errors.h" 15 #include "net/base/net_log.h" 16 #include "net/base/network_delegate.h" 17 #include "net/base/ssl_cert_request_info.h" 18 #include "net/base/upload_data.h" 19 #include "net/http/http_response_headers.h" 20 #include "net/http/http_util.h" 21 #include "net/url_request/url_request_context.h" 22 #include "net/url_request/url_request_error_job.h" 23 #include "net/url_request/url_request_job.h" 24 #include "net/url_request/url_request_job_manager.h" 25 #include "net/url_request/url_request_netlog_params.h" 26 #include "net/url_request/url_request_redirect_job.h" 27 28 using base::Time; 29 using std::string; 30 31 namespace net { 32 33 namespace { 34 35 // Max number of http redirects to follow. Same number as gecko. 36 const int kMaxRedirects = 20; 37 38 // Discard headers which have meaning in POST (Content-Length, Content-Type, 39 // Origin). 40 void StripPostSpecificHeaders(HttpRequestHeaders* headers) { 41 // These are headers that may be attached to a POST. 42 headers->RemoveHeader(HttpRequestHeaders::kContentLength); 43 headers->RemoveHeader(HttpRequestHeaders::kContentType); 44 headers->RemoveHeader(HttpRequestHeaders::kOrigin); 45 } 46 47 // This counter keeps track of the identifiers used for URL requests so far. 48 // 0 is reserved to represent an invalid ID. 49 uint64 g_next_url_request_identifier = 1; 50 51 // This lock protects g_next_url_request_identifier. 52 base::Lock g_next_url_request_identifier_lock; 53 54 // Returns an prior unused identifier for URL requests. 55 uint64 GenerateURLRequestIdentifier() { 56 base::AutoLock lock(g_next_url_request_identifier_lock); 57 return g_next_url_request_identifier++; 58 } 59 60 } // namespace 61 62 /////////////////////////////////////////////////////////////////////////////// 63 // URLRequest::Interceptor 64 65 URLRequestJob* URLRequest::Interceptor::MaybeInterceptRedirect( 66 URLRequest* request, 67 const GURL& location) { 68 return NULL; 69 } 70 71 URLRequestJob* URLRequest::Interceptor::MaybeInterceptResponse( 72 URLRequest* request) { 73 return NULL; 74 } 75 76 /////////////////////////////////////////////////////////////////////////////// 77 // URLRequest::Delegate 78 79 void URLRequest::Delegate::OnReceivedRedirect(URLRequest* request, 80 const GURL& new_url, 81 bool* defer_redirect) { 82 } 83 84 void URLRequest::Delegate::OnAuthRequired(URLRequest* request, 85 AuthChallengeInfo* auth_info) { 86 request->CancelAuth(); 87 } 88 89 void URLRequest::Delegate::OnCertificateRequested( 90 URLRequest* request, 91 SSLCertRequestInfo* cert_request_info) { 92 request->ContinueWithCertificate(NULL); 93 } 94 95 void URLRequest::Delegate::OnSSLCertificateError(URLRequest* request, 96 int cert_error, 97 X509Certificate* cert) { 98 request->Cancel(); 99 } 100 101 void URLRequest::Delegate::OnGetCookies(URLRequest* request, 102 bool blocked_by_policy) { 103 } 104 105 void URLRequest::Delegate::OnSetCookie(URLRequest* request, 106 const std::string& cookie_line, 107 const CookieOptions& options, 108 bool blocked_by_policy) { 109 } 110 111 /////////////////////////////////////////////////////////////////////////////// 112 // URLRequest 113 114 URLRequest::URLRequest(const GURL& url, Delegate* delegate) 115 : url_chain_(1, url), 116 method_("GET"), 117 load_flags_(LOAD_NORMAL), 118 delegate_(delegate), 119 is_pending_(false), 120 redirect_limit_(kMaxRedirects), 121 final_upload_progress_(0), 122 priority_(LOWEST), 123 identifier_(GenerateURLRequestIdentifier()), 124 ALLOW_THIS_IN_INITIALIZER_LIST( 125 before_request_callback_(this, &URLRequest::BeforeRequestComplete)) { 126 SIMPLE_STATS_COUNTER("URLRequestCount"); 127 128 // Sanity check out environment. 129 DCHECK(MessageLoop::current()) << 130 "The current MessageLoop must exist"; 131 DCHECK_EQ(MessageLoop::TYPE_IO, MessageLoop::current()->type()) << 132 "The current MessageLoop must be TYPE_IO"; 133 } 134 135 URLRequest::~URLRequest() { 136 if (context_ && context_->network_delegate()) 137 context_->network_delegate()->NotifyURLRequestDestroyed(this); 138 139 Cancel(); 140 141 if (job_) 142 OrphanJob(); 143 144 set_context(NULL); 145 } 146 147 // static 148 URLRequest::ProtocolFactory* URLRequest::RegisterProtocolFactory( 149 const string& scheme, ProtocolFactory* factory) { 150 return URLRequestJobManager::GetInstance()->RegisterProtocolFactory(scheme, 151 factory); 152 } 153 154 // static 155 void URLRequest::RegisterRequestInterceptor(Interceptor* interceptor) { 156 URLRequestJobManager::GetInstance()->RegisterRequestInterceptor(interceptor); 157 } 158 159 // static 160 void URLRequest::UnregisterRequestInterceptor(Interceptor* interceptor) { 161 URLRequestJobManager::GetInstance()->UnregisterRequestInterceptor( 162 interceptor); 163 } 164 165 void URLRequest::AppendBytesToUpload(const char* bytes, int bytes_len) { 166 DCHECK(bytes_len > 0 && bytes); 167 if (!upload_) 168 upload_ = new UploadData(); 169 upload_->AppendBytes(bytes, bytes_len); 170 } 171 172 void URLRequest::AppendFileRangeToUpload( 173 const FilePath& file_path, 174 uint64 offset, 175 uint64 length, 176 const base::Time& expected_modification_time) { 177 DCHECK(file_path.value().length() > 0 && length > 0); 178 if (!upload_) 179 upload_ = new UploadData(); 180 upload_->AppendFileRange(file_path, offset, length, 181 expected_modification_time); 182 } 183 184 void URLRequest::EnableChunkedUpload() { 185 DCHECK(!upload_ || upload_->is_chunked()); 186 if (!upload_) { 187 upload_ = new UploadData(); 188 upload_->set_is_chunked(true); 189 } 190 } 191 192 void URLRequest::AppendChunkToUpload(const char* bytes, 193 int bytes_len, 194 bool is_last_chunk) { 195 DCHECK(upload_); 196 DCHECK(upload_->is_chunked()); 197 DCHECK_GT(bytes_len, 0); 198 upload_->AppendChunk(bytes, bytes_len, is_last_chunk); 199 } 200 201 void URLRequest::set_upload(UploadData* upload) { 202 upload_ = upload; 203 } 204 205 // Get the upload data directly. 206 UploadData* URLRequest::get_upload() { 207 return upload_.get(); 208 } 209 210 bool URLRequest::has_upload() const { 211 return upload_ != NULL; 212 } 213 214 void URLRequest::SetExtraRequestHeaderById(int id, const string& value, 215 bool overwrite) { 216 DCHECK(!is_pending_); 217 NOTREACHED() << "implement me!"; 218 } 219 220 void URLRequest::SetExtraRequestHeaderByName(const string& name, 221 const string& value, 222 bool overwrite) { 223 DCHECK(!is_pending_); 224 NOTREACHED() << "implement me!"; 225 } 226 227 void URLRequest::SetExtraRequestHeaders( 228 const HttpRequestHeaders& headers) { 229 DCHECK(!is_pending_); 230 extra_request_headers_ = headers; 231 232 // NOTE: This method will likely become non-trivial once the other setters 233 // for request headers are implemented. 234 } 235 236 LoadState URLRequest::GetLoadState() const { 237 return job_ ? job_->GetLoadState() : LOAD_STATE_IDLE; 238 } 239 240 uint64 URLRequest::GetUploadProgress() const { 241 if (!job_) { 242 // We haven't started or the request was cancelled 243 return 0; 244 } 245 if (final_upload_progress_) { 246 // The first job completed and none of the subsequent series of 247 // GETs when following redirects will upload anything, so we return the 248 // cached results from the initial job, the POST. 249 return final_upload_progress_; 250 } 251 return job_->GetUploadProgress(); 252 } 253 254 void URLRequest::GetResponseHeaderById(int id, string* value) { 255 DCHECK(job_); 256 NOTREACHED() << "implement me!"; 257 } 258 259 void URLRequest::GetResponseHeaderByName(const string& name, string* value) { 260 DCHECK(value); 261 if (response_info_.headers) { 262 response_info_.headers->GetNormalizedHeader(name, value); 263 } else { 264 value->clear(); 265 } 266 } 267 268 void URLRequest::GetAllResponseHeaders(string* headers) { 269 DCHECK(headers); 270 if (response_info_.headers) { 271 response_info_.headers->GetNormalizedHeaders(headers); 272 } else { 273 headers->clear(); 274 } 275 } 276 277 HostPortPair URLRequest::GetSocketAddress() const { 278 DCHECK(job_); 279 return job_->GetSocketAddress(); 280 } 281 282 HttpResponseHeaders* URLRequest::response_headers() const { 283 return response_info_.headers.get(); 284 } 285 286 bool URLRequest::GetResponseCookies(ResponseCookies* cookies) { 287 DCHECK(job_); 288 return job_->GetResponseCookies(cookies); 289 } 290 291 void URLRequest::GetMimeType(string* mime_type) { 292 DCHECK(job_); 293 job_->GetMimeType(mime_type); 294 } 295 296 void URLRequest::GetCharset(string* charset) { 297 DCHECK(job_); 298 job_->GetCharset(charset); 299 } 300 301 int URLRequest::GetResponseCode() { 302 DCHECK(job_); 303 return job_->GetResponseCode(); 304 } 305 306 // static 307 bool URLRequest::IsHandledProtocol(const std::string& scheme) { 308 return URLRequestJobManager::GetInstance()->SupportsScheme(scheme); 309 } 310 311 // static 312 bool URLRequest::IsHandledURL(const GURL& url) { 313 if (!url.is_valid()) { 314 // We handle error cases. 315 return true; 316 } 317 318 return IsHandledProtocol(url.scheme()); 319 } 320 321 // static 322 void URLRequest::AllowFileAccess() { 323 URLRequestJobManager::GetInstance()->set_enable_file_access(true); 324 } 325 326 // static 327 bool URLRequest::IsFileAccessAllowed() { 328 return URLRequestJobManager::GetInstance()->enable_file_access(); 329 } 330 331 332 void URLRequest::set_first_party_for_cookies( 333 const GURL& first_party_for_cookies) { 334 first_party_for_cookies_ = first_party_for_cookies; 335 } 336 337 void URLRequest::set_method(const std::string& method) { 338 DCHECK(!is_pending_); 339 method_ = method; 340 } 341 342 void URLRequest::set_referrer(const std::string& referrer) { 343 DCHECK(!is_pending_); 344 referrer_ = referrer; 345 } 346 347 GURL URLRequest::GetSanitizedReferrer() const { 348 GURL ret(referrer()); 349 350 // Ensure that we do not send username and password fields in the referrer. 351 if (ret.has_username() || ret.has_password()) { 352 GURL::Replacements referrer_mods; 353 referrer_mods.ClearUsername(); 354 referrer_mods.ClearPassword(); 355 ret = ret.ReplaceComponents(referrer_mods); 356 } 357 358 return ret; 359 } 360 361 void URLRequest::Start() { 362 response_info_.request_time = Time::Now(); 363 364 // Only notify the delegate for the initial request. 365 if (context_ && context_->network_delegate()) { 366 if (context_->network_delegate()->NotifyBeforeURLRequest( 367 this, &before_request_callback_, &delegate_redirect_url_) == 368 net::ERR_IO_PENDING) { 369 net_log_.BeginEvent(NetLog::TYPE_URL_REQUEST_BLOCKED_ON_EXTENSION, NULL); 370 return; // paused 371 } 372 } 373 374 StartInternal(); 375 } 376 377 /////////////////////////////////////////////////////////////////////////////// 378 379 void URLRequest::BeforeRequestComplete(int error) { 380 DCHECK(!job_); 381 DCHECK_NE(ERR_IO_PENDING, error); 382 383 net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_BLOCKED_ON_EXTENSION, NULL); 384 if (error != OK) { 385 StartJob(new URLRequestErrorJob(this, error)); 386 } else if (!delegate_redirect_url_.is_empty()) { 387 GURL new_url; 388 new_url.Swap(&delegate_redirect_url_); 389 StartJob(new URLRequestRedirectJob(this, new_url)); 390 } else { 391 StartInternal(); 392 } 393 } 394 395 void URLRequest::StartInternal() { 396 StartJob(URLRequestJobManager::GetInstance()->CreateJob(this)); 397 } 398 399 void URLRequest::StartJob(URLRequestJob* job) { 400 DCHECK(!is_pending_); 401 DCHECK(!job_); 402 403 net_log_.BeginEvent( 404 NetLog::TYPE_URL_REQUEST_START_JOB, 405 make_scoped_refptr(new URLRequestStartEventParameters( 406 url(), method_, load_flags_, priority_))); 407 408 job_ = job; 409 job_->SetExtraRequestHeaders(extra_request_headers_); 410 411 if (upload_.get()) 412 job_->SetUpload(upload_.get()); 413 414 is_pending_ = true; 415 416 response_info_.was_cached = false; 417 418 // Don't allow errors to be sent from within Start(). 419 // TODO(brettw) this may cause NotifyDone to be sent synchronously, 420 // we probably don't want this: they should be sent asynchronously so 421 // the caller does not get reentered. 422 job_->Start(); 423 } 424 425 void URLRequest::Restart() { 426 // Should only be called if the original job didn't make any progress. 427 DCHECK(job_ && !job_->has_response_started()); 428 RestartWithJob(URLRequestJobManager::GetInstance()->CreateJob(this)); 429 } 430 431 void URLRequest::RestartWithJob(URLRequestJob *job) { 432 DCHECK(job->request() == this); 433 PrepareToRestart(); 434 StartJob(job); 435 } 436 437 void URLRequest::Cancel() { 438 DoCancel(ERR_ABORTED, SSLInfo()); 439 } 440 441 void URLRequest::SimulateError(int os_error) { 442 DoCancel(os_error, SSLInfo()); 443 } 444 445 void URLRequest::SimulateSSLError(int os_error, const SSLInfo& ssl_info) { 446 // This should only be called on a started request. 447 if (!is_pending_ || !job_ || job_->has_response_started()) { 448 NOTREACHED(); 449 return; 450 } 451 DoCancel(os_error, ssl_info); 452 } 453 454 void URLRequest::DoCancel(int os_error, const SSLInfo& ssl_info) { 455 DCHECK(os_error < 0); 456 457 // If the URL request already has an error status, then canceling is a no-op. 458 // Plus, we don't want to change the error status once it has been set. 459 if (status_.is_success()) { 460 status_.set_status(URLRequestStatus::CANCELED); 461 status_.set_os_error(os_error); 462 response_info_.ssl_info = ssl_info; 463 } 464 465 // There's nothing to do if we are not waiting on a Job. 466 if (!is_pending_ || !job_) 467 return; 468 469 job_->Kill(); 470 471 // The Job will call our NotifyDone method asynchronously. This is done so 472 // that the Delegate implementation can call Cancel without having to worry 473 // about being called recursively. 474 } 475 476 bool URLRequest::Read(IOBuffer* dest, int dest_size, int* bytes_read) { 477 DCHECK(job_); 478 DCHECK(bytes_read); 479 DCHECK(!job_->is_done()); 480 *bytes_read = 0; 481 482 if (dest_size == 0) { 483 // Caller is not too bright. I guess we've done what they asked. 484 return true; 485 } 486 487 // Once the request fails or is cancelled, read will just return 0 bytes 488 // to indicate end of stream. 489 if (!status_.is_success()) { 490 return true; 491 } 492 493 return job_->Read(dest, dest_size, bytes_read); 494 } 495 496 void URLRequest::StopCaching() { 497 DCHECK(job_); 498 job_->StopCaching(); 499 } 500 501 void URLRequest::ReceivedRedirect(const GURL& location, bool* defer_redirect) { 502 URLRequestJob* job = 503 URLRequestJobManager::GetInstance()->MaybeInterceptRedirect(this, 504 location); 505 if (job) { 506 RestartWithJob(job); 507 } else if (delegate_) { 508 delegate_->OnReceivedRedirect(this, location, defer_redirect); 509 } 510 } 511 512 void URLRequest::ResponseStarted() { 513 scoped_refptr<NetLog::EventParameters> params; 514 if (!status_.is_success()) 515 params = new NetLogIntegerParameter("net_error", status_.os_error()); 516 net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB, params); 517 518 URLRequestJob* job = 519 URLRequestJobManager::GetInstance()->MaybeInterceptResponse(this); 520 if (job) { 521 RestartWithJob(job); 522 } else { 523 if (context_ && context_->network_delegate()) 524 context_->network_delegate()->NotifyResponseStarted(this); 525 if (delegate_) 526 delegate_->OnResponseStarted(this); 527 } 528 } 529 530 void URLRequest::FollowDeferredRedirect() { 531 CHECK(job_); 532 CHECK(status_.is_success()); 533 534 job_->FollowDeferredRedirect(); 535 } 536 537 void URLRequest::SetAuth(const string16& username, const string16& password) { 538 DCHECK(job_); 539 DCHECK(job_->NeedsAuth()); 540 541 job_->SetAuth(username, password); 542 } 543 544 void URLRequest::CancelAuth() { 545 DCHECK(job_); 546 DCHECK(job_->NeedsAuth()); 547 548 job_->CancelAuth(); 549 } 550 551 void URLRequest::ContinueWithCertificate(X509Certificate* client_cert) { 552 DCHECK(job_); 553 554 job_->ContinueWithCertificate(client_cert); 555 } 556 557 void URLRequest::ContinueDespiteLastError() { 558 DCHECK(job_); 559 560 job_->ContinueDespiteLastError(); 561 } 562 563 void URLRequest::PrepareToRestart() { 564 DCHECK(job_); 565 566 // Close the current URL_REQUEST_START_JOB, since we will be starting a new 567 // one. 568 net_log_.EndEvent(NetLog::TYPE_URL_REQUEST_START_JOB, NULL); 569 570 OrphanJob(); 571 572 response_info_ = HttpResponseInfo(); 573 response_info_.request_time = Time::Now(); 574 status_ = URLRequestStatus(); 575 is_pending_ = false; 576 } 577 578 void URLRequest::OrphanJob() { 579 job_->Kill(); 580 job_->DetachRequest(); // ensures that the job will not call us again 581 job_ = NULL; 582 } 583 584 int URLRequest::Redirect(const GURL& location, int http_status_code) { 585 if (net_log_.IsLoggingAllEvents()) { 586 net_log_.AddEvent( 587 NetLog::TYPE_URL_REQUEST_REDIRECTED, 588 make_scoped_refptr(new NetLogStringParameter( 589 "location", location.possibly_invalid_spec()))); 590 } 591 if (redirect_limit_ <= 0) { 592 DVLOG(1) << "disallowing redirect: exceeds limit"; 593 return ERR_TOO_MANY_REDIRECTS; 594 } 595 596 if (!location.is_valid()) 597 return ERR_INVALID_URL; 598 599 if (!job_->IsSafeRedirect(location)) { 600 DVLOG(1) << "disallowing redirect: unsafe protocol"; 601 return ERR_UNSAFE_REDIRECT; 602 } 603 604 bool strip_post_specific_headers = false; 605 if (http_status_code != 307) { 606 // NOTE: Even though RFC 2616 says to preserve the request method when 607 // following a 302 redirect, normal browsers don't do that. Instead, they 608 // all convert a POST into a GET in response to a 302 and so shall we. For 609 // 307 redirects, browsers preserve the method. The RFC says to prompt the 610 // user to confirm the generation of a new POST request, but IE omits this 611 // prompt and so shall we. 612 strip_post_specific_headers = method_ == "POST"; 613 method_ = "GET"; 614 upload_ = NULL; 615 } 616 617 // Suppress the referrer if we're redirecting out of https. 618 if (GURL(referrer_).SchemeIsSecure() && !location.SchemeIsSecure()) 619 referrer_.clear(); 620 621 url_chain_.push_back(location); 622 --redirect_limit_; 623 624 if (strip_post_specific_headers) { 625 // If being switched from POST to GET, must remove headers that were 626 // specific to the POST and don't have meaning in GET. For example 627 // the inclusion of a multipart Content-Type header in GET can cause 628 // problems with some servers: 629 // http://code.google.com/p/chromium/issues/detail?id=843 630 StripPostSpecificHeaders(&extra_request_headers_); 631 } 632 633 if (!final_upload_progress_) 634 final_upload_progress_ = job_->GetUploadProgress(); 635 636 PrepareToRestart(); 637 StartInternal(); 638 return OK; 639 } 640 641 URLRequestContext* URLRequest::context() const { 642 return context_.get(); 643 } 644 645 void URLRequest::set_context(URLRequestContext* context) { 646 scoped_refptr<URLRequestContext> prev_context = context_; 647 648 context_ = context; 649 650 // If the context this request belongs to has changed, update the tracker. 651 if (prev_context != context) { 652 net_log_.EndEvent(NetLog::TYPE_REQUEST_ALIVE, NULL); 653 net_log_ = BoundNetLog(); 654 655 if (context) { 656 net_log_ = BoundNetLog::Make(context->net_log(), 657 NetLog::SOURCE_URL_REQUEST); 658 net_log_.BeginEvent(NetLog::TYPE_REQUEST_ALIVE, NULL); 659 } 660 } 661 } 662 663 int64 URLRequest::GetExpectedContentSize() const { 664 int64 expected_content_size = -1; 665 if (job_) 666 expected_content_size = job_->expected_content_size(); 667 668 return expected_content_size; 669 } 670 671 URLRequest::UserData* URLRequest::GetUserData(const void* key) const { 672 UserDataMap::const_iterator found = user_data_.find(key); 673 if (found != user_data_.end()) 674 return found->second.get(); 675 return NULL; 676 } 677 678 void URLRequest::SetUserData(const void* key, UserData* data) { 679 user_data_[key] = linked_ptr<UserData>(data); 680 } 681 682 } // namespace net 683