1 /* 2 Copyright (C) 1998 Lars Knoll (knoll (at) mpi-hd.mpg.de) 3 Copyright (C) 2001 Dirk Mueller (mueller (at) kde.org) 4 Copyright (C) 2002 Waldo Bastian (bastian (at) kde.org) 5 Copyright (C) 2006 Samuel Weinig (sam.weinig (at) gmail.com) 6 Copyright (C) 2004, 2005, 2006, 2007, 2008 Apple Inc. All rights reserved. 7 8 This library is free software; you can redistribute it and/or 9 modify it under the terms of the GNU Library General Public 10 License as published by the Free Software Foundation; either 11 version 2 of the License, or (at your option) any later version. 12 13 This library is distributed in the hope that it will be useful, 14 but WITHOUT ANY WARRANTY; without even the implied warranty of 15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 Library General Public License for more details. 17 18 You should have received a copy of the GNU Library General Public License 19 along with this library; see the file COPYING.LIB. If not, write to 20 the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 21 Boston, MA 02110-1301, USA. 22 */ 23 24 #include "config.h" 25 #include "loader.h" 26 27 #include "Cache.h" 28 #include "CachedImage.h" 29 #include "CachedResource.h" 30 #include "CString.h" 31 #include "DocLoader.h" 32 #include "Frame.h" 33 #include "FrameLoader.h" 34 #include "HTMLDocument.h" 35 #include "Request.h" 36 #include "ResourceHandle.h" 37 #include "ResourceRequest.h" 38 #include "ResourceResponse.h" 39 #include "SecurityOrigin.h" 40 #include "SubresourceLoader.h" 41 #include <wtf/Assertions.h> 42 #include <wtf/Vector.h> 43 44 #define REQUEST_MANAGEMENT_ENABLED 1 45 #define REQUEST_DEBUG 0 46 47 namespace WebCore { 48 49 #if REQUEST_MANAGEMENT_ENABLED 50 // Match the parallel connection count used by the networking layer 51 static unsigned maxRequestsInFlightPerHost; 52 // Having a limit might still help getting more important resources first 53 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 20; 54 #else 55 static const unsigned maxRequestsInFlightPerHost = 10000; 56 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 10000; 57 #endif 58 59 Loader::Loader() 60 : m_requestTimer(this, &Loader::requestTimerFired) 61 , m_isSuspendingPendingRequests(false) 62 { 63 m_nonHTTPProtocolHost = Host::create(AtomicString(), maxRequestsInFlightForNonHTTPProtocols); 64 #if REQUEST_MANAGEMENT_ENABLED 65 maxRequestsInFlightPerHost = initializeMaximumHTTPConnectionCountPerHost(); 66 #endif 67 } 68 69 Loader::~Loader() 70 { 71 ASSERT_NOT_REACHED(); 72 } 73 74 static ResourceRequest::TargetType cachedResourceTypeToTargetType(CachedResource::Type type) 75 { 76 switch (type) { 77 case CachedResource::CSSStyleSheet: 78 #if ENABLE(XSLT) 79 case CachedResource::XSLStyleSheet: 80 #endif 81 #if ENABLE(XBL) 82 case CachedResource::XBL: 83 #endif 84 return ResourceRequest::TargetIsStyleSheet; 85 case CachedResource::Script: 86 return ResourceRequest::TargetIsScript; 87 case CachedResource::FontResource: 88 return ResourceRequest::TargetIsFontResource; 89 case CachedResource::ImageResource: 90 return ResourceRequest::TargetIsImage; 91 } 92 return ResourceRequest::TargetIsSubresource; 93 } 94 95 Loader::Priority Loader::determinePriority(const CachedResource* resource) const 96 { 97 #if REQUEST_MANAGEMENT_ENABLED 98 switch (resource->type()) { 99 case CachedResource::CSSStyleSheet: 100 #if ENABLE(XSLT) 101 case CachedResource::XSLStyleSheet: 102 #endif 103 #if ENABLE(XBL) 104 case CachedResource::XBL: 105 #endif 106 return High; 107 case CachedResource::Script: 108 case CachedResource::FontResource: 109 return Medium; 110 case CachedResource::ImageResource: 111 return Low; 112 } 113 ASSERT_NOT_REACHED(); 114 return Low; 115 #else 116 return High; 117 #endif 118 } 119 120 void Loader::load(DocLoader* docLoader, CachedResource* resource, bool incremental, SecurityCheckPolicy securityCheck, bool sendResourceLoadCallbacks) 121 { 122 ASSERT(docLoader); 123 Request* request = new Request(docLoader, resource, incremental, securityCheck, sendResourceLoadCallbacks); 124 125 RefPtr<Host> host; 126 KURL url(ParsedURLString, resource->url()); 127 if (url.protocolInHTTPFamily()) { 128 m_hosts.checkConsistency(); 129 AtomicString hostName = url.host(); 130 host = m_hosts.get(hostName.impl()); 131 if (!host) { 132 host = Host::create(hostName, maxRequestsInFlightPerHost); 133 m_hosts.add(hostName.impl(), host); 134 } 135 } else 136 host = m_nonHTTPProtocolHost; 137 138 bool hadRequests = host->hasRequests(); 139 Priority priority = determinePriority(resource); 140 host->addRequest(request, priority); 141 docLoader->incrementRequestCount(); 142 143 if (priority > Low || !url.protocolInHTTPFamily() || !hadRequests) { 144 // Try to request important resources immediately 145 host->servePendingRequests(priority); 146 } else { 147 // Handle asynchronously so early low priority requests don't get scheduled before later high priority ones 148 scheduleServePendingRequests(); 149 } 150 } 151 152 void Loader::scheduleServePendingRequests() 153 { 154 if (!m_requestTimer.isActive()) 155 m_requestTimer.startOneShot(0); 156 } 157 158 void Loader::requestTimerFired(Timer<Loader>*) 159 { 160 servePendingRequests(); 161 } 162 163 void Loader::servePendingRequests(Priority minimumPriority) 164 { 165 if (m_isSuspendingPendingRequests) 166 return; 167 168 m_requestTimer.stop(); 169 170 m_nonHTTPProtocolHost->servePendingRequests(minimumPriority); 171 172 Vector<Host*> hostsToServe; 173 m_hosts.checkConsistency(); 174 HostMap::iterator i = m_hosts.begin(); 175 HostMap::iterator end = m_hosts.end(); 176 for (;i != end; ++i) 177 hostsToServe.append(i->second.get()); 178 179 for (unsigned n = 0; n < hostsToServe.size(); ++n) { 180 Host* host = hostsToServe[n]; 181 if (host->hasRequests()) 182 host->servePendingRequests(minimumPriority); 183 else if (!host->processingResource()){ 184 AtomicString name = host->name(); 185 m_hosts.remove(name.impl()); 186 } 187 } 188 } 189 190 void Loader::suspendPendingRequests() 191 { 192 ASSERT(!m_isSuspendingPendingRequests); 193 m_isSuspendingPendingRequests = true; 194 } 195 196 void Loader::resumePendingRequests() 197 { 198 ASSERT(m_isSuspendingPendingRequests); 199 m_isSuspendingPendingRequests = false; 200 if (!m_hosts.isEmpty() || m_nonHTTPProtocolHost->hasRequests()) 201 scheduleServePendingRequests(); 202 } 203 204 void Loader::nonCacheRequestInFlight(const KURL& url) 205 { 206 if (!url.protocolInHTTPFamily()) 207 return; 208 209 AtomicString hostName = url.host(); 210 m_hosts.checkConsistency(); 211 RefPtr<Host> host = m_hosts.get(hostName.impl()); 212 if (!host) { 213 host = Host::create(hostName, maxRequestsInFlightPerHost); 214 m_hosts.add(hostName.impl(), host); 215 } 216 217 host->nonCacheRequestInFlight(); 218 } 219 220 void Loader::nonCacheRequestComplete(const KURL& url) 221 { 222 if (!url.protocolInHTTPFamily()) 223 return; 224 225 AtomicString hostName = url.host(); 226 m_hosts.checkConsistency(); 227 RefPtr<Host> host = m_hosts.get(hostName.impl()); 228 ASSERT(host); 229 if (!host) 230 return; 231 232 host->nonCacheRequestComplete(); 233 } 234 235 void Loader::cancelRequests(DocLoader* docLoader) 236 { 237 docLoader->clearPendingPreloads(); 238 239 if (m_nonHTTPProtocolHost->hasRequests()) 240 m_nonHTTPProtocolHost->cancelRequests(docLoader); 241 242 Vector<Host*> hostsToCancel; 243 m_hosts.checkConsistency(); 244 HostMap::iterator i = m_hosts.begin(); 245 HostMap::iterator end = m_hosts.end(); 246 for (;i != end; ++i) 247 hostsToCancel.append(i->second.get()); 248 249 for (unsigned n = 0; n < hostsToCancel.size(); ++n) { 250 Host* host = hostsToCancel[n]; 251 if (host->hasRequests()) 252 host->cancelRequests(docLoader); 253 } 254 255 scheduleServePendingRequests(); 256 257 ASSERT(docLoader->requestCount() == (docLoader->loadInProgress() ? 1 : 0)); 258 } 259 260 Loader::Host::Host(const AtomicString& name, unsigned maxRequestsInFlight) 261 : m_name(name) 262 , m_maxRequestsInFlight(maxRequestsInFlight) 263 , m_numResourcesProcessing(0) 264 , m_nonCachedRequestsInFlight(0) 265 { 266 } 267 268 Loader::Host::~Host() 269 { 270 ASSERT(m_requestsLoading.isEmpty()); 271 for (unsigned p = 0; p <= High; p++) 272 ASSERT(m_requestsPending[p].isEmpty()); 273 } 274 275 void Loader::Host::addRequest(Request* request, Priority priority) 276 { 277 m_requestsPending[priority].append(request); 278 } 279 280 void Loader::Host::nonCacheRequestInFlight() 281 { 282 ++m_nonCachedRequestsInFlight; 283 } 284 285 void Loader::Host::nonCacheRequestComplete() 286 { 287 --m_nonCachedRequestsInFlight; 288 ASSERT(m_nonCachedRequestsInFlight >= 0); 289 } 290 291 bool Loader::Host::hasRequests() const 292 { 293 if (!m_requestsLoading.isEmpty()) 294 return true; 295 for (unsigned p = 0; p <= High; p++) { 296 if (!m_requestsPending[p].isEmpty()) 297 return true; 298 } 299 return false; 300 } 301 302 void Loader::Host::servePendingRequests(Loader::Priority minimumPriority) 303 { 304 if (cache()->loader()->isSuspendingPendingRequests()) 305 return; 306 307 bool serveMore = true; 308 for (int priority = High; priority >= minimumPriority && serveMore; --priority) 309 servePendingRequests(m_requestsPending[priority], serveMore); 310 } 311 312 void Loader::Host::servePendingRequests(RequestQueue& requestsPending, bool& serveLowerPriority) 313 { 314 while (!requestsPending.isEmpty()) { 315 Request* request = requestsPending.first(); 316 DocLoader* docLoader = request->docLoader(); 317 bool resourceIsCacheValidator = request->cachedResource()->isCacheValidator(); 318 319 // For named hosts - which are only http(s) hosts - we should always enforce the connection limit. 320 // For non-named hosts - everything but http(s) - we should only enforce the limit if the document isn't done parsing 321 // and we don't know all stylesheets yet. 322 bool shouldLimitRequests = !m_name.isNull() || docLoader->doc()->parsing() || !docLoader->doc()->haveStylesheetsLoaded(); 323 if (shouldLimitRequests && m_requestsLoading.size() + m_nonCachedRequestsInFlight >= m_maxRequestsInFlight) { 324 serveLowerPriority = false; 325 cache()->loader()->scheduleServePendingRequests(); 326 return; 327 } 328 requestsPending.removeFirst(); 329 330 ResourceRequest resourceRequest(request->cachedResource()->url()); 331 resourceRequest.setTargetType(cachedResourceTypeToTargetType(request->cachedResource()->type())); 332 333 if (!request->cachedResource()->accept().isEmpty()) 334 resourceRequest.setHTTPAccept(request->cachedResource()->accept()); 335 336 // Do not set the referrer or HTTP origin here. That's handled by SubresourceLoader::create. 337 338 if (resourceIsCacheValidator) { 339 CachedResource* resourceToRevalidate = request->cachedResource()->resourceToRevalidate(); 340 ASSERT(resourceToRevalidate->canUseCacheValidator()); 341 ASSERT(resourceToRevalidate->isLoaded()); 342 const String& lastModified = resourceToRevalidate->response().httpHeaderField("Last-Modified"); 343 const String& eTag = resourceToRevalidate->response().httpHeaderField("ETag"); 344 if (!lastModified.isEmpty() || !eTag.isEmpty()) { 345 ASSERT(docLoader->cachePolicy() != CachePolicyReload); 346 if (docLoader->cachePolicy() == CachePolicyRevalidate) 347 resourceRequest.setHTTPHeaderField("Cache-Control", "max-age=0"); 348 if (!lastModified.isEmpty()) 349 resourceRequest.setHTTPHeaderField("If-Modified-Since", lastModified); 350 if (!eTag.isEmpty()) 351 resourceRequest.setHTTPHeaderField("If-None-Match", eTag); 352 } 353 } 354 355 RefPtr<SubresourceLoader> loader = SubresourceLoader::create(docLoader->doc()->frame(), 356 this, resourceRequest, request->shouldDoSecurityCheck(), request->sendResourceLoadCallbacks()); 357 if (loader) { 358 m_requestsLoading.add(loader.release(), request); 359 request->cachedResource()->setRequestedFromNetworkingLayer(); 360 #if REQUEST_DEBUG 361 printf("HOST %s COUNT %d LOADING %s\n", resourceRequest.url().host().latin1().data(), m_requestsLoading.size(), request->cachedResource()->url().latin1().data()); 362 #endif 363 } else { 364 docLoader->decrementRequestCount(); 365 docLoader->setLoadInProgress(true); 366 request->cachedResource()->error(); 367 docLoader->setLoadInProgress(false); 368 delete request; 369 } 370 } 371 } 372 373 void Loader::Host::didFinishLoading(SubresourceLoader* loader) 374 { 375 RefPtr<Host> myProtector(this); 376 377 RequestMap::iterator i = m_requestsLoading.find(loader); 378 if (i == m_requestsLoading.end()) 379 return; 380 381 Request* request = i->second; 382 m_requestsLoading.remove(i); 383 DocLoader* docLoader = request->docLoader(); 384 // Prevent the document from being destroyed before we are done with 385 // the docLoader that it will delete when the document gets deleted. 386 RefPtr<Document> protector(docLoader->doc()); 387 if (!request->isMultipart()) 388 docLoader->decrementRequestCount(); 389 390 CachedResource* resource = request->cachedResource(); 391 ASSERT(!resource->resourceToRevalidate()); 392 393 // If we got a 4xx response, we're pretending to have received a network 394 // error, so we can't send the successful data() and finish() callbacks. 395 if (!resource->errorOccurred()) { 396 docLoader->setLoadInProgress(true); 397 resource->data(loader->resourceData(), true); 398 resource->finish(); 399 } 400 401 delete request; 402 403 docLoader->setLoadInProgress(false); 404 405 docLoader->checkForPendingPreloads(); 406 407 #if REQUEST_DEBUG 408 KURL u(ParsedURLString, resource->url()); 409 printf("HOST %s COUNT %d RECEIVED %s\n", u.host().latin1().data(), m_requestsLoading.size(), resource->url().latin1().data()); 410 #endif 411 servePendingRequests(); 412 } 413 414 void Loader::Host::didFail(SubresourceLoader* loader, const ResourceError&) 415 { 416 didFail(loader); 417 } 418 419 void Loader::Host::didFail(SubresourceLoader* loader, bool cancelled) 420 { 421 RefPtr<Host> myProtector(this); 422 423 loader->clearClient(); 424 425 RequestMap::iterator i = m_requestsLoading.find(loader); 426 if (i == m_requestsLoading.end()) 427 return; 428 429 Request* request = i->second; 430 m_requestsLoading.remove(i); 431 DocLoader* docLoader = request->docLoader(); 432 // Prevent the document from being destroyed before we are done with 433 // the docLoader that it will delete when the document gets deleted. 434 RefPtr<Document> protector(docLoader->doc()); 435 if (!request->isMultipart()) 436 docLoader->decrementRequestCount(); 437 438 CachedResource* resource = request->cachedResource(); 439 440 if (resource->resourceToRevalidate()) 441 cache()->revalidationFailed(resource); 442 443 if (!cancelled) { 444 docLoader->setLoadInProgress(true); 445 resource->error(); 446 } 447 448 docLoader->setLoadInProgress(false); 449 if (cancelled || !resource->isPreloaded()) 450 cache()->remove(resource); 451 452 delete request; 453 454 docLoader->checkForPendingPreloads(); 455 456 servePendingRequests(); 457 } 458 459 void Loader::Host::didReceiveResponse(SubresourceLoader* loader, const ResourceResponse& response) 460 { 461 RefPtr<Host> protector(this); 462 463 Request* request = m_requestsLoading.get(loader); 464 465 // FIXME: This is a workaround for <rdar://problem/5236843> 466 // If a load starts while the frame is still in the provisional state 467 // (this can be the case when loading the user style sheet), committing the load then causes all 468 // requests to be removed from the m_requestsLoading map. This means that request might be null here. 469 // In that case we just return early. 470 // ASSERT(request); 471 if (!request) 472 return; 473 474 CachedResource* resource = request->cachedResource(); 475 476 if (resource->isCacheValidator()) { 477 if (response.httpStatusCode() == 304) { 478 // 304 Not modified / Use local copy 479 m_requestsLoading.remove(loader); 480 loader->clearClient(); 481 request->docLoader()->decrementRequestCount(); 482 483 // Existing resource is ok, just use it updating the expiration time. 484 cache()->revalidationSucceeded(resource, response); 485 486 if (request->docLoader()->frame()) 487 request->docLoader()->frame()->loader()->checkCompleted(); 488 489 delete request; 490 491 servePendingRequests(); 492 return; 493 } 494 // Did not get 304 response, continue as a regular resource load. 495 cache()->revalidationFailed(resource); 496 } 497 498 resource->setResponse(response); 499 500 String encoding = response.textEncodingName(); 501 if (!encoding.isNull()) 502 resource->setEncoding(encoding); 503 504 if (request->isMultipart()) { 505 ASSERT(resource->isImage()); 506 static_cast<CachedImage*>(resource)->clear(); 507 if (request->docLoader()->frame()) 508 request->docLoader()->frame()->loader()->checkCompleted(); 509 } else if (response.isMultipart()) { 510 request->setIsMultipart(true); 511 512 // We don't count multiParts in a DocLoader's request count 513 request->docLoader()->decrementRequestCount(); 514 515 // If we get a multipart response, we must have a handle 516 ASSERT(loader->handle()); 517 if (!resource->isImage()) 518 loader->handle()->cancel(); 519 } 520 } 521 522 void Loader::Host::didReceiveData(SubresourceLoader* loader, const char* data, int size) 523 { 524 RefPtr<Host> protector(this); 525 526 Request* request = m_requestsLoading.get(loader); 527 if (!request) 528 return; 529 530 CachedResource* resource = request->cachedResource(); 531 ASSERT(!resource->isCacheValidator()); 532 533 if (resource->errorOccurred()) 534 return; 535 536 if (resource->response().httpStatusCode() / 100 == 4) { 537 // Treat a 4xx response like a network error for all resources but images (which will ignore the error and continue to load for 538 // legacy compatibility). 539 resource->httpStatusCodeError(); 540 return; 541 } 542 543 // Set the data. 544 if (request->isMultipart()) { 545 // The loader delivers the data in a multipart section all at once, send eof. 546 // The resource data will change as the next part is loaded, so we need to make a copy. 547 RefPtr<SharedBuffer> copiedData = SharedBuffer::create(data, size); 548 resource->data(copiedData.release(), true); 549 } else if (request->isIncremental()) 550 resource->data(loader->resourceData(), false); 551 } 552 553 void Loader::Host::cancelPendingRequests(RequestQueue& requestsPending, DocLoader* docLoader) 554 { 555 RequestQueue remaining; 556 RequestQueue::iterator end = requestsPending.end(); 557 for (RequestQueue::iterator it = requestsPending.begin(); it != end; ++it) { 558 Request* request = *it; 559 if (request->docLoader() == docLoader) { 560 cache()->remove(request->cachedResource()); 561 delete request; 562 docLoader->decrementRequestCount(); 563 } else 564 remaining.append(request); 565 } 566 requestsPending.swap(remaining); 567 } 568 569 void Loader::Host::cancelRequests(DocLoader* docLoader) 570 { 571 for (unsigned p = 0; p <= High; p++) 572 cancelPendingRequests(m_requestsPending[p], docLoader); 573 574 Vector<SubresourceLoader*, 256> loadersToCancel; 575 576 RequestMap::iterator end = m_requestsLoading.end(); 577 for (RequestMap::iterator i = m_requestsLoading.begin(); i != end; ++i) { 578 Request* r = i->second; 579 if (r->docLoader() == docLoader) 580 loadersToCancel.append(i->first.get()); 581 } 582 583 for (unsigned i = 0; i < loadersToCancel.size(); ++i) { 584 SubresourceLoader* loader = loadersToCancel[i]; 585 didFail(loader, true); 586 } 587 } 588 589 } //namespace WebCore 590