Home | History | Annotate | Download | only in loader 
      1 /*
      2     Copyright (C) 1998 Lars Knoll (knoll (at) mpi-hd.mpg.de)
      3     Copyright (C) 2001 Dirk Mueller (mueller (at) kde.org)
      4     Copyright (C) 2002 Waldo Bastian (bastian (at) kde.org)
      5     Copyright (C) 2006 Samuel Weinig (sam.weinig (at) gmail.com)
      6     Copyright (C) 2004, 2005, 2006, 2007, 2008 Apple Inc. All rights reserved.
      7 
      8     This library is free software; you can redistribute it and/or
      9     modify it under the terms of the GNU Library General Public
     10     License as published by the Free Software Foundation; either
     11     version 2 of the License, or (at your option) any later version.
     12 
     13     This library is distributed in the hope that it will be useful,
     14     but WITHOUT ANY WARRANTY; without even the implied warranty of
     15     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     16     Library General Public License for more details.
     17 
     18     You should have received a copy of the GNU Library General Public License
     19     along with this library; see the file COPYING.LIB.  If not, write to
     20     the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     21     Boston, MA 02110-1301, USA.
     22 */
     23 
     24 #include "config.h"
     25 #include "loader.h"
     26 
     27 #include "Cache.h"
     28 #include "CachedImage.h"
     29 #include "CachedResource.h"
     30 #include "CString.h"
     31 #include "DocLoader.h"
     32 #include "Frame.h"
     33 #include "FrameLoader.h"
     34 #include "HTMLDocument.h"
     35 #include "Request.h"
     36 #include "ResourceHandle.h"
     37 #include "ResourceRequest.h"
     38 #include "ResourceResponse.h"
     39 #include "SecurityOrigin.h"
     40 #include "SubresourceLoader.h"
     41 #include <wtf/Assertions.h>
     42 #include <wtf/Vector.h>
     43 
     44 #define REQUEST_MANAGEMENT_ENABLED 1
     45 #define REQUEST_DEBUG 0
     46 
     47 namespace WebCore {
     48 
     49 #if REQUEST_MANAGEMENT_ENABLED
     50 // Match the parallel connection count used by the networking layer
     51 static unsigned maxRequestsInFlightPerHost;
     52 // Having a limit might still help getting more important resources first
     53 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 20;
     54 #else
     55 static const unsigned maxRequestsInFlightPerHost = 10000;
     56 static const unsigned maxRequestsInFlightForNonHTTPProtocols = 10000;
     57 #endif
     58 
     59 Loader::Loader()
     60     : m_requestTimer(this, &Loader::requestTimerFired)
     61     , m_isSuspendingPendingRequests(false)
     62 {
     63     m_nonHTTPProtocolHost = Host::create(AtomicString(), maxRequestsInFlightForNonHTTPProtocols);
     64 #if REQUEST_MANAGEMENT_ENABLED
     65     maxRequestsInFlightPerHost = initializeMaximumHTTPConnectionCountPerHost();
     66 #endif
     67 }
     68 
     69 Loader::~Loader()
     70 {
     71     ASSERT_NOT_REACHED();
     72 }
     73 
     74 static ResourceRequest::TargetType cachedResourceTypeToTargetType(CachedResource::Type type)
     75 {
     76     switch (type) {
     77     case CachedResource::CSSStyleSheet:
     78 #if ENABLE(XSLT)
     79     case CachedResource::XSLStyleSheet:
     80 #endif
     81 #if ENABLE(XBL)
     82     case CachedResource::XBL:
     83 #endif
     84         return ResourceRequest::TargetIsStyleSheet;
     85     case CachedResource::Script:
     86         return ResourceRequest::TargetIsScript;
     87     case CachedResource::FontResource:
     88         return ResourceRequest::TargetIsFontResource;
     89     case CachedResource::ImageResource:
     90         return ResourceRequest::TargetIsImage;
     91     }
     92     return ResourceRequest::TargetIsSubresource;
     93 }
     94 
     95 Loader::Priority Loader::determinePriority(const CachedResource* resource) const
     96 {
     97 #if REQUEST_MANAGEMENT_ENABLED
     98     switch (resource->type()) {
     99         case CachedResource::CSSStyleSheet:
    100 #if ENABLE(XSLT)
    101         case CachedResource::XSLStyleSheet:
    102 #endif
    103 #if ENABLE(XBL)
    104         case CachedResource::XBL:
    105 #endif
    106             return High;
    107         case CachedResource::Script:
    108         case CachedResource::FontResource:
    109             return Medium;
    110         case CachedResource::ImageResource:
    111             return Low;
    112     }
    113     ASSERT_NOT_REACHED();
    114     return Low;
    115 #else
    116     return High;
    117 #endif
    118 }
    119 
    120 void Loader::load(DocLoader* docLoader, CachedResource* resource, bool incremental, SecurityCheckPolicy securityCheck, bool sendResourceLoadCallbacks)
    121 {
    122     ASSERT(docLoader);
    123     Request* request = new Request(docLoader, resource, incremental, securityCheck, sendResourceLoadCallbacks);
    124 
    125     RefPtr<Host> host;
    126     KURL url(ParsedURLString, resource->url());
    127     if (url.protocolInHTTPFamily()) {
    128         m_hosts.checkConsistency();
    129         AtomicString hostName = url.host();
    130         host = m_hosts.get(hostName.impl());
    131         if (!host) {
    132             host = Host::create(hostName, maxRequestsInFlightPerHost);
    133             m_hosts.add(hostName.impl(), host);
    134         }
    135     } else
    136         host = m_nonHTTPProtocolHost;
    137 
    138     bool hadRequests = host->hasRequests();
    139     Priority priority = determinePriority(resource);
    140     host->addRequest(request, priority);
    141     docLoader->incrementRequestCount();
    142 
    143     if (priority > Low || !url.protocolInHTTPFamily() || !hadRequests) {
    144         // Try to request important resources immediately
    145         host->servePendingRequests(priority);
    146     } else {
    147         // Handle asynchronously so early low priority requests don't get scheduled before later high priority ones
    148         scheduleServePendingRequests();
    149     }
    150 }
    151 
    152 void Loader::scheduleServePendingRequests()
    153 {
    154     if (!m_requestTimer.isActive())
    155         m_requestTimer.startOneShot(0);
    156 }
    157 
    158 void Loader::requestTimerFired(Timer<Loader>*)
    159 {
    160     servePendingRequests();
    161 }
    162 
    163 void Loader::servePendingRequests(Priority minimumPriority)
    164 {
    165     if (m_isSuspendingPendingRequests)
    166         return;
    167 
    168     m_requestTimer.stop();
    169 
    170     m_nonHTTPProtocolHost->servePendingRequests(minimumPriority);
    171 
    172     Vector<Host*> hostsToServe;
    173     m_hosts.checkConsistency();
    174     HostMap::iterator i = m_hosts.begin();
    175     HostMap::iterator end = m_hosts.end();
    176     for (;i != end; ++i)
    177         hostsToServe.append(i->second.get());
    178 
    179     for (unsigned n = 0; n < hostsToServe.size(); ++n) {
    180         Host* host = hostsToServe[n];
    181         if (host->hasRequests())
    182             host->servePendingRequests(minimumPriority);
    183         else if (!host->processingResource()){
    184             AtomicString name = host->name();
    185             m_hosts.remove(name.impl());
    186         }
    187     }
    188 }
    189 
    190 void Loader::suspendPendingRequests()
    191 {
    192     ASSERT(!m_isSuspendingPendingRequests);
    193     m_isSuspendingPendingRequests = true;
    194 }
    195 
    196 void Loader::resumePendingRequests()
    197 {
    198     ASSERT(m_isSuspendingPendingRequests);
    199     m_isSuspendingPendingRequests = false;
    200     if (!m_hosts.isEmpty() || m_nonHTTPProtocolHost->hasRequests())
    201         scheduleServePendingRequests();
    202 }
    203 
    204 void Loader::nonCacheRequestInFlight(const KURL& url)
    205 {
    206     if (!url.protocolInHTTPFamily())
    207         return;
    208 
    209     AtomicString hostName = url.host();
    210     m_hosts.checkConsistency();
    211     RefPtr<Host> host = m_hosts.get(hostName.impl());
    212     if (!host) {
    213         host = Host::create(hostName, maxRequestsInFlightPerHost);
    214         m_hosts.add(hostName.impl(), host);
    215     }
    216 
    217     host->nonCacheRequestInFlight();
    218 }
    219 
    220 void Loader::nonCacheRequestComplete(const KURL& url)
    221 {
    222     if (!url.protocolInHTTPFamily())
    223         return;
    224 
    225     AtomicString hostName = url.host();
    226     m_hosts.checkConsistency();
    227     RefPtr<Host> host = m_hosts.get(hostName.impl());
    228     ASSERT(host);
    229     if (!host)
    230         return;
    231 
    232     host->nonCacheRequestComplete();
    233 }
    234 
    235 void Loader::cancelRequests(DocLoader* docLoader)
    236 {
    237     docLoader->clearPendingPreloads();
    238 
    239     if (m_nonHTTPProtocolHost->hasRequests())
    240         m_nonHTTPProtocolHost->cancelRequests(docLoader);
    241 
    242     Vector<Host*> hostsToCancel;
    243     m_hosts.checkConsistency();
    244     HostMap::iterator i = m_hosts.begin();
    245     HostMap::iterator end = m_hosts.end();
    246     for (;i != end; ++i)
    247         hostsToCancel.append(i->second.get());
    248 
    249     for (unsigned n = 0; n < hostsToCancel.size(); ++n) {
    250         Host* host = hostsToCancel[n];
    251         if (host->hasRequests())
    252             host->cancelRequests(docLoader);
    253     }
    254 
    255     scheduleServePendingRequests();
    256 
    257     ASSERT(docLoader->requestCount() == (docLoader->loadInProgress() ? 1 : 0));
    258 }
    259 
    260 Loader::Host::Host(const AtomicString& name, unsigned maxRequestsInFlight)
    261     : m_name(name)
    262     , m_maxRequestsInFlight(maxRequestsInFlight)
    263     , m_numResourcesProcessing(0)
    264     , m_nonCachedRequestsInFlight(0)
    265 {
    266 }
    267 
    268 Loader::Host::~Host()
    269 {
    270     ASSERT(m_requestsLoading.isEmpty());
    271     for (unsigned p = 0; p <= High; p++)
    272         ASSERT(m_requestsPending[p].isEmpty());
    273 }
    274 
    275 void Loader::Host::addRequest(Request* request, Priority priority)
    276 {
    277     m_requestsPending[priority].append(request);
    278 }
    279 
    280 void Loader::Host::nonCacheRequestInFlight()
    281 {
    282     ++m_nonCachedRequestsInFlight;
    283 }
    284 
    285 void Loader::Host::nonCacheRequestComplete()
    286 {
    287     --m_nonCachedRequestsInFlight;
    288     ASSERT(m_nonCachedRequestsInFlight >= 0);
    289 }
    290 
    291 bool Loader::Host::hasRequests() const
    292 {
    293     if (!m_requestsLoading.isEmpty())
    294         return true;
    295     for (unsigned p = 0; p <= High; p++) {
    296         if (!m_requestsPending[p].isEmpty())
    297             return true;
    298     }
    299     return false;
    300 }
    301 
    302 void Loader::Host::servePendingRequests(Loader::Priority minimumPriority)
    303 {
    304     if (cache()->loader()->isSuspendingPendingRequests())
    305         return;
    306 
    307     bool serveMore = true;
    308     for (int priority = High; priority >= minimumPriority && serveMore; --priority)
    309         servePendingRequests(m_requestsPending[priority], serveMore);
    310 }
    311 
    312 void Loader::Host::servePendingRequests(RequestQueue& requestsPending, bool& serveLowerPriority)
    313 {
    314     while (!requestsPending.isEmpty()) {
    315         Request* request = requestsPending.first();
    316         DocLoader* docLoader = request->docLoader();
    317         bool resourceIsCacheValidator = request->cachedResource()->isCacheValidator();
    318 
    319         // For named hosts - which are only http(s) hosts - we should always enforce the connection limit.
    320         // For non-named hosts - everything but http(s) - we should only enforce the limit if the document isn't done parsing
    321         // and we don't know all stylesheets yet.
    322         bool shouldLimitRequests = !m_name.isNull() || docLoader->doc()->parsing() || !docLoader->doc()->haveStylesheetsLoaded();
    323         if (shouldLimitRequests && m_requestsLoading.size() + m_nonCachedRequestsInFlight >= m_maxRequestsInFlight) {
    324             serveLowerPriority = false;
    325             cache()->loader()->scheduleServePendingRequests();
    326             return;
    327         }
    328         requestsPending.removeFirst();
    329 
    330         ResourceRequest resourceRequest(request->cachedResource()->url());
    331         resourceRequest.setTargetType(cachedResourceTypeToTargetType(request->cachedResource()->type()));
    332 
    333         if (!request->cachedResource()->accept().isEmpty())
    334             resourceRequest.setHTTPAccept(request->cachedResource()->accept());
    335 
    336          // Do not set the referrer or HTTP origin here. That's handled by SubresourceLoader::create.
    337 
    338         if (resourceIsCacheValidator) {
    339             CachedResource* resourceToRevalidate = request->cachedResource()->resourceToRevalidate();
    340             ASSERT(resourceToRevalidate->canUseCacheValidator());
    341             ASSERT(resourceToRevalidate->isLoaded());
    342             const String& lastModified = resourceToRevalidate->response().httpHeaderField("Last-Modified");
    343             const String& eTag = resourceToRevalidate->response().httpHeaderField("ETag");
    344             if (!lastModified.isEmpty() || !eTag.isEmpty()) {
    345                 ASSERT(docLoader->cachePolicy() != CachePolicyReload);
    346                 if (docLoader->cachePolicy() == CachePolicyRevalidate)
    347                     resourceRequest.setHTTPHeaderField("Cache-Control", "max-age=0");
    348                 if (!lastModified.isEmpty())
    349                     resourceRequest.setHTTPHeaderField("If-Modified-Since", lastModified);
    350                 if (!eTag.isEmpty())
    351                     resourceRequest.setHTTPHeaderField("If-None-Match", eTag);
    352             }
    353         }
    354 
    355         RefPtr<SubresourceLoader> loader = SubresourceLoader::create(docLoader->doc()->frame(),
    356             this, resourceRequest, request->shouldDoSecurityCheck(), request->sendResourceLoadCallbacks());
    357         if (loader) {
    358             m_requestsLoading.add(loader.release(), request);
    359             request->cachedResource()->setRequestedFromNetworkingLayer();
    360 #if REQUEST_DEBUG
    361             printf("HOST %s COUNT %d LOADING %s\n", resourceRequest.url().host().latin1().data(), m_requestsLoading.size(), request->cachedResource()->url().latin1().data());
    362 #endif
    363         } else {
    364             docLoader->decrementRequestCount();
    365             docLoader->setLoadInProgress(true);
    366             request->cachedResource()->error();
    367             docLoader->setLoadInProgress(false);
    368             delete request;
    369         }
    370     }
    371 }
    372 
    373 void Loader::Host::didFinishLoading(SubresourceLoader* loader)
    374 {
    375     RefPtr<Host> myProtector(this);
    376 
    377     RequestMap::iterator i = m_requestsLoading.find(loader);
    378     if (i == m_requestsLoading.end())
    379         return;
    380 
    381     Request* request = i->second;
    382     m_requestsLoading.remove(i);
    383     DocLoader* docLoader = request->docLoader();
    384     // Prevent the document from being destroyed before we are done with
    385     // the docLoader that it will delete when the document gets deleted.
    386     RefPtr<Document> protector(docLoader->doc());
    387     if (!request->isMultipart())
    388         docLoader->decrementRequestCount();
    389 
    390     CachedResource* resource = request->cachedResource();
    391     ASSERT(!resource->resourceToRevalidate());
    392 
    393     // If we got a 4xx response, we're pretending to have received a network
    394     // error, so we can't send the successful data() and finish() callbacks.
    395     if (!resource->errorOccurred()) {
    396         docLoader->setLoadInProgress(true);
    397         resource->data(loader->resourceData(), true);
    398         resource->finish();
    399     }
    400 
    401     delete request;
    402 
    403     docLoader->setLoadInProgress(false);
    404 
    405     docLoader->checkForPendingPreloads();
    406 
    407 #if REQUEST_DEBUG
    408     KURL u(ParsedURLString, resource->url());
    409     printf("HOST %s COUNT %d RECEIVED %s\n", u.host().latin1().data(), m_requestsLoading.size(), resource->url().latin1().data());
    410 #endif
    411     servePendingRequests();
    412 }
    413 
    414 void Loader::Host::didFail(SubresourceLoader* loader, const ResourceError&)
    415 {
    416     didFail(loader);
    417 }
    418 
    419 void Loader::Host::didFail(SubresourceLoader* loader, bool cancelled)
    420 {
    421     RefPtr<Host> myProtector(this);
    422 
    423     loader->clearClient();
    424 
    425     RequestMap::iterator i = m_requestsLoading.find(loader);
    426     if (i == m_requestsLoading.end())
    427         return;
    428 
    429     Request* request = i->second;
    430     m_requestsLoading.remove(i);
    431     DocLoader* docLoader = request->docLoader();
    432     // Prevent the document from being destroyed before we are done with
    433     // the docLoader that it will delete when the document gets deleted.
    434     RefPtr<Document> protector(docLoader->doc());
    435     if (!request->isMultipart())
    436         docLoader->decrementRequestCount();
    437 
    438     CachedResource* resource = request->cachedResource();
    439 
    440     if (resource->resourceToRevalidate())
    441         cache()->revalidationFailed(resource);
    442 
    443     if (!cancelled) {
    444         docLoader->setLoadInProgress(true);
    445         resource->error();
    446     }
    447 
    448     docLoader->setLoadInProgress(false);
    449     if (cancelled || !resource->isPreloaded())
    450         cache()->remove(resource);
    451 
    452     delete request;
    453 
    454     docLoader->checkForPendingPreloads();
    455 
    456     servePendingRequests();
    457 }
    458 
    459 void Loader::Host::didReceiveResponse(SubresourceLoader* loader, const ResourceResponse& response)
    460 {
    461     RefPtr<Host> protector(this);
    462 
    463     Request* request = m_requestsLoading.get(loader);
    464 
    465     // FIXME: This is a workaround for <rdar://problem/5236843>
    466     // If a load starts while the frame is still in the provisional state
    467     // (this can be the case when loading the user style sheet), committing the load then causes all
    468     // requests to be removed from the m_requestsLoading map. This means that request might be null here.
    469     // In that case we just return early.
    470     // ASSERT(request);
    471     if (!request)
    472         return;
    473 
    474     CachedResource* resource = request->cachedResource();
    475 
    476     if (resource->isCacheValidator()) {
    477         if (response.httpStatusCode() == 304) {
    478             // 304 Not modified / Use local copy
    479             m_requestsLoading.remove(loader);
    480             loader->clearClient();
    481             request->docLoader()->decrementRequestCount();
    482 
    483             // Existing resource is ok, just use it updating the expiration time.
    484             cache()->revalidationSucceeded(resource, response);
    485 
    486             if (request->docLoader()->frame())
    487                 request->docLoader()->frame()->loader()->checkCompleted();
    488 
    489             delete request;
    490 
    491             servePendingRequests();
    492             return;
    493         }
    494         // Did not get 304 response, continue as a regular resource load.
    495         cache()->revalidationFailed(resource);
    496     }
    497 
    498     resource->setResponse(response);
    499 
    500     String encoding = response.textEncodingName();
    501     if (!encoding.isNull())
    502         resource->setEncoding(encoding);
    503 
    504     if (request->isMultipart()) {
    505         ASSERT(resource->isImage());
    506         static_cast<CachedImage*>(resource)->clear();
    507         if (request->docLoader()->frame())
    508             request->docLoader()->frame()->loader()->checkCompleted();
    509     } else if (response.isMultipart()) {
    510         request->setIsMultipart(true);
    511 
    512         // We don't count multiParts in a DocLoader's request count
    513         request->docLoader()->decrementRequestCount();
    514 
    515         // If we get a multipart response, we must have a handle
    516         ASSERT(loader->handle());
    517         if (!resource->isImage())
    518             loader->handle()->cancel();
    519     }
    520 }
    521 
    522 void Loader::Host::didReceiveData(SubresourceLoader* loader, const char* data, int size)
    523 {
    524     RefPtr<Host> protector(this);
    525 
    526     Request* request = m_requestsLoading.get(loader);
    527     if (!request)
    528         return;
    529 
    530     CachedResource* resource = request->cachedResource();
    531     ASSERT(!resource->isCacheValidator());
    532 
    533     if (resource->errorOccurred())
    534         return;
    535 
    536     if (resource->response().httpStatusCode() / 100 == 4) {
    537         // Treat a 4xx response like a network error for all resources but images (which will ignore the error and continue to load for
    538         // legacy compatibility).
    539         resource->httpStatusCodeError();
    540         return;
    541     }
    542 
    543     // Set the data.
    544     if (request->isMultipart()) {
    545         // The loader delivers the data in a multipart section all at once, send eof.
    546         // The resource data will change as the next part is loaded, so we need to make a copy.
    547         RefPtr<SharedBuffer> copiedData = SharedBuffer::create(data, size);
    548         resource->data(copiedData.release(), true);
    549     } else if (request->isIncremental())
    550         resource->data(loader->resourceData(), false);
    551 }
    552 
    553 void Loader::Host::cancelPendingRequests(RequestQueue& requestsPending, DocLoader* docLoader)
    554 {
    555     RequestQueue remaining;
    556     RequestQueue::iterator end = requestsPending.end();
    557     for (RequestQueue::iterator it = requestsPending.begin(); it != end; ++it) {
    558         Request* request = *it;
    559         if (request->docLoader() == docLoader) {
    560             cache()->remove(request->cachedResource());
    561             delete request;
    562             docLoader->decrementRequestCount();
    563         } else
    564             remaining.append(request);
    565     }
    566     requestsPending.swap(remaining);
    567 }
    568 
    569 void Loader::Host::cancelRequests(DocLoader* docLoader)
    570 {
    571     for (unsigned p = 0; p <= High; p++)
    572         cancelPendingRequests(m_requestsPending[p], docLoader);
    573 
    574     Vector<SubresourceLoader*, 256> loadersToCancel;
    575 
    576     RequestMap::iterator end = m_requestsLoading.end();
    577     for (RequestMap::iterator i = m_requestsLoading.begin(); i != end; ++i) {
    578         Request* r = i->second;
    579         if (r->docLoader() == docLoader)
    580             loadersToCancel.append(i->first.get());
    581     }
    582 
    583     for (unsigned i = 0; i < loadersToCancel.size(); ++i) {
    584         SubresourceLoader* loader = loadersToCancel[i];
    585         didFail(loader, true);
    586     }
    587 }
    588 
    589 } //namespace WebCore
    590