Home | History | Annotate | Download | only in predictors 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #ifndef CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
      6 #define CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
      7 
      8 #include <map>
      9 #include <string>
     10 #include <vector>
     11 
     12 #include "base/gtest_prod_util.h"
     13 #include "base/memory/linked_ptr.h"
     14 #include "base/memory/scoped_ptr.h"
     15 #include "base/memory/weak_ptr.h"
     16 #include "base/time/time.h"
     17 #include "chrome/browser/common/cancelable_request.h"
     18 #include "chrome/browser/history/history_types.h"
     19 #include "chrome/browser/predictors/resource_prefetch_common.h"
     20 #include "chrome/browser/predictors/resource_prefetch_predictor_tables.h"
     21 #include "chrome/browser/predictors/resource_prefetcher.h"
     22 #include "components/browser_context_keyed_service/browser_context_keyed_service.h"
     23 #include "content/public/browser/notification_observer.h"
     24 #include "content/public/browser/notification_registrar.h"
     25 #include "url/gurl.h"
     26 #include "webkit/common/resource_type.h"
     27 
     28 class PredictorsHandler;
     29 class Profile;
     30 
     31 namespace content {
     32 class WebContents;
     33 }
     34 
     35 namespace net {
     36 class URLRequest;
     37 }
     38 
     39 namespace predictors {
     40 
     41 class ResourcePrefetcherManager;
     42 
     43 // Contains logic for learning what can be prefetched and for kicking off
     44 // speculative prefetching.
     45 // - The class is a profile keyed service owned by the profile.
     46 // - All the non-static methods of this class need to be called on the UI
     47 //   thread.
     48 //
     49 // The overall flow of the resource prefetching algorithm is as follows:
     50 //
     51 // * ResourcePrefetchPredictorObserver - Listens for URL requests, responses and
     52 //   redirects on the IO thread(via RDHostDelegate) and post tasks to the
     53 //   ResourcePrefetchPredictor on the UI thread. This is owned by the
     54 //   ProfileIOData for the profile.
     55 // * ResourcePrefetchPredictorTables - Persists ResourcePrefetchPredictor data
     56 //   to a sql database. Runs entirely on the DB thread. Owned by the
     57 //   PredictorDatabase.
     58 // * ResourcePrefetchPredictor - Learns about resource requirements per URL in
     59 //   the UI thread through the ResourcePrefetchPredictorObserver and perisists
     60 //   it to disk in the DB thread through the ResourcePrefetchPredictorTables. It
     61 //   initiates resource prefetching using the ResourcePrefetcherManager. Owned
     62 //   by profile.
     63 // * ResourcePrefetcherManager - Manages the ResourcePrefetchers that do the
     64 //   prefetching on the IO thread. The manager is owned by the
     65 //   ResourcePrefetchPredictor and interfaces between the predictor on the UI
     66 //   thread and the prefetchers on the IO thread.
     67 // * ResourcePrefetcher - Lives entirely on the IO thread, owned by the
     68 //   ResourcePrefetcherManager, and issues net::URLRequest to fetch resources.
     69 //
     70 // TODO(shishir): Do speculative prefetching for https resources and/or https
     71 // main frame urls.
     72 class ResourcePrefetchPredictor
     73     : public BrowserContextKeyedService,
     74       public content::NotificationObserver,
     75       public base::SupportsWeakPtr<ResourcePrefetchPredictor> {
     76  public:
     77   // Stores the data that we need to get from the URLRequest.
     78   struct URLRequestSummary {
     79     URLRequestSummary();
     80     URLRequestSummary(const URLRequestSummary& other);
     81     ~URLRequestSummary();
     82 
     83     NavigationID navigation_id;
     84     GURL resource_url;
     85     ResourceType::Type resource_type;
     86 
     87     // Only for responses.
     88     std::string mime_type;
     89     bool was_cached;
     90     GURL redirect_url;  // Empty unless request was redirected to a valid url.
     91   };
     92 
     93   ResourcePrefetchPredictor(const ResourcePrefetchPredictorConfig& config,
     94                             Profile* profile);
     95   virtual ~ResourcePrefetchPredictor();
     96 
     97   // Thread safe.
     98   static bool ShouldRecordRequest(net::URLRequest* request,
     99                                   ResourceType::Type resource_type);
    100   static bool ShouldRecordResponse(net::URLRequest* response);
    101   static bool ShouldRecordRedirect(net::URLRequest* response);
    102 
    103   // Determines the ResourceType from the mime type, defaulting to the
    104   // |fallback| if the ResourceType could not be determined.
    105   static ResourceType::Type GetResourceTypeFromMimeType(
    106       const std::string& mime_type,
    107       ResourceType::Type fallback);
    108 
    109   // 'ResourcePrefetchPredictorObserver' calls the below functions to inform the
    110   // predictor of main frame and resource requests. Should only be called if the
    111   // corresponding Should* functions return true.
    112   void RecordURLRequest(const URLRequestSummary& request);
    113   void RecordUrlResponse(const URLRequestSummary& response);
    114   void RecordUrlRedirect(const URLRequestSummary& response);
    115 
    116   // Called by ResourcePrefetcherManager to notify that prefetching has finished
    117   // for a navigation. Should take ownership of |requests|.
    118   virtual void FinishedPrefetchForNavigation(
    119       const NavigationID& navigation_id,
    120       PrefetchKeyType key_type,
    121       ResourcePrefetcher::RequestVector* requests);
    122 
    123  private:
    124   friend class ::PredictorsHandler;
    125   friend class ResourcePrefetchPredictorTest;
    126 
    127   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, DeleteUrls);
    128   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
    129                            LazilyInitializeEmpty);
    130   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
    131                            LazilyInitializeWithData);
    132   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
    133                            NavigationNotRecorded);
    134   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, NavigationUrlInDB);
    135   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, NavigationUrlNotInDB);
    136   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
    137                            NavigationUrlNotInDBAndDBFull);
    138   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, OnMainFrameRequest);
    139   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest, OnMainFrameRedirect);
    140   FRIEND_TEST_ALL_PREFIXES(ResourcePrefetchPredictorTest,
    141                            OnSubresourceResponse);
    142 
    143   enum InitializationState {
    144     NOT_INITIALIZED = 0,
    145     INITIALIZING = 1,
    146     INITIALIZED = 2
    147   };
    148 
    149   // Stores prefetching results.
    150   struct Result {
    151     // Takes ownership of requests.
    152     Result(PrefetchKeyType key_type,
    153            ResourcePrefetcher::RequestVector* requests);
    154     ~Result();
    155 
    156     PrefetchKeyType key_type;
    157     scoped_ptr<ResourcePrefetcher::RequestVector> requests;
    158 
    159    private:
    160     DISALLOW_COPY_AND_ASSIGN(Result);
    161   };
    162 
    163   typedef ResourcePrefetchPredictorTables::ResourceRow ResourceRow;
    164   typedef ResourcePrefetchPredictorTables::ResourceRows ResourceRows;
    165   typedef ResourcePrefetchPredictorTables::PrefetchData PrefetchData;
    166   typedef ResourcePrefetchPredictorTables::PrefetchDataMap PrefetchDataMap;
    167   typedef std::map<NavigationID, linked_ptr<std::vector<URLRequestSummary> > >
    168       NavigationMap;
    169   typedef std::map<NavigationID, Result*> ResultsMap;
    170 
    171   // Returns true if the main page request is supported for prediction.
    172   static bool IsHandledMainPage(net::URLRequest* request);
    173 
    174   // Returns true if the subresource request is supported for prediction.
    175   static bool IsHandledSubresource(net::URLRequest* request);
    176 
    177   // Returns true if the request (should have a response in it) is cacheable.
    178   static bool IsCacheable(const net::URLRequest* request);
    179 
    180   // content::NotificationObserver methods OVERRIDE.
    181   virtual void Observe(int type,
    182                        const content::NotificationSource& source,
    183                        const content::NotificationDetails& details) OVERRIDE;
    184 
    185   // BrowserContextKeyedService methods OVERRIDE.
    186   virtual void Shutdown() OVERRIDE;
    187 
    188   // Functions called on different network events pertaining to the loading of
    189   // main frame resource or sub resources.
    190   void OnMainFrameRequest(const URLRequestSummary& request);
    191   void OnMainFrameResponse(const URLRequestSummary& response);
    192   void OnMainFrameRedirect(const URLRequestSummary& response);
    193   void OnSubresourceResponse(const URLRequestSummary& response);
    194   void OnSubresourceLoadedFromMemory(const NavigationID& navigation_id,
    195                                      const GURL& resource_url,
    196                                      const std::string& mime_type,
    197                                      ResourceType::Type resource_type);
    198 
    199   // Called when onload completes for a navigation. We treat this point as the
    200   // "completion" of the navigation. The resources requested by the page upto
    201   // this point are the only ones considered for prefetching.
    202   void OnNavigationComplete(const NavigationID& navigation_id);
    203 
    204   // Returns true if there is PrefetchData that can be used for the
    205   // navigation and fills in the |prefetch_data| to resources that need to be
    206   // prefetched.
    207   bool GetPrefetchData(const NavigationID& navigation_id,
    208                        ResourcePrefetcher::RequestVector* prefetch_requests,
    209                        PrefetchKeyType* key_type);
    210 
    211   // Converts a PrefetchData into a ResourcePrefetcher::RequestVector.
    212   void PopulatePrefetcherRequest(const PrefetchData& data,
    213                                  ResourcePrefetcher::RequestVector* requests);
    214 
    215   // Starts prefetching if it is enabled and prefetching data exists for the
    216   // NavigationID either at the URL or at the host level.
    217   void StartPrefetching(const NavigationID& navigation_id);
    218 
    219   // Stops prefetching that may be in progress corresponding to |navigation_id|.
    220   void StopPrefetching(const NavigationID& navigation_id);
    221 
    222   // Starts initialization by posting a task to the DB thread to read the
    223   // predictor database.
    224   void StartInitialization();
    225 
    226   // Callback for task to read predictor database. Takes ownership of
    227   // |url_data_map| and |host_data_map|.
    228   void CreateCaches(scoped_ptr<PrefetchDataMap> url_data_map,
    229                     scoped_ptr<PrefetchDataMap> host_data_map);
    230 
    231   // Called during initialization when history is read and the predictor
    232   // database has been read.
    233   void OnHistoryAndCacheLoaded();
    234 
    235   // Removes data for navigations where the onload never fired. Will cleanup
    236   // inflight_navigations_ and results_map_.
    237   void CleanupAbandonedNavigations(const NavigationID& navigation_id);
    238 
    239   // Deletes all URLs from the predictor database, the caches and removes all
    240   // inflight navigations.
    241   void DeleteAllUrls();
    242 
    243   // Deletes data for the input |urls| and their corresponding hosts from the
    244   // predictor database and caches.
    245   void DeleteUrls(const history::URLRows& urls);
    246 
    247   // Callback for GetUrlVisitCountTask.
    248   void OnVisitCountLookup(int visit_count,
    249                           const NavigationID& navigation_id,
    250                           const std::vector<URLRequestSummary>& requests);
    251 
    252   // Removes the oldest entry in the input |data_map|, also deleting it from the
    253   // predictor database.
    254   void RemoveOldestEntryInPrefetchDataMap(PrefetchKeyType key_type,
    255                                           PrefetchDataMap* data_map);
    256 
    257   // Merges resources in |new_resources| into the |data_map| and correspondingly
    258   // updates the predictor database.
    259   void LearnNavigation(const std::string& key,
    260                        PrefetchKeyType key_type,
    261                        const std::vector<URLRequestSummary>& new_resources,
    262                        int max_data_map_size,
    263                        PrefetchDataMap* data_map);
    264 
    265   // Reports accuracy by comparing prefetched resources with resources that are
    266   // actually used by the page.
    267   void ReportAccuracyStats(PrefetchKeyType key_type,
    268                            const std::vector<URLRequestSummary>& actual,
    269                            ResourcePrefetcher::RequestVector* prefetched) const;
    270 
    271   // Reports predicted accuracy i.e. by comparing resources that are actually
    272   // used by the page with those that may have been prefetched.
    273   void ReportPredictedAccuracyStats(
    274       PrefetchKeyType key_type,
    275       const std::vector<URLRequestSummary>& actual,
    276       const ResourcePrefetcher::RequestVector& predicted) const;
    277   void ReportPredictedAccuracyStatsHelper(
    278       PrefetchKeyType key_type,
    279       const ResourcePrefetcher::RequestVector& predicted,
    280       const std::map<GURL, bool>& actual,
    281       int total_resources_fetched_from_network,
    282       int max_assumed_prefetched) const;
    283 
    284   // Used for testing to inject mock tables.
    285   void set_mock_tables(scoped_refptr<ResourcePrefetchPredictorTables> tables) {
    286     tables_ = tables;
    287   }
    288 
    289   Profile* const profile_;
    290   ResourcePrefetchPredictorConfig const config_;
    291   InitializationState initialization_state_;
    292   scoped_refptr<ResourcePrefetchPredictorTables> tables_;
    293   scoped_refptr<ResourcePrefetcherManager> prefetch_manager_;
    294   content::NotificationRegistrar notification_registrar_;
    295   CancelableRequestConsumer history_lookup_consumer_;
    296 
    297   // Map of all the navigations in flight to their resource requests.
    298   NavigationMap inflight_navigations_;
    299 
    300   // Copy of the data in the predictor tables.
    301   scoped_ptr<PrefetchDataMap> url_table_cache_;
    302   scoped_ptr<PrefetchDataMap> host_table_cache_;
    303 
    304   ResultsMap results_map_;
    305   STLValueDeleter<ResultsMap> results_map_deleter_;
    306 
    307   DISALLOW_COPY_AND_ASSIGN(ResourcePrefetchPredictor);
    308 };
    309 
    310 }  // namespace predictors
    311 
    312 #endif  // CHROME_BROWSER_PREDICTORS_RESOURCE_PREFETCH_PREDICTOR_H_
    313