1 // Copyright (c) 2011 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 // Provides global database of differential decompression dictionaries for the 6 // SDCH filter (processes sdch enconded content). 7 8 // Exactly one instance of SdchManager is built, and all references are made 9 // into that collection. 10 // 11 // The SdchManager maintains a collection of memory resident dictionaries. It 12 // can find a dictionary (based on a server specification of a hash), store a 13 // dictionary, and make judgements about what URLs can use, set, etc. a 14 // dictionary. 15 16 // These dictionaries are acquired over the net, and include a header 17 // (containing metadata) as well as a VCDIFF dictionary (for use by a VCDIFF 18 // module) to decompress data. 19 20 #ifndef NET_BASE_SDCH_MANAGER_H_ 21 #define NET_BASE_SDCH_MANAGER_H_ 22 23 #include <map> 24 #include <set> 25 #include <string> 26 27 #include "base/gtest_prod_util.h" 28 #include "base/memory/ref_counted.h" 29 #include "base/memory/scoped_ptr.h" 30 #include "base/threading/non_thread_safe.h" 31 #include "base/time/time.h" 32 #include "net/base/net_export.h" 33 #include "url/gurl.h" 34 35 namespace net { 36 37 //------------------------------------------------------------------------------ 38 // Create a public interface to help us load SDCH dictionaries. 39 // The SdchManager class allows registration to support this interface. 40 // A browser may register a fetcher that is used by the dictionary managers to 41 // get data from a specified URL. This allows us to use very high level browser 42 // functionality in this base (when the functionaity can be provided). 43 class NET_EXPORT SdchFetcher { 44 public: 45 SdchFetcher() {} 46 virtual ~SdchFetcher() {} 47 48 // The Schedule() method is called when there is a need to get a dictionary 49 // from a server. The callee is responsible for getting that dictionary_text, 50 // and then calling back to AddSdchDictionary() to the SdchManager instance. 51 virtual void Schedule(const GURL& dictionary_url) = 0; 52 53 // The Cancel() method is called to cancel all pending dictionary fetches. 54 // This is used for implementation of ClearData() below. 55 virtual void Cancel() = 0; 56 57 private: 58 DISALLOW_COPY_AND_ASSIGN(SdchFetcher); 59 }; 60 61 //------------------------------------------------------------------------------ 62 63 class NET_EXPORT SdchManager : public NON_EXPORTED_BASE(base::NonThreadSafe) { 64 public: 65 // A list of errors that appeared and were either resolved, or used to turn 66 // off sdch encoding. 67 enum ProblemCodes { 68 MIN_PROBLEM_CODE, 69 70 // Content-encoding correction problems. 71 ADDED_CONTENT_ENCODING = 1, 72 FIXED_CONTENT_ENCODING = 2, 73 FIXED_CONTENT_ENCODINGS = 3, 74 75 // Content decoding errors. 76 DECODE_HEADER_ERROR = 4, 77 DECODE_BODY_ERROR = 5, 78 79 // More content-encoding correction problems. 80 OPTIONAL_GUNZIP_ENCODING_ADDED = 6, 81 82 // Content encoding correction when we're not even tagged as HTML!?! 83 BINARY_ADDED_CONTENT_ENCODING = 7, 84 BINARY_FIXED_CONTENT_ENCODING = 8, 85 BINARY_FIXED_CONTENT_ENCODINGS = 9, 86 87 // Dictionary selection for use problems. 88 DICTIONARY_FOUND_HAS_WRONG_DOMAIN = 10, 89 DICTIONARY_FOUND_HAS_WRONG_PORT_LIST = 11, 90 DICTIONARY_FOUND_HAS_WRONG_PATH = 12, 91 DICTIONARY_FOUND_HAS_WRONG_SCHEME = 13, 92 DICTIONARY_HASH_NOT_FOUND = 14, 93 DICTIONARY_HASH_MALFORMED = 15, 94 95 // Dictionary saving problems. 96 DICTIONARY_HAS_NO_HEADER = 20, 97 DICTIONARY_HEADER_LINE_MISSING_COLON = 21, 98 DICTIONARY_MISSING_DOMAIN_SPECIFIER = 22, 99 DICTIONARY_SPECIFIES_TOP_LEVEL_DOMAIN = 23, 100 DICTIONARY_DOMAIN_NOT_MATCHING_SOURCE_URL = 24, 101 DICTIONARY_PORT_NOT_MATCHING_SOURCE_URL = 25, 102 DICTIONARY_HAS_NO_TEXT = 26, 103 DICTIONARY_REFERER_URL_HAS_DOT_IN_PREFIX = 27, 104 105 // Dictionary loading problems. 106 DICTIONARY_LOAD_ATTEMPT_FROM_DIFFERENT_HOST = 30, 107 DICTIONARY_SELECTED_FOR_SSL = 31, 108 DICTIONARY_ALREADY_LOADED = 32, 109 DICTIONARY_SELECTED_FROM_NON_HTTP = 33, 110 DICTIONARY_IS_TOO_LARGE= 34, 111 DICTIONARY_COUNT_EXCEEDED = 35, 112 DICTIONARY_ALREADY_SCHEDULED_TO_DOWNLOAD = 36, 113 DICTIONARY_ALREADY_TRIED_TO_DOWNLOAD = 37, 114 115 // Failsafe hack. 116 ATTEMPT_TO_DECODE_NON_HTTP_DATA = 40, 117 118 119 // Content-Encoding problems detected, with no action taken. 120 MULTIENCODING_FOR_NON_SDCH_REQUEST = 50, 121 SDCH_CONTENT_ENCODE_FOR_NON_SDCH_REQUEST = 51, 122 123 // Dictionary manager issues. 124 DOMAIN_BLACKLIST_INCLUDES_TARGET = 61, 125 126 // Problematic decode recovery methods. 127 META_REFRESH_RECOVERY = 70, // Dictionary not found. 128 // defunct = 71, // Almost the same as META_REFRESH_UNSUPPORTED. 129 // defunct = 72, // Almost the same as CACHED_META_REFRESH_UNSUPPORTED. 130 // defunct = 73, // PASSING_THROUGH_NON_SDCH plus DISCARD_TENTATIVE_SDCH. 131 META_REFRESH_UNSUPPORTED = 74, // Unrecoverable error. 132 CACHED_META_REFRESH_UNSUPPORTED = 75, // As above, but pulled from cache. 133 PASSING_THROUGH_NON_SDCH = 76, // Tagged sdch but missing dictionary-hash. 134 INCOMPLETE_SDCH_CONTENT = 77, // Last window was not completely decoded. 135 PASS_THROUGH_404_CODE = 78, // URL not found message passing through. 136 137 // This next report is very common, and not really an error scenario, but 138 // it exercises the error recovery logic. 139 PASS_THROUGH_OLD_CACHED = 79, // Back button got pre-SDCH cached content. 140 141 // Common decoded recovery methods. 142 META_REFRESH_CACHED_RECOVERY = 80, // Probably startup tab loading. 143 DISCARD_TENTATIVE_SDCH = 81, // Server decided not to use sdch. 144 145 // Non SDCH problems, only accounted for to make stat counting complete 146 // (i.e., be able to be sure all dictionary advertisements are accounted 147 // for). 148 149 UNFLUSHED_CONTENT = 90, // Possible error in filter chaining. 150 // defunct = 91, // MISSING_TIME_STATS (Should never happen.) 151 CACHE_DECODED = 92, // No timing stats recorded. 152 // defunct = 93, // OVER_10_MINUTES (No timing stats recorded.) 153 UNINITIALIZED = 94, // Filter never even got initialized. 154 PRIOR_TO_DICTIONARY = 95, // We hadn't even parsed a dictionary selector. 155 DECODE_ERROR = 96, // Something went wrong during decode. 156 157 // Problem during the latency test. 158 LATENCY_TEST_DISALLOWED = 100, // SDCH now failing, but it worked before! 159 160 MAX_PROBLEM_CODE // Used to bound histogram. 161 }; 162 163 // Use the following static limits to block DOS attacks until we implement 164 // a cached dictionary evicition strategy. 165 static const size_t kMaxDictionarySize; 166 static const size_t kMaxDictionaryCount; 167 168 // There is one instance of |Dictionary| for each memory-cached SDCH 169 // dictionary. 170 class NET_EXPORT_PRIVATE Dictionary : public base::RefCounted<Dictionary> { 171 public: 172 // Sdch filters can get our text to use in decoding compressed data. 173 const std::string& text() const { return text_; } 174 175 private: 176 friend class base::RefCounted<Dictionary>; 177 friend class SdchManager; // Only manager can construct an instance. 178 FRIEND_TEST_ALL_PREFIXES(SdchManagerTest, PathMatch); 179 180 // Construct a vc-diff usable dictionary from the dictionary_text starting 181 // at the given offset. The supplied client_hash should be used to 182 // advertise the dictionary's availability relative to the suppplied URL. 183 Dictionary(const std::string& dictionary_text, 184 size_t offset, 185 const std::string& client_hash, 186 const GURL& url, 187 const std::string& domain, 188 const std::string& path, 189 const base::Time& expiration, 190 const std::set<int>& ports); 191 ~Dictionary(); 192 193 const GURL& url() const { return url_; } 194 const std::string& client_hash() const { return client_hash_; } 195 196 // Security method to check if we can advertise this dictionary for use 197 // if the |target_url| returns SDCH compressed data. 198 bool CanAdvertise(const GURL& target_url); 199 200 // Security methods to check if we can establish a new dictionary with the 201 // given data, that arrived in response to get of dictionary_url. 202 static bool CanSet(const std::string& domain, const std::string& path, 203 const std::set<int>& ports, const GURL& dictionary_url); 204 205 // Security method to check if we can use a dictionary to decompress a 206 // target that arrived with a reference to this dictionary. 207 bool CanUse(const GURL& referring_url); 208 209 // Compare paths to see if they "match" for dictionary use. 210 static bool PathMatch(const std::string& path, 211 const std::string& restriction); 212 213 // Compare domains to see if the "match" for dictionary use. 214 static bool DomainMatch(const GURL& url, const std::string& restriction); 215 216 217 // The actual text of the dictionary. 218 std::string text_; 219 220 // Part of the hash of text_ that the client uses to advertise the fact that 221 // it has a specific dictionary pre-cached. 222 std::string client_hash_; 223 224 // The GURL that arrived with the text_ in a URL request to specify where 225 // this dictionary may be used. 226 const GURL url_; 227 228 // Metadate "headers" in before dictionary text contained the following: 229 // Each dictionary payload consists of several headers, followed by the text 230 // of the dictionary. The following are the known headers. 231 const std::string domain_; 232 const std::string path_; 233 const base::Time expiration_; // Implied by max-age. 234 const std::set<int> ports_; 235 236 DISALLOW_COPY_AND_ASSIGN(Dictionary); 237 }; 238 239 SdchManager(); 240 ~SdchManager(); 241 242 // Clear data (for browser data removal). 243 void ClearData(); 244 245 // Record stats on various errors. 246 static void SdchErrorRecovery(ProblemCodes problem); 247 248 // Register a fetcher that this class can use to obtain dictionaries. 249 void set_sdch_fetcher(SdchFetcher* fetcher); 250 251 // Enables or disables SDCH compression. 252 static void EnableSdchSupport(bool enabled); 253 254 static bool sdch_enabled() { return g_sdch_enabled_; } 255 256 // Enables or disables SDCH compression over secure connection. 257 static void EnableSecureSchemeSupport(bool enabled); 258 259 static bool secure_scheme_supported() { return g_secure_scheme_supported_; } 260 261 // Briefly prevent further advertising of SDCH on this domain (if SDCH is 262 // enabled). After enough calls to IsInSupportedDomain() the blacklisting 263 // will be removed. Additional blacklists take exponentially more calls 264 // to IsInSupportedDomain() before the blacklisting is undone. 265 // Used when filter errors are found from a given domain, but it is plausible 266 // that the cause is temporary (such as application startup, where cached 267 // entries are used, but a dictionary is not yet loaded). 268 void BlacklistDomain(const GURL& url); 269 270 // Used when SEVERE filter errors are found from a given domain, to prevent 271 // further use of SDCH on that domain. 272 void BlacklistDomainForever(const GURL& url); 273 274 // Unit test only, this function resets enabling of sdch, and clears the 275 // blacklist. 276 void ClearBlacklistings(); 277 278 // Unit test only, this function resets the blacklisting count for a domain. 279 void ClearDomainBlacklisting(const std::string& domain); 280 281 // Unit test only: indicate how many more times a domain will be blacklisted. 282 int BlackListDomainCount(const std::string& domain); 283 284 // Unit test only: Indicate what current blacklist increment is for a domain. 285 int BlacklistDomainExponential(const std::string& domain); 286 287 // Check to see if SDCH is enabled (globally), and the given URL is in a 288 // supported domain (i.e., not blacklisted, and either the specific supported 289 // domain, or all domains were assumed supported). If it is blacklist, reduce 290 // by 1 the number of times it will be reported as blacklisted. 291 bool IsInSupportedDomain(const GURL& url); 292 293 // Schedule the URL fetching to load a dictionary. This will always return 294 // before the dictionary is actually loaded and added. 295 // After the implied task does completes, the dictionary will have been 296 // cached in memory. 297 void FetchDictionary(const GURL& request_url, const GURL& dictionary_url); 298 299 // Security test function used before initiating a FetchDictionary. 300 // Return true if fetch is legal. 301 bool CanFetchDictionary(const GURL& referring_url, 302 const GURL& dictionary_url) const; 303 304 // Add an SDCH dictionary to our list of availible dictionaries. This addition 305 // will fail (return false) if addition is illegal (data in the dictionary is 306 // not acceptable from the dictionary_url; dictionary already added, etc.). 307 bool AddSdchDictionary(const std::string& dictionary_text, 308 const GURL& dictionary_url); 309 310 // Find the vcdiff dictionary (the body of the sdch dictionary that appears 311 // after the meta-data headers like Domain:...) with the given |server_hash| 312 // to use to decompreses data that arrived as SDCH encoded content. Check to 313 // be sure the returned |dictionary| can be used for decoding content supplied 314 // in response to a request for |referring_url|. 315 // Return null in |dictionary| if there is no matching legal dictionary. 316 void GetVcdiffDictionary(const std::string& server_hash, 317 const GURL& referring_url, 318 scoped_refptr<Dictionary>* dictionary); 319 320 // Get list of available (pre-cached) dictionaries that we have already loaded 321 // into memory. The list is a comma separated list of (client) hashes per 322 // the SDCH spec. 323 void GetAvailDictionaryList(const GURL& target_url, std::string* list); 324 325 // Construct the pair of hashes for client and server to identify an SDCH 326 // dictionary. This is only made public to facilitate unit testing, but is 327 // otherwise private 328 static void GenerateHash(const std::string& dictionary_text, 329 std::string* client_hash, std::string* server_hash); 330 331 // For Latency testing only, we need to know if we've succeeded in doing a 332 // round trip before starting our comparative tests. If ever we encounter 333 // problems with SDCH, we opt-out of the test unless/until we perform a 334 // complete SDCH decoding. 335 bool AllowLatencyExperiment(const GURL& url) const; 336 337 void SetAllowLatencyExperiment(const GURL& url, bool enable); 338 339 private: 340 typedef std::map<std::string, int> DomainCounter; 341 typedef std::set<std::string> ExperimentSet; 342 343 // A map of dictionaries info indexed by the hash that the server provides. 344 typedef std::map<std::string, scoped_refptr<Dictionary> > DictionaryMap; 345 346 // Support SDCH compression, by advertising in headers. 347 static bool g_sdch_enabled_; 348 349 // Support SDCH compression for HTTPS requests and responses. When supported, 350 // HTTPS applicable dictionaries MUST have been acquired securely via HTTPS. 351 static bool g_secure_scheme_supported_; 352 353 // A simple implementation of a RFC 3548 "URL safe" base64 encoder. 354 static void UrlSafeBase64Encode(const std::string& input, 355 std::string* output); 356 DictionaryMap dictionaries_; 357 358 // An instance that can fetch a dictionary given a URL. 359 scoped_ptr<SdchFetcher> fetcher_; 360 361 // List domains where decode failures have required disabling sdch, along with 362 // count of how many additonal uses should be blacklisted. 363 DomainCounter blacklisted_domains_; 364 365 // Support exponential backoff in number of domain accesses before 366 // blacklisting expires. 367 DomainCounter exponential_blacklist_count_; 368 369 // List of hostnames for which a latency experiment is allowed (because a 370 // round trip test has recently passed). 371 ExperimentSet allow_latency_experiment_; 372 373 DISALLOW_COPY_AND_ASSIGN(SdchManager); 374 }; 375 376 } // namespace net 377 378 #endif // NET_BASE_SDCH_MANAGER_H_ 379