Home | History | Annotate | Download | only in browser 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "components/policy/core/browser/url_blacklist_manager.h"
      6 
      7 #include "base/bind.h"
      8 #include "base/files/file_path.h"
      9 #include "base/location.h"
     10 #include "base/message_loop/message_loop_proxy.h"
     11 #include "base/prefs/pref_service.h"
     12 #include "base/sequenced_task_runner.h"
     13 #include "base/stl_util.h"
     14 #include "base/strings/string_number_conversions.h"
     15 #include "base/task_runner_util.h"
     16 #include "base/values.h"
     17 #include "components/policy/core/common/policy_pref_names.h"
     18 #include "components/pref_registry/pref_registry_syncable.h"
     19 #include "net/base/filename_util.h"
     20 #include "net/base/load_flags.h"
     21 #include "net/base/net_errors.h"
     22 #include "net/url_request/url_request.h"
     23 #include "url/url_constants.h"
     24 #include "url/url_parse.h"
     25 
     26 using url_matcher::URLMatcher;
     27 using url_matcher::URLMatcherCondition;
     28 using url_matcher::URLMatcherConditionFactory;
     29 using url_matcher::URLMatcherConditionSet;
     30 using url_matcher::URLMatcherPortFilter;
     31 using url_matcher::URLMatcherSchemeFilter;
     32 using url_matcher::URLQueryElementMatcherCondition;
     33 
     34 namespace policy {
     35 
     36 namespace {
     37 
     38 // List of schemes of URLs that should not be blocked by the "*" wildcard in
     39 // the blacklist. Note that URLs with these schemes can still be blocked with
     40 // a more specific filter e.g. "chrome-extension://*".
     41 // The schemes are hardcoded here to avoid dependencies on //extensions and
     42 // //chrome.
     43 const char* kBypassBlacklistWildcardForSchemes[] = {
     44   // For internal extension URLs e.g. the Bookmark Manager and the File
     45   // Manager on Chrome OS.
     46   "chrome-extension",
     47 
     48   // NTP on Android.
     49   "chrome-native",
     50 
     51   // NTP on other platforms.
     52   "chrome-search",
     53 };
     54 
     55 // Maximum filters per policy. Filters over this index are ignored.
     56 const size_t kMaxFiltersPerPolicy = 1000;
     57 
     58 // A task that builds the blacklist on a background thread.
     59 scoped_ptr<URLBlacklist> BuildBlacklist(
     60     scoped_ptr<base::ListValue> block,
     61     scoped_ptr<base::ListValue> allow,
     62     URLBlacklist::SegmentURLCallback segment_url) {
     63   scoped_ptr<URLBlacklist> blacklist(new URLBlacklist(segment_url));
     64   blacklist->Block(block.get());
     65   blacklist->Allow(allow.get());
     66   return blacklist.Pass();
     67 }
     68 
     69 // Tokenise the parameter |query| and add appropriate query element matcher
     70 // conditions to the |query_conditions|.
     71 void ProcessQueryToConditions(
     72     url_matcher::URLMatcherConditionFactory* condition_factory,
     73     const std::string& query,
     74     bool allow,
     75     std::set<URLQueryElementMatcherCondition>* query_conditions) {
     76   url::Component query_left = url::MakeRange(0, query.length());
     77   url::Component key;
     78   url::Component value;
     79   // Depending on the filter type being black-list or white-list, the matcher
     80   // choose any or every match. The idea is a URL should be black-listed if
     81   // there is any occurrence of the key value pair. It should be white-listed
     82   // only if every occurrence of the key is followed by the value. This avoids
     83   // situations such as a user appending a white-listed video parameter in the
     84   // end of the query and watching a video of his choice (the last parameter is
     85   // ignored by some web servers like youtube's).
     86   URLQueryElementMatcherCondition::Type match_type =
     87       allow ? URLQueryElementMatcherCondition::MATCH_ALL
     88             : URLQueryElementMatcherCondition::MATCH_ANY;
     89 
     90   while (ExtractQueryKeyValue(query.data(), &query_left, &key, &value)) {
     91     URLQueryElementMatcherCondition::QueryElementType query_element_type =
     92         value.len ? URLQueryElementMatcherCondition::ELEMENT_TYPE_KEY_VALUE
     93                   : URLQueryElementMatcherCondition::ELEMENT_TYPE_KEY;
     94     URLQueryElementMatcherCondition::QueryValueMatchType query_value_match_type;
     95     if (!value.len && key.len && query[key.end() - 1] == '*') {
     96       --key.len;
     97       query_value_match_type =
     98           URLQueryElementMatcherCondition::QUERY_VALUE_MATCH_PREFIX;
     99     } else if (value.len && query[value.end() - 1] == '*') {
    100       --value.len;
    101       query_value_match_type =
    102           URLQueryElementMatcherCondition::QUERY_VALUE_MATCH_PREFIX;
    103     } else {
    104       query_value_match_type =
    105           URLQueryElementMatcherCondition::QUERY_VALUE_MATCH_EXACT;
    106     }
    107     query_conditions->insert(
    108         URLQueryElementMatcherCondition(query.substr(key.begin, key.len),
    109                                         query.substr(value.begin, value.len),
    110                                         query_value_match_type,
    111                                         query_element_type,
    112                                         match_type,
    113                                         condition_factory));
    114   }
    115 }
    116 
    117 bool BypassBlacklistWildcardForURL(const GURL& url) {
    118   const std::string& scheme = url.scheme();
    119   for (size_t i = 0; i < arraysize(kBypassBlacklistWildcardForSchemes); ++i) {
    120     if (scheme == kBypassBlacklistWildcardForSchemes[i])
    121       return true;
    122   }
    123   return false;
    124 }
    125 
    126 }  // namespace
    127 
    128 struct URLBlacklist::FilterComponents {
    129   FilterComponents() : port(0), match_subdomains(true), allow(true) {}
    130   ~FilterComponents() {}
    131 
    132   // Returns true if |this| represents the "*" filter in the blacklist.
    133   bool IsBlacklistWildcard() const {
    134     return !allow && host.empty() && scheme.empty() && path.empty() &&
    135            query.empty() && port == 0 && number_of_key_value_pairs == 0 &&
    136            match_subdomains;
    137   }
    138 
    139   std::string scheme;
    140   std::string host;
    141   uint16 port;
    142   std::string path;
    143   std::string query;
    144   int number_of_key_value_pairs;
    145   bool match_subdomains;
    146   bool allow;
    147 };
    148 
    149 URLBlacklist::URLBlacklist(SegmentURLCallback segment_url)
    150     : segment_url_(segment_url), id_(0), url_matcher_(new URLMatcher) {}
    151 
    152 URLBlacklist::~URLBlacklist() {}
    153 
    154 void URLBlacklist::AddFilters(bool allow,
    155                               const base::ListValue* list) {
    156   URLMatcherConditionSet::Vector all_conditions;
    157   size_t size = std::min(kMaxFiltersPerPolicy, list->GetSize());
    158   for (size_t i = 0; i < size; ++i) {
    159     std::string pattern;
    160     bool success = list->GetString(i, &pattern);
    161     DCHECK(success);
    162     FilterComponents components;
    163     components.allow = allow;
    164     if (!FilterToComponents(segment_url_,
    165                             pattern,
    166                             &components.scheme,
    167                             &components.host,
    168                             &components.match_subdomains,
    169                             &components.port,
    170                             &components.path,
    171                             &components.query)) {
    172       LOG(ERROR) << "Invalid pattern " << pattern;
    173       continue;
    174     }
    175 
    176     scoped_refptr<URLMatcherConditionSet> condition_set =
    177         CreateConditionSet(url_matcher_.get(),
    178                            ++id_,
    179                            components.scheme,
    180                            components.host,
    181                            components.match_subdomains,
    182                            components.port,
    183                            components.path,
    184                            components.query,
    185                            allow);
    186     components.number_of_key_value_pairs =
    187         condition_set->query_conditions().size();
    188     all_conditions.push_back(condition_set);
    189     filters_[id_] = components;
    190   }
    191   url_matcher_->AddConditionSets(all_conditions);
    192 }
    193 
    194 void URLBlacklist::Block(const base::ListValue* filters) {
    195   AddFilters(false, filters);
    196 }
    197 
    198 void URLBlacklist::Allow(const base::ListValue* filters) {
    199   AddFilters(true, filters);
    200 }
    201 
    202 bool URLBlacklist::IsURLBlocked(const GURL& url) const {
    203   std::set<URLMatcherConditionSet::ID> matching_ids =
    204       url_matcher_->MatchURL(url);
    205 
    206   const FilterComponents* max = NULL;
    207   for (std::set<URLMatcherConditionSet::ID>::iterator id = matching_ids.begin();
    208        id != matching_ids.end(); ++id) {
    209     std::map<int, FilterComponents>::const_iterator it = filters_.find(*id);
    210     DCHECK(it != filters_.end());
    211     const FilterComponents& filter = it->second;
    212     if (!max || FilterTakesPrecedence(filter, *max))
    213       max = &filter;
    214   }
    215 
    216   // Default to allow.
    217   if (!max)
    218     return false;
    219 
    220   // Some of the internal Chrome URLs are not affected by the "*" in the
    221   // blacklist. Note that the "*" is the lowest priority filter possible, so
    222   // any higher priority filter will be applied first.
    223   if (max->IsBlacklistWildcard() && BypassBlacklistWildcardForURL(url))
    224     return false;
    225 
    226   return !max->allow;
    227 }
    228 
    229 size_t URLBlacklist::Size() const {
    230   return filters_.size();
    231 }
    232 
    233 // static
    234 bool URLBlacklist::FilterToComponents(SegmentURLCallback segment_url,
    235                                       const std::string& filter,
    236                                       std::string* scheme,
    237                                       std::string* host,
    238                                       bool* match_subdomains,
    239                                       uint16* port,
    240                                       std::string* path,
    241                                       std::string* query) {
    242   url::Parsed parsed;
    243 
    244   if (segment_url(filter, &parsed) == url::kFileScheme) {
    245     base::FilePath file_path;
    246     if (!net::FileURLToFilePath(GURL(filter), &file_path))
    247       return false;
    248 
    249     *scheme = url::kFileScheme;
    250     host->clear();
    251     *match_subdomains = true;
    252     *port = 0;
    253     // Special path when the |filter| is 'file://*'.
    254     *path = (filter == "file://*") ? "" : file_path.AsUTF8Unsafe();
    255 #if defined(FILE_PATH_USES_WIN_SEPARATORS)
    256     // Separators have to be canonicalized on Windows.
    257     std::replace(path->begin(), path->end(), '\\', '/');
    258     *path = "/" + *path;
    259 #endif
    260     return true;
    261   }
    262 
    263   if (!parsed.host.is_nonempty())
    264     return false;
    265 
    266   if (parsed.scheme.is_nonempty())
    267     scheme->assign(filter, parsed.scheme.begin, parsed.scheme.len);
    268   else
    269     scheme->clear();
    270 
    271   host->assign(filter, parsed.host.begin, parsed.host.len);
    272   // Special '*' host, matches all hosts.
    273   if (*host == "*") {
    274     host->clear();
    275     *match_subdomains = true;
    276   } else if ((*host)[0] == '.') {
    277     // A leading dot in the pattern syntax means that we don't want to match
    278     // subdomains.
    279     host->erase(0, 1);
    280     *match_subdomains = false;
    281   } else {
    282     url::RawCanonOutputT<char> output;
    283     url::CanonHostInfo host_info;
    284     url::CanonicalizeHostVerbose(filter.c_str(), parsed.host, &output,
    285                                  &host_info);
    286     if (host_info.family == url::CanonHostInfo::NEUTRAL) {
    287       // We want to match subdomains. Add a dot in front to make sure we only
    288       // match at domain component boundaries.
    289       *host = "." + *host;
    290       *match_subdomains = true;
    291     } else {
    292       *match_subdomains = false;
    293     }
    294   }
    295 
    296   if (parsed.port.is_nonempty()) {
    297     int int_port;
    298     if (!base::StringToInt(filter.substr(parsed.port.begin, parsed.port.len),
    299                            &int_port)) {
    300       return false;
    301     }
    302     if (int_port <= 0 || int_port > kuint16max)
    303       return false;
    304     *port = int_port;
    305   } else {
    306     // Match any port.
    307     *port = 0;
    308   }
    309 
    310   if (parsed.path.is_nonempty())
    311     path->assign(filter, parsed.path.begin, parsed.path.len);
    312   else
    313     path->clear();
    314 
    315   if (query) {
    316     if (parsed.query.is_nonempty())
    317       query->assign(filter, parsed.query.begin, parsed.query.len);
    318     else
    319       query->clear();
    320   }
    321 
    322   return true;
    323 }
    324 
    325 // static
    326 scoped_refptr<URLMatcherConditionSet> URLBlacklist::CreateConditionSet(
    327     URLMatcher* url_matcher,
    328     int id,
    329     const std::string& scheme,
    330     const std::string& host,
    331     bool match_subdomains,
    332     uint16 port,
    333     const std::string& path,
    334     const std::string& query,
    335     bool allow) {
    336   URLMatcherConditionFactory* condition_factory =
    337       url_matcher->condition_factory();
    338   std::set<URLMatcherCondition> conditions;
    339   conditions.insert(match_subdomains ?
    340       condition_factory->CreateHostSuffixPathPrefixCondition(host, path) :
    341       condition_factory->CreateHostEqualsPathPrefixCondition(host, path));
    342 
    343   std::set<URLQueryElementMatcherCondition> query_conditions;
    344   if (!query.empty()) {
    345     ProcessQueryToConditions(
    346         condition_factory, query, allow, &query_conditions);
    347   }
    348 
    349   scoped_ptr<URLMatcherSchemeFilter> scheme_filter;
    350   if (!scheme.empty())
    351     scheme_filter.reset(new URLMatcherSchemeFilter(scheme));
    352 
    353   scoped_ptr<URLMatcherPortFilter> port_filter;
    354   if (port != 0) {
    355     std::vector<URLMatcherPortFilter::Range> ranges;
    356     ranges.push_back(URLMatcherPortFilter::CreateRange(port));
    357     port_filter.reset(new URLMatcherPortFilter(ranges));
    358   }
    359 
    360   return new URLMatcherConditionSet(id,
    361                                     conditions,
    362                                     query_conditions,
    363                                     scheme_filter.Pass(),
    364                                     port_filter.Pass());
    365 }
    366 
    367 // static
    368 bool URLBlacklist::FilterTakesPrecedence(const FilterComponents& lhs,
    369                                          const FilterComponents& rhs) {
    370   // The "*" wildcard is the lowest priority filter.
    371   if (rhs.IsBlacklistWildcard())
    372     return true;
    373 
    374   if (lhs.match_subdomains && !rhs.match_subdomains)
    375     return false;
    376   if (!lhs.match_subdomains && rhs.match_subdomains)
    377     return true;
    378 
    379   size_t host_length = lhs.host.length();
    380   size_t other_host_length = rhs.host.length();
    381   if (host_length != other_host_length)
    382     return host_length > other_host_length;
    383 
    384   size_t path_length = lhs.path.length();
    385   size_t other_path_length = rhs.path.length();
    386   if (path_length != other_path_length)
    387     return path_length > other_path_length;
    388 
    389   if (lhs.number_of_key_value_pairs != rhs.number_of_key_value_pairs)
    390     return lhs.number_of_key_value_pairs > rhs.number_of_key_value_pairs;
    391 
    392   if (lhs.allow && !rhs.allow)
    393     return true;
    394 
    395   return false;
    396 }
    397 
    398 URLBlacklistManager::URLBlacklistManager(
    399     PrefService* pref_service,
    400     const scoped_refptr<base::SequencedTaskRunner>& background_task_runner,
    401     const scoped_refptr<base::SequencedTaskRunner>& io_task_runner,
    402     URLBlacklist::SegmentURLCallback segment_url,
    403     OverrideBlacklistCallback override_blacklist)
    404     : pref_service_(pref_service),
    405       background_task_runner_(background_task_runner),
    406       io_task_runner_(io_task_runner),
    407       segment_url_(segment_url),
    408       override_blacklist_(override_blacklist),
    409       ui_task_runner_(base::MessageLoopProxy::current()),
    410       blacklist_(new URLBlacklist(segment_url)),
    411       ui_weak_ptr_factory_(this),
    412       io_weak_ptr_factory_(this) {
    413   pref_change_registrar_.Init(pref_service_);
    414   base::Closure callback = base::Bind(&URLBlacklistManager::ScheduleUpdate,
    415                                       base::Unretained(this));
    416   pref_change_registrar_.Add(policy_prefs::kUrlBlacklist, callback);
    417   pref_change_registrar_.Add(policy_prefs::kUrlWhitelist, callback);
    418 
    419   // Start enforcing the policies without a delay when they are present at
    420   // startup.
    421   if (pref_service_->HasPrefPath(policy_prefs::kUrlBlacklist))
    422     Update();
    423 }
    424 
    425 void URLBlacklistManager::ShutdownOnUIThread() {
    426   DCHECK(ui_task_runner_->RunsTasksOnCurrentThread());
    427   // Cancel any pending updates, and stop listening for pref change updates.
    428   ui_weak_ptr_factory_.InvalidateWeakPtrs();
    429   pref_change_registrar_.RemoveAll();
    430 }
    431 
    432 URLBlacklistManager::~URLBlacklistManager() {
    433 }
    434 
    435 void URLBlacklistManager::ScheduleUpdate() {
    436   DCHECK(ui_task_runner_->RunsTasksOnCurrentThread());
    437   // Cancel pending updates, if any. This can happen if two preferences that
    438   // change the blacklist are updated in one message loop cycle. In those cases,
    439   // only rebuild the blacklist after all the preference updates are processed.
    440   ui_weak_ptr_factory_.InvalidateWeakPtrs();
    441   ui_task_runner_->PostTask(
    442       FROM_HERE,
    443       base::Bind(&URLBlacklistManager::Update,
    444                  ui_weak_ptr_factory_.GetWeakPtr()));
    445 }
    446 
    447 void URLBlacklistManager::Update() {
    448   DCHECK(ui_task_runner_->RunsTasksOnCurrentThread());
    449 
    450   // The preferences can only be read on the UI thread.
    451   scoped_ptr<base::ListValue> block(
    452       pref_service_->GetList(policy_prefs::kUrlBlacklist)->DeepCopy());
    453   scoped_ptr<base::ListValue> allow(
    454       pref_service_->GetList(policy_prefs::kUrlWhitelist)->DeepCopy());
    455 
    456   // Go through the IO thread to grab a WeakPtr to |this|. This is safe from
    457   // here, since this task will always execute before a potential deletion of
    458   // ProfileIOData on IO.
    459   io_task_runner_->PostTask(FROM_HERE,
    460                             base::Bind(&URLBlacklistManager::UpdateOnIO,
    461                                        base::Unretained(this),
    462                                        base::Passed(&block),
    463                                        base::Passed(&allow)));
    464 }
    465 
    466 void URLBlacklistManager::UpdateOnIO(scoped_ptr<base::ListValue> block,
    467                                      scoped_ptr<base::ListValue> allow) {
    468   DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
    469   // The URLBlacklist is built on a worker thread. Once it's ready, it is passed
    470   // to the URLBlacklistManager on IO.
    471   base::PostTaskAndReplyWithResult(
    472       background_task_runner_.get(),
    473       FROM_HERE,
    474       base::Bind(&BuildBlacklist,
    475                  base::Passed(&block),
    476                  base::Passed(&allow),
    477                  segment_url_),
    478       base::Bind(&URLBlacklistManager::SetBlacklist,
    479                  io_weak_ptr_factory_.GetWeakPtr()));
    480 }
    481 
    482 void URLBlacklistManager::SetBlacklist(scoped_ptr<URLBlacklist> blacklist) {
    483   DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
    484   blacklist_ = blacklist.Pass();
    485 }
    486 
    487 bool URLBlacklistManager::IsURLBlocked(const GURL& url) const {
    488   DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
    489   return blacklist_->IsURLBlocked(url);
    490 }
    491 
    492 bool URLBlacklistManager::IsRequestBlocked(
    493     const net::URLRequest& request, int* reason) const {
    494   DCHECK(io_task_runner_->RunsTasksOnCurrentThread());
    495 #if !defined(OS_IOS)
    496   // TODO(joaodasilva): iOS doesn't set these flags. http://crbug.com/338283
    497   int filter_flags = net::LOAD_MAIN_FRAME | net::LOAD_SUB_FRAME;
    498   if ((request.load_flags() & filter_flags) == 0)
    499     return false;
    500 #endif
    501 
    502   bool block = false;
    503   if (override_blacklist_.Run(request.url(), &block, reason))
    504     return block;
    505 
    506   *reason = net::ERR_BLOCKED_BY_ADMINISTRATOR;
    507   return IsURLBlocked(request.url());
    508 }
    509 
    510 // static
    511 void URLBlacklistManager::RegisterProfilePrefs(
    512     user_prefs::PrefRegistrySyncable* registry) {
    513   registry->RegisterListPref(policy_prefs::kUrlBlacklist,
    514                              user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
    515   registry->RegisterListPref(policy_prefs::kUrlWhitelist,
    516                              user_prefs::PrefRegistrySyncable::UNSYNCABLE_PREF);
    517 }
    518 
    519 }  // namespace policy
    520