Home | History | Annotate | Download | only in browser 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "components/data_reduction_proxy/browser/data_reduction_proxy_metrics.h"
      6 
      7 #include "base/metrics/histogram.h"
      8 #include "base/prefs/pref_service.h"
      9 #include "base/prefs/scoped_user_pref_update.h"
     10 #include "base/strings/string_number_conversions.h"
     11 #include "base/strings/string_util.h"
     12 #include "components/data_reduction_proxy/browser/data_reduction_proxy_settings.h"
     13 #include "components/data_reduction_proxy/common/data_reduction_proxy_headers.h"
     14 #include "components/data_reduction_proxy/common/data_reduction_proxy_pref_names.h"
     15 #include "net/base/host_port_pair.h"
     16 #include "net/http/http_response_headers.h"
     17 #include "net/proxy/proxy_retry_info.h"
     18 #include "net/proxy/proxy_service.h"
     19 #include "net/url_request/url_request_context.h"
     20 
     21 namespace data_reduction_proxy {
     22 
     23 namespace {
     24 
     25 // A bypass delay more than this is treated as a long delay.
     26 const int kLongBypassDelayInSeconds = 30 * 60;
     27 
     28 // Increments an int64, stored as a string, in a ListPref at the specified
     29 // index.  The value must already exist and be a string representation of a
     30 // number.
     31 void AddInt64ToListPref(size_t index,
     32                         int64 length,
     33                         base::ListValue* list_update) {
     34   int64 value = 0;
     35   std::string old_string_value;
     36   bool rv = list_update->GetString(index, &old_string_value);
     37   DCHECK(rv);
     38   if (rv) {
     39     rv = base::StringToInt64(old_string_value, &value);
     40     DCHECK(rv);
     41   }
     42   value += length;
     43   list_update->Set(index, new base::StringValue(base::Int64ToString(value)));
     44 }
     45 
     46 int64 ListPrefInt64Value(const base::ListValue& list_update, size_t index) {
     47   std::string string_value;
     48   if (!list_update.GetString(index, &string_value)) {
     49     NOTREACHED();
     50     return 0;
     51   }
     52 
     53   int64 value = 0;
     54   bool rv = base::StringToInt64(string_value, &value);
     55   DCHECK(rv);
     56   return value;
     57 }
     58 
     59 // Report UMA metrics for daily data reductions.
     60 void RecordDailyContentLengthHistograms(
     61     int64 original_length,
     62     int64 received_length,
     63     int64 original_length_with_data_reduction_enabled,
     64     int64 received_length_with_data_reduction_enabled,
     65     int64 original_length_via_data_reduction_proxy,
     66     int64 received_length_via_data_reduction_proxy,
     67     int64 https_length_with_data_reduction_enabled,
     68     int64 short_bypass_length_with_data_reduction_enabled,
     69     int64 long_bypass_length_with_data_reduction_enabled,
     70     int64 unknown_length_with_data_reduction_enabled) {
     71   // Report daily UMA only for days having received content.
     72   if (original_length <= 0 || received_length <= 0)
     73     return;
     74 
     75   // Record metrics in KB.
     76   UMA_HISTOGRAM_COUNTS(
     77       "Net.DailyOriginalContentLength", original_length >> 10);
     78   UMA_HISTOGRAM_COUNTS(
     79       "Net.DailyContentLength", received_length >> 10);
     80   int percent = 0;
     81   // UMA percentage cannot be negative.
     82   if (original_length > received_length) {
     83     percent = (100 * (original_length - received_length)) / original_length;
     84   }
     85   UMA_HISTOGRAM_PERCENTAGE("Net.DailyContentSavingPercent", percent);
     86 
     87   if (original_length_with_data_reduction_enabled <= 0 ||
     88       received_length_with_data_reduction_enabled <= 0) {
     89     return;
     90   }
     91 
     92   UMA_HISTOGRAM_COUNTS(
     93       "Net.DailyOriginalContentLength_DataReductionProxyEnabled",
     94       original_length_with_data_reduction_enabled >> 10);
     95   UMA_HISTOGRAM_COUNTS(
     96       "Net.DailyContentLength_DataReductionProxyEnabled",
     97       received_length_with_data_reduction_enabled >> 10);
     98 
     99   int percent_data_reduction_proxy_enabled = 0;
    100   // UMA percentage cannot be negative.
    101   if (original_length_with_data_reduction_enabled >
    102       received_length_with_data_reduction_enabled) {
    103     percent_data_reduction_proxy_enabled =
    104         100 * (original_length_with_data_reduction_enabled -
    105                received_length_with_data_reduction_enabled) /
    106         original_length_with_data_reduction_enabled;
    107   }
    108   UMA_HISTOGRAM_PERCENTAGE(
    109       "Net.DailyContentSavingPercent_DataReductionProxyEnabled",
    110       percent_data_reduction_proxy_enabled);
    111 
    112   UMA_HISTOGRAM_PERCENTAGE(
    113       "Net.DailyContentPercent_DataReductionProxyEnabled",
    114       (100 * received_length_with_data_reduction_enabled) / received_length);
    115 
    116   DCHECK_GE(https_length_with_data_reduction_enabled, 0);
    117   UMA_HISTOGRAM_COUNTS(
    118       "Net.DailyContentLength_DataReductionProxyEnabled_Https",
    119       https_length_with_data_reduction_enabled >> 10);
    120   UMA_HISTOGRAM_PERCENTAGE(
    121       "Net.DailyContentPercent_DataReductionProxyEnabled_Https",
    122       (100 * https_length_with_data_reduction_enabled) / received_length);
    123 
    124   DCHECK_GE(short_bypass_length_with_data_reduction_enabled, 0);
    125   UMA_HISTOGRAM_COUNTS(
    126       "Net.DailyContentLength_DataReductionProxyEnabled_ShortBypass",
    127       short_bypass_length_with_data_reduction_enabled >> 10);
    128   UMA_HISTOGRAM_PERCENTAGE(
    129       "Net.DailyContentPercent_DataReductionProxyEnabled_ShortBypass",
    130       ((100 * short_bypass_length_with_data_reduction_enabled) /
    131        received_length));
    132 
    133   DCHECK_GE(long_bypass_length_with_data_reduction_enabled, 0);
    134   UMA_HISTOGRAM_COUNTS(
    135       "Net.DailyContentLength_DataReductionProxyEnabled_LongBypass",
    136       long_bypass_length_with_data_reduction_enabled >> 10);
    137   UMA_HISTOGRAM_PERCENTAGE(
    138       "Net.DailyContentPercent_DataReductionProxyEnabled_LongBypass",
    139       ((100 * long_bypass_length_with_data_reduction_enabled) /
    140        received_length));
    141 
    142   DCHECK_GE(unknown_length_with_data_reduction_enabled, 0);
    143   UMA_HISTOGRAM_COUNTS(
    144       "Net.DailyContentLength_DataReductionProxyEnabled_Unknown",
    145       unknown_length_with_data_reduction_enabled >> 10);
    146   UMA_HISTOGRAM_PERCENTAGE(
    147       "Net.DailyContentPercent_DataReductionProxyEnabled_Unknown",
    148       ((100 * unknown_length_with_data_reduction_enabled) /
    149        received_length));
    150 
    151   DCHECK_GE(original_length_via_data_reduction_proxy, 0);
    152   UMA_HISTOGRAM_COUNTS(
    153       "Net.DailyOriginalContentLength_ViaDataReductionProxy",
    154       original_length_via_data_reduction_proxy >> 10);
    155   DCHECK_GE(received_length_via_data_reduction_proxy, 0);
    156   UMA_HISTOGRAM_COUNTS(
    157       "Net.DailyContentLength_ViaDataReductionProxy",
    158       received_length_via_data_reduction_proxy >> 10);
    159   int percent_via_data_reduction_proxy = 0;
    160   if (original_length_via_data_reduction_proxy >
    161       received_length_via_data_reduction_proxy) {
    162     percent_via_data_reduction_proxy =
    163         100 * (original_length_via_data_reduction_proxy -
    164                received_length_via_data_reduction_proxy) /
    165         original_length_via_data_reduction_proxy;
    166   }
    167   UMA_HISTOGRAM_PERCENTAGE(
    168       "Net.DailyContentSavingPercent_ViaDataReductionProxy",
    169       percent_via_data_reduction_proxy);
    170   UMA_HISTOGRAM_PERCENTAGE(
    171       "Net.DailyContentPercent_ViaDataReductionProxy",
    172       (100 * received_length_via_data_reduction_proxy) / received_length);
    173 }
    174 
    175 // Ensure list has exactly |length| elements, either by truncating at the
    176 // front, or appending "0"'s to the back.
    177 void MaintainContentLengthPrefsWindow(base::ListValue* list, size_t length) {
    178   // Remove data for old days from the front.
    179   while (list->GetSize() > length)
    180     list->Remove(0, NULL);
    181   // Newly added lists are empty. Add entries to back to fill the window,
    182   // each initialized to zero.
    183   while (list->GetSize() < length)
    184     list->AppendString(base::Int64ToString(0));
    185   DCHECK_EQ(length, list->GetSize());
    186 }
    187 
    188 // DailyContentLengthUpdate maintains a data saving pref. The pref is a list
    189 // of |kNumDaysInHistory| elements of daily total content lengths for the past
    190 // |kNumDaysInHistory| days.
    191 class DailyContentLengthUpdate {
    192  public:
    193   DailyContentLengthUpdate(
    194       const char* pref,
    195       PrefService* pref_service)
    196       : update_(pref_service, pref) {
    197   }
    198 
    199   void UpdateForDataChange(int days_since_last_update) {
    200     // New empty lists may have been created. Maintain the invariant that
    201     // there should be exactly |kNumDaysInHistory| days in the histories.
    202     MaintainContentLengthPrefsWindow(update_.Get(), kNumDaysInHistory);
    203     if (days_since_last_update) {
    204       MaintainContentLengthPrefForDateChange(days_since_last_update);
    205     }
    206   }
    207 
    208   // Update the lengths for the current day.
    209   void Add(int content_length) {
    210     AddInt64ToListPref(kNumDaysInHistory - 1, content_length, update_.Get());
    211   }
    212 
    213   int64 GetListPrefValue(size_t index) {
    214     return ListPrefInt64Value(*update_, index);
    215   }
    216 
    217  private:
    218   // Update the list for date change and ensure the list has exactly |length|
    219   // elements. The last entry in the list will be for the current day after
    220   // the update.
    221   void MaintainContentLengthPrefForDateChange(int days_since_last_update) {
    222     if (days_since_last_update == -1) {
    223       // The system may go backwards in time by up to a day for legitimate
    224       // reasons, such as with changes to the time zone. In such cases, we
    225       // keep adding to the current day.
    226       // Note: we accept the fact that some reported data is shifted to
    227       // the adjacent day if users travel back and forth across time zones.
    228       days_since_last_update = 0;
    229     } else if (days_since_last_update < -1) {
    230       // Erase all entries if the system went backwards in time by more than
    231       // a day.
    232       update_->Clear();
    233 
    234       days_since_last_update = kNumDaysInHistory;
    235     }
    236     DCHECK_GE(days_since_last_update, 0);
    237 
    238     // Add entries for days since last update event. This will make the
    239     // lists longer than kNumDaysInHistory. The additional items will be cut off
    240     // from the head of the lists by |MaintainContentLengthPrefsWindow|, below.
    241     for (int i = 0;
    242          i < days_since_last_update && i < static_cast<int>(kNumDaysInHistory);
    243          ++i) {
    244       update_->AppendString(base::Int64ToString(0));
    245     }
    246 
    247     // Entries for new days may have been appended. Maintain the invariant that
    248     // there should be exactly |kNumDaysInHistory| days in the histories.
    249     MaintainContentLengthPrefsWindow(update_.Get(), kNumDaysInHistory);
    250   }
    251 
    252   ListPrefUpdate update_;
    253 };
    254 
    255 // DailyDataSavingUpdate maintains a pair of data saving prefs, original_update_
    256 // and received_update_. pref_original is a list of |kNumDaysInHistory| elements
    257 // of daily total original content lengths for the past |kNumDaysInHistory|
    258 // days. pref_received is the corresponding list of the daily total received
    259 // content lengths.
    260 class DailyDataSavingUpdate {
    261  public:
    262   DailyDataSavingUpdate(
    263       const char* pref_original,
    264       const char* pref_received,
    265       PrefService* pref_service)
    266       : original_(pref_original, pref_service),
    267         received_(pref_received, pref_service) {
    268   }
    269 
    270   void UpdateForDataChange(int days_since_last_update) {
    271     original_.UpdateForDataChange(days_since_last_update);
    272     received_.UpdateForDataChange(days_since_last_update);
    273   }
    274 
    275   // Update the lengths for the current day.
    276   void Add(int original_content_length, int received_content_length) {
    277     original_.Add(original_content_length);
    278     received_.Add(received_content_length);
    279   }
    280 
    281   int64 GetOriginalListPrefValue(size_t index) {
    282     return original_.GetListPrefValue(index);
    283   }
    284   int64 GetReceivedListPrefValue(size_t index) {
    285     return received_.GetListPrefValue(index);
    286   }
    287 
    288  private:
    289   DailyContentLengthUpdate original_;
    290   DailyContentLengthUpdate received_;
    291 };
    292 
    293 // Returns true if the request is bypassed by all configured data reduction
    294 // proxies. It returns the bypass delay in delay_seconds (if not NULL). If
    295 // the request is bypassed by more than one proxy, delay_seconds returns
    296 // shortest delay.
    297 bool IsBypassRequest(const net::URLRequest* request, int64* delay_seconds) {
    298   // TODO(bengr): Add support for other data reduction proxy configurations.
    299 #if defined(SPDY_PROXY_AUTH_ORIGIN)
    300   DataReductionProxyParams params(
    301       DataReductionProxyParams::kAllowed |
    302       DataReductionProxyParams::kFallbackAllowed |
    303       DataReductionProxyParams::kPromoAllowed);
    304   DataReductionProxyParams::DataReductionProxyList proxies =
    305       params.GetAllowedProxies();
    306   if (proxies.size() == 0)
    307     return false;
    308 
    309   if (request == NULL || request->context() == NULL ||
    310       request->context()->proxy_service() == NULL) {
    311     return false;
    312   }
    313 
    314   const net::ProxyRetryInfoMap& retry_map =
    315       request->context()->proxy_service()->proxy_retry_info();
    316   if (retry_map.size() == 0)
    317     return false;
    318 
    319   int64 shortest_delay = 0;
    320   // The request is bypassed if all configured proxies are in the retry map.
    321   for (size_t i = 0; i < proxies.size(); ++i) {
    322     std::string proxy = net::HostPortPair::FromURL(proxies[i]).ToString();
    323     // The retry list has the scheme prefix for https but not for http.
    324     if (proxies[i].SchemeIs("https"))
    325       proxy = std::string("https://") + proxy;
    326 
    327     net::ProxyRetryInfoMap::const_iterator found = retry_map.find(proxy);
    328     if (found == retry_map.end())
    329       return false;
    330     if (shortest_delay == 0 ||
    331         shortest_delay > found->second.current_delay.InSeconds()) {
    332       shortest_delay = found->second.current_delay.InSeconds();
    333     }
    334   }
    335   if (delay_seconds != NULL)
    336     *delay_seconds = shortest_delay;
    337   return true;
    338 #else
    339   return false;
    340 #endif
    341 }
    342 
    343 }  // namespace
    344 
    345 DataReductionProxyRequestType GetDataReductionProxyRequestType(
    346     const net::URLRequest* request) {
    347   if (request->url().SchemeIs("https"))
    348     return HTTPS;
    349   if (!request->url().SchemeIs("http")) {
    350     NOTREACHED();
    351     return UNKNOWN_TYPE;
    352   }
    353   int64 bypass_delay = 0;
    354   if (IsBypassRequest(request, &bypass_delay)) {
    355     return (bypass_delay > kLongBypassDelayInSeconds) ?
    356       LONG_BYPASS : SHORT_BYPASS;
    357   }
    358   if (request->response_info().headers &&
    359       HasDataReductionProxyViaHeader(request->response_info().headers)) {
    360     return VIA_DATA_REDUCTION_PROXY;
    361   }
    362   return UNKNOWN_TYPE;
    363 }
    364 
    365 int64 GetAdjustedOriginalContentLength(
    366     DataReductionProxyRequestType request_type,
    367     int64 original_content_length,
    368     int64 received_content_length) {
    369   // Since there was no indication of the original content length, presume
    370   // it is no different from the number of bytes read.
    371   if (original_content_length == -1 ||
    372       request_type != VIA_DATA_REDUCTION_PROXY) {
    373     return received_content_length;
    374   }
    375   return original_content_length;
    376 }
    377 
    378 void UpdateContentLengthPrefsForDataReductionProxy(
    379     int received_content_length,
    380     int original_content_length,
    381     bool with_data_reduction_proxy_enabled,
    382     DataReductionProxyRequestType request_type,
    383     base::Time now, PrefService* prefs) {
    384   // TODO(bengr): Remove this check once the underlying cause of
    385   // http://crbug.com/287821 is fixed. For now, only continue if the current
    386   // year is reported as being between 1972 and 2970.
    387   base::TimeDelta time_since_unix_epoch = now - base::Time::UnixEpoch();
    388   const int kMinDaysSinceUnixEpoch = 365 * 2;  // 2 years.
    389   const int kMaxDaysSinceUnixEpoch = 365 * 1000;  // 1000 years.
    390   if (time_since_unix_epoch.InDays() < kMinDaysSinceUnixEpoch ||
    391       time_since_unix_epoch.InDays() > kMaxDaysSinceUnixEpoch) {
    392     return;
    393   }
    394 
    395   // Determine how many days it has been since the last update.
    396   int64 then_internal = prefs->GetInt64(
    397       data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate);
    398 
    399 #if defined(OS_WIN)
    400   base::Time then_midnight = base::Time::FromInternalValue(then_internal);
    401   base::Time midnight =
    402       base::Time::FromInternalValue(
    403           (now.ToInternalValue() / base::Time::kMicrosecondsPerDay) *
    404               base::Time::kMicrosecondsPerDay);
    405 #else
    406   // Local midnight could have been shifted due to time zone change.
    407   base::Time then_midnight =
    408       base::Time::FromInternalValue(then_internal).LocalMidnight();
    409   base::Time midnight = now.LocalMidnight();
    410 #endif
    411 
    412   int days_since_last_update = (midnight - then_midnight).InDays();
    413 
    414   // Each day, we calculate the total number of bytes received and the total
    415   // size of all corresponding resources before any data-reducing recompression
    416   // is applied. These values are used to compute the data savings realized
    417   // by applying our compression techniques. Totals for the last
    418   // |kNumDaysInHistory| days are maintained.
    419   DailyDataSavingUpdate total(
    420       data_reduction_proxy::prefs::kDailyHttpOriginalContentLength,
    421       data_reduction_proxy::prefs::kDailyHttpReceivedContentLength,
    422       prefs);
    423   total.UpdateForDataChange(days_since_last_update);
    424 
    425   DailyDataSavingUpdate proxy_enabled(
    426       data_reduction_proxy::prefs::
    427           kDailyOriginalContentLengthWithDataReductionProxyEnabled,
    428       data_reduction_proxy::prefs::
    429           kDailyContentLengthWithDataReductionProxyEnabled,
    430       prefs);
    431   proxy_enabled.UpdateForDataChange(days_since_last_update);
    432 
    433   DailyDataSavingUpdate via_proxy(
    434       data_reduction_proxy::prefs::
    435           kDailyOriginalContentLengthViaDataReductionProxy,
    436       data_reduction_proxy::prefs::
    437           kDailyContentLengthViaDataReductionProxy,
    438       prefs);
    439   via_proxy.UpdateForDataChange(days_since_last_update);
    440 
    441   DailyContentLengthUpdate https(
    442       data_reduction_proxy::prefs::
    443           kDailyContentLengthHttpsWithDataReductionProxyEnabled,
    444       prefs);
    445   https.UpdateForDataChange(days_since_last_update);
    446 
    447   DailyContentLengthUpdate short_bypass(
    448       data_reduction_proxy::prefs::
    449           kDailyContentLengthShortBypassWithDataReductionProxyEnabled,
    450       prefs);
    451   short_bypass.UpdateForDataChange(days_since_last_update);
    452 
    453   DailyContentLengthUpdate long_bypass(
    454       data_reduction_proxy::prefs::
    455           kDailyContentLengthLongBypassWithDataReductionProxyEnabled,
    456       prefs);
    457   long_bypass.UpdateForDataChange(days_since_last_update);
    458 
    459   DailyContentLengthUpdate unknown(
    460       data_reduction_proxy::prefs::
    461           kDailyContentLengthUnknownWithDataReductionProxyEnabled,
    462       prefs);
    463   unknown.UpdateForDataChange(days_since_last_update);
    464 
    465   total.Add(original_content_length, received_content_length);
    466   if (with_data_reduction_proxy_enabled) {
    467     proxy_enabled.Add(original_content_length, received_content_length);
    468     // Ignore data source cases, if exist, when
    469     // "with_data_reduction_proxy_enabled == false"
    470     switch (request_type) {
    471       case VIA_DATA_REDUCTION_PROXY:
    472         via_proxy.Add(original_content_length, received_content_length);
    473         break;
    474       case HTTPS:
    475         https.Add(received_content_length);
    476         break;
    477       case SHORT_BYPASS:
    478         short_bypass.Add(received_content_length);
    479         break;
    480       case LONG_BYPASS:
    481         long_bypass.Add(received_content_length);
    482         break;
    483       case UNKNOWN_TYPE:
    484         unknown.Add(received_content_length);
    485         break;
    486     }
    487   }
    488 
    489   if (days_since_last_update) {
    490     // Record the last update time in microseconds in UTC.
    491     prefs->SetInt64(
    492         data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate,
    493         midnight.ToInternalValue());
    494 
    495     // A new day. Report the previous day's data if exists. We'll lose usage
    496     // data if the last time Chrome was run was more than a day ago.
    497     // Here, we prefer collecting less data but the collected data is
    498     // associated with an accurate date.
    499     if (days_since_last_update == 1) {
    500       // The previous day's data point is the second one from the tail.
    501       // Therefore (kNumDaysInHistory - 2) below.
    502       RecordDailyContentLengthHistograms(
    503           total.GetOriginalListPrefValue(kNumDaysInHistory - 2),
    504           total.GetReceivedListPrefValue(kNumDaysInHistory - 2),
    505           proxy_enabled.GetOriginalListPrefValue(kNumDaysInHistory - 2),
    506           proxy_enabled.GetReceivedListPrefValue(kNumDaysInHistory - 2),
    507           via_proxy.GetOriginalListPrefValue(kNumDaysInHistory - 2),
    508           via_proxy.GetReceivedListPrefValue(kNumDaysInHistory - 2),
    509           https.GetListPrefValue(kNumDaysInHistory - 2),
    510           short_bypass.GetListPrefValue(kNumDaysInHistory - 2),
    511           long_bypass.GetListPrefValue(kNumDaysInHistory - 2),
    512           unknown.GetListPrefValue(kNumDaysInHistory - 2));
    513     }
    514   }
    515 }
    516 
    517 void UpdateContentLengthPrefs(
    518     int received_content_length,
    519     int original_content_length,
    520     bool with_data_reduction_proxy_enabled,
    521     DataReductionProxyRequestType request_type,
    522     PrefService* prefs) {
    523   int64 total_received = prefs->GetInt64(
    524       data_reduction_proxy::prefs::kHttpReceivedContentLength);
    525   int64 total_original = prefs->GetInt64(
    526       data_reduction_proxy::prefs::kHttpOriginalContentLength);
    527   total_received += received_content_length;
    528   total_original += original_content_length;
    529   prefs->SetInt64(data_reduction_proxy::prefs::kHttpReceivedContentLength,
    530                   total_received);
    531   prefs->SetInt64(data_reduction_proxy::prefs::kHttpOriginalContentLength,
    532                   total_original);
    533 
    534   UpdateContentLengthPrefsForDataReductionProxy(
    535       received_content_length,
    536       original_content_length,
    537       with_data_reduction_proxy_enabled,
    538       request_type,
    539       base::Time::Now(),
    540       prefs);
    541 }
    542 
    543 }  // namespace data_reduction_proxy
    544