Home | History | Annotate | Download | only in browser 
      1 // Copyright 2014 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "components/data_reduction_proxy/browser/data_reduction_proxy_metrics.h"
      6 
      7 #include "base/metrics/histogram.h"
      8 #include "base/prefs/pref_service.h"
      9 #include "base/strings/string_number_conversions.h"
     10 #include "base/strings/string_util.h"
     11 #include "components/data_reduction_proxy/browser/data_reduction_proxy_settings.h"
     12 #include "components/data_reduction_proxy/browser/data_reduction_proxy_statistics_prefs.h"
     13 #include "components/data_reduction_proxy/common/data_reduction_proxy_headers.h"
     14 #include "components/data_reduction_proxy/common/data_reduction_proxy_pref_names.h"
     15 #include "net/base/host_port_pair.h"
     16 #include "net/http/http_response_headers.h"
     17 #include "net/proxy/proxy_retry_info.h"
     18 #include "net/proxy/proxy_service.h"
     19 #include "net/url_request/url_request_context.h"
     20 
     21 namespace data_reduction_proxy {
     22 
     23 namespace {
     24 
     25 // A bypass delay more than this is treated as a long delay.
     26 const int kLongBypassDelayInSeconds = 30 * 60;
     27 
     28 // Increments an int64, stored as a string, in a ListPref at the specified
     29 // index.  The value must already exist and be a string representation of a
     30 // number.
     31 void AddInt64ToListPref(size_t index,
     32                         int64 length,
     33                         base::ListValue* list_update) {
     34   int64 value = 0;
     35   std::string old_string_value;
     36   bool rv = list_update->GetString(index, &old_string_value);
     37   DCHECK(rv);
     38   if (rv) {
     39     rv = base::StringToInt64(old_string_value, &value);
     40     DCHECK(rv);
     41   }
     42   value += length;
     43   list_update->Set(index, new base::StringValue(base::Int64ToString(value)));
     44 }
     45 
     46 int64 ListPrefInt64Value(const base::ListValue& list_update, size_t index) {
     47   std::string string_value;
     48   if (!list_update.GetString(index, &string_value)) {
     49     NOTREACHED();
     50     return 0;
     51   }
     52 
     53   int64 value = 0;
     54   bool rv = base::StringToInt64(string_value, &value);
     55   DCHECK(rv);
     56   return value;
     57 }
     58 
     59 // Report UMA metrics for daily data reductions.
     60 void RecordDailyContentLengthHistograms(
     61     int64 original_length,
     62     int64 received_length,
     63     int64 original_length_with_data_reduction_enabled,
     64     int64 received_length_with_data_reduction_enabled,
     65     int64 original_length_via_data_reduction_proxy,
     66     int64 received_length_via_data_reduction_proxy,
     67     int64 https_length_with_data_reduction_enabled,
     68     int64 short_bypass_length_with_data_reduction_enabled,
     69     int64 long_bypass_length_with_data_reduction_enabled,
     70     int64 unknown_length_with_data_reduction_enabled) {
     71   // Report daily UMA only for days having received content.
     72   if (original_length <= 0 || received_length <= 0)
     73     return;
     74 
     75   // Record metrics in KB.
     76   UMA_HISTOGRAM_COUNTS(
     77       "Net.DailyOriginalContentLength", original_length >> 10);
     78   UMA_HISTOGRAM_COUNTS(
     79       "Net.DailyContentLength", received_length >> 10);
     80   int percent = 0;
     81   // UMA percentage cannot be negative.
     82   if (original_length > received_length) {
     83     percent = (100 * (original_length - received_length)) / original_length;
     84   }
     85   UMA_HISTOGRAM_PERCENTAGE("Net.DailyContentSavingPercent", percent);
     86 
     87   if (original_length_with_data_reduction_enabled <= 0 ||
     88       received_length_with_data_reduction_enabled <= 0) {
     89     return;
     90   }
     91 
     92   UMA_HISTOGRAM_COUNTS(
     93       "Net.DailyOriginalContentLength_DataReductionProxyEnabled",
     94       original_length_with_data_reduction_enabled >> 10);
     95   UMA_HISTOGRAM_COUNTS(
     96       "Net.DailyContentLength_DataReductionProxyEnabled",
     97       received_length_with_data_reduction_enabled >> 10);
     98 
     99   int percent_data_reduction_proxy_enabled = 0;
    100   // UMA percentage cannot be negative.
    101   if (original_length_with_data_reduction_enabled >
    102       received_length_with_data_reduction_enabled) {
    103     percent_data_reduction_proxy_enabled =
    104         100 * (original_length_with_data_reduction_enabled -
    105                received_length_with_data_reduction_enabled) /
    106         original_length_with_data_reduction_enabled;
    107   }
    108   UMA_HISTOGRAM_PERCENTAGE(
    109       "Net.DailyContentSavingPercent_DataReductionProxyEnabled",
    110       percent_data_reduction_proxy_enabled);
    111 
    112   UMA_HISTOGRAM_PERCENTAGE(
    113       "Net.DailyContentPercent_DataReductionProxyEnabled",
    114       (100 * received_length_with_data_reduction_enabled) / received_length);
    115 
    116   DCHECK_GE(https_length_with_data_reduction_enabled, 0);
    117   UMA_HISTOGRAM_COUNTS(
    118       "Net.DailyContentLength_DataReductionProxyEnabled_Https",
    119       https_length_with_data_reduction_enabled >> 10);
    120   UMA_HISTOGRAM_PERCENTAGE(
    121       "Net.DailyContentPercent_DataReductionProxyEnabled_Https",
    122       (100 * https_length_with_data_reduction_enabled) / received_length);
    123 
    124   DCHECK_GE(short_bypass_length_with_data_reduction_enabled, 0);
    125   UMA_HISTOGRAM_COUNTS(
    126       "Net.DailyContentLength_DataReductionProxyEnabled_ShortBypass",
    127       short_bypass_length_with_data_reduction_enabled >> 10);
    128   UMA_HISTOGRAM_PERCENTAGE(
    129       "Net.DailyContentPercent_DataReductionProxyEnabled_ShortBypass",
    130       ((100 * short_bypass_length_with_data_reduction_enabled) /
    131        received_length));
    132 
    133   DCHECK_GE(long_bypass_length_with_data_reduction_enabled, 0);
    134   UMA_HISTOGRAM_COUNTS(
    135       "Net.DailyContentLength_DataReductionProxyEnabled_LongBypass",
    136       long_bypass_length_with_data_reduction_enabled >> 10);
    137   UMA_HISTOGRAM_PERCENTAGE(
    138       "Net.DailyContentPercent_DataReductionProxyEnabled_LongBypass",
    139       ((100 * long_bypass_length_with_data_reduction_enabled) /
    140        received_length));
    141 
    142   DCHECK_GE(unknown_length_with_data_reduction_enabled, 0);
    143   UMA_HISTOGRAM_COUNTS(
    144       "Net.DailyContentLength_DataReductionProxyEnabled_Unknown",
    145       unknown_length_with_data_reduction_enabled >> 10);
    146   UMA_HISTOGRAM_PERCENTAGE(
    147       "Net.DailyContentPercent_DataReductionProxyEnabled_Unknown",
    148       ((100 * unknown_length_with_data_reduction_enabled) /
    149        received_length));
    150 
    151   DCHECK_GE(original_length_via_data_reduction_proxy, 0);
    152   UMA_HISTOGRAM_COUNTS(
    153       "Net.DailyOriginalContentLength_ViaDataReductionProxy",
    154       original_length_via_data_reduction_proxy >> 10);
    155   DCHECK_GE(received_length_via_data_reduction_proxy, 0);
    156   UMA_HISTOGRAM_COUNTS(
    157       "Net.DailyContentLength_ViaDataReductionProxy",
    158       received_length_via_data_reduction_proxy >> 10);
    159   int percent_via_data_reduction_proxy = 0;
    160   if (original_length_via_data_reduction_proxy >
    161       received_length_via_data_reduction_proxy) {
    162     percent_via_data_reduction_proxy =
    163         100 * (original_length_via_data_reduction_proxy -
    164                received_length_via_data_reduction_proxy) /
    165         original_length_via_data_reduction_proxy;
    166   }
    167   UMA_HISTOGRAM_PERCENTAGE(
    168       "Net.DailyContentSavingPercent_ViaDataReductionProxy",
    169       percent_via_data_reduction_proxy);
    170   UMA_HISTOGRAM_PERCENTAGE(
    171       "Net.DailyContentPercent_ViaDataReductionProxy",
    172       (100 * received_length_via_data_reduction_proxy) / received_length);
    173 }
    174 
    175 // Ensure list has exactly |length| elements, either by truncating at the
    176 // front, or appending "0"'s to the back.
    177 void MaintainContentLengthPrefsWindow(base::ListValue* list, size_t length) {
    178   // Remove data for old days from the front.
    179   while (list->GetSize() > length)
    180     list->Remove(0, NULL);
    181   // Newly added lists are empty. Add entries to back to fill the window,
    182   // each initialized to zero.
    183   while (list->GetSize() < length)
    184     list->AppendString(base::Int64ToString(0));
    185   DCHECK_EQ(length, list->GetSize());
    186 }
    187 
    188 // DailyContentLengthUpdate maintains a data saving pref. The pref is a list
    189 // of |kNumDaysInHistory| elements of daily total content lengths for the past
    190 // |kNumDaysInHistory| days.
    191 class DailyContentLengthUpdate {
    192  public:
    193   DailyContentLengthUpdate(const char* pref,
    194                            DataReductionProxyStatisticsPrefs* pref_service)
    195       : update_(pref_service->GetList(pref)) {
    196   }
    197 
    198   void UpdateForDataChange(int days_since_last_update) {
    199     // New empty lists may have been created. Maintain the invariant that
    200     // there should be exactly |kNumDaysInHistory| days in the histories.
    201     MaintainContentLengthPrefsWindow(update_, kNumDaysInHistory);
    202     if (days_since_last_update) {
    203       MaintainContentLengthPrefForDateChange(days_since_last_update);
    204     }
    205   }
    206 
    207   // Update the lengths for the current day.
    208   void Add(int content_length) {
    209     AddInt64ToListPref(kNumDaysInHistory - 1, content_length, update_);
    210   }
    211 
    212   int64 GetListPrefValue(size_t index) {
    213     return ListPrefInt64Value(*update_, index);
    214   }
    215 
    216  private:
    217   // Update the list for date change and ensure the list has exactly |length|
    218   // elements. The last entry in the list will be for the current day after
    219   // the update.
    220   void MaintainContentLengthPrefForDateChange(int days_since_last_update) {
    221     if (days_since_last_update == -1) {
    222       // The system may go backwards in time by up to a day for legitimate
    223       // reasons, such as with changes to the time zone. In such cases, we
    224       // keep adding to the current day.
    225       // Note: we accept the fact that some reported data is shifted to
    226       // the adjacent day if users travel back and forth across time zones.
    227       days_since_last_update = 0;
    228     } else if (days_since_last_update < -1) {
    229       // Erase all entries if the system went backwards in time by more than
    230       // a day.
    231       update_->Clear();
    232 
    233       days_since_last_update = kNumDaysInHistory;
    234     }
    235     DCHECK_GE(days_since_last_update, 0);
    236 
    237     // Add entries for days since last update event. This will make the
    238     // lists longer than kNumDaysInHistory. The additional items will be cut off
    239     // from the head of the lists by |MaintainContentLengthPrefsWindow|, below.
    240     for (int i = 0;
    241          i < days_since_last_update && i < static_cast<int>(kNumDaysInHistory);
    242          ++i) {
    243       update_->AppendString(base::Int64ToString(0));
    244     }
    245 
    246     // Entries for new days may have been appended. Maintain the invariant that
    247     // there should be exactly |kNumDaysInHistory| days in the histories.
    248     MaintainContentLengthPrefsWindow(update_, kNumDaysInHistory);
    249   }
    250 
    251   base::ListValue* update_;
    252 };
    253 
    254 // DailyDataSavingUpdate maintains a pair of data saving prefs, original_update_
    255 // and received_update_. pref_original is a list of |kNumDaysInHistory| elements
    256 // of daily total original content lengths for the past |kNumDaysInHistory|
    257 // days. pref_received is the corresponding list of the daily total received
    258 // content lengths.
    259 class DailyDataSavingUpdate {
    260  public:
    261   DailyDataSavingUpdate(const char* pref_original,
    262                         const char* pref_received,
    263                         DataReductionProxyStatisticsPrefs* prefs)
    264       : original_(pref_original, prefs),
    265         received_(pref_received, prefs) {
    266   }
    267 
    268   void UpdateForDataChange(int days_since_last_update) {
    269     original_.UpdateForDataChange(days_since_last_update);
    270     received_.UpdateForDataChange(days_since_last_update);
    271   }
    272 
    273   // Update the lengths for the current day.
    274   void Add(int original_content_length, int received_content_length) {
    275     original_.Add(original_content_length);
    276     received_.Add(received_content_length);
    277   }
    278 
    279   int64 GetOriginalListPrefValue(size_t index) {
    280     return original_.GetListPrefValue(index);
    281   }
    282   int64 GetReceivedListPrefValue(size_t index) {
    283     return received_.GetListPrefValue(index);
    284   }
    285 
    286  private:
    287   DailyContentLengthUpdate original_;
    288   DailyContentLengthUpdate received_;
    289 };
    290 
    291 }  // namespace
    292 
    293 DataReductionProxyRequestType GetDataReductionProxyRequestType(
    294     const net::URLRequest* request) {
    295   if (request->url().SchemeIs("https"))
    296     return HTTPS;
    297   if (!request->url().SchemeIs("http")) {
    298     NOTREACHED();
    299     return UNKNOWN_TYPE;
    300   }
    301   DataReductionProxyParams params(
    302         DataReductionProxyParams::kAllowed |
    303         DataReductionProxyParams::kFallbackAllowed |
    304         DataReductionProxyParams::kPromoAllowed);
    305   base::TimeDelta bypass_delay;
    306   if (params.AreDataReductionProxiesBypassed(*request, &bypass_delay)) {
    307     if (bypass_delay > base::TimeDelta::FromSeconds(kLongBypassDelayInSeconds))
    308       return LONG_BYPASS;
    309     return SHORT_BYPASS;
    310   }
    311   if (request->response_info().headers.get() &&
    312       HasDataReductionProxyViaHeader(request->response_info().headers.get(),
    313                                      NULL)) {
    314     return VIA_DATA_REDUCTION_PROXY;
    315   }
    316   return UNKNOWN_TYPE;
    317 }
    318 
    319 int64 GetAdjustedOriginalContentLength(
    320     DataReductionProxyRequestType request_type,
    321     int64 original_content_length,
    322     int64 received_content_length) {
    323   // Since there was no indication of the original content length, presume
    324   // it is no different from the number of bytes read.
    325   if (original_content_length == -1 ||
    326       request_type != VIA_DATA_REDUCTION_PROXY) {
    327     return received_content_length;
    328   }
    329   return original_content_length;
    330 }
    331 
    332 void UpdateContentLengthPrefsForDataReductionProxy(
    333     int received_content_length,
    334     int original_content_length,
    335     bool with_data_reduction_proxy_enabled,
    336     DataReductionProxyRequestType request_type,
    337     base::Time now,
    338     DataReductionProxyStatisticsPrefs* prefs) {
    339   // TODO(bengr): Remove this check once the underlying cause of
    340   // http://crbug.com/287821 is fixed. For now, only continue if the current
    341   // year is reported as being between 1972 and 2970.
    342   base::TimeDelta time_since_unix_epoch = now - base::Time::UnixEpoch();
    343   const int kMinDaysSinceUnixEpoch = 365 * 2;  // 2 years.
    344   const int kMaxDaysSinceUnixEpoch = 365 * 1000;  // 1000 years.
    345   if (time_since_unix_epoch.InDays() < kMinDaysSinceUnixEpoch ||
    346       time_since_unix_epoch.InDays() > kMaxDaysSinceUnixEpoch) {
    347     return;
    348   }
    349 
    350   // Determine how many days it has been since the last update.
    351   int64 then_internal = prefs->GetInt64(
    352       data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate);
    353 
    354   // Local midnight could have been shifted due to time zone change.
    355   // If time is null then don't care if midnight will be wrong shifted due to
    356   // time zone change because it's still too much time ago.
    357   base::Time then_midnight = base::Time::FromInternalValue(then_internal);
    358   if (!then_midnight.is_null()) {
    359     then_midnight = then_midnight.LocalMidnight();
    360   }
    361   base::Time midnight = now.LocalMidnight();
    362 
    363   int days_since_last_update = (midnight - then_midnight).InDays();
    364 
    365   // Each day, we calculate the total number of bytes received and the total
    366   // size of all corresponding resources before any data-reducing recompression
    367   // is applied. These values are used to compute the data savings realized
    368   // by applying our compression techniques. Totals for the last
    369   // |kNumDaysInHistory| days are maintained.
    370   DailyDataSavingUpdate total(
    371       data_reduction_proxy::prefs::kDailyHttpOriginalContentLength,
    372       data_reduction_proxy::prefs::kDailyHttpReceivedContentLength,
    373       prefs);
    374   total.UpdateForDataChange(days_since_last_update);
    375 
    376   DailyDataSavingUpdate proxy_enabled(
    377       data_reduction_proxy::prefs::
    378           kDailyOriginalContentLengthWithDataReductionProxyEnabled,
    379       data_reduction_proxy::prefs::
    380           kDailyContentLengthWithDataReductionProxyEnabled,
    381       prefs);
    382   proxy_enabled.UpdateForDataChange(days_since_last_update);
    383 
    384   DailyDataSavingUpdate via_proxy(
    385       data_reduction_proxy::prefs::
    386           kDailyOriginalContentLengthViaDataReductionProxy,
    387       data_reduction_proxy::prefs::
    388           kDailyContentLengthViaDataReductionProxy,
    389       prefs);
    390   via_proxy.UpdateForDataChange(days_since_last_update);
    391 
    392   DailyContentLengthUpdate https(
    393       data_reduction_proxy::prefs::
    394           kDailyContentLengthHttpsWithDataReductionProxyEnabled,
    395       prefs);
    396   https.UpdateForDataChange(days_since_last_update);
    397 
    398   DailyContentLengthUpdate short_bypass(
    399       data_reduction_proxy::prefs::
    400           kDailyContentLengthShortBypassWithDataReductionProxyEnabled,
    401       prefs);
    402   short_bypass.UpdateForDataChange(days_since_last_update);
    403 
    404   DailyContentLengthUpdate long_bypass(
    405       data_reduction_proxy::prefs::
    406           kDailyContentLengthLongBypassWithDataReductionProxyEnabled,
    407       prefs);
    408   long_bypass.UpdateForDataChange(days_since_last_update);
    409 
    410   DailyContentLengthUpdate unknown(
    411       data_reduction_proxy::prefs::
    412           kDailyContentLengthUnknownWithDataReductionProxyEnabled,
    413       prefs);
    414   unknown.UpdateForDataChange(days_since_last_update);
    415 
    416   total.Add(original_content_length, received_content_length);
    417   if (with_data_reduction_proxy_enabled) {
    418     proxy_enabled.Add(original_content_length, received_content_length);
    419     // Ignore data source cases, if exist, when
    420     // "with_data_reduction_proxy_enabled == false"
    421     switch (request_type) {
    422       case VIA_DATA_REDUCTION_PROXY:
    423         via_proxy.Add(original_content_length, received_content_length);
    424         break;
    425       case HTTPS:
    426         https.Add(received_content_length);
    427         break;
    428       case SHORT_BYPASS:
    429         short_bypass.Add(received_content_length);
    430         break;
    431       case LONG_BYPASS:
    432         long_bypass.Add(received_content_length);
    433         break;
    434       case UNKNOWN_TYPE:
    435         unknown.Add(received_content_length);
    436         break;
    437     }
    438   }
    439 
    440   if (days_since_last_update) {
    441     // Record the last update time in microseconds in UTC.
    442     prefs->SetInt64(
    443         data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate,
    444         midnight.ToInternalValue());
    445 
    446     // A new day. Report the previous day's data if exists. We'll lose usage
    447     // data if the last time Chrome was run was more than a day ago.
    448     // Here, we prefer collecting less data but the collected data is
    449     // associated with an accurate date.
    450     if (days_since_last_update == 1) {
    451       // The previous day's data point is the second one from the tail.
    452       // Therefore (kNumDaysInHistory - 2) below.
    453       RecordDailyContentLengthHistograms(
    454           total.GetOriginalListPrefValue(kNumDaysInHistory - 2),
    455           total.GetReceivedListPrefValue(kNumDaysInHistory - 2),
    456           proxy_enabled.GetOriginalListPrefValue(kNumDaysInHistory - 2),
    457           proxy_enabled.GetReceivedListPrefValue(kNumDaysInHistory - 2),
    458           via_proxy.GetOriginalListPrefValue(kNumDaysInHistory - 2),
    459           via_proxy.GetReceivedListPrefValue(kNumDaysInHistory - 2),
    460           https.GetListPrefValue(kNumDaysInHistory - 2),
    461           short_bypass.GetListPrefValue(kNumDaysInHistory - 2),
    462           long_bypass.GetListPrefValue(kNumDaysInHistory - 2),
    463           unknown.GetListPrefValue(kNumDaysInHistory - 2));
    464     }
    465   }
    466 }
    467 
    468 void UpdateContentLengthPrefs(int received_content_length,
    469                               int original_content_length,
    470                               PrefService* profile_prefs,
    471                               DataReductionProxyRequestType request_type,
    472                               DataReductionProxyStatisticsPrefs* prefs) {
    473   int64 total_received = prefs->GetInt64(
    474       data_reduction_proxy::prefs::kHttpReceivedContentLength);
    475   int64 total_original = prefs->GetInt64(
    476       data_reduction_proxy::prefs::kHttpOriginalContentLength);
    477   total_received += received_content_length;
    478   total_original += original_content_length;
    479   prefs->SetInt64(
    480       data_reduction_proxy::prefs::kHttpReceivedContentLength,
    481       total_received);
    482   prefs->SetInt64(
    483       data_reduction_proxy::prefs::kHttpOriginalContentLength,
    484       total_original);
    485 
    486   bool with_data_reduction_proxy_enabled =
    487       profile_prefs->GetBoolean(
    488           data_reduction_proxy::prefs::kDataReductionProxyEnabled);
    489   UpdateContentLengthPrefsForDataReductionProxy(
    490       received_content_length,
    491       original_content_length,
    492       with_data_reduction_proxy_enabled,
    493       request_type,
    494       base::Time::Now(),
    495       prefs);
    496 }
    497 
    498 }  // namespace data_reduction_proxy
    499