1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "components/data_reduction_proxy/browser/data_reduction_proxy_metrics.h" 6 7 #include "base/metrics/histogram.h" 8 #include "base/prefs/pref_service.h" 9 #include "base/prefs/scoped_user_pref_update.h" 10 #include "base/strings/string_number_conversions.h" 11 #include "base/strings/string_util.h" 12 #include "components/data_reduction_proxy/browser/data_reduction_proxy_settings.h" 13 #include "components/data_reduction_proxy/common/data_reduction_proxy_headers.h" 14 #include "components/data_reduction_proxy/common/data_reduction_proxy_pref_names.h" 15 #include "net/base/host_port_pair.h" 16 #include "net/http/http_response_headers.h" 17 #include "net/proxy/proxy_retry_info.h" 18 #include "net/proxy/proxy_service.h" 19 #include "net/url_request/url_request_context.h" 20 21 namespace data_reduction_proxy { 22 23 namespace { 24 25 // A bypass delay more than this is treated as a long delay. 26 const int kLongBypassDelayInSeconds = 30 * 60; 27 28 // Increments an int64, stored as a string, in a ListPref at the specified 29 // index. The value must already exist and be a string representation of a 30 // number. 31 void AddInt64ToListPref(size_t index, 32 int64 length, 33 base::ListValue* list_update) { 34 int64 value = 0; 35 std::string old_string_value; 36 bool rv = list_update->GetString(index, &old_string_value); 37 DCHECK(rv); 38 if (rv) { 39 rv = base::StringToInt64(old_string_value, &value); 40 DCHECK(rv); 41 } 42 value += length; 43 list_update->Set(index, new base::StringValue(base::Int64ToString(value))); 44 } 45 46 int64 ListPrefInt64Value(const base::ListValue& list_update, size_t index) { 47 std::string string_value; 48 if (!list_update.GetString(index, &string_value)) { 49 NOTREACHED(); 50 return 0; 51 } 52 53 int64 value = 0; 54 bool rv = base::StringToInt64(string_value, &value); 55 DCHECK(rv); 56 return value; 57 } 58 59 // Report UMA metrics for daily data reductions. 60 void RecordDailyContentLengthHistograms( 61 int64 original_length, 62 int64 received_length, 63 int64 original_length_with_data_reduction_enabled, 64 int64 received_length_with_data_reduction_enabled, 65 int64 original_length_via_data_reduction_proxy, 66 int64 received_length_via_data_reduction_proxy, 67 int64 https_length_with_data_reduction_enabled, 68 int64 short_bypass_length_with_data_reduction_enabled, 69 int64 long_bypass_length_with_data_reduction_enabled, 70 int64 unknown_length_with_data_reduction_enabled) { 71 // Report daily UMA only for days having received content. 72 if (original_length <= 0 || received_length <= 0) 73 return; 74 75 // Record metrics in KB. 76 UMA_HISTOGRAM_COUNTS( 77 "Net.DailyOriginalContentLength", original_length >> 10); 78 UMA_HISTOGRAM_COUNTS( 79 "Net.DailyContentLength", received_length >> 10); 80 int percent = 0; 81 // UMA percentage cannot be negative. 82 if (original_length > received_length) { 83 percent = (100 * (original_length - received_length)) / original_length; 84 } 85 UMA_HISTOGRAM_PERCENTAGE("Net.DailyContentSavingPercent", percent); 86 87 if (original_length_with_data_reduction_enabled <= 0 || 88 received_length_with_data_reduction_enabled <= 0) { 89 return; 90 } 91 92 UMA_HISTOGRAM_COUNTS( 93 "Net.DailyOriginalContentLength_DataReductionProxyEnabled", 94 original_length_with_data_reduction_enabled >> 10); 95 UMA_HISTOGRAM_COUNTS( 96 "Net.DailyContentLength_DataReductionProxyEnabled", 97 received_length_with_data_reduction_enabled >> 10); 98 99 int percent_data_reduction_proxy_enabled = 0; 100 // UMA percentage cannot be negative. 101 if (original_length_with_data_reduction_enabled > 102 received_length_with_data_reduction_enabled) { 103 percent_data_reduction_proxy_enabled = 104 100 * (original_length_with_data_reduction_enabled - 105 received_length_with_data_reduction_enabled) / 106 original_length_with_data_reduction_enabled; 107 } 108 UMA_HISTOGRAM_PERCENTAGE( 109 "Net.DailyContentSavingPercent_DataReductionProxyEnabled", 110 percent_data_reduction_proxy_enabled); 111 112 UMA_HISTOGRAM_PERCENTAGE( 113 "Net.DailyContentPercent_DataReductionProxyEnabled", 114 (100 * received_length_with_data_reduction_enabled) / received_length); 115 116 DCHECK_GE(https_length_with_data_reduction_enabled, 0); 117 UMA_HISTOGRAM_COUNTS( 118 "Net.DailyContentLength_DataReductionProxyEnabled_Https", 119 https_length_with_data_reduction_enabled >> 10); 120 UMA_HISTOGRAM_PERCENTAGE( 121 "Net.DailyContentPercent_DataReductionProxyEnabled_Https", 122 (100 * https_length_with_data_reduction_enabled) / received_length); 123 124 DCHECK_GE(short_bypass_length_with_data_reduction_enabled, 0); 125 UMA_HISTOGRAM_COUNTS( 126 "Net.DailyContentLength_DataReductionProxyEnabled_ShortBypass", 127 short_bypass_length_with_data_reduction_enabled >> 10); 128 UMA_HISTOGRAM_PERCENTAGE( 129 "Net.DailyContentPercent_DataReductionProxyEnabled_ShortBypass", 130 ((100 * short_bypass_length_with_data_reduction_enabled) / 131 received_length)); 132 133 DCHECK_GE(long_bypass_length_with_data_reduction_enabled, 0); 134 UMA_HISTOGRAM_COUNTS( 135 "Net.DailyContentLength_DataReductionProxyEnabled_LongBypass", 136 long_bypass_length_with_data_reduction_enabled >> 10); 137 UMA_HISTOGRAM_PERCENTAGE( 138 "Net.DailyContentPercent_DataReductionProxyEnabled_LongBypass", 139 ((100 * long_bypass_length_with_data_reduction_enabled) / 140 received_length)); 141 142 DCHECK_GE(unknown_length_with_data_reduction_enabled, 0); 143 UMA_HISTOGRAM_COUNTS( 144 "Net.DailyContentLength_DataReductionProxyEnabled_Unknown", 145 unknown_length_with_data_reduction_enabled >> 10); 146 UMA_HISTOGRAM_PERCENTAGE( 147 "Net.DailyContentPercent_DataReductionProxyEnabled_Unknown", 148 ((100 * unknown_length_with_data_reduction_enabled) / 149 received_length)); 150 151 DCHECK_GE(original_length_via_data_reduction_proxy, 0); 152 UMA_HISTOGRAM_COUNTS( 153 "Net.DailyOriginalContentLength_ViaDataReductionProxy", 154 original_length_via_data_reduction_proxy >> 10); 155 DCHECK_GE(received_length_via_data_reduction_proxy, 0); 156 UMA_HISTOGRAM_COUNTS( 157 "Net.DailyContentLength_ViaDataReductionProxy", 158 received_length_via_data_reduction_proxy >> 10); 159 int percent_via_data_reduction_proxy = 0; 160 if (original_length_via_data_reduction_proxy > 161 received_length_via_data_reduction_proxy) { 162 percent_via_data_reduction_proxy = 163 100 * (original_length_via_data_reduction_proxy - 164 received_length_via_data_reduction_proxy) / 165 original_length_via_data_reduction_proxy; 166 } 167 UMA_HISTOGRAM_PERCENTAGE( 168 "Net.DailyContentSavingPercent_ViaDataReductionProxy", 169 percent_via_data_reduction_proxy); 170 UMA_HISTOGRAM_PERCENTAGE( 171 "Net.DailyContentPercent_ViaDataReductionProxy", 172 (100 * received_length_via_data_reduction_proxy) / received_length); 173 } 174 175 // Ensure list has exactly |length| elements, either by truncating at the 176 // front, or appending "0"'s to the back. 177 void MaintainContentLengthPrefsWindow(base::ListValue* list, size_t length) { 178 // Remove data for old days from the front. 179 while (list->GetSize() > length) 180 list->Remove(0, NULL); 181 // Newly added lists are empty. Add entries to back to fill the window, 182 // each initialized to zero. 183 while (list->GetSize() < length) 184 list->AppendString(base::Int64ToString(0)); 185 DCHECK_EQ(length, list->GetSize()); 186 } 187 188 // DailyContentLengthUpdate maintains a data saving pref. The pref is a list 189 // of |kNumDaysInHistory| elements of daily total content lengths for the past 190 // |kNumDaysInHistory| days. 191 class DailyContentLengthUpdate { 192 public: 193 DailyContentLengthUpdate( 194 const char* pref, 195 PrefService* pref_service) 196 : update_(pref_service, pref) { 197 } 198 199 void UpdateForDataChange(int days_since_last_update) { 200 // New empty lists may have been created. Maintain the invariant that 201 // there should be exactly |kNumDaysInHistory| days in the histories. 202 MaintainContentLengthPrefsWindow(update_.Get(), kNumDaysInHistory); 203 if (days_since_last_update) { 204 MaintainContentLengthPrefForDateChange(days_since_last_update); 205 } 206 } 207 208 // Update the lengths for the current day. 209 void Add(int content_length) { 210 AddInt64ToListPref(kNumDaysInHistory - 1, content_length, update_.Get()); 211 } 212 213 int64 GetListPrefValue(size_t index) { 214 return ListPrefInt64Value(*update_, index); 215 } 216 217 private: 218 // Update the list for date change and ensure the list has exactly |length| 219 // elements. The last entry in the list will be for the current day after 220 // the update. 221 void MaintainContentLengthPrefForDateChange(int days_since_last_update) { 222 if (days_since_last_update == -1) { 223 // The system may go backwards in time by up to a day for legitimate 224 // reasons, such as with changes to the time zone. In such cases, we 225 // keep adding to the current day. 226 // Note: we accept the fact that some reported data is shifted to 227 // the adjacent day if users travel back and forth across time zones. 228 days_since_last_update = 0; 229 } else if (days_since_last_update < -1) { 230 // Erase all entries if the system went backwards in time by more than 231 // a day. 232 update_->Clear(); 233 234 days_since_last_update = kNumDaysInHistory; 235 } 236 DCHECK_GE(days_since_last_update, 0); 237 238 // Add entries for days since last update event. This will make the 239 // lists longer than kNumDaysInHistory. The additional items will be cut off 240 // from the head of the lists by |MaintainContentLengthPrefsWindow|, below. 241 for (int i = 0; 242 i < days_since_last_update && i < static_cast<int>(kNumDaysInHistory); 243 ++i) { 244 update_->AppendString(base::Int64ToString(0)); 245 } 246 247 // Entries for new days may have been appended. Maintain the invariant that 248 // there should be exactly |kNumDaysInHistory| days in the histories. 249 MaintainContentLengthPrefsWindow(update_.Get(), kNumDaysInHistory); 250 } 251 252 ListPrefUpdate update_; 253 }; 254 255 // DailyDataSavingUpdate maintains a pair of data saving prefs, original_update_ 256 // and received_update_. pref_original is a list of |kNumDaysInHistory| elements 257 // of daily total original content lengths for the past |kNumDaysInHistory| 258 // days. pref_received is the corresponding list of the daily total received 259 // content lengths. 260 class DailyDataSavingUpdate { 261 public: 262 DailyDataSavingUpdate( 263 const char* pref_original, 264 const char* pref_received, 265 PrefService* pref_service) 266 : original_(pref_original, pref_service), 267 received_(pref_received, pref_service) { 268 } 269 270 void UpdateForDataChange(int days_since_last_update) { 271 original_.UpdateForDataChange(days_since_last_update); 272 received_.UpdateForDataChange(days_since_last_update); 273 } 274 275 // Update the lengths for the current day. 276 void Add(int original_content_length, int received_content_length) { 277 original_.Add(original_content_length); 278 received_.Add(received_content_length); 279 } 280 281 int64 GetOriginalListPrefValue(size_t index) { 282 return original_.GetListPrefValue(index); 283 } 284 int64 GetReceivedListPrefValue(size_t index) { 285 return received_.GetListPrefValue(index); 286 } 287 288 private: 289 DailyContentLengthUpdate original_; 290 DailyContentLengthUpdate received_; 291 }; 292 293 // Returns true if the request is bypassed by all configured data reduction 294 // proxies. It returns the bypass delay in delay_seconds (if not NULL). If 295 // the request is bypassed by more than one proxy, delay_seconds returns 296 // shortest delay. 297 bool IsBypassRequest(const net::URLRequest* request, int64* delay_seconds) { 298 // TODO(bengr): Add support for other data reduction proxy configurations. 299 #if defined(SPDY_PROXY_AUTH_ORIGIN) 300 DataReductionProxyParams params( 301 DataReductionProxyParams::kAllowed | 302 DataReductionProxyParams::kFallbackAllowed | 303 DataReductionProxyParams::kPromoAllowed); 304 DataReductionProxyParams::DataReductionProxyList proxies = 305 params.GetAllowedProxies(); 306 if (proxies.size() == 0) 307 return false; 308 309 if (request == NULL || request->context() == NULL || 310 request->context()->proxy_service() == NULL) { 311 return false; 312 } 313 314 const net::ProxyRetryInfoMap& retry_map = 315 request->context()->proxy_service()->proxy_retry_info(); 316 if (retry_map.size() == 0) 317 return false; 318 319 int64 shortest_delay = 0; 320 // The request is bypassed if all configured proxies are in the retry map. 321 for (size_t i = 0; i < proxies.size(); ++i) { 322 std::string proxy = net::HostPortPair::FromURL(proxies[i]).ToString(); 323 // The retry list has the scheme prefix for https but not for http. 324 if (proxies[i].SchemeIs("https")) 325 proxy = std::string("https://") + proxy; 326 327 net::ProxyRetryInfoMap::const_iterator found = retry_map.find(proxy); 328 if (found == retry_map.end()) 329 return false; 330 if (shortest_delay == 0 || 331 shortest_delay > found->second.current_delay.InSeconds()) { 332 shortest_delay = found->second.current_delay.InSeconds(); 333 } 334 } 335 if (delay_seconds != NULL) 336 *delay_seconds = shortest_delay; 337 return true; 338 #else 339 return false; 340 #endif 341 } 342 343 } // namespace 344 345 DataReductionProxyRequestType GetDataReductionProxyRequestType( 346 const net::URLRequest* request) { 347 if (request->url().SchemeIs("https")) 348 return HTTPS; 349 if (!request->url().SchemeIs("http")) { 350 NOTREACHED(); 351 return UNKNOWN_TYPE; 352 } 353 int64 bypass_delay = 0; 354 if (IsBypassRequest(request, &bypass_delay)) { 355 return (bypass_delay > kLongBypassDelayInSeconds) ? 356 LONG_BYPASS : SHORT_BYPASS; 357 } 358 if (request->response_info().headers && 359 HasDataReductionProxyViaHeader(request->response_info().headers)) { 360 return VIA_DATA_REDUCTION_PROXY; 361 } 362 return UNKNOWN_TYPE; 363 } 364 365 int64 GetAdjustedOriginalContentLength( 366 DataReductionProxyRequestType request_type, 367 int64 original_content_length, 368 int64 received_content_length) { 369 // Since there was no indication of the original content length, presume 370 // it is no different from the number of bytes read. 371 if (original_content_length == -1 || 372 request_type != VIA_DATA_REDUCTION_PROXY) { 373 return received_content_length; 374 } 375 return original_content_length; 376 } 377 378 void UpdateContentLengthPrefsForDataReductionProxy( 379 int received_content_length, 380 int original_content_length, 381 bool with_data_reduction_proxy_enabled, 382 DataReductionProxyRequestType request_type, 383 base::Time now, PrefService* prefs) { 384 // TODO(bengr): Remove this check once the underlying cause of 385 // http://crbug.com/287821 is fixed. For now, only continue if the current 386 // year is reported as being between 1972 and 2970. 387 base::TimeDelta time_since_unix_epoch = now - base::Time::UnixEpoch(); 388 const int kMinDaysSinceUnixEpoch = 365 * 2; // 2 years. 389 const int kMaxDaysSinceUnixEpoch = 365 * 1000; // 1000 years. 390 if (time_since_unix_epoch.InDays() < kMinDaysSinceUnixEpoch || 391 time_since_unix_epoch.InDays() > kMaxDaysSinceUnixEpoch) { 392 return; 393 } 394 395 // Determine how many days it has been since the last update. 396 int64 then_internal = prefs->GetInt64( 397 data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate); 398 399 #if defined(OS_WIN) 400 base::Time then_midnight = base::Time::FromInternalValue(then_internal); 401 base::Time midnight = 402 base::Time::FromInternalValue( 403 (now.ToInternalValue() / base::Time::kMicrosecondsPerDay) * 404 base::Time::kMicrosecondsPerDay); 405 #else 406 // Local midnight could have been shifted due to time zone change. 407 base::Time then_midnight = 408 base::Time::FromInternalValue(then_internal).LocalMidnight(); 409 base::Time midnight = now.LocalMidnight(); 410 #endif 411 412 int days_since_last_update = (midnight - then_midnight).InDays(); 413 414 // Each day, we calculate the total number of bytes received and the total 415 // size of all corresponding resources before any data-reducing recompression 416 // is applied. These values are used to compute the data savings realized 417 // by applying our compression techniques. Totals for the last 418 // |kNumDaysInHistory| days are maintained. 419 DailyDataSavingUpdate total( 420 data_reduction_proxy::prefs::kDailyHttpOriginalContentLength, 421 data_reduction_proxy::prefs::kDailyHttpReceivedContentLength, 422 prefs); 423 total.UpdateForDataChange(days_since_last_update); 424 425 DailyDataSavingUpdate proxy_enabled( 426 data_reduction_proxy::prefs:: 427 kDailyOriginalContentLengthWithDataReductionProxyEnabled, 428 data_reduction_proxy::prefs:: 429 kDailyContentLengthWithDataReductionProxyEnabled, 430 prefs); 431 proxy_enabled.UpdateForDataChange(days_since_last_update); 432 433 DailyDataSavingUpdate via_proxy( 434 data_reduction_proxy::prefs:: 435 kDailyOriginalContentLengthViaDataReductionProxy, 436 data_reduction_proxy::prefs:: 437 kDailyContentLengthViaDataReductionProxy, 438 prefs); 439 via_proxy.UpdateForDataChange(days_since_last_update); 440 441 DailyContentLengthUpdate https( 442 data_reduction_proxy::prefs:: 443 kDailyContentLengthHttpsWithDataReductionProxyEnabled, 444 prefs); 445 https.UpdateForDataChange(days_since_last_update); 446 447 DailyContentLengthUpdate short_bypass( 448 data_reduction_proxy::prefs:: 449 kDailyContentLengthShortBypassWithDataReductionProxyEnabled, 450 prefs); 451 short_bypass.UpdateForDataChange(days_since_last_update); 452 453 DailyContentLengthUpdate long_bypass( 454 data_reduction_proxy::prefs:: 455 kDailyContentLengthLongBypassWithDataReductionProxyEnabled, 456 prefs); 457 long_bypass.UpdateForDataChange(days_since_last_update); 458 459 DailyContentLengthUpdate unknown( 460 data_reduction_proxy::prefs:: 461 kDailyContentLengthUnknownWithDataReductionProxyEnabled, 462 prefs); 463 unknown.UpdateForDataChange(days_since_last_update); 464 465 total.Add(original_content_length, received_content_length); 466 if (with_data_reduction_proxy_enabled) { 467 proxy_enabled.Add(original_content_length, received_content_length); 468 // Ignore data source cases, if exist, when 469 // "with_data_reduction_proxy_enabled == false" 470 switch (request_type) { 471 case VIA_DATA_REDUCTION_PROXY: 472 via_proxy.Add(original_content_length, received_content_length); 473 break; 474 case HTTPS: 475 https.Add(received_content_length); 476 break; 477 case SHORT_BYPASS: 478 short_bypass.Add(received_content_length); 479 break; 480 case LONG_BYPASS: 481 long_bypass.Add(received_content_length); 482 break; 483 case UNKNOWN_TYPE: 484 unknown.Add(received_content_length); 485 break; 486 } 487 } 488 489 if (days_since_last_update) { 490 // Record the last update time in microseconds in UTC. 491 prefs->SetInt64( 492 data_reduction_proxy::prefs::kDailyHttpContentLengthLastUpdateDate, 493 midnight.ToInternalValue()); 494 495 // A new day. Report the previous day's data if exists. We'll lose usage 496 // data if the last time Chrome was run was more than a day ago. 497 // Here, we prefer collecting less data but the collected data is 498 // associated with an accurate date. 499 if (days_since_last_update == 1) { 500 // The previous day's data point is the second one from the tail. 501 // Therefore (kNumDaysInHistory - 2) below. 502 RecordDailyContentLengthHistograms( 503 total.GetOriginalListPrefValue(kNumDaysInHistory - 2), 504 total.GetReceivedListPrefValue(kNumDaysInHistory - 2), 505 proxy_enabled.GetOriginalListPrefValue(kNumDaysInHistory - 2), 506 proxy_enabled.GetReceivedListPrefValue(kNumDaysInHistory - 2), 507 via_proxy.GetOriginalListPrefValue(kNumDaysInHistory - 2), 508 via_proxy.GetReceivedListPrefValue(kNumDaysInHistory - 2), 509 https.GetListPrefValue(kNumDaysInHistory - 2), 510 short_bypass.GetListPrefValue(kNumDaysInHistory - 2), 511 long_bypass.GetListPrefValue(kNumDaysInHistory - 2), 512 unknown.GetListPrefValue(kNumDaysInHistory - 2)); 513 } 514 } 515 } 516 517 void UpdateContentLengthPrefs( 518 int received_content_length, 519 int original_content_length, 520 bool with_data_reduction_proxy_enabled, 521 DataReductionProxyRequestType request_type, 522 PrefService* prefs) { 523 int64 total_received = prefs->GetInt64( 524 data_reduction_proxy::prefs::kHttpReceivedContentLength); 525 int64 total_original = prefs->GetInt64( 526 data_reduction_proxy::prefs::kHttpOriginalContentLength); 527 total_received += received_content_length; 528 total_original += original_content_length; 529 prefs->SetInt64(data_reduction_proxy::prefs::kHttpReceivedContentLength, 530 total_received); 531 prefs->SetInt64(data_reduction_proxy::prefs::kHttpOriginalContentLength, 532 total_original); 533 534 UpdateContentLengthPrefsForDataReductionProxy( 535 received_content_length, 536 original_content_length, 537 with_data_reduction_proxy_enabled, 538 request_type, 539 base::Time::Now(), 540 prefs); 541 } 542 543 } // namespace data_reduction_proxy 544