1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/performance_monitor/database.h" 6 7 #include "base/file_util.h" 8 #include "base/files/file_path.h" 9 #include "base/json/json_reader.h" 10 #include "base/json/json_writer.h" 11 #include "base/logging.h" 12 #include "base/path_service.h" 13 #include "base/stl_util.h" 14 #include "base/strings/string_number_conversions.h" 15 #include "base/strings/utf_string_conversions.h" 16 #include "base/time/time.h" 17 #include "chrome/browser/performance_monitor/key_builder.h" 18 #include "chrome/common/chrome_paths.h" 19 #include "content/public/browser/browser_thread.h" 20 #include "third_party/leveldatabase/src/include/leveldb/db.h" 21 #include "third_party/leveldatabase/src/include/leveldb/iterator.h" 22 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h" 23 24 namespace performance_monitor { 25 namespace { 26 const char kDbDir[] = "Performance Monitor Databases"; 27 const char kRecentDb[] = "Recent Metrics"; 28 const char kMaxValueDb[] = "Max Value Metrics"; 29 const char kEventDb[] = "Events"; 30 const char kStateDb[] = "Configuration"; 31 const char kActiveIntervalDb[] = "Active Interval"; 32 const char kMetricDb[] = "Metrics"; 33 const double kDefaultMaxValue = 0.0; 34 35 // If the db is quiet for this number of minutes, then it is considered down. 36 const base::TimeDelta kActiveIntervalTimeout() { 37 return base::TimeDelta::FromMinutes(5); 38 } 39 40 TimeRange ActiveIntervalToTimeRange(const std::string& start_time, 41 const std::string& end_time) { 42 int64 start_time_int = 0; 43 int64 end_time_int = 0; 44 base::StringToInt64(start_time, &start_time_int); 45 base::StringToInt64(end_time, &end_time_int); 46 return TimeRange(base::Time::FromInternalValue(start_time_int), 47 base::Time::FromInternalValue(end_time_int)); 48 } 49 50 double StringToDouble(const std::string& s) { 51 double value = 0.0; 52 if (!base::StringToDouble(s, &value)) 53 LOG(ERROR) << "Failed to convert " << s << " to double."; 54 return value; 55 } 56 57 // Returns an event from the given JSON string; the scoped_ptr will be NULL if 58 // we are unable to properly parse the JSON. 59 scoped_ptr<Event> EventFromJSON(const std::string& data) { 60 base::Value* value = base::JSONReader::Read(data); 61 base::DictionaryValue* dict = NULL; 62 if (!value || !value->GetAsDictionary(&dict)) 63 return scoped_ptr<Event>(); 64 65 return Event::FromValue(scoped_ptr<base::DictionaryValue>(dict)); 66 } 67 68 } // namespace 69 70 const char Database::kDatabaseSequenceToken[] = 71 "_performance_monitor_db_sequence_token_"; 72 73 TimeRange::TimeRange() { 74 } 75 76 TimeRange::TimeRange(base::Time start_time, base::Time end_time) 77 : start(start_time), 78 end(end_time) { 79 } 80 81 TimeRange::~TimeRange() { 82 } 83 84 base::Time Database::SystemClock::GetTime() { 85 return base::Time::Now(); 86 } 87 88 // Static 89 scoped_ptr<Database> Database::Create(base::FilePath path) { 90 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 91 if (path.empty()) { 92 CHECK(PathService::Get(chrome::DIR_USER_DATA, &path)); 93 path = path.AppendASCII(kDbDir); 94 } 95 scoped_ptr<Database> database; 96 if (!base::DirectoryExists(path) && !base::CreateDirectory(path)) 97 return database.Pass(); 98 database.reset(new Database(path)); 99 100 // If the database did not initialize correctly, return a NULL scoped_ptr. 101 if (!database->valid_) 102 database.reset(); 103 return database.Pass(); 104 } 105 106 bool Database::AddStateValue(const std::string& key, const std::string& value) { 107 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 108 UpdateActiveInterval(); 109 leveldb::Status insert_status = state_db_->Put(write_options_, key, value); 110 return insert_status.ok(); 111 } 112 113 std::string Database::GetStateValue(const std::string& key) { 114 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 115 std::string result; 116 state_db_->Get(read_options_, key, &result); 117 return result; 118 } 119 120 bool Database::AddEvent(const Event& event) { 121 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 122 UpdateActiveInterval(); 123 std::string value; 124 base::JSONWriter::Write(event.data(), &value); 125 std::string key = key_builder_->CreateEventKey(event.time(), event.type()); 126 leveldb::Status status = event_db_->Put(write_options_, key, value); 127 return status.ok(); 128 } 129 130 std::vector<TimeRange> Database::GetActiveIntervals(const base::Time& start, 131 const base::Time& end) { 132 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 133 std::vector<TimeRange> results; 134 std::string start_key = key_builder_->CreateActiveIntervalKey(start); 135 std::string end_key = key_builder_->CreateActiveIntervalKey(end); 136 scoped_ptr<leveldb::Iterator> it(active_interval_db_->NewIterator( 137 read_options_)); 138 it->Seek(start_key); 139 // If the interator is valid, we check the previous value in case we jumped 140 // into the middle of an active interval. If the iterator is not valid, then 141 // the key may be in the current active interval. 142 if (it->Valid()) 143 it->Prev(); 144 else 145 it->SeekToLast(); 146 if (it->Valid() && it->value().ToString() > start_key) { 147 results.push_back(ActiveIntervalToTimeRange(it->key().ToString(), 148 it->value().ToString())); 149 } 150 151 for (it->Seek(start_key); 152 it->Valid() && it->key().ToString() < end_key; 153 it->Next()) { 154 results.push_back(ActiveIntervalToTimeRange(it->key().ToString(), 155 it->value().ToString())); 156 } 157 return results; 158 } 159 160 Database::EventVector Database::GetEvents(EventType type, 161 const base::Time& start, 162 const base::Time& end) { 163 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 164 EventVector events; 165 std::string start_key = 166 key_builder_->CreateEventKey(start, EVENT_UNDEFINED); 167 std::string end_key = 168 key_builder_->CreateEventKey(end, EVENT_NUMBER_OF_EVENTS); 169 leveldb::WriteBatch invalid_entries; 170 scoped_ptr<leveldb::Iterator> it(event_db_->NewIterator(read_options_)); 171 for (it->Seek(start_key); 172 it->Valid() && it->key().ToString() <= end_key; 173 it->Next()) { 174 if (type != EVENT_UNDEFINED) { 175 EventType key_type = 176 key_builder_->EventKeyToEventType(it->key().ToString()); 177 if (key_type != type) 178 continue; 179 } 180 scoped_ptr<Event> event = EventFromJSON(it->value().ToString()); 181 if (!event.get()) { 182 invalid_entries.Delete(it->key()); 183 LOG(ERROR) << "Found invalid event in the database. JSON: '" 184 << it->value().ToString() 185 << "'. Erasing event from the database."; 186 continue; 187 } 188 events.push_back(linked_ptr<Event>(event.release())); 189 } 190 event_db_->Write(write_options_, &invalid_entries); 191 return events; 192 } 193 194 Database::EventTypeSet Database::GetEventTypes(const base::Time& start, 195 const base::Time& end) { 196 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 197 EventTypeSet results; 198 std::string start_key = 199 key_builder_->CreateEventKey(start, EVENT_UNDEFINED); 200 std::string end_key = 201 key_builder_->CreateEventKey(end, EVENT_NUMBER_OF_EVENTS); 202 scoped_ptr<leveldb::Iterator> it(event_db_->NewIterator(read_options_)); 203 for (it->Seek(start_key); 204 it->Valid() && it->key().ToString() <= end_key; 205 it->Next()) { 206 EventType key_type = 207 key_builder_->EventKeyToEventType(it->key().ToString()); 208 results.insert(key_type); 209 } 210 return results; 211 } 212 213 bool Database::AddMetric(const std::string& activity, 214 const Metric& metric) { 215 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 216 if (!metric.IsValid()) { 217 DLOG(ERROR) << "Metric to be added is invalid. Type: " << metric.type 218 << ", Time: " << metric.time.ToInternalValue() 219 << ", Value: " << metric.value << ". Ignoring."; 220 return false; 221 } 222 223 UpdateActiveInterval(); 224 std::string recent_key = 225 key_builder_->CreateRecentKey(metric.time, metric.type, activity); 226 std::string metric_key = 227 key_builder_->CreateMetricKey(metric.time, metric.type, activity); 228 std::string recent_map_key = 229 key_builder_->CreateRecentMapKey(metric.type, activity); 230 // Use recent_map_ to quickly find the key that must be removed. 231 RecentMap::iterator old_it = recent_map_.find(recent_map_key); 232 if (old_it != recent_map_.end()) 233 recent_db_->Delete(write_options_, old_it->second); 234 recent_map_[recent_map_key] = recent_key; 235 leveldb::Status recent_status = 236 recent_db_->Put(write_options_, recent_key, metric.ValueAsString()); 237 leveldb::Status metric_status = 238 metric_db_->Put(write_options_, metric_key, metric.ValueAsString()); 239 240 bool max_value_success = 241 UpdateMaxValue(activity, metric.type, metric.ValueAsString()); 242 return recent_status.ok() && metric_status.ok() && max_value_success; 243 } 244 245 bool Database::UpdateMaxValue(const std::string& activity, 246 MetricType metric, 247 const std::string& value) { 248 std::string max_value_key( 249 key_builder_->CreateMaxValueKey(metric, activity)); 250 bool has_key = ContainsKey(max_value_map_, max_value_key); 251 if ((has_key && StringToDouble(value) > max_value_map_[max_value_key]) || 252 !has_key) { 253 max_value_map_[max_value_key] = StringToDouble(value); 254 return max_value_db_->Put(write_options_, max_value_key, value).ok(); 255 } 256 257 return true; 258 } 259 260 Database::MetricTypeSet Database::GetActiveMetrics(const base::Time& start, 261 const base::Time& end) { 262 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 263 std::string recent_start_key = key_builder_->CreateRecentKey( 264 start, static_cast<MetricType>(0), std::string()); 265 std::string recent_end_key = key_builder_->CreateRecentKey( 266 end, METRIC_NUMBER_OF_METRICS, std::string()); 267 std::string recent_end_of_time_key = key_builder_->CreateRecentKey( 268 clock_->GetTime(), METRIC_NUMBER_OF_METRICS, std::string()); 269 270 MetricTypeSet active_metrics; 271 // Get all the guaranteed metrics. 272 scoped_ptr<leveldb::Iterator> recent_it( 273 recent_db_->NewIterator(read_options_)); 274 for (recent_it->Seek(recent_start_key); 275 recent_it->Valid() && recent_it->key().ToString() <= recent_end_key; 276 recent_it->Next()) { 277 RecentKey split_key = 278 key_builder_->SplitRecentKey(recent_it->key().ToString()); 279 active_metrics.insert(split_key.type); 280 } 281 // Get all the possible metrics (metrics that may have been updated after 282 // |end|). 283 MetricTypeSet possible_metrics; 284 for (recent_it->Seek(recent_end_key); 285 recent_it->Valid() && 286 recent_it->key().ToString() <= recent_end_of_time_key; 287 recent_it->Next()) { 288 RecentKey split_key = 289 key_builder_->SplitRecentKey(recent_it->key().ToString()); 290 possible_metrics.insert(split_key.type); 291 } 292 MetricTypeSet::iterator possible_it; 293 scoped_ptr<leveldb::Iterator> metric_it( 294 metric_db_->NewIterator(read_options_)); 295 for (possible_it = possible_metrics.begin(); 296 possible_it != possible_metrics.end(); 297 ++possible_it) { 298 std::string metric_start_key = 299 key_builder_->CreateMetricKey(start, *possible_it,std::string()); 300 std::string metric_end_key = 301 key_builder_->CreateMetricKey(end, *possible_it, std::string()); 302 metric_it->Seek(metric_start_key); 303 // Stats in the timerange from any activity makes the metric active. 304 if (metric_it->Valid() && metric_it->key().ToString() <= metric_end_key) { 305 active_metrics.insert(*possible_it); 306 } 307 } 308 309 return active_metrics; 310 } 311 312 std::set<std::string> Database::GetActiveActivities(MetricType metric_type, 313 const base::Time& start) { 314 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 315 std::set<std::string> results; 316 std::string start_key = key_builder_->CreateRecentKey( 317 start, static_cast<MetricType>(0), std::string()); 318 scoped_ptr<leveldb::Iterator> it(recent_db_->NewIterator(read_options_)); 319 for (it->Seek(start_key); it->Valid(); it->Next()) { 320 RecentKey split_key = 321 key_builder_->SplitRecentKey(it->key().ToString()); 322 if (split_key.type == metric_type) 323 results.insert(split_key.activity); 324 } 325 return results; 326 } 327 328 double Database::GetMaxStatsForActivityAndMetric(const std::string& activity, 329 MetricType metric) { 330 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 331 std::string max_value_key( 332 key_builder_->CreateMaxValueKey(metric, activity)); 333 if (ContainsKey(max_value_map_, max_value_key)) 334 return max_value_map_[max_value_key]; 335 return kDefaultMaxValue; 336 } 337 338 bool Database::GetRecentStatsForActivityAndMetric(const std::string& activity, 339 MetricType metric_type, 340 Metric* metric) { 341 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 342 std::string recent_map_key = 343 key_builder_->CreateRecentMapKey(metric_type, activity); 344 if (!ContainsKey(recent_map_, recent_map_key)) 345 return false; 346 std::string recent_key = recent_map_[recent_map_key]; 347 348 std::string result; 349 leveldb::Status status = recent_db_->Get(read_options_, recent_key, &result); 350 if (status.ok()) 351 *metric = Metric(metric_type, 352 key_builder_->SplitRecentKey(recent_key).time, 353 result); 354 return status.ok(); 355 } 356 357 scoped_ptr<Database::MetricVector> Database::GetStatsForActivityAndMetric( 358 const std::string& activity, 359 MetricType metric_type, 360 const base::Time& start, 361 const base::Time& end) { 362 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 363 scoped_ptr<MetricVector> results(new MetricVector()); 364 std::string start_key = 365 key_builder_->CreateMetricKey(start, metric_type, activity); 366 std::string end_key = 367 key_builder_->CreateMetricKey(end, metric_type, activity); 368 leveldb::WriteBatch invalid_entries; 369 scoped_ptr<leveldb::Iterator> it(metric_db_->NewIterator(read_options_)); 370 for (it->Seek(start_key); 371 it->Valid() && it->key().ToString() <= end_key; 372 it->Next()) { 373 MetricKey split_key = 374 key_builder_->SplitMetricKey(it->key().ToString()); 375 if (split_key.activity == activity) { 376 Metric metric(metric_type, split_key.time, it->value().ToString()); 377 if (!metric.IsValid()) { 378 invalid_entries.Delete(it->key()); 379 LOG(ERROR) << "Found bad metric in the database. Type: " 380 << metric.type << ", Time: " << metric.time.ToInternalValue() 381 << ", Value: " << metric.value 382 << ". Erasing metric from database."; 383 continue; 384 } 385 results->push_back(metric); 386 } 387 } 388 metric_db_->Write(write_options_, &invalid_entries); 389 return results.Pass(); 390 } 391 392 Database::MetricVectorMap Database::GetStatsForMetricByActivity( 393 MetricType metric_type, 394 const base::Time& start, 395 const base::Time& end) { 396 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 397 MetricVectorMap results; 398 std::string start_key = 399 key_builder_->CreateMetricKey(start, metric_type, std::string()); 400 std::string end_key = 401 key_builder_->CreateMetricKey(end, metric_type, std::string()); 402 leveldb::WriteBatch invalid_entries; 403 scoped_ptr<leveldb::Iterator> it(metric_db_->NewIterator(read_options_)); 404 for (it->Seek(start_key); 405 it->Valid() && it->key().ToString() <= end_key; 406 it->Next()) { 407 MetricKey split_key = key_builder_->SplitMetricKey(it->key().ToString()); 408 if (!results[split_key.activity].get()) { 409 results[split_key.activity] = 410 linked_ptr<MetricVector >(new MetricVector()); 411 } 412 Metric metric(metric_type, split_key.time, it->value().ToString()); 413 if (!metric.IsValid()) { 414 invalid_entries.Delete(it->key()); 415 LOG(ERROR) << "Found bad metric in the database. Type: " 416 << metric.type << ", Time: " << metric.time.ToInternalValue() 417 << ", Value: " << metric.value 418 << ". Erasing metric from database."; 419 continue; 420 } 421 results[split_key.activity]->push_back(metric); 422 } 423 metric_db_->Write(write_options_, &invalid_entries); 424 return results; 425 } 426 427 Database::Database(const base::FilePath& path) 428 : key_builder_(new KeyBuilder()), 429 path_(path), 430 read_options_(leveldb::ReadOptions()), 431 write_options_(leveldb::WriteOptions()), 432 valid_(false) { 433 if (!InitDBs()) 434 return; 435 LoadRecents(); 436 LoadMaxValues(); 437 clock_ = scoped_ptr<Clock>(new SystemClock()); 438 valid_ = true; 439 } 440 441 Database::~Database() { 442 } 443 444 bool Database::InitDBs() { 445 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 446 leveldb::Options open_options; 447 open_options.max_open_files = 0; // Use minimum. 448 open_options.create_if_missing = true; 449 450 // TODO (rdevlin.cronin): This code is ugly. Fix it. 451 recent_db_ = SafelyOpenDatabase(open_options, 452 kRecentDb, 453 true); // fix if damaged 454 max_value_db_ = SafelyOpenDatabase(open_options, 455 kMaxValueDb, 456 true); // fix if damaged 457 state_db_ = SafelyOpenDatabase(open_options, 458 kStateDb, 459 true); // fix if damaged 460 active_interval_db_ = SafelyOpenDatabase(open_options, 461 kActiveIntervalDb, 462 true); // fix if damaged 463 metric_db_ = SafelyOpenDatabase(open_options, 464 kMetricDb, 465 true); // fix if damaged 466 event_db_ = SafelyOpenDatabase(open_options, 467 kEventDb, 468 true); // fix if damaged 469 return recent_db_ && max_value_db_ && state_db_ && 470 active_interval_db_ && metric_db_ && event_db_; 471 } 472 473 scoped_ptr<leveldb::DB> Database::SafelyOpenDatabase( 474 const leveldb::Options& options, 475 const std::string& path, 476 bool fix_if_damaged) { 477 #if defined(OS_POSIX) 478 std::string name = path_.AppendASCII(path).value(); 479 #elif defined(OS_WIN) 480 std::string name = base::WideToUTF8(path_.AppendASCII(path).value()); 481 #endif 482 483 leveldb::DB* database; 484 leveldb::Status status = leveldb::DB::Open(options, name, &database); 485 // If all goes well, return the database. 486 if (status.ok()) 487 return scoped_ptr<leveldb::DB>(database); 488 489 // Return NULL and print the error if we either didn't find the database and 490 // don't want to create it, or if we don't want to try to fix it. 491 if ((status.IsNotFound() && !options.create_if_missing) || !fix_if_damaged) { 492 LOG(ERROR) << status.ToString(); 493 return scoped_ptr<leveldb::DB>(); 494 } 495 // Otherwise, we have an error (corruption, io error, or a not found error 496 // even if we tried to create it). 497 // 498 // First, we try again. 499 LOG(ERROR) << "Database error: " << status.ToString() << ". Trying again."; 500 status = leveldb::DB::Open(options, name, &database); 501 // If we fail on corruption, we can try to repair it. 502 if (status.IsCorruption()) { 503 LOG(ERROR) << "Database corrupt (second attempt). Trying to repair."; 504 status = leveldb::RepairDB(name, options); 505 // If the repair succeeds and we can open the database, return the 506 // database. Otherwise, continue on. 507 if (status.ok()) { 508 status = leveldb::DB::Open(options, name, &database); 509 if (status.ok()) 510 return scoped_ptr<leveldb::DB>(database); 511 } 512 LOG(ERROR) << "Repair failed. Deleting database."; 513 } 514 // Next, try to delete and recreate the database. Return NULL if we fail 515 // on either of these steps. 516 status = leveldb::DestroyDB(name, options); 517 if (!status.ok()) { 518 LOG(ERROR) << "Failed to delete database. " << status.ToString(); 519 return scoped_ptr<leveldb::DB>(); 520 } 521 // If we don't have the create_if_missing option, add it (it's safe to 522 // assume this is okay, since we have permission to |fix_if_damaged|). 523 if (!options.create_if_missing) { 524 leveldb::Options create_options(options); 525 create_options.create_if_missing = true; 526 status = leveldb::DB::Open(create_options, name, &database); 527 } else { 528 status = leveldb::DB::Open(options, name, &database); 529 } 530 // There's nothing else we can try at this point. 531 if (status.ok()) 532 return scoped_ptr<leveldb::DB>(database); 533 // Return the database if we succeeded, or NULL on failure. 534 LOG(ERROR) << "Failed to recreate database. " << status.ToString(); 535 return scoped_ptr<leveldb::DB>(); 536 } 537 538 bool Database::Close() { 539 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 540 metric_db_.reset(); 541 event_db_.reset(); 542 recent_db_.reset(); 543 max_value_db_.reset(); 544 state_db_.reset(); 545 active_interval_db_.reset(); 546 start_time_key_.clear(); 547 return true; 548 } 549 550 void Database::LoadRecents() { 551 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 552 recent_map_.clear(); 553 scoped_ptr<leveldb::Iterator> it(recent_db_->NewIterator(read_options_)); 554 for (it->SeekToFirst(); it->Valid(); it->Next()) { 555 RecentKey split_key = key_builder_->SplitRecentKey(it->key().ToString()); 556 recent_map_[key_builder_-> 557 CreateRecentMapKey(split_key.type, split_key.activity)] = 558 it->key().ToString(); 559 } 560 } 561 562 void Database::LoadMaxValues() { 563 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 564 max_value_map_.clear(); 565 scoped_ptr<leveldb::Iterator> it(max_value_db_->NewIterator(read_options_)); 566 for (it->SeekToFirst(); it->Valid(); it->Next()) { 567 max_value_map_[it->key().ToString()] = 568 StringToDouble(it->value().ToString()); 569 } 570 } 571 572 // TODO(chebert): Only update the active interval under certian circumstances 573 // eg. every 10 times or when forced. 574 void Database::UpdateActiveInterval() { 575 CHECK(!content::BrowserThread::CurrentlyOn(content::BrowserThread::UI)); 576 base::Time current_time = clock_->GetTime(); 577 std::string end_time; 578 // If the last update was too long ago. 579 if (start_time_key_.empty() || 580 current_time - last_update_time_ > kActiveIntervalTimeout()) { 581 start_time_key_ = key_builder_->CreateActiveIntervalKey(current_time); 582 end_time = start_time_key_; 583 } else { 584 end_time = key_builder_->CreateActiveIntervalKey(clock_->GetTime()); 585 } 586 last_update_time_ = current_time; 587 active_interval_db_->Put(write_options_, start_time_key_, end_time); 588 } 589 590 } // namespace performance_monitor 591
