1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/chromeos/drive/resource_metadata_storage.h" 6 7 #include "base/bind.h" 8 #include "base/file_util.h" 9 #include "base/location.h" 10 #include "base/logging.h" 11 #include "base/metrics/histogram.h" 12 #include "base/sequenced_task_runner.h" 13 #include "base/threading/thread_restrictions.h" 14 #include "chrome/browser/chromeos/drive/drive.pb.h" 15 #include "third_party/leveldatabase/src/include/leveldb/db.h" 16 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h" 17 18 namespace drive { 19 namespace internal { 20 21 namespace { 22 23 // Enum to describe DB initialization status. 24 enum DBInitStatus { 25 DB_INIT_SUCCESS, 26 DB_INIT_NOT_FOUND, 27 DB_INIT_CORRUPTION, 28 DB_INIT_IO_ERROR, 29 DB_INIT_FAILED, 30 DB_INIT_INCOMPATIBLE, 31 DB_INIT_BROKEN, 32 DB_INIT_MAX_VALUE, 33 }; 34 35 const base::FilePath::CharType kResourceMapDBName[] = 36 FILE_PATH_LITERAL("resource_metadata_resource_map.db"); 37 const base::FilePath::CharType kChildMapDBName[] = 38 FILE_PATH_LITERAL("resource_metadata_child_map.db"); 39 40 // Meant to be a character which never happen to be in real resource IDs. 41 const char kDBKeyDelimeter = '\0'; 42 43 // String used as a suffix of a key for a cache entry. 44 const char kCacheEntryKeySuffix[] = "CACHE"; 45 46 // Returns a string to be used as the key for the header. 47 std::string GetHeaderDBKey() { 48 std::string key; 49 key.push_back(kDBKeyDelimeter); 50 key.append("HEADER"); 51 return key; 52 } 53 54 // Returns true if |key| is a key for a child entry. 55 bool IsChildEntryKey(const leveldb::Slice& key) { 56 return !key.empty() && key[key.size() - 1] == kDBKeyDelimeter; 57 } 58 59 // Returns a string to be used as a key for a cache entry. 60 std::string GetCacheEntryKey(const std::string& resource_id) { 61 std::string key(resource_id); 62 key.push_back(kDBKeyDelimeter); 63 key.append(kCacheEntryKeySuffix); 64 return key; 65 } 66 67 // Returns true if |key| is a key for a cache entry. 68 bool IsCacheEntryKey(const leveldb::Slice& key) { 69 // A cache entry key should end with |kDBKeyDelimeter + kCacheEntryKeySuffix|. 70 const leveldb::Slice expected_suffix(kCacheEntryKeySuffix, 71 arraysize(kCacheEntryKeySuffix) - 1); 72 if (key.size() < 1 + expected_suffix.size() || 73 key[key.size() - expected_suffix.size() - 1] != kDBKeyDelimeter) 74 return false; 75 76 const leveldb::Slice key_substring( 77 key.data() + key.size() - expected_suffix.size(), expected_suffix.size()); 78 return key_substring.compare(expected_suffix) == 0; 79 } 80 81 // Converts leveldb::Status to DBInitStatus. 82 DBInitStatus LevelDBStatusToDBInitStatus(const leveldb::Status status) { 83 if (status.ok()) 84 return DB_INIT_SUCCESS; 85 if (status.IsNotFound()) 86 return DB_INIT_NOT_FOUND; 87 if (status.IsCorruption()) 88 return DB_INIT_CORRUPTION; 89 if (status.IsIOError()) 90 return DB_INIT_IO_ERROR; 91 return DB_INIT_FAILED; 92 } 93 94 } // namespace 95 96 ResourceMetadataStorage::Iterator::Iterator(scoped_ptr<leveldb::Iterator> it) 97 : it_(it.Pass()) { 98 base::ThreadRestrictions::AssertIOAllowed(); 99 DCHECK(it_); 100 101 // Skip the header entry. 102 // Note: The header entry comes before all other entries because its key 103 // starts with kDBKeyDelimeter. (i.e. '\0') 104 it_->Seek(leveldb::Slice(GetHeaderDBKey())); 105 106 Advance(); 107 } 108 109 ResourceMetadataStorage::Iterator::~Iterator() { 110 base::ThreadRestrictions::AssertIOAllowed(); 111 } 112 113 bool ResourceMetadataStorage::Iterator::IsAtEnd() const { 114 base::ThreadRestrictions::AssertIOAllowed(); 115 return !it_->Valid(); 116 } 117 118 const ResourceEntry& ResourceMetadataStorage::Iterator::Get() const { 119 base::ThreadRestrictions::AssertIOAllowed(); 120 DCHECK(!IsAtEnd()); 121 return entry_; 122 } 123 124 bool ResourceMetadataStorage::Iterator::GetCacheEntry( 125 FileCacheEntry* cache_entry) { 126 base::ThreadRestrictions::AssertIOAllowed(); 127 DCHECK(!IsAtEnd()); 128 129 // Try to seek to the cache entry. 130 std::string current_key = it_->key().ToString(); 131 std::string cache_entry_key = GetCacheEntryKey(current_key); 132 it_->Seek(leveldb::Slice(cache_entry_key)); 133 134 bool success = it_->Valid() && 135 it_->key().compare(cache_entry_key) == 0 && 136 cache_entry->ParseFromArray(it_->value().data(), it_->value().size()); 137 138 // Seek back to the original position. 139 it_->Seek(leveldb::Slice(current_key)); 140 DCHECK(!IsAtEnd()); 141 DCHECK_EQ(current_key, it_->key().ToString()); 142 143 return success; 144 } 145 146 void ResourceMetadataStorage::Iterator::Advance() { 147 base::ThreadRestrictions::AssertIOAllowed(); 148 DCHECK(!IsAtEnd()); 149 150 for (it_->Next() ; it_->Valid(); it_->Next()) { 151 if (!IsChildEntryKey(it_->key()) && 152 !IsCacheEntryKey(it_->key()) && 153 entry_.ParseFromArray(it_->value().data(), it_->value().size())) 154 break; 155 } 156 } 157 158 bool ResourceMetadataStorage::Iterator::HasError() const { 159 base::ThreadRestrictions::AssertIOAllowed(); 160 return !it_->status().ok(); 161 } 162 163 ResourceMetadataStorage::CacheEntryIterator::CacheEntryIterator( 164 scoped_ptr<leveldb::Iterator> it) : it_(it.Pass()) { 165 base::ThreadRestrictions::AssertIOAllowed(); 166 DCHECK(it_); 167 168 it_->SeekToFirst(); 169 AdvanceInternal(); 170 } 171 172 ResourceMetadataStorage::CacheEntryIterator::~CacheEntryIterator() { 173 base::ThreadRestrictions::AssertIOAllowed(); 174 } 175 176 bool ResourceMetadataStorage::CacheEntryIterator::IsAtEnd() const { 177 base::ThreadRestrictions::AssertIOAllowed(); 178 return !it_->Valid(); 179 } 180 181 const std::string& ResourceMetadataStorage::CacheEntryIterator::GetID() const { 182 base::ThreadRestrictions::AssertIOAllowed(); 183 DCHECK(!IsAtEnd()); 184 return resource_id_; 185 } 186 187 const FileCacheEntry& 188 ResourceMetadataStorage::CacheEntryIterator::GetValue() const { 189 base::ThreadRestrictions::AssertIOAllowed(); 190 DCHECK(!IsAtEnd()); 191 return entry_; 192 } 193 194 void ResourceMetadataStorage::CacheEntryIterator::Advance() { 195 base::ThreadRestrictions::AssertIOAllowed(); 196 DCHECK(!IsAtEnd()); 197 198 it_->Next(); 199 AdvanceInternal(); 200 } 201 202 bool ResourceMetadataStorage::CacheEntryIterator::HasError() const { 203 base::ThreadRestrictions::AssertIOAllowed(); 204 return !it_->status().ok(); 205 } 206 207 void ResourceMetadataStorage::CacheEntryIterator::AdvanceInternal() { 208 for (; it_->Valid(); it_->Next()) { 209 // Skip unparsable broken entries. 210 // TODO(hashimoto): Broken entries should be cleaned up at some point. 211 if (IsCacheEntryKey(it_->key()) && 212 entry_.ParseFromArray(it_->value().data(), it_->value().size())) { 213 // Drop the suffix |kDBKeyDelimeter + kCacheEntryKeySuffix| from the key. 214 const size_t kSuffixLength = arraysize(kCacheEntryKeySuffix) - 1; 215 const int id_length = it_->key().size() - 1 - kSuffixLength; 216 resource_id_.assign(it_->key().data(), id_length); 217 break; 218 } 219 } 220 } 221 222 ResourceMetadataStorage::ResourceMetadataStorage( 223 const base::FilePath& directory_path, 224 base::SequencedTaskRunner* blocking_task_runner) 225 : directory_path_(directory_path), 226 opened_existing_db_(false), 227 blocking_task_runner_(blocking_task_runner) { 228 } 229 230 void ResourceMetadataStorage::Destroy() { 231 blocking_task_runner_->PostTask( 232 FROM_HERE, 233 base::Bind(&ResourceMetadataStorage::DestroyOnBlockingPool, 234 base::Unretained(this))); 235 } 236 237 bool ResourceMetadataStorage::Initialize() { 238 base::ThreadRestrictions::AssertIOAllowed(); 239 240 // Remove unused child map DB. 241 const base::FilePath child_map_path = directory_path_.Append(kChildMapDBName); 242 base::DeleteFile(child_map_path, true /* recursive */); 243 244 resource_map_.reset(); 245 246 const base::FilePath resource_map_path = 247 directory_path_.Append(kResourceMapDBName); 248 249 // Try to open the existing DB. 250 leveldb::DB* db = NULL; 251 leveldb::Options options; 252 options.max_open_files = 64; // Use minimum. 253 options.create_if_missing = false; 254 255 DBInitStatus open_existing_result = DB_INIT_NOT_FOUND; 256 if (base::PathExists(resource_map_path)) { 257 leveldb::Status status = 258 leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db); 259 open_existing_result = LevelDBStatusToDBInitStatus(status); 260 } 261 262 if (open_existing_result == DB_INIT_SUCCESS) { 263 resource_map_.reset(db); 264 265 // Check the validity of existing DB. 266 ResourceMetadataHeader header; 267 if (!GetHeader(&header) || header.version() != kDBVersion) { 268 open_existing_result = DB_INIT_INCOMPATIBLE; 269 LOG(INFO) << "Reject incompatible DB."; 270 } else if (!CheckValidity()) { 271 open_existing_result = DB_INIT_BROKEN; 272 LOG(ERROR) << "Reject invalid DB."; 273 } 274 275 if (open_existing_result == DB_INIT_SUCCESS) 276 opened_existing_db_ = true; 277 else 278 resource_map_.reset(); 279 } 280 281 UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBOpenExistingResult", 282 open_existing_result, 283 DB_INIT_MAX_VALUE); 284 285 DBInitStatus init_result = DB_INIT_SUCCESS; 286 287 // Failed to open the existing DB, create new DB. 288 if (!resource_map_) { 289 resource_map_.reset(); 290 291 // Clean up the destination. 292 const bool kRecursive = true; 293 base::DeleteFile(resource_map_path, kRecursive); 294 295 // Create DB. 296 options.max_open_files = 64; // Use minimum. 297 options.create_if_missing = true; 298 299 leveldb::Status status = 300 leveldb::DB::Open(options, resource_map_path.AsUTF8Unsafe(), &db); 301 if (status.ok()) { 302 resource_map_.reset(db); 303 304 // Set up header. 305 ResourceMetadataHeader header; 306 header.set_version(kDBVersion); 307 if (!PutHeader(header)) { 308 init_result = DB_INIT_FAILED; 309 resource_map_.reset(); 310 } 311 } else { 312 LOG(ERROR) << "Failed to create resource map DB: " << status.ToString(); 313 init_result = LevelDBStatusToDBInitStatus(status); 314 } 315 } 316 317 UMA_HISTOGRAM_ENUMERATION("Drive.MetadataDBInitResult", 318 init_result, 319 DB_INIT_MAX_VALUE); 320 return resource_map_; 321 } 322 323 bool ResourceMetadataStorage::SetLargestChangestamp( 324 int64 largest_changestamp) { 325 base::ThreadRestrictions::AssertIOAllowed(); 326 327 ResourceMetadataHeader header; 328 if (!GetHeader(&header)) { 329 DLOG(ERROR) << "Failed to get the header."; 330 return false; 331 } 332 header.set_largest_changestamp(largest_changestamp); 333 return PutHeader(header); 334 } 335 336 int64 ResourceMetadataStorage::GetLargestChangestamp() { 337 base::ThreadRestrictions::AssertIOAllowed(); 338 ResourceMetadataHeader header; 339 if (!GetHeader(&header)) { 340 DLOG(ERROR) << "Failed to get the header."; 341 return 0; 342 } 343 return header.largest_changestamp(); 344 } 345 346 bool ResourceMetadataStorage::PutEntry(const ResourceEntry& entry) { 347 base::ThreadRestrictions::AssertIOAllowed(); 348 DCHECK(!entry.resource_id().empty()); 349 350 std::string serialized_entry; 351 if (!entry.SerializeToString(&serialized_entry)) { 352 DLOG(ERROR) << "Failed to serialize the entry: " << entry.resource_id(); 353 return false; 354 } 355 356 leveldb::WriteBatch batch; 357 358 // Remove from the old parent. 359 ResourceEntry old_entry; 360 if (GetEntry(entry.resource_id(), &old_entry) && 361 !old_entry.parent_resource_id().empty()) { 362 batch.Delete(GetChildEntryKey(old_entry.parent_resource_id(), 363 old_entry.base_name())); 364 } 365 366 // Add to the new parent. 367 if (!entry.parent_resource_id().empty()) { 368 batch.Put(GetChildEntryKey(entry.parent_resource_id(), entry.base_name()), 369 entry.resource_id()); 370 } 371 372 // Put the entry itself. 373 batch.Put(entry.resource_id(), serialized_entry); 374 375 const leveldb::Status status = resource_map_->Write(leveldb::WriteOptions(), 376 &batch); 377 return status.ok(); 378 } 379 380 bool ResourceMetadataStorage::GetEntry(const std::string& resource_id, 381 ResourceEntry* out_entry) { 382 base::ThreadRestrictions::AssertIOAllowed(); 383 DCHECK(!resource_id.empty()); 384 385 std::string serialized_entry; 386 const leveldb::Status status = resource_map_->Get(leveldb::ReadOptions(), 387 leveldb::Slice(resource_id), 388 &serialized_entry); 389 return status.ok() && out_entry->ParseFromString(serialized_entry); 390 } 391 392 bool ResourceMetadataStorage::RemoveEntry(const std::string& resource_id) { 393 base::ThreadRestrictions::AssertIOAllowed(); 394 DCHECK(!resource_id.empty()); 395 396 ResourceEntry entry; 397 if (!GetEntry(resource_id, &entry)) 398 return false; 399 400 leveldb::WriteBatch batch; 401 402 // Remove from the parent. 403 if (!entry.parent_resource_id().empty()) { 404 batch.Delete(GetChildEntryKey(entry.parent_resource_id(), 405 entry.base_name())); 406 } 407 // Remove the entry itself. 408 batch.Delete(resource_id); 409 410 const leveldb::Status status = resource_map_->Write(leveldb::WriteOptions(), 411 &batch); 412 return status.ok(); 413 } 414 415 scoped_ptr<ResourceMetadataStorage::Iterator> 416 ResourceMetadataStorage::GetIterator() { 417 base::ThreadRestrictions::AssertIOAllowed(); 418 419 scoped_ptr<leveldb::Iterator> it( 420 resource_map_->NewIterator(leveldb::ReadOptions())); 421 return make_scoped_ptr(new Iterator(it.Pass())); 422 } 423 424 std::string ResourceMetadataStorage::GetChild( 425 const std::string& parent_resource_id, 426 const std::string& child_name) { 427 base::ThreadRestrictions::AssertIOAllowed(); 428 429 std::string child_resource_id; 430 resource_map_->Get( 431 leveldb::ReadOptions(), 432 leveldb::Slice(GetChildEntryKey(parent_resource_id, child_name)), 433 &child_resource_id); 434 return child_resource_id; 435 } 436 437 void ResourceMetadataStorage::GetChildren(const std::string& parent_resource_id, 438 std::vector<std::string>* children) { 439 base::ThreadRestrictions::AssertIOAllowed(); 440 441 // Iterate over all entries with keys starting with |parent_resource_id|. 442 scoped_ptr<leveldb::Iterator> it( 443 resource_map_->NewIterator(leveldb::ReadOptions())); 444 for (it->Seek(parent_resource_id); 445 it->Valid() && it->key().starts_with(leveldb::Slice(parent_resource_id)); 446 it->Next()) { 447 if (IsChildEntryKey(it->key())) 448 children->push_back(it->value().ToString()); 449 } 450 DCHECK(it->status().ok()); 451 } 452 453 bool ResourceMetadataStorage::PutCacheEntry(const std::string& resource_id, 454 const FileCacheEntry& entry) { 455 base::ThreadRestrictions::AssertIOAllowed(); 456 DCHECK(!resource_id.empty()); 457 458 std::string serialized_entry; 459 if (!entry.SerializeToString(&serialized_entry)) { 460 DLOG(ERROR) << "Failed to serialize the entry."; 461 return false; 462 } 463 464 const leveldb::Status status = resource_map_->Put( 465 leveldb::WriteOptions(), 466 leveldb::Slice(GetCacheEntryKey(resource_id)), 467 leveldb::Slice(serialized_entry)); 468 return status.ok(); 469 } 470 471 bool ResourceMetadataStorage::GetCacheEntry(const std::string& resource_id, 472 FileCacheEntry* out_entry) { 473 base::ThreadRestrictions::AssertIOAllowed(); 474 DCHECK(!resource_id.empty()); 475 476 std::string serialized_entry; 477 const leveldb::Status status = resource_map_->Get( 478 leveldb::ReadOptions(), 479 leveldb::Slice(GetCacheEntryKey(resource_id)), 480 &serialized_entry); 481 return status.ok() && out_entry->ParseFromString(serialized_entry); 482 } 483 484 bool ResourceMetadataStorage::RemoveCacheEntry(const std::string& resource_id) { 485 base::ThreadRestrictions::AssertIOAllowed(); 486 DCHECK(!resource_id.empty()); 487 488 const leveldb::Status status = resource_map_->Delete( 489 leveldb::WriteOptions(), 490 leveldb::Slice(GetCacheEntryKey(resource_id))); 491 return status.ok(); 492 } 493 494 scoped_ptr<ResourceMetadataStorage::CacheEntryIterator> 495 ResourceMetadataStorage::GetCacheEntryIterator() { 496 base::ThreadRestrictions::AssertIOAllowed(); 497 498 scoped_ptr<leveldb::Iterator> it( 499 resource_map_->NewIterator(leveldb::ReadOptions())); 500 return make_scoped_ptr(new CacheEntryIterator(it.Pass())); 501 } 502 503 ResourceMetadataStorage::~ResourceMetadataStorage() { 504 base::ThreadRestrictions::AssertIOAllowed(); 505 } 506 507 void ResourceMetadataStorage::DestroyOnBlockingPool() { 508 delete this; 509 } 510 511 // static 512 std::string ResourceMetadataStorage::GetChildEntryKey( 513 const std::string& parent_resource_id, 514 const std::string& child_name) { 515 std::string key = parent_resource_id; 516 key.push_back(kDBKeyDelimeter); 517 key.append(child_name); 518 key.push_back(kDBKeyDelimeter); 519 return key; 520 } 521 522 bool ResourceMetadataStorage::PutHeader( 523 const ResourceMetadataHeader& header) { 524 base::ThreadRestrictions::AssertIOAllowed(); 525 526 std::string serialized_header; 527 if (!header.SerializeToString(&serialized_header)) { 528 DLOG(ERROR) << "Failed to serialize the header"; 529 return false; 530 } 531 532 const leveldb::Status status = resource_map_->Put( 533 leveldb::WriteOptions(), 534 leveldb::Slice(GetHeaderDBKey()), 535 leveldb::Slice(serialized_header)); 536 return status.ok(); 537 } 538 539 bool ResourceMetadataStorage::GetHeader(ResourceMetadataHeader* header) { 540 base::ThreadRestrictions::AssertIOAllowed(); 541 542 std::string serialized_header; 543 const leveldb::Status status = resource_map_->Get( 544 leveldb::ReadOptions(), 545 leveldb::Slice(GetHeaderDBKey()), 546 &serialized_header); 547 return status.ok() && header->ParseFromString(serialized_header); 548 } 549 550 bool ResourceMetadataStorage::CheckValidity() { 551 base::ThreadRestrictions::AssertIOAllowed(); 552 553 // Perform read with checksums verification enalbed. 554 leveldb::ReadOptions options; 555 options.verify_checksums = true; 556 557 scoped_ptr<leveldb::Iterator> it(resource_map_->NewIterator(options)); 558 it->SeekToFirst(); 559 560 // DB is organized like this: 561 // 562 // <key> : <value> 563 // "\0HEADER" : ResourceMetadataHeader 564 // "|ID of A|" : ResourceEntry for entry A. 565 // "|ID of A|\0CACHE" : FileCacheEntry for entry A. 566 // "|ID of A|\0|child name 1|\0" : ID of the 1st child entry of entry A. 567 // "|ID of A|\0|child name 2|\0" : ID of the 2nd child entry of entry A. 568 // ... 569 // "|ID of A|\0|child name n|\0" : ID of the nth child entry of entry A. 570 // "|ID of B|" : ResourceEntry for entry B. 571 // "|ID of B|\0CACHE" : FileCacheEntry for entry B. 572 // ... 573 574 // Check the header. 575 ResourceMetadataHeader header; 576 if (!it->Valid() || 577 it->key() != GetHeaderDBKey() || // Header entry must come first. 578 !header.ParseFromArray(it->value().data(), it->value().size()) || 579 header.version() != kDBVersion) { 580 DLOG(ERROR) << "Invalid header detected. version = " << header.version(); 581 return false; 582 } 583 584 // Check all entries. 585 size_t num_entries_with_parent = 0; 586 size_t num_child_entries = 0; 587 ResourceEntry entry; 588 std::string serialized_parent_entry; 589 std::string child_resource_id; 590 for (it->Next(); it->Valid(); it->Next()) { 591 // Count child entries. 592 if (IsChildEntryKey(it->key())) { 593 ++num_child_entries; 594 continue; 595 } 596 597 // Ignore cache entries. 598 if (IsCacheEntryKey(it->key())) 599 continue; 600 601 // Check if stored data is broken. 602 if (!entry.ParseFromArray(it->value().data(), it->value().size()) || 603 entry.resource_id() != it->key()) { 604 DLOG(ERROR) << "Broken entry detected"; 605 return false; 606 } 607 608 if (!entry.parent_resource_id().empty()) { 609 // Check if the parent entry is stored. 610 leveldb::Status status = resource_map_->Get( 611 options, 612 leveldb::Slice(entry.parent_resource_id()), 613 &serialized_parent_entry); 614 if (!status.ok()) { 615 DLOG(ERROR) << "Can't get parent entry. status = " << status.ToString(); 616 return false; 617 } 618 619 // Check if parent-child relationship is stored correctly. 620 status = resource_map_->Get( 621 options, 622 leveldb::Slice(GetChildEntryKey(entry.parent_resource_id(), 623 entry.base_name())), 624 &child_resource_id); 625 if (!status.ok() || child_resource_id != entry.resource_id()) { 626 DLOG(ERROR) << "Child map is broken. status = " << status.ToString(); 627 return false; 628 } 629 ++num_entries_with_parent; 630 } 631 } 632 if (!it->status().ok() || num_child_entries != num_entries_with_parent) { 633 DLOG(ERROR) << "Error during checking resource map. status = " 634 << it->status().ToString(); 635 return false; 636 } 637 return true; 638 } 639 640 } // namespace internal 641 } // namespace drive 642