1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/sync_file_system/local/local_file_change_tracker.h" 6 7 #include <queue> 8 9 #include "base/location.h" 10 #include "base/logging.h" 11 #include "base/sequenced_task_runner.h" 12 #include "base/stl_util.h" 13 #include "chrome/browser/sync_file_system/local/local_file_sync_status.h" 14 #include "chrome/browser/sync_file_system/syncable_file_system_util.h" 15 #include "third_party/leveldatabase/src/helpers/memenv/memenv.h" 16 #include "third_party/leveldatabase/src/include/leveldb/db.h" 17 #include "third_party/leveldatabase/src/include/leveldb/env.h" 18 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h" 19 #include "webkit/browser/fileapi/file_system_context.h" 20 #include "webkit/browser/fileapi/file_system_file_util.h" 21 #include "webkit/browser/fileapi/file_system_operation_context.h" 22 #include "webkit/common/fileapi/file_system_util.h" 23 24 using fileapi::FileSystemContext; 25 using fileapi::FileSystemFileUtil; 26 using fileapi::FileSystemOperationContext; 27 using fileapi::FileSystemURL; 28 using fileapi::FileSystemURLSet; 29 30 namespace sync_file_system { 31 32 namespace { 33 const base::FilePath::CharType kDatabaseName[] = 34 FILE_PATH_LITERAL("LocalFileChangeTracker"); 35 const char kMark[] = "d"; 36 } // namespace 37 38 // A database class that stores local file changes in a local database. This 39 // object must be destructed on file_task_runner. 40 class LocalFileChangeTracker::TrackerDB { 41 public: 42 TrackerDB(const base::FilePath& base_path, 43 leveldb::Env* env_override); 44 45 SyncStatusCode MarkDirty(const std::string& url); 46 SyncStatusCode ClearDirty(const std::string& url); 47 SyncStatusCode GetDirtyEntries( 48 std::queue<FileSystemURL>* dirty_files); 49 SyncStatusCode WriteBatch(scoped_ptr<leveldb::WriteBatch> batch); 50 51 private: 52 enum RecoveryOption { 53 REPAIR_ON_CORRUPTION, 54 FAIL_ON_CORRUPTION, 55 }; 56 57 SyncStatusCode Init(RecoveryOption recovery_option); 58 SyncStatusCode Repair(const std::string& db_path); 59 void HandleError(const tracked_objects::Location& from_here, 60 const leveldb::Status& status); 61 62 const base::FilePath base_path_; 63 leveldb::Env* env_override_; 64 scoped_ptr<leveldb::DB> db_; 65 SyncStatusCode db_status_; 66 67 DISALLOW_COPY_AND_ASSIGN(TrackerDB); 68 }; 69 70 LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {} 71 LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {} 72 73 // LocalFileChangeTracker ------------------------------------------------------ 74 75 LocalFileChangeTracker::LocalFileChangeTracker( 76 const base::FilePath& base_path, 77 leveldb::Env* env_override, 78 base::SequencedTaskRunner* file_task_runner) 79 : initialized_(false), 80 file_task_runner_(file_task_runner), 81 tracker_db_(new TrackerDB(base_path, env_override)), 82 current_change_seq_(0), 83 num_changes_(0) { 84 } 85 86 LocalFileChangeTracker::~LocalFileChangeTracker() { 87 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 88 tracker_db_.reset(); 89 } 90 91 void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL& url) { 92 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 93 if (ContainsKey(changes_, url)) 94 return; 95 // TODO(nhiroki): propagate the error code (see http://crbug.com/152127). 96 MarkDirtyOnDatabase(url); 97 } 98 99 void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL& url) {} 100 101 void LocalFileChangeTracker::OnCreateFile(const FileSystemURL& url) { 102 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 103 SYNC_FILE_TYPE_FILE)); 104 } 105 106 void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL& url, 107 const FileSystemURL& src) { 108 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 109 SYNC_FILE_TYPE_FILE)); 110 } 111 112 void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL& url) { 113 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE, 114 SYNC_FILE_TYPE_FILE)); 115 } 116 117 void LocalFileChangeTracker::OnModifyFile(const FileSystemURL& url) { 118 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 119 SYNC_FILE_TYPE_FILE)); 120 } 121 122 void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL& url) { 123 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 124 SYNC_FILE_TYPE_DIRECTORY)); 125 } 126 127 void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL& url) { 128 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE, 129 SYNC_FILE_TYPE_DIRECTORY)); 130 } 131 132 void LocalFileChangeTracker::GetNextChangedURLs( 133 std::deque<FileSystemURL>* urls, int max_urls) { 134 DCHECK(urls); 135 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 136 urls->clear(); 137 // Mildly prioritizes the URLs that older changes and have not been updated 138 // for a while. 139 for (ChangeSeqMap::iterator iter = change_seqs_.begin(); 140 iter != change_seqs_.end() && 141 (max_urls == 0 || urls->size() < static_cast<size_t>(max_urls)); 142 ++iter) { 143 urls->push_back(iter->second); 144 } 145 } 146 147 void LocalFileChangeTracker::GetChangesForURL( 148 const FileSystemURL& url, FileChangeList* changes) { 149 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 150 DCHECK(changes); 151 changes->clear(); 152 FileChangeMap::iterator found = changes_.find(url); 153 if (found == changes_.end()) { 154 found = demoted_changes_.find(url); 155 if (found == demoted_changes_.end()) 156 return; 157 } 158 *changes = found->second.change_list; 159 } 160 161 void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL& url) { 162 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 163 ClearDirtyOnDatabase(url); 164 mirror_changes_.erase(url); 165 demoted_changes_.erase(url); 166 FileChangeMap::iterator found = changes_.find(url); 167 if (found == changes_.end()) 168 return; 169 change_seqs_.erase(found->second.change_seq); 170 changes_.erase(found); 171 UpdateNumChanges(); 172 } 173 174 void LocalFileChangeTracker::CreateFreshMirrorForURL( 175 const fileapi::FileSystemURL& url) { 176 DCHECK(!ContainsKey(mirror_changes_, url)); 177 mirror_changes_[url] = ChangeInfo(); 178 } 179 180 void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL( 181 const fileapi::FileSystemURL& url) { 182 FileChangeMap::iterator found = mirror_changes_.find(url); 183 if (found == mirror_changes_.end()) 184 return; 185 mirror_changes_.erase(found); 186 187 if (ContainsKey(changes_, url)) 188 MarkDirtyOnDatabase(url); 189 else 190 ClearDirtyOnDatabase(url); 191 UpdateNumChanges(); 192 } 193 194 void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL( 195 const fileapi::FileSystemURL& url) { 196 FileChangeMap::iterator found = mirror_changes_.find(url); 197 if (found == mirror_changes_.end() || found->second.change_list.empty()) { 198 ClearChangesForURL(url); 199 return; 200 } 201 const ChangeInfo& info = found->second; 202 change_seqs_[info.change_seq] = url; 203 changes_[url] = info; 204 RemoveMirrorAndCommitChangesForURL(url); 205 } 206 207 void LocalFileChangeTracker::DemoteChangesForURL( 208 const fileapi::FileSystemURL& url) { 209 FileChangeMap::iterator found = changes_.find(url); 210 if (found == changes_.end()) 211 return; 212 FileChangeList changes = found->second.change_list; 213 214 mirror_changes_.erase(url); 215 change_seqs_.erase(found->second.change_seq); 216 changes_.erase(found); 217 218 FileChangeList::List change_list = changes.list(); 219 while (!change_list.empty()) { 220 RecordChangeToChangeMaps(url, change_list.front(), 0, 221 &demoted_changes_, NULL); 222 change_list.pop_front(); 223 } 224 } 225 226 bool LocalFileChangeTracker::PromoteDemotedChanges() { 227 if (demoted_changes_.empty()) 228 return false; 229 for (FileChangeMap::iterator iter = demoted_changes_.begin(); 230 iter != demoted_changes_.end();) { 231 fileapi::FileSystemURL url = iter->first; 232 FileChangeList::List change_list = iter->second.change_list.list(); 233 demoted_changes_.erase(iter++); 234 235 // Make sure that this URL is in no queues. 236 DCHECK(!ContainsKey(changes_, url)); 237 DCHECK(!ContainsKey(demoted_changes_, url)); 238 DCHECK(!ContainsKey(mirror_changes_, url)); 239 240 while (!change_list.empty()) { 241 RecordChange(url, change_list.front()); 242 change_list.pop_front(); 243 } 244 } 245 demoted_changes_.clear(); 246 return true; 247 } 248 249 SyncStatusCode LocalFileChangeTracker::Initialize( 250 FileSystemContext* file_system_context) { 251 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 252 DCHECK(!initialized_); 253 DCHECK(file_system_context); 254 255 SyncStatusCode status = CollectLastDirtyChanges(file_system_context); 256 if (status == SYNC_STATUS_OK) 257 initialized_ = true; 258 return status; 259 } 260 261 void LocalFileChangeTracker::ResetForFileSystem( 262 const GURL& origin, 263 fileapi::FileSystemType type) { 264 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 265 scoped_ptr<leveldb::WriteBatch> batch(new leveldb::WriteBatch); 266 for (FileChangeMap::iterator iter = changes_.begin(); 267 iter != changes_.end();) { 268 fileapi::FileSystemURL url = iter->first; 269 if (url.origin() != origin || url.type() != type) { 270 ++iter; 271 continue; 272 } 273 mirror_changes_.erase(url); 274 demoted_changes_.erase(url); 275 change_seqs_.erase(iter->second.change_seq); 276 changes_.erase(iter++); 277 278 std::string serialized_url; 279 const bool should_success = 280 SerializeSyncableFileSystemURL(url, &serialized_url); 281 if (!should_success) { 282 NOTREACHED() << "Failed to serialize: " << url.DebugString(); 283 continue; 284 } 285 batch->Delete(serialized_url); 286 } 287 // Fail to apply batch to database wouldn't have critical effect, they'll be 288 // just marked deleted on next relaunch. 289 tracker_db_->WriteBatch(batch.Pass()); 290 UpdateNumChanges(); 291 } 292 293 void LocalFileChangeTracker::UpdateNumChanges() { 294 base::AutoLock lock(num_changes_lock_); 295 num_changes_ = static_cast<int64>(change_seqs_.size()); 296 } 297 298 void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet* urls) { 299 std::deque<FileSystemURL> url_deque; 300 GetNextChangedURLs(&url_deque, 0); 301 urls->clear(); 302 urls->insert(url_deque.begin(), url_deque.end()); 303 } 304 305 void LocalFileChangeTracker::DropAllChanges() { 306 changes_.clear(); 307 change_seqs_.clear(); 308 mirror_changes_.clear(); 309 } 310 311 SyncStatusCode LocalFileChangeTracker::MarkDirtyOnDatabase( 312 const FileSystemURL& url) { 313 std::string serialized_url; 314 if (!SerializeSyncableFileSystemURL(url, &serialized_url)) 315 return SYNC_FILE_ERROR_INVALID_URL; 316 317 return tracker_db_->MarkDirty(serialized_url); 318 } 319 320 SyncStatusCode LocalFileChangeTracker::ClearDirtyOnDatabase( 321 const FileSystemURL& url) { 322 std::string serialized_url; 323 if (!SerializeSyncableFileSystemURL(url, &serialized_url)) 324 return SYNC_FILE_ERROR_INVALID_URL; 325 326 return tracker_db_->ClearDirty(serialized_url); 327 } 328 329 SyncStatusCode LocalFileChangeTracker::CollectLastDirtyChanges( 330 FileSystemContext* file_system_context) { 331 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 332 333 std::queue<FileSystemURL> dirty_files; 334 const SyncStatusCode status = tracker_db_->GetDirtyEntries(&dirty_files); 335 if (status != SYNC_STATUS_OK) 336 return status; 337 338 FileSystemFileUtil* file_util = 339 file_system_context->sandbox_delegate()->sync_file_util(); 340 DCHECK(file_util); 341 scoped_ptr<FileSystemOperationContext> context( 342 new FileSystemOperationContext(file_system_context)); 343 344 base::File::Info file_info; 345 base::FilePath platform_path; 346 347 while (!dirty_files.empty()) { 348 const FileSystemURL url = dirty_files.front(); 349 dirty_files.pop(); 350 DCHECK_EQ(url.type(), fileapi::kFileSystemTypeSyncable); 351 352 switch (file_util->GetFileInfo(context.get(), url, 353 &file_info, &platform_path)) { 354 case base::File::FILE_OK: { 355 if (!file_info.is_directory) { 356 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 357 SYNC_FILE_TYPE_FILE)); 358 break; 359 } 360 361 RecordChange(url, FileChange( 362 FileChange::FILE_CHANGE_ADD_OR_UPDATE, 363 SYNC_FILE_TYPE_DIRECTORY)); 364 365 // Push files and directories in this directory into |dirty_files|. 366 scoped_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator( 367 file_util->CreateFileEnumerator(context.get(), url)); 368 base::FilePath path_each; 369 while (!(path_each = enumerator->Next()).empty()) { 370 dirty_files.push(CreateSyncableFileSystemURL( 371 url.origin(), path_each)); 372 } 373 break; 374 } 375 case base::File::FILE_ERROR_NOT_FOUND: { 376 // File represented by |url| has already been deleted. Since we cannot 377 // figure out if this file was directory or not from the URL, file 378 // type is treated as SYNC_FILE_TYPE_UNKNOWN. 379 // 380 // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is 381 // also treated as FILE_CHANGE_DELETE. 382 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE, 383 SYNC_FILE_TYPE_UNKNOWN)); 384 break; 385 } 386 case base::File::FILE_ERROR_FAILED: 387 default: 388 // TODO(nhiroki): handle file access error (http://crbug.com/155251). 389 LOG(WARNING) << "Failed to access local file."; 390 break; 391 } 392 } 393 return SYNC_STATUS_OK; 394 } 395 396 void LocalFileChangeTracker::RecordChange( 397 const FileSystemURL& url, const FileChange& change) { 398 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 399 if (ContainsKey(demoted_changes_, url)) { 400 RecordChangeToChangeMaps(url, change, 0, &demoted_changes_, NULL); 401 return; 402 } 403 int change_seq = current_change_seq_++; 404 RecordChangeToChangeMaps(url, change, change_seq, &changes_, &change_seqs_); 405 if (ContainsKey(mirror_changes_, url)) 406 RecordChangeToChangeMaps(url, change, change_seq, &mirror_changes_, NULL); 407 UpdateNumChanges(); 408 } 409 410 void LocalFileChangeTracker::RecordChangeToChangeMaps( 411 const FileSystemURL& url, 412 const FileChange& change, 413 int new_change_seq, 414 FileChangeMap* changes, 415 ChangeSeqMap* change_seqs) { 416 ChangeInfo& info = (*changes)[url]; 417 if (info.change_seq >= 0 && change_seqs) 418 change_seqs->erase(info.change_seq); 419 info.change_list.Update(change); 420 if (info.change_list.empty()) { 421 changes->erase(url); 422 return; 423 } 424 info.change_seq = new_change_seq; 425 if (change_seqs) 426 (*change_seqs)[info.change_seq] = url; 427 } 428 429 // TrackerDB ------------------------------------------------------------------- 430 431 LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath& base_path, 432 leveldb::Env* env_override) 433 : base_path_(base_path), 434 env_override_(env_override), 435 db_status_(SYNC_STATUS_OK) {} 436 437 SyncStatusCode LocalFileChangeTracker::TrackerDB::Init( 438 RecoveryOption recovery_option) { 439 if (db_.get() && db_status_ == SYNC_STATUS_OK) 440 return SYNC_STATUS_OK; 441 442 std::string path = fileapi::FilePathToString( 443 base_path_.Append(kDatabaseName)); 444 leveldb::Options options; 445 options.max_open_files = 0; // Use minimum. 446 options.create_if_missing = true; 447 if (env_override_) 448 options.env = env_override_; 449 leveldb::DB* db; 450 leveldb::Status status = leveldb::DB::Open(options, path, &db); 451 if (status.ok()) { 452 db_.reset(db); 453 return SYNC_STATUS_OK; 454 } 455 456 HandleError(FROM_HERE, status); 457 if (!status.IsCorruption()) 458 return LevelDBStatusToSyncStatusCode(status); 459 460 // Try to repair the corrupted DB. 461 switch (recovery_option) { 462 case FAIL_ON_CORRUPTION: 463 return SYNC_DATABASE_ERROR_CORRUPTION; 464 case REPAIR_ON_CORRUPTION: 465 return Repair(path); 466 } 467 NOTREACHED(); 468 return SYNC_DATABASE_ERROR_FAILED; 469 } 470 471 SyncStatusCode LocalFileChangeTracker::TrackerDB::Repair( 472 const std::string& db_path) { 473 DCHECK(!db_.get()); 474 LOG(WARNING) << "Attempting to repair TrackerDB."; 475 476 leveldb::Options options; 477 options.max_open_files = 0; // Use minimum. 478 if (leveldb::RepairDB(db_path, options).ok() && 479 Init(FAIL_ON_CORRUPTION) == SYNC_STATUS_OK) { 480 // TODO(nhiroki): perform some consistency checks between TrackerDB and 481 // syncable file system. 482 LOG(WARNING) << "Repairing TrackerDB completed."; 483 return SYNC_STATUS_OK; 484 } 485 486 LOG(WARNING) << "Failed to repair TrackerDB."; 487 return SYNC_DATABASE_ERROR_CORRUPTION; 488 } 489 490 // TODO(nhiroki): factor out the common methods into somewhere else. 491 void LocalFileChangeTracker::TrackerDB::HandleError( 492 const tracked_objects::Location& from_here, 493 const leveldb::Status& status) { 494 LOG(ERROR) << "LocalFileChangeTracker::TrackerDB failed at: " 495 << from_here.ToString() << " with error: " << status.ToString(); 496 } 497 498 SyncStatusCode LocalFileChangeTracker::TrackerDB::MarkDirty( 499 const std::string& url) { 500 if (db_status_ != SYNC_STATUS_OK) 501 return db_status_; 502 503 db_status_ = Init(REPAIR_ON_CORRUPTION); 504 if (db_status_ != SYNC_STATUS_OK) { 505 db_.reset(); 506 return db_status_; 507 } 508 509 leveldb::Status status = db_->Put(leveldb::WriteOptions(), url, kMark); 510 if (!status.ok()) { 511 HandleError(FROM_HERE, status); 512 db_status_ = LevelDBStatusToSyncStatusCode(status); 513 db_.reset(); 514 return db_status_; 515 } 516 return SYNC_STATUS_OK; 517 } 518 519 SyncStatusCode LocalFileChangeTracker::TrackerDB::ClearDirty( 520 const std::string& url) { 521 if (db_status_ != SYNC_STATUS_OK) 522 return db_status_; 523 524 // Should not reach here before initializing the database. The database should 525 // be cleared after read, and should be initialized during read if 526 // uninitialized. 527 DCHECK(db_.get()); 528 529 leveldb::Status status = db_->Delete(leveldb::WriteOptions(), url); 530 if (!status.ok() && !status.IsNotFound()) { 531 HandleError(FROM_HERE, status); 532 db_status_ = LevelDBStatusToSyncStatusCode(status); 533 db_.reset(); 534 return db_status_; 535 } 536 return SYNC_STATUS_OK; 537 } 538 539 SyncStatusCode LocalFileChangeTracker::TrackerDB::GetDirtyEntries( 540 std::queue<FileSystemURL>* dirty_files) { 541 if (db_status_ != SYNC_STATUS_OK) 542 return db_status_; 543 544 db_status_ = Init(REPAIR_ON_CORRUPTION); 545 if (db_status_ != SYNC_STATUS_OK) { 546 db_.reset(); 547 return db_status_; 548 } 549 550 scoped_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions())); 551 iter->SeekToFirst(); 552 FileSystemURL url; 553 while (iter->Valid()) { 554 if (!DeserializeSyncableFileSystemURL(iter->key().ToString(), &url)) { 555 LOG(WARNING) << "Failed to deserialize an URL. " 556 << "TrackerDB might be corrupted."; 557 db_status_ = SYNC_DATABASE_ERROR_CORRUPTION; 558 db_.reset(); 559 return db_status_; 560 } 561 dirty_files->push(url); 562 iter->Next(); 563 } 564 return SYNC_STATUS_OK; 565 } 566 567 SyncStatusCode LocalFileChangeTracker::TrackerDB::WriteBatch( 568 scoped_ptr<leveldb::WriteBatch> batch) { 569 if (db_status_ != SYNC_STATUS_OK) 570 return db_status_; 571 572 leveldb::Status status = db_->Write(leveldb::WriteOptions(), batch.get()); 573 if (!status.ok() && !status.IsNotFound()) { 574 HandleError(FROM_HERE, status); 575 db_status_ = LevelDBStatusToSyncStatusCode(status); 576 db_.reset(); 577 return db_status_; 578 } 579 return SYNC_STATUS_OK; 580 } 581 582 } // namespace sync_file_system 583
