1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/sync_file_system/local/local_file_change_tracker.h" 6 7 #include <queue> 8 9 #include "base/location.h" 10 #include "base/logging.h" 11 #include "base/sequenced_task_runner.h" 12 #include "base/stl_util.h" 13 #include "chrome/browser/sync_file_system/local/local_file_sync_status.h" 14 #include "chrome/browser/sync_file_system/syncable_file_system_util.h" 15 #include "third_party/leveldatabase/src/include/leveldb/db.h" 16 #include "third_party/leveldatabase/src/include/leveldb/write_batch.h" 17 #include "webkit/browser/fileapi/file_system_context.h" 18 #include "webkit/browser/fileapi/file_system_file_util.h" 19 #include "webkit/browser/fileapi/file_system_operation_context.h" 20 #include "webkit/common/fileapi/file_system_util.h" 21 22 using fileapi::FileSystemContext; 23 using fileapi::FileSystemFileUtil; 24 using fileapi::FileSystemOperationContext; 25 using fileapi::FileSystemURL; 26 using fileapi::FileSystemURLSet; 27 28 namespace sync_file_system { 29 30 namespace { 31 const base::FilePath::CharType kDatabaseName[] = 32 FILE_PATH_LITERAL("LocalFileChangeTracker"); 33 const char kMark[] = "d"; 34 } // namespace 35 36 // A database class that stores local file changes in a local database. This 37 // object must be destructed on file_task_runner. 38 class LocalFileChangeTracker::TrackerDB { 39 public: 40 explicit TrackerDB(const base::FilePath& base_path); 41 42 SyncStatusCode MarkDirty(const std::string& url); 43 SyncStatusCode ClearDirty(const std::string& url); 44 SyncStatusCode GetDirtyEntries( 45 std::queue<FileSystemURL>* dirty_files); 46 SyncStatusCode WriteBatch(scoped_ptr<leveldb::WriteBatch> batch); 47 48 private: 49 enum RecoveryOption { 50 REPAIR_ON_CORRUPTION, 51 FAIL_ON_CORRUPTION, 52 }; 53 54 SyncStatusCode Init(RecoveryOption recovery_option); 55 SyncStatusCode Repair(const std::string& db_path); 56 void HandleError(const tracked_objects::Location& from_here, 57 const leveldb::Status& status); 58 59 const base::FilePath base_path_; 60 scoped_ptr<leveldb::DB> db_; 61 SyncStatusCode db_status_; 62 63 DISALLOW_COPY_AND_ASSIGN(TrackerDB); 64 }; 65 66 LocalFileChangeTracker::ChangeInfo::ChangeInfo() : change_seq(-1) {} 67 LocalFileChangeTracker::ChangeInfo::~ChangeInfo() {} 68 69 // LocalFileChangeTracker ------------------------------------------------------ 70 71 LocalFileChangeTracker::LocalFileChangeTracker( 72 const base::FilePath& base_path, 73 base::SequencedTaskRunner* file_task_runner) 74 : initialized_(false), 75 file_task_runner_(file_task_runner), 76 tracker_db_(new TrackerDB(base_path)), 77 current_change_seq_(0), 78 num_changes_(0) { 79 } 80 81 LocalFileChangeTracker::~LocalFileChangeTracker() { 82 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 83 tracker_db_.reset(); 84 } 85 86 void LocalFileChangeTracker::OnStartUpdate(const FileSystemURL& url) { 87 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 88 if (ContainsKey(changes_, url)) 89 return; 90 // TODO(nhiroki): propagate the error code (see http://crbug.com/152127). 91 MarkDirtyOnDatabase(url); 92 } 93 94 void LocalFileChangeTracker::OnEndUpdate(const FileSystemURL& url) {} 95 96 void LocalFileChangeTracker::OnCreateFile(const FileSystemURL& url) { 97 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 98 SYNC_FILE_TYPE_FILE)); 99 } 100 101 void LocalFileChangeTracker::OnCreateFileFrom(const FileSystemURL& url, 102 const FileSystemURL& src) { 103 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 104 SYNC_FILE_TYPE_FILE)); 105 } 106 107 void LocalFileChangeTracker::OnRemoveFile(const FileSystemURL& url) { 108 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE, 109 SYNC_FILE_TYPE_FILE)); 110 } 111 112 void LocalFileChangeTracker::OnModifyFile(const FileSystemURL& url) { 113 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 114 SYNC_FILE_TYPE_FILE)); 115 } 116 117 void LocalFileChangeTracker::OnCreateDirectory(const FileSystemURL& url) { 118 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 119 SYNC_FILE_TYPE_DIRECTORY)); 120 } 121 122 void LocalFileChangeTracker::OnRemoveDirectory(const FileSystemURL& url) { 123 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE, 124 SYNC_FILE_TYPE_DIRECTORY)); 125 } 126 127 void LocalFileChangeTracker::GetNextChangedURLs( 128 std::deque<FileSystemURL>* urls, int max_urls) { 129 DCHECK(urls); 130 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 131 urls->clear(); 132 // Mildly prioritizes the URLs that older changes and have not been updated 133 // for a while. 134 for (ChangeSeqMap::iterator iter = change_seqs_.begin(); 135 iter != change_seqs_.end() && 136 (max_urls == 0 || urls->size() < static_cast<size_t>(max_urls)); 137 ++iter) { 138 urls->push_back(iter->second); 139 } 140 } 141 142 void LocalFileChangeTracker::GetChangesForURL( 143 const FileSystemURL& url, FileChangeList* changes) { 144 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 145 DCHECK(changes); 146 changes->clear(); 147 FileChangeMap::iterator found = changes_.find(url); 148 if (found == changes_.end()) 149 return; 150 *changes = found->second.change_list; 151 } 152 153 void LocalFileChangeTracker::ClearChangesForURL(const FileSystemURL& url) { 154 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 155 ClearDirtyOnDatabase(url); 156 mirror_changes_.erase(url); 157 FileChangeMap::iterator found = changes_.find(url); 158 if (found == changes_.end()) 159 return; 160 change_seqs_.erase(found->second.change_seq); 161 changes_.erase(found); 162 UpdateNumChanges(); 163 } 164 165 void LocalFileChangeTracker::CreateFreshMirrorForURL( 166 const fileapi::FileSystemURL& url) { 167 DCHECK(!ContainsKey(mirror_changes_, url)); 168 mirror_changes_[url] = ChangeInfo(); 169 } 170 171 void LocalFileChangeTracker::RemoveMirrorAndCommitChangesForURL( 172 const fileapi::FileSystemURL& url) { 173 FileChangeMap::iterator found = mirror_changes_.find(url); 174 if (found == mirror_changes_.end()) 175 return; 176 mirror_changes_.erase(found); 177 178 if (ContainsKey(changes_, url)) 179 MarkDirtyOnDatabase(url); 180 else 181 ClearDirtyOnDatabase(url); 182 UpdateNumChanges(); 183 } 184 185 void LocalFileChangeTracker::ResetToMirrorAndCommitChangesForURL( 186 const fileapi::FileSystemURL& url) { 187 FileChangeMap::iterator found = mirror_changes_.find(url); 188 if (found == mirror_changes_.end() || found->second.change_list.empty()) { 189 ClearChangesForURL(url); 190 return; 191 } 192 const ChangeInfo& info = found->second; 193 change_seqs_[info.change_seq] = url; 194 changes_[url] = info; 195 RemoveMirrorAndCommitChangesForURL(url); 196 } 197 198 void LocalFileChangeTracker::DemoteChangesForURL( 199 const fileapi::FileSystemURL& url) { 200 FileChangeMap::iterator found = changes_.find(url); 201 if (found == changes_.end()) 202 return; 203 FileChangeList changes = found->second.change_list; 204 205 mirror_changes_.erase(url); 206 change_seqs_.erase(found->second.change_seq); 207 changes_.erase(found); 208 209 FileChangeList::List change_list = changes.list(); 210 while (!change_list.empty()) { 211 RecordChange(url, change_list.front()); 212 change_list.pop_front(); 213 } 214 } 215 216 SyncStatusCode LocalFileChangeTracker::Initialize( 217 FileSystemContext* file_system_context) { 218 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 219 DCHECK(!initialized_); 220 DCHECK(file_system_context); 221 222 SyncStatusCode status = CollectLastDirtyChanges(file_system_context); 223 if (status == SYNC_STATUS_OK) 224 initialized_ = true; 225 return status; 226 } 227 228 void LocalFileChangeTracker::ResetForFileSystem( 229 const GURL& origin, 230 fileapi::FileSystemType type) { 231 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 232 scoped_ptr<leveldb::WriteBatch> batch(new leveldb::WriteBatch); 233 for (FileChangeMap::iterator iter = changes_.begin(); 234 iter != changes_.end();) { 235 fileapi::FileSystemURL url = iter->first; 236 if (url.origin() != origin || url.type() != type) { 237 ++iter; 238 continue; 239 } 240 mirror_changes_.erase(url); 241 change_seqs_.erase(iter->second.change_seq); 242 changes_.erase(iter++); 243 244 std::string serialized_url; 245 const bool should_success = 246 SerializeSyncableFileSystemURL(url, &serialized_url); 247 if (!should_success) { 248 NOTREACHED() << "Failed to serialize: " << url.DebugString(); 249 continue; 250 } 251 batch->Delete(serialized_url); 252 } 253 // Fail to apply batch to database wouldn't have critical effect, they'll be 254 // just marked deleted on next relaunch. 255 tracker_db_->WriteBatch(batch.Pass()); 256 UpdateNumChanges(); 257 } 258 259 void LocalFileChangeTracker::UpdateNumChanges() { 260 base::AutoLock lock(num_changes_lock_); 261 num_changes_ = static_cast<int64>(change_seqs_.size()); 262 } 263 264 void LocalFileChangeTracker::GetAllChangedURLs(FileSystemURLSet* urls) { 265 std::deque<FileSystemURL> url_deque; 266 GetNextChangedURLs(&url_deque, 0); 267 urls->clear(); 268 urls->insert(url_deque.begin(), url_deque.end()); 269 } 270 271 void LocalFileChangeTracker::DropAllChanges() { 272 changes_.clear(); 273 change_seqs_.clear(); 274 mirror_changes_.clear(); 275 } 276 277 SyncStatusCode LocalFileChangeTracker::MarkDirtyOnDatabase( 278 const FileSystemURL& url) { 279 std::string serialized_url; 280 if (!SerializeSyncableFileSystemURL(url, &serialized_url)) 281 return SYNC_FILE_ERROR_INVALID_URL; 282 283 return tracker_db_->MarkDirty(serialized_url); 284 } 285 286 SyncStatusCode LocalFileChangeTracker::ClearDirtyOnDatabase( 287 const FileSystemURL& url) { 288 std::string serialized_url; 289 if (!SerializeSyncableFileSystemURL(url, &serialized_url)) 290 return SYNC_FILE_ERROR_INVALID_URL; 291 292 return tracker_db_->ClearDirty(serialized_url); 293 } 294 295 SyncStatusCode LocalFileChangeTracker::CollectLastDirtyChanges( 296 FileSystemContext* file_system_context) { 297 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 298 299 std::queue<FileSystemURL> dirty_files; 300 const SyncStatusCode status = tracker_db_->GetDirtyEntries(&dirty_files); 301 if (status != SYNC_STATUS_OK) 302 return status; 303 304 FileSystemFileUtil* file_util = 305 file_system_context->sandbox_delegate()->sync_file_util(); 306 DCHECK(file_util); 307 scoped_ptr<FileSystemOperationContext> context( 308 new FileSystemOperationContext(file_system_context)); 309 310 base::PlatformFileInfo file_info; 311 base::FilePath platform_path; 312 313 while (!dirty_files.empty()) { 314 const FileSystemURL url = dirty_files.front(); 315 dirty_files.pop(); 316 DCHECK_EQ(url.type(), fileapi::kFileSystemTypeSyncable); 317 318 switch (file_util->GetFileInfo(context.get(), url, 319 &file_info, &platform_path)) { 320 case base::PLATFORM_FILE_OK: { 321 if (!file_info.is_directory) { 322 RecordChange(url, FileChange(FileChange::FILE_CHANGE_ADD_OR_UPDATE, 323 SYNC_FILE_TYPE_FILE)); 324 break; 325 } 326 327 RecordChange(url, FileChange( 328 FileChange::FILE_CHANGE_ADD_OR_UPDATE, 329 SYNC_FILE_TYPE_DIRECTORY)); 330 331 // Push files and directories in this directory into |dirty_files|. 332 scoped_ptr<FileSystemFileUtil::AbstractFileEnumerator> enumerator( 333 file_util->CreateFileEnumerator(context.get(), url)); 334 base::FilePath path_each; 335 while (!(path_each = enumerator->Next()).empty()) { 336 dirty_files.push(CreateSyncableFileSystemURL( 337 url.origin(), path_each)); 338 } 339 break; 340 } 341 case base::PLATFORM_FILE_ERROR_NOT_FOUND: { 342 // File represented by |url| has already been deleted. Since we cannot 343 // figure out if this file was directory or not from the URL, file 344 // type is treated as SYNC_FILE_TYPE_UNKNOWN. 345 // 346 // NOTE: Directory to have been reverted (that is, ADD -> DELETE) is 347 // also treated as FILE_CHANGE_DELETE. 348 RecordChange(url, FileChange(FileChange::FILE_CHANGE_DELETE, 349 SYNC_FILE_TYPE_UNKNOWN)); 350 break; 351 } 352 case base::PLATFORM_FILE_ERROR_FAILED: 353 default: 354 // TODO(nhiroki): handle file access error (http://crbug.com/155251). 355 LOG(WARNING) << "Failed to access local file."; 356 break; 357 } 358 } 359 return SYNC_STATUS_OK; 360 } 361 362 void LocalFileChangeTracker::RecordChange( 363 const FileSystemURL& url, const FileChange& change) { 364 DCHECK(file_task_runner_->RunsTasksOnCurrentThread()); 365 int change_seq = current_change_seq_++; 366 RecordChangeToChangeMaps(url, change, change_seq, &changes_, &change_seqs_); 367 if (ContainsKey(mirror_changes_, url)) 368 RecordChangeToChangeMaps(url, change, change_seq, &mirror_changes_, NULL); 369 UpdateNumChanges(); 370 } 371 372 void LocalFileChangeTracker::RecordChangeToChangeMaps( 373 const FileSystemURL& url, 374 const FileChange& change, 375 int new_change_seq, 376 FileChangeMap* changes, 377 ChangeSeqMap* change_seqs) { 378 ChangeInfo& info = (*changes)[url]; 379 if (info.change_seq >= 0 && change_seqs) 380 change_seqs->erase(info.change_seq); 381 info.change_list.Update(change); 382 if (info.change_list.empty()) { 383 changes->erase(url); 384 return; 385 } 386 info.change_seq = new_change_seq; 387 if (change_seqs) 388 (*change_seqs)[info.change_seq] = url; 389 } 390 391 // TrackerDB ------------------------------------------------------------------- 392 393 LocalFileChangeTracker::TrackerDB::TrackerDB(const base::FilePath& base_path) 394 : base_path_(base_path), 395 db_status_(SYNC_STATUS_OK) {} 396 397 SyncStatusCode LocalFileChangeTracker::TrackerDB::Init( 398 RecoveryOption recovery_option) { 399 if (db_.get() && db_status_ == SYNC_STATUS_OK) 400 return SYNC_STATUS_OK; 401 402 std::string path = fileapi::FilePathToString( 403 base_path_.Append(kDatabaseName)); 404 leveldb::Options options; 405 options.max_open_files = 0; // Use minimum. 406 options.create_if_missing = true; 407 leveldb::DB* db; 408 leveldb::Status status = leveldb::DB::Open(options, path, &db); 409 if (status.ok()) { 410 db_.reset(db); 411 return SYNC_STATUS_OK; 412 } 413 414 HandleError(FROM_HERE, status); 415 if (!status.IsCorruption()) 416 return LevelDBStatusToSyncStatusCode(status); 417 418 // Try to repair the corrupted DB. 419 switch (recovery_option) { 420 case FAIL_ON_CORRUPTION: 421 return SYNC_DATABASE_ERROR_CORRUPTION; 422 case REPAIR_ON_CORRUPTION: 423 return Repair(path); 424 } 425 NOTREACHED(); 426 return SYNC_DATABASE_ERROR_FAILED; 427 } 428 429 SyncStatusCode LocalFileChangeTracker::TrackerDB::Repair( 430 const std::string& db_path) { 431 DCHECK(!db_.get()); 432 LOG(WARNING) << "Attempting to repair TrackerDB."; 433 434 leveldb::Options options; 435 options.max_open_files = 0; // Use minimum. 436 if (leveldb::RepairDB(db_path, options).ok() && 437 Init(FAIL_ON_CORRUPTION) == SYNC_STATUS_OK) { 438 // TODO(nhiroki): perform some consistency checks between TrackerDB and 439 // syncable file system. 440 LOG(WARNING) << "Repairing TrackerDB completed."; 441 return SYNC_STATUS_OK; 442 } 443 444 LOG(WARNING) << "Failed to repair TrackerDB."; 445 return SYNC_DATABASE_ERROR_CORRUPTION; 446 } 447 448 // TODO(nhiroki): factor out the common methods into somewhere else. 449 void LocalFileChangeTracker::TrackerDB::HandleError( 450 const tracked_objects::Location& from_here, 451 const leveldb::Status& status) { 452 LOG(ERROR) << "LocalFileChangeTracker::TrackerDB failed at: " 453 << from_here.ToString() << " with error: " << status.ToString(); 454 } 455 456 SyncStatusCode LocalFileChangeTracker::TrackerDB::MarkDirty( 457 const std::string& url) { 458 if (db_status_ != SYNC_STATUS_OK) 459 return db_status_; 460 461 db_status_ = Init(REPAIR_ON_CORRUPTION); 462 if (db_status_ != SYNC_STATUS_OK) { 463 db_.reset(); 464 return db_status_; 465 } 466 467 leveldb::Status status = db_->Put(leveldb::WriteOptions(), url, kMark); 468 if (!status.ok()) { 469 HandleError(FROM_HERE, status); 470 db_status_ = LevelDBStatusToSyncStatusCode(status); 471 db_.reset(); 472 return db_status_; 473 } 474 return SYNC_STATUS_OK; 475 } 476 477 SyncStatusCode LocalFileChangeTracker::TrackerDB::ClearDirty( 478 const std::string& url) { 479 if (db_status_ != SYNC_STATUS_OK) 480 return db_status_; 481 482 // Should not reach here before initializing the database. The database should 483 // be cleared after read, and should be initialized during read if 484 // uninitialized. 485 DCHECK(db_.get()); 486 487 leveldb::Status status = db_->Delete(leveldb::WriteOptions(), url); 488 if (!status.ok() && !status.IsNotFound()) { 489 HandleError(FROM_HERE, status); 490 db_status_ = LevelDBStatusToSyncStatusCode(status); 491 db_.reset(); 492 return db_status_; 493 } 494 return SYNC_STATUS_OK; 495 } 496 497 SyncStatusCode LocalFileChangeTracker::TrackerDB::GetDirtyEntries( 498 std::queue<FileSystemURL>* dirty_files) { 499 if (db_status_ != SYNC_STATUS_OK) 500 return db_status_; 501 502 db_status_ = Init(REPAIR_ON_CORRUPTION); 503 if (db_status_ != SYNC_STATUS_OK) { 504 db_.reset(); 505 return db_status_; 506 } 507 508 scoped_ptr<leveldb::Iterator> iter(db_->NewIterator(leveldb::ReadOptions())); 509 iter->SeekToFirst(); 510 FileSystemURL url; 511 while (iter->Valid()) { 512 if (!DeserializeSyncableFileSystemURL(iter->key().ToString(), &url)) { 513 LOG(WARNING) << "Failed to deserialize an URL. " 514 << "TrackerDB might be corrupted."; 515 db_status_ = SYNC_DATABASE_ERROR_CORRUPTION; 516 db_.reset(); 517 return db_status_; 518 } 519 dirty_files->push(url); 520 iter->Next(); 521 } 522 return SYNC_STATUS_OK; 523 } 524 525 SyncStatusCode LocalFileChangeTracker::TrackerDB::WriteBatch( 526 scoped_ptr<leveldb::WriteBatch> batch) { 527 if (db_status_ != SYNC_STATUS_OK) 528 return db_status_; 529 530 leveldb::Status status = db_->Write(leveldb::WriteOptions(), batch.get()); 531 if (!status.ok() && !status.IsNotFound()) { 532 HandleError(FROM_HERE, status); 533 db_status_ = LevelDBStatusToSyncStatusCode(status); 534 db_.reset(); 535 return db_status_; 536 } 537 return SYNC_STATUS_OK; 538 } 539 540 } // namespace sync_file_system 541
