1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/chromeos/drive/change_list_processor.h" 6 7 #include "base/metrics/histogram.h" 8 #include "base/strings/string_number_conversions.h" 9 #include "chrome/browser/chromeos/drive/drive.pb.h" 10 #include "chrome/browser/chromeos/drive/file_system_util.h" 11 #include "chrome/browser/chromeos/drive/resource_entry_conversion.h" 12 #include "chrome/browser/chromeos/drive/resource_metadata.h" 13 #include "chrome/browser/drive/drive_api_util.h" 14 #include "google_apis/drive/drive_api_parser.h" 15 16 namespace drive { 17 namespace internal { 18 19 namespace { 20 21 class ChangeListToEntryMapUMAStats { 22 public: 23 ChangeListToEntryMapUMAStats() 24 : num_regular_files_(0), 25 num_hosted_documents_(0), 26 num_shared_with_me_entries_(0) { 27 } 28 29 // Increments number of files. 30 void IncrementNumFiles(bool is_hosted_document) { 31 is_hosted_document ? num_hosted_documents_++ : num_regular_files_++; 32 } 33 34 // Increments number of shared-with-me entries. 35 void IncrementNumSharedWithMeEntries() { 36 num_shared_with_me_entries_++; 37 } 38 39 // Updates UMA histograms with file counts. 40 void UpdateFileCountUmaHistograms() { 41 const int num_total_files = num_hosted_documents_ + num_regular_files_; 42 UMA_HISTOGRAM_COUNTS("Drive.NumberOfRegularFiles", num_regular_files_); 43 UMA_HISTOGRAM_COUNTS("Drive.NumberOfHostedDocuments", 44 num_hosted_documents_); 45 UMA_HISTOGRAM_COUNTS("Drive.NumberOfTotalFiles", num_total_files); 46 UMA_HISTOGRAM_COUNTS("Drive.NumberOfSharedWithMeEntries", 47 num_shared_with_me_entries_); 48 } 49 50 private: 51 int num_regular_files_; 52 int num_hosted_documents_; 53 int num_shared_with_me_entries_; 54 }; 55 56 // Returns true if it's OK to overwrite the local entry with the remote one. 57 bool ShouldApplyChange(const ResourceEntry& local_entry, 58 const ResourceEntry& remote_entry) { 59 if (local_entry.metadata_edit_state() == ResourceEntry::CLEAN) 60 return true; 61 return base::Time::FromInternalValue(remote_entry.modification_date()) > 62 base::Time::FromInternalValue(local_entry.modification_date()); 63 } 64 65 } // namespace 66 67 std::string DirectoryFetchInfo::ToString() const { 68 return ("local_id: " + local_id_ + 69 ", resource_id: " + resource_id_ + 70 ", changestamp: " + base::Int64ToString(changestamp_)); 71 } 72 73 ChangeList::ChangeList() {} 74 75 ChangeList::ChangeList(const google_apis::ChangeList& change_list) 76 : next_url_(change_list.next_link()), 77 largest_changestamp_(change_list.largest_change_id()) { 78 const ScopedVector<google_apis::ChangeResource>& items = change_list.items(); 79 entries_.resize(items.size()); 80 parent_resource_ids_.resize(items.size()); 81 size_t entries_index = 0; 82 for (size_t i = 0; i < items.size(); ++i) { 83 if (ConvertChangeResourceToResourceEntry( 84 *items[i], 85 &entries_[entries_index], 86 &parent_resource_ids_[entries_index])) { 87 ++entries_index; 88 } 89 } 90 entries_.resize(entries_index); 91 parent_resource_ids_.resize(entries_index); 92 } 93 94 ChangeList::ChangeList(const google_apis::FileList& file_list) 95 : next_url_(file_list.next_link()), 96 largest_changestamp_(0) { 97 const ScopedVector<google_apis::FileResource>& items = file_list.items(); 98 entries_.resize(items.size()); 99 parent_resource_ids_.resize(items.size()); 100 size_t entries_index = 0; 101 for (size_t i = 0; i < items.size(); ++i) { 102 if (ConvertFileResourceToResourceEntry( 103 *items[i], 104 &entries_[entries_index], 105 &parent_resource_ids_[entries_index])) { 106 ++entries_index; 107 } 108 } 109 entries_.resize(entries_index); 110 parent_resource_ids_.resize(entries_index); 111 } 112 113 ChangeList::~ChangeList() {} 114 115 ChangeListProcessor::ChangeListProcessor(ResourceMetadata* resource_metadata) 116 : resource_metadata_(resource_metadata) { 117 } 118 119 ChangeListProcessor::~ChangeListProcessor() { 120 } 121 122 FileError ChangeListProcessor::Apply( 123 scoped_ptr<google_apis::AboutResource> about_resource, 124 ScopedVector<ChangeList> change_lists, 125 bool is_delta_update) { 126 DCHECK(about_resource); 127 128 int64 largest_changestamp = 0; 129 if (is_delta_update) { 130 if (!change_lists.empty()) { 131 // The changestamp appears in the first page of the change list. 132 // The changestamp does not appear in the full resource list. 133 largest_changestamp = change_lists[0]->largest_changestamp(); 134 DCHECK_GE(change_lists[0]->largest_changestamp(), 0); 135 } 136 } else { 137 largest_changestamp = about_resource->largest_change_id(); 138 139 DVLOG(1) << "Root folder ID is " << about_resource->root_folder_id(); 140 DCHECK(!about_resource->root_folder_id().empty()); 141 } 142 143 // Convert ChangeList to map. 144 ChangeListToEntryMapUMAStats uma_stats; 145 for (size_t i = 0; i < change_lists.size(); ++i) { 146 ChangeList* change_list = change_lists[i]; 147 148 std::vector<ResourceEntry>* entries = change_list->mutable_entries(); 149 for (size_t i = 0; i < entries->size(); ++i) { 150 ResourceEntry* entry = &(*entries)[i]; 151 152 // Count the number of files. 153 if (!entry->file_info().is_directory()) { 154 uma_stats.IncrementNumFiles( 155 entry->file_specific_info().is_hosted_document()); 156 if (entry->shared_with_me()) 157 uma_stats.IncrementNumSharedWithMeEntries(); 158 } 159 parent_resource_id_map_[entry->resource_id()] = 160 change_list->parent_resource_ids()[i]; 161 entry_map_[entry->resource_id()].Swap(entry); 162 LOG_IF(WARNING, !entry->resource_id().empty()) 163 << "Found duplicated file: " << entry->base_name(); 164 } 165 } 166 167 // Add the largest changestamp for directories. 168 for (ResourceEntryMap::iterator it = entry_map_.begin(); 169 it != entry_map_.end(); ++it) { 170 if (it->second.file_info().is_directory()) { 171 it->second.mutable_directory_specific_info()->set_changestamp( 172 largest_changestamp); 173 } 174 } 175 176 FileError error = ApplyEntryMap(largest_changestamp, about_resource.Pass()); 177 if (error != FILE_ERROR_OK) { 178 DLOG(ERROR) << "ApplyEntryMap failed: " << FileErrorToString(error); 179 return error; 180 } 181 182 // Update changestamp. 183 error = resource_metadata_->SetLargestChangestamp(largest_changestamp); 184 if (error != FILE_ERROR_OK) { 185 DLOG(ERROR) << "SetLargestChangeStamp failed: " << FileErrorToString(error); 186 return error; 187 } 188 189 // Shouldn't record histograms when processing delta update. 190 if (!is_delta_update) 191 uma_stats.UpdateFileCountUmaHistograms(); 192 193 return FILE_ERROR_OK; 194 } 195 196 FileError ChangeListProcessor::ApplyEntryMap( 197 int64 changestamp, 198 scoped_ptr<google_apis::AboutResource> about_resource) { 199 DCHECK(about_resource); 200 201 // Create the entry for "My Drive" directory with the latest changestamp. 202 ResourceEntry root; 203 FileError error = resource_metadata_->GetResourceEntryByPath( 204 util::GetDriveMyDriveRootPath(), &root); 205 if (error != FILE_ERROR_OK) { 206 LOG(ERROR) << "Failed to get root entry: " << FileErrorToString(error); 207 return error; 208 } 209 210 root.mutable_directory_specific_info()->set_changestamp(changestamp); 211 root.set_resource_id(about_resource->root_folder_id()); 212 error = resource_metadata_->RefreshEntry(root); 213 if (error != FILE_ERROR_OK) { 214 LOG(ERROR) << "Failed to update root entry: " << FileErrorToString(error); 215 return error; 216 } 217 218 // Gather the set of changes in the old path. 219 // Note that we want to notify the change in both old and new paths (suppose 220 // /a/b/c is moved to /x/y/c. We want to notify both "/a/b" and "/x/y".) 221 // The old paths must be calculated before we apply any actual changes. 222 // The new paths are calculated after each change is applied. It correctly 223 // sets the new path because we apply changes in such an order (see below). 224 for (ResourceEntryMap::iterator it = entry_map_.begin(); 225 it != entry_map_.end(); ++it) { 226 UpdateChangedDirs(it->second); 227 } 228 229 // Apply all entries except deleted ones to the metadata. 230 std::vector<std::string> deleted_resource_ids; 231 while (!entry_map_.empty()) { 232 ResourceEntryMap::iterator it = entry_map_.begin(); 233 234 // Process deleted entries later to avoid deleting moved entries under it. 235 if (it->second.deleted()) { 236 deleted_resource_ids.push_back(it->first); 237 entry_map_.erase(it); 238 continue; 239 } 240 241 // Start from entry_map_.begin() and traverse ancestors using the 242 // parent-child relationships in the result (after this apply) tree. 243 // Then apply the topmost change first. 244 // 245 // By doing this, assuming the result tree does not contain any cycles, we 246 // can guarantee that no cycle is made during this apply (i.e. no entry gets 247 // moved under any of its descendants) because the following conditions are 248 // always satisfied in any move: 249 // - The new parent entry is not a descendant of the moved entry. 250 // - The new parent and its ancestors will no longer move during this apply. 251 std::vector<ResourceEntryMap::iterator> entries; 252 for (ResourceEntryMap::iterator it = entry_map_.begin(); 253 it != entry_map_.end();) { 254 entries.push_back(it); 255 256 DCHECK(parent_resource_id_map_.count(it->first)) << it->first; 257 const std::string& parent_resource_id = 258 parent_resource_id_map_[it->first]; 259 260 if (parent_resource_id.empty()) // This entry has no parent. 261 break; 262 263 ResourceEntryMap::iterator it_parent = 264 entry_map_.find(parent_resource_id); 265 if (it_parent == entry_map_.end()) { 266 // Current entry's parent is already updated or not going to be updated, 267 // get the parent from the local tree. 268 std::string parent_local_id; 269 FileError error = resource_metadata_->GetIdByResourceId( 270 parent_resource_id, &parent_local_id); 271 if (error != FILE_ERROR_OK) { 272 // See crbug.com/326043. In some complicated situations, parent folder 273 // for shared entries may be accessible (and hence its resource id is 274 // included), but not in the change/file list. 275 // In such a case, clear the parent and move it to drive/other. 276 if (error == FILE_ERROR_NOT_FOUND) { 277 parent_resource_id_map_[it->first] = ""; 278 } else { 279 LOG(ERROR) << "Failed to get local ID: " << parent_resource_id 280 << ", error = " << FileErrorToString(error); 281 } 282 break; 283 } 284 ResourceEntry parent_entry; 285 while (it_parent == entry_map_.end() && !parent_local_id.empty()) { 286 error = resource_metadata_->GetResourceEntryById( 287 parent_local_id, &parent_entry); 288 if (error != FILE_ERROR_OK) { 289 LOG(ERROR) << "Failed to get local entry: " 290 << FileErrorToString(error); 291 break; 292 } 293 it_parent = entry_map_.find(parent_entry.resource_id()); 294 parent_local_id = parent_entry.parent_local_id(); 295 } 296 } 297 it = it_parent; 298 } 299 300 // Apply the parent first. 301 std::reverse(entries.begin(), entries.end()); 302 for (size_t i = 0; i < entries.size(); ++i) { 303 // Skip root entry in the change list. We don't expect servers to send 304 // root entry, but we should better be defensive (see crbug.com/297259). 305 ResourceEntryMap::iterator it = entries[i]; 306 if (it->first != root.resource_id()) { 307 FileError error = ApplyEntry(it->second); 308 if (error != FILE_ERROR_OK) { 309 LOG(ERROR) << "ApplyEntry failed: " << FileErrorToString(error) 310 << ", title = " << it->second.title(); 311 return error; 312 } 313 } 314 entry_map_.erase(it); 315 } 316 } 317 318 // Apply deleted entries. 319 for (size_t i = 0; i < deleted_resource_ids.size(); ++i) { 320 std::string local_id; 321 FileError error = resource_metadata_->GetIdByResourceId( 322 deleted_resource_ids[i], &local_id); 323 switch (error) { 324 case FILE_ERROR_OK: 325 error = resource_metadata_->RemoveEntry(local_id); 326 break; 327 case FILE_ERROR_NOT_FOUND: 328 error = FILE_ERROR_OK; 329 break; 330 default: 331 break; 332 } 333 if (error != FILE_ERROR_OK) { 334 LOG(ERROR) << "Failed to delete: " << FileErrorToString(error) 335 << ", resource_id = " << deleted_resource_ids[i]; 336 return error; 337 } 338 } 339 340 return FILE_ERROR_OK; 341 } 342 343 FileError ChangeListProcessor::ApplyEntry(const ResourceEntry& entry) { 344 DCHECK(!entry.deleted()); 345 DCHECK(parent_resource_id_map_.count(entry.resource_id())); 346 const std::string& parent_resource_id = 347 parent_resource_id_map_[entry.resource_id()]; 348 349 ResourceEntry new_entry(entry); 350 FileError error = SetParentLocalIdOfEntry(resource_metadata_, &new_entry, 351 parent_resource_id); 352 if (error != FILE_ERROR_OK) 353 return error; 354 355 // Lookup the entry. 356 std::string local_id; 357 error = resource_metadata_->GetIdByResourceId(entry.resource_id(), &local_id); 358 359 ResourceEntry existing_entry; 360 if (error == FILE_ERROR_OK) 361 error = resource_metadata_->GetResourceEntryById(local_id, &existing_entry); 362 363 switch (error) { 364 case FILE_ERROR_OK: 365 if (ShouldApplyChange(existing_entry, new_entry)) { 366 // Entry exists and needs to be refreshed. 367 new_entry.set_local_id(local_id); 368 error = resource_metadata_->RefreshEntry(new_entry); 369 } else { 370 if (entry.file_info().is_directory()) { 371 // No need to refresh, but update the changestamp. 372 new_entry = existing_entry; 373 new_entry.mutable_directory_specific_info()->set_changestamp( 374 new_entry.directory_specific_info().changestamp()); 375 error = resource_metadata_->RefreshEntry(new_entry); 376 } 377 DVLOG(1) << "Change was discarded for: " << entry.resource_id(); 378 } 379 break; 380 case FILE_ERROR_NOT_FOUND: { // Adding a new entry. 381 std::string local_id; 382 error = resource_metadata_->AddEntry(new_entry, &local_id); 383 break; 384 } 385 default: 386 return error; 387 } 388 if (error != FILE_ERROR_OK) 389 return error; 390 391 UpdateChangedDirs(entry); 392 return FILE_ERROR_OK; 393 } 394 395 // static 396 FileError ChangeListProcessor::RefreshDirectory( 397 ResourceMetadata* resource_metadata, 398 const DirectoryFetchInfo& directory_fetch_info, 399 scoped_ptr<ChangeList> change_list, 400 std::vector<ResourceEntry>* out_refreshed_entries) { 401 DCHECK(!directory_fetch_info.empty()); 402 403 ResourceEntry directory; 404 FileError error = resource_metadata->GetResourceEntryById( 405 directory_fetch_info.local_id(), &directory); 406 if (error != FILE_ERROR_OK) 407 return error; 408 409 if (!directory.file_info().is_directory()) 410 return FILE_ERROR_NOT_A_DIRECTORY; 411 412 std::vector<ResourceEntry>* entries = change_list->mutable_entries(); 413 for (size_t i = 0; i < entries->size(); ++i) { 414 ResourceEntry* entry = &(*entries)[i]; 415 const std::string& parent_resource_id = 416 change_list->parent_resource_ids()[i]; 417 418 // Skip if the parent resource ID does not match. This is needed to 419 // handle entries with multiple parents. For such entries, the first 420 // parent is picked and other parents are ignored, hence some entries may 421 // have a parent resource ID which does not match the target directory's. 422 if (parent_resource_id != directory_fetch_info.resource_id()) { 423 DVLOG(1) << "Wrong-parent entry rejected: " << entry->resource_id(); 424 continue; 425 } 426 427 entry->set_parent_local_id(directory_fetch_info.local_id()); 428 429 std::string local_id; 430 error = resource_metadata->GetIdByResourceId(entry->resource_id(), 431 &local_id); 432 if (error == FILE_ERROR_OK) { 433 entry->set_local_id(local_id); 434 error = resource_metadata->RefreshEntry(*entry); 435 } 436 437 if (error == FILE_ERROR_NOT_FOUND) { // If refreshing fails, try adding. 438 entry->clear_local_id(); 439 error = resource_metadata->AddEntry(*entry, &local_id); 440 } 441 442 if (error != FILE_ERROR_OK) 443 return error; 444 445 ResourceEntry result_entry; 446 error = resource_metadata->GetResourceEntryById(local_id, &result_entry); 447 if (error != FILE_ERROR_OK) 448 return error; 449 out_refreshed_entries->push_back(result_entry); 450 } 451 return FILE_ERROR_OK; 452 } 453 454 // static 455 FileError ChangeListProcessor::SetParentLocalIdOfEntry( 456 ResourceMetadata* resource_metadata, 457 ResourceEntry* entry, 458 const std::string& parent_resource_id) { 459 std::string parent_local_id; 460 if (parent_resource_id.empty()) { 461 // Entries without parents should go under "other" directory. 462 parent_local_id = util::kDriveOtherDirLocalId; 463 } else { 464 FileError error = resource_metadata->GetIdByResourceId( 465 parent_resource_id, &parent_local_id); 466 if (error != FILE_ERROR_OK) 467 return error; 468 } 469 entry->set_parent_local_id(parent_local_id); 470 return FILE_ERROR_OK; 471 } 472 473 void ChangeListProcessor::UpdateChangedDirs(const ResourceEntry& entry) { 474 DCHECK(!entry.resource_id().empty()); 475 476 std::string local_id; 477 base::FilePath file_path; 478 if (resource_metadata_->GetIdByResourceId( 479 entry.resource_id(), &local_id) == FILE_ERROR_OK) 480 resource_metadata_->GetFilePath(local_id, &file_path); 481 482 if (!file_path.empty()) { 483 // Notify parent. 484 changed_dirs_.insert(file_path.DirName()); 485 486 if (entry.file_info().is_directory()) { 487 // Notify self if entry is a directory. 488 changed_dirs_.insert(file_path); 489 490 // Notify all descendants if it is a directory deletion. 491 if (entry.deleted()) { 492 std::set<base::FilePath> sub_directories; 493 resource_metadata_->GetSubDirectoriesRecursively(local_id, 494 &sub_directories); 495 changed_dirs_.insert(sub_directories.begin(), sub_directories.end()); 496 } 497 } 498 } 499 } 500 501 } // namespace internal 502 } // namespace drive 503