1 // Copyright 2014 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "chrome/browser/media_galleries/media_scan_manager.h" 6 7 #include "base/files/file_enumerator.h" 8 #include "base/files/file_util.h" 9 #include "base/logging.h" 10 #include "base/metrics/histogram.h" 11 #include "base/time/time.h" 12 #include "chrome/browser/extensions/extension_service.h" 13 #include "chrome/browser/media_galleries/media_galleries_preferences.h" 14 #include "chrome/browser/media_galleries/media_galleries_preferences_factory.h" 15 #include "chrome/browser/media_galleries/media_scan_manager_observer.h" 16 #include "chrome/browser/profiles/profile.h" 17 #include "chrome/common/extensions/api/media_galleries.h" 18 #include "content/public/browser/browser_thread.h" 19 #include "extensions/browser/extension_registry.h" 20 #include "extensions/browser/extension_system.h" 21 #include "extensions/common/extension.h" 22 23 using extensions::ExtensionRegistry; 24 25 namespace media_galleries = extensions::api::media_galleries; 26 27 namespace { 28 29 typedef std::set<std::string /*extension id*/> ScanningExtensionIdSet; 30 31 // When multiple scan results have the same parent, sometimes it makes sense 32 // to combine them into a single scan result at the parent. This constant 33 // governs when that happens; kContainerDirectoryMinimumPercent percent of the 34 // directories in the parent directory must be scan results. 35 const int kContainerDirectoryMinimumPercent = 80; 36 37 // How long after a completed media scan can we provide the cached results. 38 const int kScanResultsExpiryTimeInHours = 24; 39 40 struct LocationInfo { 41 LocationInfo() 42 : pref_id(kInvalidMediaGalleryPrefId), 43 type(MediaGalleryPrefInfo::kInvalidType) {} 44 LocationInfo(MediaGalleryPrefId pref_id, MediaGalleryPrefInfo::Type type, 45 base::FilePath path) 46 : pref_id(pref_id), type(type), path(path) {} 47 // Highest priority comparison by path, next by type (scan result last), 48 // then by pref id (invalid last). 49 bool operator<(const LocationInfo& rhs) const { 50 if (path.value() == rhs.path.value()) { 51 if (type == rhs.type) { 52 return pref_id > rhs.pref_id; 53 } 54 return rhs.type == MediaGalleryPrefInfo::kScanResult; 55 } 56 return path.value() < rhs.path.value(); 57 } 58 59 MediaGalleryPrefId pref_id; 60 MediaGalleryPrefInfo::Type type; 61 base::FilePath path; 62 MediaGalleryScanResult file_counts; 63 }; 64 65 // Finds new scan results that are shadowed (the same location, or a child) by 66 // existing locations and moves them from |found_folders| to |child_folders|. 67 // Also moves new scan results that are shadowed by other new scan results 68 // to |child_folders|. 69 void PartitionChildScanResults( 70 MediaGalleriesPreferences* preferences, 71 MediaFolderFinder::MediaFolderFinderResults* found_folders, 72 MediaFolderFinder::MediaFolderFinderResults* child_folders) { 73 // Construct a list with everything in it. 74 std::vector<LocationInfo> all_locations; 75 for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it = 76 found_folders->begin(); it != found_folders->end(); ++it) { 77 all_locations.push_back(LocationInfo(kInvalidMediaGalleryPrefId, 78 MediaGalleryPrefInfo::kScanResult, 79 it->first)); 80 all_locations.back().file_counts = it->second; 81 } 82 const MediaGalleriesPrefInfoMap& known_galleries = 83 preferences->known_galleries(); 84 for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin(); 85 it != known_galleries.end(); 86 ++it) { 87 all_locations.push_back(LocationInfo(it->second.pref_id, it->second.type, 88 it->second.AbsolutePath())); 89 } 90 // Sorting on path should put all paths that are prefixes of other paths 91 // next to each other, with the shortest one first. 92 std::sort(all_locations.begin(), all_locations.end()); 93 94 size_t previous_parent_index = 0; 95 for (size_t i = 1; i < all_locations.size(); i++) { 96 const LocationInfo& current = all_locations[i]; 97 const LocationInfo& previous_parent = all_locations[previous_parent_index]; 98 bool is_child = previous_parent.path.IsParent(current.path); 99 if (current.type == MediaGalleryPrefInfo::kScanResult && 100 current.pref_id == kInvalidMediaGalleryPrefId && 101 (is_child || previous_parent.path == current.path)) { 102 // Move new scan results that are shadowed. 103 (*child_folders)[current.path] = current.file_counts; 104 found_folders->erase(current.path); 105 } else if (!is_child) { 106 previous_parent_index = i; 107 } 108 } 109 } 110 111 MediaGalleryScanResult SumFilesUnderPath( 112 const base::FilePath& path, 113 const MediaFolderFinder::MediaFolderFinderResults& candidates) { 114 MediaGalleryScanResult results; 115 for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it = 116 candidates.begin(); it != candidates.end(); ++it) { 117 if (it->first == path || path.IsParent(it->first)) { 118 results.audio_count += it->second.audio_count; 119 results.image_count += it->second.image_count; 120 results.video_count += it->second.video_count; 121 } 122 } 123 return results; 124 } 125 126 void AddScanResultsForProfile( 127 MediaGalleriesPreferences* preferences, 128 const MediaFolderFinder::MediaFolderFinderResults& found_folders) { 129 // First, remove any existing scan results where no app has been granted 130 // permission - either it is gone, or is already in the new scan results. 131 // This burns some pref ids, but not at an appreciable rate. 132 MediaGalleryPrefIdSet to_remove; 133 const MediaGalleriesPrefInfoMap& known_galleries = 134 preferences->known_galleries(); 135 for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin(); 136 it != known_galleries.end(); 137 ++it) { 138 if (it->second.type == MediaGalleryPrefInfo::kScanResult && 139 !preferences->NonAutoGalleryHasPermission(it->first)) { 140 to_remove.insert(it->first); 141 } 142 } 143 for (MediaGalleryPrefIdSet::const_iterator it = to_remove.begin(); 144 it != to_remove.end(); 145 ++it) { 146 preferences->EraseGalleryById(*it); 147 } 148 149 MediaFolderFinder::MediaFolderFinderResults child_folders; 150 MediaFolderFinder::MediaFolderFinderResults 151 unique_found_folders(found_folders); 152 PartitionChildScanResults(preferences, &unique_found_folders, &child_folders); 153 154 // Updating prefs while iterating them will invalidate the pointer, so 155 // calculate the changes first and then apply them. 156 std::map<MediaGalleryPrefId, MediaGalleryScanResult> to_update; 157 for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin(); 158 it != known_galleries.end(); 159 ++it) { 160 const MediaGalleryPrefInfo& gallery = it->second; 161 if (!gallery.IsBlackListedType()) { 162 MediaGalleryScanResult file_counts = 163 SumFilesUnderPath(gallery.AbsolutePath(), child_folders); 164 if (gallery.audio_count != file_counts.audio_count || 165 gallery.image_count != file_counts.image_count || 166 gallery.video_count != file_counts.video_count) { 167 to_update[it->first] = file_counts; 168 } 169 } 170 } 171 172 for (std::map<MediaGalleryPrefId, 173 MediaGalleryScanResult>::const_iterator it = to_update.begin(); 174 it != to_update.end(); 175 ++it) { 176 const MediaGalleryPrefInfo& gallery = 177 preferences->known_galleries().find(it->first)->second; 178 preferences->AddGallery(gallery.device_id, gallery.path, gallery.type, 179 gallery.volume_label, gallery.vendor_name, 180 gallery.model_name, gallery.total_size_in_bytes, 181 gallery.last_attach_time, 182 it->second.audio_count, 183 it->second.image_count, 184 it->second.video_count); 185 } 186 187 // Add new scan results. 188 for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it = 189 unique_found_folders.begin(); 190 it != unique_found_folders.end(); 191 ++it) { 192 MediaGalleryScanResult file_counts = 193 SumFilesUnderPath(it->first, child_folders); 194 // The top level scan result is not in |child_folders|. Add it in as well. 195 file_counts.audio_count += it->second.audio_count; 196 file_counts.image_count += it->second.image_count; 197 file_counts.video_count += it->second.video_count; 198 199 MediaGalleryPrefInfo gallery; 200 bool existing = preferences->LookUpGalleryByPath(it->first, &gallery); 201 DCHECK(!existing); 202 preferences->AddGallery(gallery.device_id, gallery.path, 203 MediaGalleryPrefInfo::kScanResult, 204 gallery.volume_label, gallery.vendor_name, 205 gallery.model_name, gallery.total_size_in_bytes, 206 gallery.last_attach_time, file_counts.audio_count, 207 file_counts.image_count, file_counts.video_count); 208 } 209 UMA_HISTOGRAM_COUNTS_10000("MediaGalleries.ScanGalleriesPopulated", 210 unique_found_folders.size() + to_update.size()); 211 } 212 213 int CountScanResultsForExtension(MediaGalleriesPreferences* preferences, 214 const extensions::Extension* extension, 215 MediaGalleryScanResult* file_counts) { 216 int gallery_count = 0; 217 218 MediaGalleryPrefIdSet permitted_galleries = 219 preferences->GalleriesForExtension(*extension); 220 const MediaGalleriesPrefInfoMap& known_galleries = 221 preferences->known_galleries(); 222 for (MediaGalleriesPrefInfoMap::const_iterator it = known_galleries.begin(); 223 it != known_galleries.end(); 224 ++it) { 225 if (it->second.type == MediaGalleryPrefInfo::kScanResult && 226 !ContainsKey(permitted_galleries, it->first)) { 227 gallery_count++; 228 file_counts->audio_count += it->second.audio_count; 229 file_counts->image_count += it->second.image_count; 230 file_counts->video_count += it->second.video_count; 231 } 232 } 233 return gallery_count; 234 } 235 236 int CountDirectoryEntries(const base::FilePath& path) { 237 base::FileEnumerator dir_counter( 238 path, false /*recursive*/, base::FileEnumerator::DIRECTORIES); 239 int count = 0; 240 base::FileEnumerator::FileInfo info; 241 for (base::FilePath name = dir_counter.Next(); !name.empty(); 242 name = dir_counter.Next()) { 243 if (!base::IsLink(name)) 244 ++count; 245 } 246 return count; 247 } 248 249 struct ContainerCount { 250 int seen_count, entries_count; 251 bool is_qualified; 252 253 ContainerCount() : seen_count(0), entries_count(-1), is_qualified(false) {} 254 }; 255 256 typedef std::map<base::FilePath, ContainerCount> ContainerCandidates; 257 258 } // namespace 259 260 MediaScanManager::MediaScanManager() 261 : scoped_extension_registry_observer_(this), 262 weak_factory_(this) { 263 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 264 } 265 266 MediaScanManager::~MediaScanManager() { 267 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 268 } 269 270 void MediaScanManager::AddObserver(Profile* profile, 271 MediaScanManagerObserver* observer) { 272 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 273 DCHECK(!ContainsKey(observers_, profile)); 274 observers_[profile].observer = observer; 275 } 276 277 void MediaScanManager::RemoveObserver(Profile* profile) { 278 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 279 bool scan_in_progress = ScanInProgress(); 280 observers_.erase(profile); 281 DCHECK_EQ(scan_in_progress, ScanInProgress()); 282 } 283 284 void MediaScanManager::CancelScansForProfile(Profile* profile) { 285 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 286 observers_[profile].scanning_extensions.clear(); 287 288 if (!ScanInProgress()) 289 folder_finder_.reset(); 290 } 291 292 void MediaScanManager::StartScan(Profile* profile, 293 const extensions::Extension* extension, 294 bool user_gesture) { 295 DCHECK(extension); 296 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 297 298 ScanObserverMap::iterator scans_for_profile = observers_.find(profile); 299 // We expect that an MediaScanManagerObserver has already been registered. 300 DCHECK(scans_for_profile != observers_.end()); 301 bool scan_in_progress = ScanInProgress(); 302 // Ignore requests for extensions that are already scanning. 303 ScanningExtensionIdSet* scanning_extensions; 304 scanning_extensions = &scans_for_profile->second.scanning_extensions; 305 if (scan_in_progress && ContainsKey(*scanning_extensions, extension->id())) 306 return; 307 308 // Provide cached result if there is not already a scan in progress, 309 // there is no user gesture, and the previous results are unexpired. 310 MediaGalleriesPreferences* preferences = 311 MediaGalleriesPreferencesFactory::GetForProfile(profile); 312 base::TimeDelta time_since_last_scan = 313 base::Time::Now() - preferences->GetLastScanCompletionTime(); 314 if (!scan_in_progress && !user_gesture && time_since_last_scan < 315 base::TimeDelta::FromHours(kScanResultsExpiryTimeInHours)) { 316 MediaGalleryScanResult file_counts; 317 int gallery_count = 318 CountScanResultsForExtension(preferences, extension, &file_counts); 319 scans_for_profile->second.observer->OnScanStarted(extension->id()); 320 scans_for_profile->second.observer->OnScanFinished(extension->id(), 321 gallery_count, 322 file_counts); 323 return; 324 } 325 326 // On first scan for the |profile|, register to listen for extension unload. 327 if (scanning_extensions->empty()) 328 scoped_extension_registry_observer_.Add(ExtensionRegistry::Get(profile)); 329 330 scanning_extensions->insert(extension->id()); 331 scans_for_profile->second.observer->OnScanStarted(extension->id()); 332 333 if (folder_finder_) 334 return; 335 336 MediaFolderFinder::MediaFolderFinderResultsCallback callback = 337 base::Bind(&MediaScanManager::OnScanCompleted, 338 weak_factory_.GetWeakPtr()); 339 if (testing_folder_finder_factory_.is_null()) { 340 folder_finder_.reset(new MediaFolderFinder(callback)); 341 } else { 342 folder_finder_.reset(testing_folder_finder_factory_.Run(callback)); 343 } 344 scan_start_time_ = base::Time::Now(); 345 folder_finder_->StartScan(); 346 } 347 348 void MediaScanManager::CancelScan(Profile* profile, 349 const extensions::Extension* extension) { 350 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 351 352 // Erases the logical scan if found, early exit otherwise. 353 ScanObserverMap::iterator scans_for_profile = observers_.find(profile); 354 if (scans_for_profile == observers_.end() || 355 !scans_for_profile->second.scanning_extensions.erase(extension->id())) { 356 return; 357 } 358 359 scans_for_profile->second.observer->OnScanCancelled(extension->id()); 360 361 // No more scanning extensions for |profile|, so stop listening for unloads. 362 if (scans_for_profile->second.scanning_extensions.empty()) 363 scoped_extension_registry_observer_.Remove(ExtensionRegistry::Get(profile)); 364 365 if (!ScanInProgress()) { 366 folder_finder_.reset(); 367 DCHECK(!scan_start_time_.is_null()); 368 UMA_HISTOGRAM_LONG_TIMES("MediaGalleries.ScanCancelTime", 369 base::Time::Now() - scan_start_time_); 370 scan_start_time_ = base::Time(); 371 } 372 } 373 374 void MediaScanManager::SetMediaFolderFinderFactory( 375 const MediaFolderFinderFactory& factory) { 376 testing_folder_finder_factory_ = factory; 377 } 378 379 // A single directory may contain many folders with media in them, without 380 // containing any media itself. In fact, the primary purpose of that directory 381 // may be to contain media directories. This function tries to find those 382 // container directories. 383 MediaFolderFinder::MediaFolderFinderResults 384 MediaScanManager::FindContainerScanResults( 385 const MediaFolderFinder::MediaFolderFinderResults& found_folders, 386 const std::vector<base::FilePath>& sensitive_locations) { 387 DCHECK_CURRENTLY_ON(content::BrowserThread::FILE); 388 std::vector<base::FilePath> abs_sensitive_locations; 389 for (size_t i = 0; i < sensitive_locations.size(); ++i) { 390 base::FilePath path = base::MakeAbsoluteFilePath(sensitive_locations[i]); 391 if (!path.empty()) 392 abs_sensitive_locations.push_back(path); 393 } 394 // Recursively find parent directories with majority of media directories, 395 // or container directories. 396 // |candidates| keeps track of directories which might have enough 397 // such directories to have us return them. 398 typedef std::map<base::FilePath, ContainerCount> ContainerCandidates; 399 ContainerCandidates candidates; 400 for (MediaFolderFinder::MediaFolderFinderResults::const_iterator it = 401 found_folders.begin(); 402 it != found_folders.end(); 403 ++it) { 404 base::FilePath child_directory = it->first; 405 base::FilePath parent_directory = child_directory.DirName(); 406 407 // Parent of root is root. 408 while (!parent_directory.empty() && child_directory != parent_directory) { 409 // Skip sensitive folders and their ancestors. 410 base::FilePath abs_parent_directory = 411 base::MakeAbsoluteFilePath(parent_directory); 412 if (abs_parent_directory.empty()) 413 break; 414 bool is_sensitive = false; 415 for (size_t i = 0; i < abs_sensitive_locations.size(); ++i) { 416 if (abs_parent_directory == abs_sensitive_locations[i] || 417 abs_parent_directory.IsParent(abs_sensitive_locations[i])) { 418 is_sensitive = true; 419 break; 420 } 421 } 422 if (is_sensitive) 423 break; 424 425 // Don't bother with ones we already have. 426 if (found_folders.find(parent_directory) != found_folders.end()) 427 continue; 428 429 ContainerCandidates::iterator parent_it = 430 candidates.find(parent_directory); 431 if (parent_it == candidates.end()) { 432 ContainerCount count; 433 count.seen_count = 1; 434 count.entries_count = CountDirectoryEntries(parent_directory); 435 parent_it = 436 candidates.insert(std::make_pair(parent_directory, count)).first; 437 } else { 438 ++candidates[parent_directory].seen_count; 439 } 440 // If previously sufficient, or not sufficient, bail. 441 if (parent_it->second.is_qualified || 442 parent_it->second.seen_count * 100 / parent_it->second.entries_count < 443 kContainerDirectoryMinimumPercent) { 444 break; 445 } 446 // Otherwise, mark qualified and check parent. 447 parent_it->second.is_qualified = true; 448 child_directory = parent_directory; 449 parent_directory = child_directory.DirName(); 450 } 451 } 452 MediaFolderFinder::MediaFolderFinderResults result; 453 // Copy and return worthy results. 454 for (ContainerCandidates::const_iterator it = candidates.begin(); 455 it != candidates.end(); 456 ++it) { 457 if (it->second.is_qualified && it->second.seen_count >= 2) 458 result[it->first] = MediaGalleryScanResult(); 459 } 460 return result; 461 } 462 463 MediaScanManager::ScanObservers::ScanObservers() : observer(NULL) {} 464 MediaScanManager::ScanObservers::~ScanObservers() {} 465 466 void MediaScanManager::OnExtensionUnloaded( 467 content::BrowserContext* browser_context, 468 const extensions::Extension* extension, 469 extensions::UnloadedExtensionInfo::Reason reason) { 470 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 471 CancelScan(Profile::FromBrowserContext(browser_context), extension); 472 } 473 474 bool MediaScanManager::ScanInProgress() const { 475 for (ScanObserverMap::const_iterator it = observers_.begin(); 476 it != observers_.end(); 477 ++it) { 478 if (!it->second.scanning_extensions.empty()) 479 return true; 480 } 481 return false; 482 } 483 484 void MediaScanManager::OnScanCompleted( 485 bool success, 486 const MediaFolderFinder::MediaFolderFinderResults& found_folders) { 487 DCHECK_CURRENTLY_ON(content::BrowserThread::UI); 488 if (!folder_finder_ || !success) { 489 folder_finder_.reset(); 490 return; 491 } 492 493 UMA_HISTOGRAM_COUNTS_10000("MediaGalleries.ScanDirectoriesFound", 494 found_folders.size()); 495 DCHECK(!scan_start_time_.is_null()); 496 UMA_HISTOGRAM_LONG_TIMES("MediaGalleries.ScanFinishedTime", 497 base::Time::Now() - scan_start_time_); 498 scan_start_time_ = base::Time(); 499 500 content::BrowserThread::PostTaskAndReplyWithResult( 501 content::BrowserThread::FILE, FROM_HERE, 502 base::Bind(FindContainerScanResults, 503 found_folders, 504 folder_finder_->graylisted_folders()), 505 base::Bind(&MediaScanManager::OnFoundContainerDirectories, 506 weak_factory_.GetWeakPtr(), 507 found_folders)); 508 } 509 510 void MediaScanManager::OnFoundContainerDirectories( 511 const MediaFolderFinder::MediaFolderFinderResults& found_folders, 512 const MediaFolderFinder::MediaFolderFinderResults& container_folders) { 513 MediaFolderFinder::MediaFolderFinderResults folders; 514 folders.insert(found_folders.begin(), found_folders.end()); 515 folders.insert(container_folders.begin(), container_folders.end()); 516 517 for (ScanObserverMap::iterator scans_for_profile = observers_.begin(); 518 scans_for_profile != observers_.end(); 519 ++scans_for_profile) { 520 if (scans_for_profile->second.scanning_extensions.empty()) 521 continue; 522 Profile* profile = scans_for_profile->first; 523 MediaGalleriesPreferences* preferences = 524 MediaGalleriesPreferencesFactory::GetForProfile(profile); 525 ExtensionService* extension_service = 526 extensions::ExtensionSystem::Get(profile)->extension_service(); 527 if (!extension_service) 528 continue; 529 530 AddScanResultsForProfile(preferences, folders); 531 532 ScanningExtensionIdSet* scanning_extensions = 533 &scans_for_profile->second.scanning_extensions; 534 for (ScanningExtensionIdSet::const_iterator extension_id_it = 535 scanning_extensions->begin(); 536 extension_id_it != scanning_extensions->end(); 537 ++extension_id_it) { 538 const extensions::Extension* extension = 539 extension_service->GetExtensionById(*extension_id_it, false); 540 if (extension) { 541 MediaGalleryScanResult file_counts; 542 int gallery_count = CountScanResultsForExtension(preferences, extension, 543 &file_counts); 544 scans_for_profile->second.observer->OnScanFinished(*extension_id_it, 545 gallery_count, 546 file_counts); 547 } 548 } 549 scanning_extensions->clear(); 550 preferences->SetLastScanCompletionTime(base::Time::Now()); 551 } 552 scoped_extension_registry_observer_.RemoveAll(); 553 folder_finder_.reset(); 554 } 555
