Home | History | Annotate | Download | only in drive_backend
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #ifndef CHROME_BROWSER_SYNC_FILE_SYSTEM_DRIVE_BACKEND_METADATA_DATABASE_H_
      6 #define CHROME_BROWSER_SYNC_FILE_SYSTEM_DRIVE_BACKEND_METADATA_DATABASE_H_
      7 
      8 #include <map>
      9 #include <set>
     10 #include <string>
     11 #include <vector>
     12 
     13 #include "base/callback_forward.h"
     14 #include "base/memory/scoped_ptr.h"
     15 #include "base/memory/scoped_vector.h"
     16 #include "base/memory/weak_ptr.h"
     17 #include "base/values.h"
     18 #include "chrome/browser/sync_file_system/drive_backend/tracker_set.h"
     19 #include "chrome/browser/sync_file_system/sync_callbacks.h"
     20 #include "chrome/browser/sync_file_system/sync_status_code.h"
     21 
     22 namespace base {
     23 class FilePath;
     24 class SequencedTaskRunner;
     25 class SingleThreadTaskRunner;
     26 }
     27 
     28 namespace leveldb {
     29 class DB;
     30 class WriteBatch;
     31 }
     32 
     33 namespace google_apis {
     34 class ChangeResource;
     35 class FileResource;
     36 class ResourceEntry;
     37 }
     38 
     39 namespace tracked_objects {
     40 class Location;
     41 }
     42 
     43 namespace sync_file_system {
     44 namespace drive_backend {
     45 
     46 class FileDetails;
     47 class FileMetadata;
     48 class FileTracker;
     49 class ServiceMetadata;
     50 struct DatabaseContents;
     51 
     52 // MetadataDatabase holds and maintains a LevelDB instance and its indexes,
     53 // which holds 1)ServiceMetadata, 2)FileMetadata and 3)FileTracker.
     54 // 1) ServiceMetadata is a singleton in the database which holds information for
     55 //    the backend.
     56 // 2) FileMetadata represents a remote-side file and holds latest known
     57 //    metadata of the remote file.
     58 // 3) FileTracker represents a synced or to-be-synced file and maintains
     59 //    the local-side folder tree.
     60 //
     61 // The term "file" includes files, folders and other resources on Drive.
     62 //
     63 // FileTrackers form a tree structure on the database, which represents the
     64 // FileSystem trees of SyncFileSystem.  The tree has a FileTracker named
     65 // sync-root as its root node, and a set of FileTracker named app-root.  An
     66 // app-root represents a remote folder for an installed Chrome App and holds all
     67 // synced contents for the App.
     68 //
     69 // One FileMetadata is created for each tracked remote file, which is identified
     70 // by FileID.
     71 // One FileTracker is created for every different {parent tracker, FileID} pair,
     72 // excluding non-app-root inactive parent trackers. Multiple trackers may be
     73 // associated to one FileID when the file has multiple parents. Multiple
     74 // trackers may have the same {parent tracker, title} pair when the associated
     75 // remote files have the same title.
     76 //
     77 // Files have following state:
     78 //   - Unknown file
     79 //     - Has a dirty inactive tracker and empty synced_details.
     80 //     - Is initial state of a tracker, only file_id and parent_tracker_id field
     81 //       are known.
     82 //   - Folder
     83 //     - Is either one of sync-root folder, app-root folder or a regular folder.
     84 //     - Sync-root folder holds app-root folders as its direct children, and
     85 //       holds entire SyncFileSystem files as its descentants.  Its tracker
     86 //       should be stored in ServiceMetadata by its tracker_id.
     87 //     - App-root folder holds all files for an application as its descendants.
     88 //   - File
     89 //   - Unsupported file
     90 //     - Represents unsupported files such as hosted documents. Must be
     91 //       inactive.
     92 //
     93 // Invariants:
     94 //   - Any tracker in the database must either:
     95 //     - be sync-root,
     96 //     - have an app-root as its parent tracker, or
     97 //     - have an active tracker as its parent.
     98 //   That is, all trackers must be reachable from sync-root via app-root folders
     99 //   and active trackers.
    100 //
    101 //   - Any active tracker must either:
    102 //     - have |needs_folder_listing| flag and dirty flag, or
    103 //     - have all children at the stored largest change ID.
    104 //
    105 //   - If multiple trackers have the same parent tracker and same title, they
    106 //     must not have same |file_id|, and at most one of them may be active.
    107 //   - If multiple trackers have the same |file_id|, at most one of them may be
    108 //     active.
    109 //
    110 class MetadataDatabase {
    111  public:
    112   typedef std::map<std::string, FileMetadata*> FileByID;
    113   typedef std::map<int64, FileTracker*> TrackerByID;
    114   typedef std::map<std::string, TrackerSet> TrackersByFileID;
    115   typedef std::map<std::string, TrackerSet> TrackersByTitle;
    116   typedef std::map<int64, TrackersByTitle> TrackersByParentAndTitle;
    117   typedef std::map<std::string, FileTracker*> TrackerByAppID;
    118   typedef std::vector<std::string> FileIDList;
    119 
    120   typedef base::Callback<
    121       void(SyncStatusCode status, scoped_ptr<MetadataDatabase> instance)>
    122       CreateCallback;
    123 
    124   // The entry point of the MetadataDatabase for production code.
    125   static void Create(base::SequencedTaskRunner* task_runner,
    126                      const base::FilePath& database_path,
    127                      const CreateCallback& callback);
    128   ~MetadataDatabase();
    129 
    130   static void ClearDatabase(scoped_ptr<MetadataDatabase> metadata_database);
    131 
    132   int64 GetLargestFetchedChangeID() const;
    133   int64 GetSyncRootTrackerID() const;
    134   bool HasSyncRoot() const;
    135 
    136   // Returns all file metadata for the given |app_id|.
    137   scoped_ptr<base::ListValue> DumpFiles(const std::string& app_id);
    138 
    139   // Returns all database data.
    140   scoped_ptr<base::ListValue> DumpDatabase();
    141 
    142   // TODO(tzik): Move GetLargestKnownChangeID() to private section, and hide its
    143   // handling in the class, instead of letting user do.
    144   //
    145   // Gets / updates the largest known change ID.
    146   // The largest known change ID is on-memory and not persist over restart.
    147   // This is supposed to use when a task fetches ChangeList in parallel to other
    148   // operation.  When a task starts fetching paged ChangeList one by one, it
    149   // should update the largest known change ID on the first round and background
    150   // remaining fetch job.
    151   // Then, when other tasks that update FileMetadata by UpdateByFileResource,
    152   // it should use largest known change ID as the |change_id| that prevents
    153   // FileMetadata from overwritten by ChangeList.
    154   // Also if other tasks try to update a remote resource whose change is not yet
    155   // retrieved the task should fail due to etag check, so we should be fine.
    156   int64 GetLargestKnownChangeID() const;
    157   void UpdateLargestKnownChangeID(int64 change_id);
    158 
    159   // Populates empty database with initial data.
    160   // Adds a file metadata and a file tracker for |sync_root_folder|, and adds
    161   // file metadata and file trackers for each |app_root_folders|.
    162   // Newly added tracker for |sync_root_folder| is active and non-dirty.
    163   // Newly added trackers for |app_root_folders| are inactive and non-dirty.
    164   // Trackers for |app_root_folders| are not yet registered as app-roots, but
    165   // are ready to register.
    166   void PopulateInitialData(
    167       int64 largest_change_id,
    168       const google_apis::FileResource& sync_root_folder,
    169       const ScopedVector<google_apis::FileResource>& app_root_folders,
    170       const SyncStatusCallback& callback);
    171 
    172   // Returns true if the folder associated to |app_id| is enabled.
    173   bool IsAppEnabled(const std::string& app_id) const;
    174 
    175   // Registers existing folder as the app-root for |app_id|.  The folder
    176   // must be an inactive folder that does not yet associated to any App.
    177   // This method associates the folder with |app_id| and activates it.
    178   void RegisterApp(const std::string& app_id,
    179                    const std::string& folder_id,
    180                    const SyncStatusCallback& callback);
    181 
    182   // Inactivates the folder associated to the app to disable |app_id|.
    183   // Does nothing if |app_id| is already disabled.
    184   void DisableApp(const std::string& app_id,
    185                   const SyncStatusCallback& callback);
    186 
    187   // Activates the folder associated to |app_id| to enable |app_id|.
    188   // Does nothing if |app_id| is already enabled.
    189   void EnableApp(const std::string& app_id,
    190                  const SyncStatusCallback& callback);
    191 
    192   // Unregisters the folder as the app-root for |app_id|.  If |app_id| does not
    193   // exist, does nothing.  The folder is left as an inactive regular folder.
    194   // Note that the inactivation drops all descendant files since they are no
    195   // longer reachable from sync-root via active folder or app-root.
    196   void UnregisterApp(const std::string& app_id,
    197                      const SyncStatusCallback& callback);
    198 
    199   // Finds the app-root folder for |app_id|.  Returns true if exists.
    200   // Copies the result to |tracker| if it is non-NULL.
    201   bool FindAppRootTracker(const std::string& app_id,
    202                           FileTracker* tracker) const;
    203 
    204   // Finds the file identified by |file_id|.  Returns true if the file is found.
    205   // Copies the metadata identified by |file_id| into |file| if exists and
    206   // |file| is non-NULL.
    207   bool FindFileByFileID(const std::string& file_id, FileMetadata* file) const;
    208 
    209   // Finds the tracker identified by |tracker_id|.  Returns true if the tracker
    210   // is found.
    211   // Copies the tracker identified by |tracker_id| into |tracker| if exists and
    212   // |tracker| is non-NULL.
    213   bool FindTrackerByTrackerID(int64 tracker_id, FileTracker* tracker) const;
    214 
    215   // Finds the trackers tracking |file_id|.  Returns true if the trackers are
    216   // found.
    217   bool FindTrackersByFileID(const std::string& file_id,
    218                             TrackerSet* trackers) const;
    219 
    220   // Finds the set of trackers whose parent's tracker ID is |parent_tracker_id|,
    221   // and who has |title| as its title in the synced_details.
    222   // Copies the tracker set to |trackers| if it is non-NULL.
    223   // Returns true if the trackers are found.
    224   bool FindTrackersByParentAndTitle(
    225       int64 parent_tracker_id,
    226       const std::string& title,
    227       TrackerSet* trackers) const;
    228 
    229   // Builds the file path for the given tracker.  Returns true on success.
    230   // |path| can be NULL.
    231   // The file path is relative to app-root and have a leading path separator.
    232   bool BuildPathForTracker(int64 tracker_id, base::FilePath* path) const;
    233 
    234   // Builds the file path for the given tracker for display purpose.
    235   // This may return a path ending with '<unknown>' if the given tracker does
    236   // not have title information (yet). This may return an empty path.
    237   base::FilePath BuildDisplayPathForTracker(const FileTracker& tracker) const;
    238 
    239   // Returns false if no registered app exists associated to |app_id|.
    240   // If |full_path| is active, assigns the tracker of |full_path| to |tracker|.
    241   // Otherwise, assigns the nearest active ancestor to |full_path| to |tracker|.
    242   // Also, assigns the full path of |tracker| to |path|.
    243   bool FindNearestActiveAncestor(const std::string& app_id,
    244                                  const base::FilePath& full_path,
    245                                  FileTracker* tracker,
    246                                  base::FilePath* path) const;
    247 
    248   // Updates database by |changes|.
    249   // Marks each tracker for modified file as dirty and adds new trackers if
    250   // needed.
    251   void UpdateByChangeList(int64 largest_change_id,
    252                           ScopedVector<google_apis::ChangeResource> changes,
    253                           const SyncStatusCallback& callback);
    254 
    255   // Updates database by |resource|.
    256   // Marks each tracker for modified file as dirty and adds new trackers if
    257   // needed.
    258   void UpdateByFileResource(const google_apis::FileResource& resource,
    259                             const SyncStatusCallback& callback);
    260   void UpdateByFileResourceList(
    261       ScopedVector<google_apis::FileResource> resources,
    262       const SyncStatusCallback& callback);
    263 
    264   void UpdateByDeletedRemoteFile(const std::string& file_id,
    265                                  const SyncStatusCallback& callback);
    266 
    267   // TODO(tzik): Drop |change_id| parameter.
    268   // Adds new FileTracker and FileMetadata.  The database must not have
    269   // |resource| beforehand.
    270   // The newly added tracker under |parent_tracker_id| is active and non-dirty.
    271   // Deactivates existing active tracker if exists that has the same title and
    272   // parent_tracker to the newly added tracker.
    273   void ReplaceActiveTrackerWithNewResource(
    274       int64 parent_tracker_id,
    275       const google_apis::FileResource& resource,
    276       const SyncStatusCallback& callback);
    277 
    278   // Adds |child_file_ids| to |folder_id| as its children.
    279   // This method affects the active tracker only.
    280   // If the tracker has no further change to sync, unmarks its dirty flag.
    281   void PopulateFolderByChildList(const std::string& folder_id,
    282                                  const FileIDList& child_file_ids,
    283                                  const SyncStatusCallback& callback);
    284 
    285   // Updates |synced_details| of the tracker with |updated_details|.
    286   void UpdateTracker(int64 tracker_id,
    287                      const FileDetails& updated_details,
    288                      const SyncStatusCallback& callback);
    289 
    290   // Returns true if a tracker of the file can be safely activated without
    291   // deactivating any other trackers.  In this case, tries to activate the
    292   // tracker, and invokes |callback| upon completion.
    293   // Returns false otherwise.  In false case, |callback| will not be invoked.
    294   bool TryNoSideEffectActivation(int64 parent_tracker_id,
    295                                  const std::string& file_id,
    296                                  const SyncStatusCallback& callback);
    297 
    298 
    299   // Changes the priority of the tracker to low.
    300   void LowerTrackerPriority(int64 tracker_id);
    301   void PromoteLowerPriorityTrackersToNormal();
    302 
    303   // Returns true if there is a normal priority dirty tracker.
    304   // Assigns the dirty tracker if exists and |tracker| is non-NULL.
    305   bool GetNormalPriorityDirtyTracker(FileTracker* tracker) const;
    306 
    307   // Returns true if there is a low priority dirty tracker.
    308   // Assigns the dirty tracker if exists and |tracker| is non-NULL.
    309   bool GetLowPriorityDirtyTracker(FileTracker* tracker) const;
    310 
    311   bool HasDirtyTracker() const {
    312     return !dirty_trackers_.empty() || !low_priority_dirty_trackers_.empty();
    313   }
    314 
    315   size_t GetDirtyTrackerCount() {
    316     return dirty_trackers_.size();
    317   }
    318 
    319   bool GetMultiParentFileTrackers(std::string* file_id,
    320                                   TrackerSet* trackers);
    321   bool GetConflictingTrackers(TrackerSet* trackers);
    322 
    323   // Sets |app_ids| to a list of all registered app ids.
    324   void GetRegisteredAppIDs(std::vector<std::string>* app_ids);
    325 
    326  private:
    327   friend class ListChangesTaskTest;
    328   friend class MetadataDatabaseTest;
    329   friend class RegisterAppTaskTest;
    330   friend class SyncEngineInitializerTest;
    331 
    332   struct DirtyTrackerComparator {
    333     bool operator()(const FileTracker* left,
    334                     const FileTracker* right) const;
    335   };
    336 
    337   typedef std::set<FileTracker*, DirtyTrackerComparator> DirtyTrackers;
    338 
    339   MetadataDatabase(base::SequencedTaskRunner* task_runner,
    340                    const base::FilePath& database_path);
    341   static void CreateOnTaskRunner(base::SingleThreadTaskRunner* callback_runner,
    342                                  base::SequencedTaskRunner* task_runner,
    343                                  const base::FilePath& database_path,
    344                                  const CreateCallback& callback);
    345   static SyncStatusCode CreateForTesting(
    346       scoped_ptr<leveldb::DB> db,
    347       scoped_ptr<MetadataDatabase>* metadata_database_out);
    348   SyncStatusCode InitializeOnTaskRunner();
    349   void BuildIndexes(DatabaseContents* contents);
    350 
    351   // Database manipulation methods.
    352   void RegisterTrackerAsAppRoot(const std::string& app_id,
    353                                 int64 tracker_id,
    354                                 leveldb::WriteBatch* batch);
    355   void MakeTrackerActive(int64 tracker_id, leveldb::WriteBatch* batch);
    356   void MakeTrackerInactive(int64 tracker_id, leveldb::WriteBatch* batch);
    357   void MakeAppRootDisabled(int64 tracker_id, leveldb::WriteBatch* batch);
    358   void MakeAppRootEnabled(int64 tracker_id, leveldb::WriteBatch* batch);
    359 
    360   void UnregisterTrackerAsAppRoot(const std::string& app_id,
    361                                   leveldb::WriteBatch* batch);
    362   void RemoveAllDescendantTrackers(int64 root_tracker_id,
    363                                    leveldb::WriteBatch* batch);
    364 
    365   void CreateTrackerForParentAndFileID(const FileTracker& parent_tracker,
    366                                        const std::string& file_id,
    367                                        leveldb::WriteBatch* batch);
    368   void CreateTrackerForParentAndFileMetadata(const FileTracker& parent_tracker,
    369                                              const FileMetadata& file_metadata,
    370                                              leveldb::WriteBatch* batch);
    371   void CreateTrackerInternal(const FileTracker& parent_tracker,
    372                              const std::string& file_id,
    373                              const FileDetails* details,
    374                              leveldb::WriteBatch* batch);
    375 
    376   void RemoveTracker(int64 tracker_id, leveldb::WriteBatch* batch);
    377   void RemoveTrackerIgnoringSameTitle(int64 tracker_id,
    378                                       leveldb::WriteBatch* batch);
    379   void RemoveTrackerInternal(int64 tracker_id,
    380                              leveldb::WriteBatch* batch,
    381                              bool ignoring_same_title);
    382   void MaybeAddTrackersForNewFile(const FileMetadata& file,
    383                                   leveldb::WriteBatch* batch);
    384 
    385   void MarkSingleTrackerDirty(FileTracker* tracker,
    386                               leveldb::WriteBatch* batch);
    387   void MarkTrackerSetDirty(TrackerSet* trackers,
    388                            leveldb::WriteBatch* batch);
    389   void MarkTrackersDirtyByFileID(const std::string& file_id,
    390                                  leveldb::WriteBatch* batch);
    391   void MarkTrackersDirtyByPath(int64 parent_tracker_id,
    392                                const std::string& title,
    393                                leveldb::WriteBatch* batch);
    394 
    395   void EraseTrackerFromFileIDIndex(FileTracker* tracker,
    396                                    leveldb::WriteBatch* batch);
    397   void EraseTrackerFromPathIndex(FileTracker* tracker);
    398   void EraseFileFromDatabase(const std::string& file_id,
    399                              leveldb::WriteBatch* batch);
    400 
    401   int64 GetNextTrackerID(leveldb::WriteBatch* batch);
    402 
    403   void RecursiveMarkTrackerAsDirty(int64 root_tracker_id,
    404                                    leveldb::WriteBatch* batch);
    405   bool CanActivateTracker(const FileTracker& tracker);
    406   bool ShouldKeepDirty(const FileTracker& tracker) const;
    407 
    408   bool HasDisabledAppRoot(const FileTracker& tracker) const;
    409   bool HasActiveTrackerForFileID(const std::string& file_id) const;
    410   bool HasActiveTrackerForPath(int64 parent_tracker,
    411                                const std::string& title) const;
    412 
    413   void UpdateByFileMetadata(const tracked_objects::Location& from_where,
    414                             scoped_ptr<FileMetadata> file,
    415                             leveldb::WriteBatch* batch);
    416 
    417   void WriteToDatabase(scoped_ptr<leveldb::WriteBatch> batch,
    418                        const SyncStatusCallback& callback);
    419 
    420   bool HasNewerFileMetadata(const std::string& file_id, int64 change_id);
    421 
    422   scoped_ptr<base::ListValue> DumpTrackers();
    423   scoped_ptr<base::ListValue> DumpMetadata();
    424 
    425   scoped_refptr<base::SequencedTaskRunner> task_runner_;
    426   base::FilePath database_path_;
    427   scoped_ptr<leveldb::DB> db_;
    428 
    429   scoped_ptr<ServiceMetadata> service_metadata_;
    430   int64 largest_known_change_id_;
    431 
    432   FileByID file_by_id_;  // Owned.
    433   TrackerByID tracker_by_id_;  // Owned.
    434 
    435   // Maps FileID to trackers.  The active tracker must be unique per FileID.
    436   // This must be updated when updating |active| field of a tracker.
    437   TrackersByFileID trackers_by_file_id_;  // Not owned.
    438 
    439   // Maps AppID to the app-root tracker.
    440   // This must be updated when a tracker is registered/unregistered as an
    441   // app-root.
    442   TrackerByAppID app_root_by_app_id_;  // Not owned.
    443 
    444   // Maps |tracker_id| to its children grouped by their |title|.
    445   // If the title is unknown for a tracker, treats its title as empty. Empty
    446   // titled file must not be active.
    447   // The active tracker must be unique per its parent_tracker and its title.
    448   // This must be updated when updating |title|, |active| or
    449   // |parent_tracker_id|.
    450   TrackersByParentAndTitle trackers_by_parent_and_title_;
    451 
    452   // Holds all trackers which marked as dirty.
    453   // This must be updated when updating |dirty| field of a tracker.
    454   DirtyTrackers dirty_trackers_;  // Not owned.
    455   DirtyTrackers low_priority_dirty_trackers_;  // Not owned.
    456 
    457   base::WeakPtrFactory<MetadataDatabase> weak_ptr_factory_;
    458 
    459   DISALLOW_COPY_AND_ASSIGN(MetadataDatabase);
    460 };
    461 
    462 }  // namespace drive_backend
    463 }  // namespace sync_file_system
    464 
    465 #endif  // CHROME_BROWSER_SYNC_FILE_SYSTEM_DRIVE_BACKEND_METADATA_DATABASE_H_
    466