Home | History | Annotate | Download | only in server2 
      1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 from file_system import FileSystem, StatInfo, FileNotFoundError
      6 from future import Future
      7 from object_store_creator import ObjectStoreCreator
      8 
      9 class _AsyncUncachedFuture(object):
     10   def __init__(self,
     11                uncached_read_futures,
     12                stats_for_uncached,
     13                current_results,
     14                file_system,
     15                object_store):
     16     self._uncached_read_futures = uncached_read_futures
     17     self._stats_for_uncached = stats_for_uncached
     18     self._current_results = current_results
     19     self._file_system = file_system
     20     self._object_store = object_store
     21 
     22   def Get(self):
     23     new_results = self._uncached_read_futures.Get()
     24     # Update the cached data in the object store. This is a path -> (read,
     25     # version) mapping.
     26     self._object_store.SetMulti(dict(
     27         (path, (new_result, self._stats_for_uncached[path].version))
     28         for path, new_result in new_results.iteritems()))
     29     new_results.update(self._current_results)
     30     return new_results
     31 
     32 class CachingFileSystem(FileSystem):
     33   '''FileSystem which implements a caching layer on top of |file_system|. It's
     34   smart, using Stat() to decided whether to skip Read()ing from |file_system|,
     35   and only Stat()ing directories never files.
     36   '''
     37   def __init__(self, file_system, object_store_creator):
     38     self._file_system = file_system
     39     def create_object_store(category, **optargs):
     40       return object_store_creator.Create(
     41           CachingFileSystem,
     42           category='%s/%s' % (file_system.GetIdentity(), category),
     43           **optargs)
     44     self._stat_object_store = create_object_store('stat')
     45     # The read caches can both (a) start populated and (b) be shared with all
     46     # other app versions, because the data changing is detected by the stat.
     47     # Without this optimisation, bumping app version is extremely slow.
     48     self._read_object_store = create_object_store(
     49         'read', start_empty=False, app_version=None)
     50     self._read_binary_object_store = create_object_store(
     51         'read-binary', start_empty=False, app_version=None)
     52 
     53   def Stat(self, path):
     54     '''Stats the directory given, or if a file is given, stats the file's parent
     55     directory to get info about the file.
     56     '''
     57     # Always stat the parent directory, since it will have the stat of the child
     58     # anyway, and this gives us an entire directory's stat info at once.
     59     if path.endswith('/'):
     60       dir_path = path
     61     else:
     62       dir_path, file_path = path.rsplit('/', 1)
     63       dir_path += '/'
     64 
     65     # ... and we only ever need to cache the dir stat, too.
     66     dir_stat = self._stat_object_store.Get(dir_path).Get()
     67     if dir_stat is None:
     68       dir_stat = self._file_system.Stat(dir_path)
     69       assert dir_stat is not None  # should raise a FileNotFoundError
     70       self._stat_object_store.Set(dir_path, dir_stat)
     71 
     72     if path == dir_path:
     73       stat_info = dir_stat
     74     else:
     75       file_version = dir_stat.child_versions.get(file_path)
     76       if file_version is None:
     77         raise FileNotFoundError('No stat found for %s in %s' % (path, dir_path))
     78       stat_info = StatInfo(file_version)
     79 
     80     return stat_info
     81 
     82   def Read(self, paths, binary=False):
     83     '''Reads a list of files. If a file is in memcache and it is not out of
     84     date, it is returned. Otherwise, the file is retrieved from the file system.
     85     '''
     86     read_object_store = (self._read_binary_object_store if binary else
     87                          self._read_object_store)
     88     read_values = read_object_store.GetMulti(paths).Get()
     89     stat_values = self._stat_object_store.GetMulti(paths).Get()
     90     results = {}  # maps path to read value
     91     uncached = {}  # maps path to stat value
     92     for path in paths:
     93       stat_value = stat_values.get(path)
     94       if stat_value is None:
     95         # TODO(cduvall): do a concurrent Stat with the missing stat values.
     96         stat_value = self.Stat(path)
     97       read_value = read_values.get(path)
     98       if read_value is None:
     99         uncached[path] = stat_value
    100         continue
    101       read_data, read_version = read_value
    102       if stat_value.version != read_version:
    103         uncached[path] = stat_value
    104         continue
    105       results[path] = read_data
    106 
    107     if not uncached:
    108       return Future(value=results)
    109 
    110     return Future(delegate=_AsyncUncachedFuture(
    111         self._file_system.Read(uncached.keys(), binary=binary),
    112         uncached,
    113         results,
    114         self,
    115         read_object_store))
    116 
    117   def GetIdentity(self):
    118     return self._file_system.GetIdentity()
    119