1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 from file_system import FileSystem, StatInfo, FileNotFoundError 6 from future import Future 7 from object_store_creator import ObjectStoreCreator 8 9 class _AsyncUncachedFuture(object): 10 def __init__(self, 11 uncached_read_futures, 12 stats_for_uncached, 13 current_results, 14 file_system, 15 object_store): 16 self._uncached_read_futures = uncached_read_futures 17 self._stats_for_uncached = stats_for_uncached 18 self._current_results = current_results 19 self._file_system = file_system 20 self._object_store = object_store 21 22 def Get(self): 23 new_results = self._uncached_read_futures.Get() 24 # Update the cached data in the object store. This is a path -> (read, 25 # version) mapping. 26 self._object_store.SetMulti(dict( 27 (path, (new_result, self._stats_for_uncached[path].version)) 28 for path, new_result in new_results.iteritems())) 29 new_results.update(self._current_results) 30 return new_results 31 32 class CachingFileSystem(FileSystem): 33 '''FileSystem which implements a caching layer on top of |file_system|. It's 34 smart, using Stat() to decided whether to skip Read()ing from |file_system|, 35 and only Stat()ing directories never files. 36 ''' 37 def __init__(self, file_system, object_store_creator): 38 self._file_system = file_system 39 def create_object_store(category, **optargs): 40 return object_store_creator.Create( 41 CachingFileSystem, 42 category='%s/%s' % (file_system.GetIdentity(), category), 43 **optargs) 44 self._stat_object_store = create_object_store('stat') 45 # The read caches can both (a) start populated and (b) be shared with all 46 # other app versions, because the data changing is detected by the stat. 47 # Without this optimisation, bumping app version is extremely slow. 48 self._read_object_store = create_object_store( 49 'read', start_empty=False, app_version=None) 50 self._read_binary_object_store = create_object_store( 51 'read-binary', start_empty=False, app_version=None) 52 53 def Stat(self, path): 54 '''Stats the directory given, or if a file is given, stats the file's parent 55 directory to get info about the file. 56 ''' 57 # Always stat the parent directory, since it will have the stat of the child 58 # anyway, and this gives us an entire directory's stat info at once. 59 if path.endswith('/'): 60 dir_path = path 61 else: 62 dir_path, file_path = path.rsplit('/', 1) 63 dir_path += '/' 64 65 # ... and we only ever need to cache the dir stat, too. 66 dir_stat = self._stat_object_store.Get(dir_path).Get() 67 if dir_stat is None: 68 dir_stat = self._file_system.Stat(dir_path) 69 assert dir_stat is not None # should raise a FileNotFoundError 70 self._stat_object_store.Set(dir_path, dir_stat) 71 72 if path == dir_path: 73 stat_info = dir_stat 74 else: 75 file_version = dir_stat.child_versions.get(file_path) 76 if file_version is None: 77 raise FileNotFoundError('No stat found for %s in %s' % (path, dir_path)) 78 stat_info = StatInfo(file_version) 79 80 return stat_info 81 82 def Read(self, paths, binary=False): 83 '''Reads a list of files. If a file is in memcache and it is not out of 84 date, it is returned. Otherwise, the file is retrieved from the file system. 85 ''' 86 read_object_store = (self._read_binary_object_store if binary else 87 self._read_object_store) 88 read_values = read_object_store.GetMulti(paths).Get() 89 stat_values = self._stat_object_store.GetMulti(paths).Get() 90 results = {} # maps path to read value 91 uncached = {} # maps path to stat value 92 for path in paths: 93 stat_value = stat_values.get(path) 94 if stat_value is None: 95 # TODO(cduvall): do a concurrent Stat with the missing stat values. 96 stat_value = self.Stat(path) 97 read_value = read_values.get(path) 98 if read_value is None: 99 uncached[path] = stat_value 100 continue 101 read_data, read_version = read_value 102 if stat_value.version != read_version: 103 uncached[path] = stat_value 104 continue 105 results[path] = read_data 106 107 if not uncached: 108 return Future(value=results) 109 110 return Future(delegate=_AsyncUncachedFuture( 111 self._file_system.Read(uncached.keys(), binary=binary), 112 uncached, 113 results, 114 self, 115 read_object_store)) 116 117 def GetIdentity(self): 118 return self._file_system.GetIdentity() 119