1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import sys 6 7 from docs_server_utils import ToUnicode 8 from file_system import FileNotFoundError 9 from future import Future 10 from path_util import AssertIsDirectory, AssertIsFile, ToDirectory 11 from third_party.json_schema_compiler import json_parse 12 from third_party.json_schema_compiler.memoize import memoize 13 from third_party.motemplate import Motemplate 14 15 16 _CACHEABLE_FUNCTIONS = set() 17 _SINGLE_FILE_FUNCTIONS = set() 18 19 20 def _GetUnboundFunction(fn): 21 '''Functions bound to an object are separate from the unbound 22 defintion. This causes issues when checking for cache membership, 23 so always get the unbound function, if possible. 24 ''' 25 return getattr(fn, 'im_func', fn) 26 27 28 def Cache(fn): 29 '''A decorator which can be applied to the compilation function 30 passed to CompiledFileSystem.Create, indicating that file/list data 31 should be cached. 32 33 This decorator should be listed first in any list of decorators, along 34 with the SingleFile decorator below. 35 ''' 36 _CACHEABLE_FUNCTIONS.add(_GetUnboundFunction(fn)) 37 return fn 38 39 40 def SingleFile(fn): 41 '''A decorator which can be optionally applied to the compilation function 42 passed to CompiledFileSystem.Create, indicating that the function only 43 needs access to the file which is given in the function's callback. When 44 this is the case some optimisations can be done. 45 46 Note that this decorator must be listed first in any list of decorators to 47 have any effect. 48 ''' 49 _SINGLE_FILE_FUNCTIONS.add(_GetUnboundFunction(fn)) 50 return fn 51 52 53 def Unicode(fn): 54 '''A decorator which can be optionally applied to the compilation function 55 passed to CompiledFileSystem.Create, indicating that the function processes 56 the file's data as Unicode text. 57 ''' 58 59 # The arguments passed to fn can be (self, path, data) or (path, data). In 60 # either case the last argument is |data|, which should be converted to 61 # Unicode. 62 def convert_args(args): 63 args = list(args) 64 args[-1] = ToUnicode(args[-1]) 65 return args 66 67 return lambda *args: fn(*convert_args(args)) 68 69 70 class _CacheEntry(object): 71 def __init__(self, cache_data, version): 72 73 self.cache_data = cache_data 74 self.version = version 75 76 77 class CompiledFileSystem(object): 78 '''This class caches FileSystem data that has been processed. 79 ''' 80 81 class Factory(object): 82 '''A class to build a CompiledFileSystem backed by |file_system|. 83 ''' 84 85 def __init__(self, object_store_creator): 86 self._object_store_creator = object_store_creator 87 88 def Create(self, file_system, compilation_function, cls, category=None): 89 '''Creates a CompiledFileSystem view over |file_system| that populates 90 its cache by calling |compilation_function| with (path, data), where 91 |data| is the data that was fetched from |path| in |file_system|. 92 93 The namespace for the compiled file system is derived similar to 94 ObjectStoreCreator: from |cls| along with an optional |category|. 95 ''' 96 assert isinstance(cls, type) 97 assert not cls.__name__[0].islower() # guard against non-class types 98 full_name = [cls.__name__, file_system.GetIdentity()] 99 if category is not None: 100 full_name.append(category) 101 def create_object_store(my_category): 102 # The read caches can start populated (start_empty=False) because file 103 # updates are picked up by the stat - but only if the compilation 104 # function is affected by a single file. If the compilation function is 105 # affected by other files (e.g. compiling a list of APIs available to 106 # extensions may be affected by both a features file and the list of 107 # files in the API directory) then this optimisation won't work. 108 return self._object_store_creator.Create( 109 CompiledFileSystem, 110 category='/'.join(full_name + [my_category]), 111 start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS) 112 return CompiledFileSystem(file_system, 113 compilation_function, 114 create_object_store('file'), 115 create_object_store('list')) 116 117 @memoize 118 def ForJson(self, file_system): 119 '''A CompiledFileSystem specifically for parsing JSON configuration data. 120 These are memoized over file systems tied to different branches. 121 ''' 122 return self.Create(file_system, 123 Cache(SingleFile(lambda _, data: 124 json_parse.Parse(ToUnicode(data)))), 125 CompiledFileSystem, 126 category='json') 127 128 @memoize 129 def ForTemplates(self, file_system): 130 '''Creates a CompiledFileSystem for parsing templates. 131 ''' 132 return self.Create( 133 file_system, 134 SingleFile(lambda path, text: Motemplate(ToUnicode(text), name=path)), 135 CompiledFileSystem) 136 137 @memoize 138 def ForUnicode(self, file_system): 139 '''Creates a CompiledFileSystem for Unicode text processing. 140 ''' 141 return self.Create( 142 file_system, 143 SingleFile(lambda _, text: ToUnicode(text)), 144 CompiledFileSystem, 145 category='text') 146 147 def __init__(self, 148 file_system, 149 compilation_function, 150 file_object_store, 151 list_object_store): 152 self._file_system = file_system 153 self._compilation_function = compilation_function 154 self._file_object_store = file_object_store 155 self._list_object_store = list_object_store 156 157 def _Get(self, store, key): 158 if _GetUnboundFunction(self._compilation_function) in _CACHEABLE_FUNCTIONS: 159 return store.Get(key) 160 return Future(value=None) 161 162 def _Set(self, store, key, value): 163 if _GetUnboundFunction(self._compilation_function) in _CACHEABLE_FUNCTIONS: 164 store.Set(key, value) 165 166 def _RecursiveList(self, path): 167 '''Returns a Future containing the recursive directory listing of |path| as 168 a flat list of paths. 169 ''' 170 def split_dirs_from_files(paths): 171 '''Returns a tuple (dirs, files) where |dirs| contains the directory 172 names in |paths| and |files| contains the files. 173 ''' 174 result = [], [] 175 for path in paths: 176 result[0 if path.endswith('/') else 1].append(path) 177 return result 178 179 def add_prefix(prefix, paths): 180 return [prefix + path for path in paths] 181 182 # Read in the initial list of files. Do this eagerly (i.e. not part of the 183 # asynchronous Future contract) because there's a greater chance to 184 # parallelise fetching with the second layer (can fetch multiple paths). 185 try: 186 first_layer_dirs, first_layer_files = split_dirs_from_files( 187 self._file_system.ReadSingle(path).Get()) 188 except FileNotFoundError: 189 return Future(exc_info=sys.exc_info()) 190 191 if not first_layer_dirs: 192 return Future(value=first_layer_files) 193 194 def get_from_future_listing(listings): 195 '''Recursively lists files from directory listing |futures|. 196 ''' 197 dirs, files = [], [] 198 for dir_name, listing in listings.iteritems(): 199 new_dirs, new_files = split_dirs_from_files(listing) 200 # |dirs| are paths for reading. Add the full prefix relative to 201 # |path| so that |file_system| can find the files. 202 dirs += add_prefix(dir_name, new_dirs) 203 # |files| are not for reading, they are for returning to the caller. 204 # This entire function set (i.e. GetFromFileListing) is defined to 205 # not include the fetched-path in the result, however, |dir_name| 206 # will be prefixed with |path|. Strip it. 207 assert dir_name.startswith(path) 208 files += add_prefix(dir_name[len(path):], new_files) 209 if dirs: 210 files += self._file_system.Read(dirs).Then( 211 get_from_future_listing).Get() 212 return files 213 214 return self._file_system.Read(add_prefix(path, first_layer_dirs)).Then( 215 lambda results: first_layer_files + get_from_future_listing(results)) 216 217 def GetFromFile(self, path, skip_not_found=False): 218 '''Calls |compilation_function| on the contents of the file at |path|. 219 If |skip_not_found| is True, then None is passed to |compilation_function|. 220 ''' 221 AssertIsFile(path) 222 223 try: 224 version = self._file_system.Stat(path).version 225 except FileNotFoundError: 226 if skip_not_found: 227 version = None 228 else: 229 return Future(exc_info=sys.exc_info()) 230 231 cache_entry = self._Get(self._file_object_store, path).Get() 232 if (cache_entry is not None) and (version == cache_entry.version): 233 return Future(value=cache_entry.cache_data) 234 235 def compile_(files): 236 cache_data = self._compilation_function(path, files) 237 self._Set(self._file_object_store, path, _CacheEntry(cache_data, version)) 238 return cache_data 239 240 return self._file_system.ReadSingle( 241 path, skip_not_found=skip_not_found).Then(compile_) 242 243 def GetFromFileListing(self, path): 244 '''Calls |compilation_function| on the listing of the files at |path|. 245 Assumes that the path given is to a directory. 246 ''' 247 AssertIsDirectory(path) 248 249 try: 250 version = self._file_system.Stat(path).version 251 except FileNotFoundError: 252 return Future(exc_info=sys.exc_info()) 253 254 cache_entry = self._Get(self._list_object_store, path).Get() 255 if (cache_entry is not None) and (version == cache_entry.version): 256 return Future(value=cache_entry.cache_data) 257 258 def compile_(files): 259 cache_data = self._compilation_function(path, files) 260 self._Set(self._list_object_store, path, _CacheEntry(cache_data, version)) 261 return cache_data 262 return self._RecursiveList(path).Then(compile_) 263 264 # _GetFileVersionFromCache and _GetFileListingVersionFromCache are exposed 265 # *only* so that ChainedCompiledFileSystem can optimise its caches. *Do not* 266 # use these methods otherwise, they don't do what you want. Use 267 # FileSystem.Stat on the FileSystem that this CompiledFileSystem uses. 268 269 def _GetFileVersionFromCache(self, path): 270 cache_entry = self._Get(self._file_object_store, path).Get() 271 if cache_entry is not None: 272 return Future(value=cache_entry.version) 273 stat_future = self._file_system.StatAsync(path) 274 return Future(callback=lambda: stat_future.Get().version) 275 276 def _GetFileListingVersionFromCache(self, path): 277 path = ToDirectory(path) 278 cache_entry = self._Get(self._list_object_store, path).Get() 279 if cache_entry is not None: 280 return Future(value=cache_entry.version) 281 stat_future = self._file_system.StatAsync(path) 282 return Future(callback=lambda: stat_future.Get().version) 283 284 def GetIdentity(self): 285 return self._file_system.GetIdentity() 286
