1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import sys 6 7 import schema_util 8 from docs_server_utils import ToUnicode 9 from file_system import FileNotFoundError 10 from future import Future 11 from path_util import AssertIsDirectory, AssertIsFile 12 from third_party.handlebar import Handlebar 13 from third_party.json_schema_compiler import json_parse 14 from third_party.json_schema_compiler.memoize import memoize 15 16 17 _SINGLE_FILE_FUNCTIONS = set() 18 19 20 def SingleFile(fn): 21 '''A decorator which can be optionally applied to the compilation function 22 passed to CompiledFileSystem.Create, indicating that the function only 23 needs access to the file which is given in the function's callback. When 24 this is the case some optimisations can be done. 25 26 Note that this decorator must be listed first in any list of decorators to 27 have any effect. 28 ''' 29 _SINGLE_FILE_FUNCTIONS.add(fn) 30 return fn 31 32 33 def Unicode(fn): 34 '''A decorator which can be optionally applied to the compilation function 35 passed to CompiledFileSystem.Create, indicating that the function processes 36 the file's data as Unicode text. 37 ''' 38 39 # The arguments passed to fn can be (self, path, data) or (path, data). In 40 # either case the last argument is |data|, which should be converted to 41 # Unicode. 42 def convert_args(args): 43 args = list(args) 44 args[-1] = ToUnicode(args[-1]) 45 return args 46 47 return lambda *args: fn(*convert_args(args)) 48 49 50 class _CacheEntry(object): 51 def __init__(self, cache_data, version): 52 53 self._cache_data = cache_data 54 self.version = version 55 56 57 class CompiledFileSystem(object): 58 '''This class caches FileSystem data that has been processed. 59 ''' 60 61 class Factory(object): 62 '''A class to build a CompiledFileSystem backed by |file_system|. 63 ''' 64 65 def __init__(self, object_store_creator): 66 self._object_store_creator = object_store_creator 67 68 def Create(self, file_system, compilation_function, cls, category=None): 69 '''Creates a CompiledFileSystem view over |file_system| that populates 70 its cache by calling |compilation_function| with (path, data), where 71 |data| is the data that was fetched from |path| in |file_system|. 72 73 The namespace for the compiled file system is derived similar to 74 ObjectStoreCreator: from |cls| along with an optional |category|. 75 ''' 76 assert isinstance(cls, type) 77 assert not cls.__name__[0].islower() # guard against non-class types 78 full_name = [cls.__name__, file_system.GetIdentity()] 79 if category is not None: 80 full_name.append(category) 81 def create_object_store(my_category): 82 # The read caches can start populated (start_empty=False) because file 83 # updates are picked up by the stat - but only if the compilation 84 # function is affected by a single file. If the compilation function is 85 # affected by other files (e.g. compiling a list of APIs available to 86 # extensions may be affected by both a features file and the list of 87 # files in the API directory) then this optimisation won't work. 88 return self._object_store_creator.Create( 89 CompiledFileSystem, 90 category='/'.join(full_name + [my_category]), 91 start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS) 92 return CompiledFileSystem(file_system, 93 compilation_function, 94 create_object_store('file'), 95 create_object_store('list')) 96 97 @memoize 98 def ForJson(self, file_system): 99 '''A CompiledFileSystem specifically for parsing JSON configuration data. 100 These are memoized over file systems tied to different branches. 101 ''' 102 return self.Create(file_system, 103 SingleFile(lambda _, data: 104 json_parse.Parse(ToUnicode(data))), 105 CompiledFileSystem, 106 category='json') 107 108 @memoize 109 def ForAPISchema(self, file_system): 110 '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema 111 data and formatting it so that it can be used by other classes, such 112 as Model and APISchemaGraph. 113 ''' 114 return self.Create(file_system, 115 SingleFile(Unicode(schema_util.ProcessSchema)), 116 CompiledFileSystem, 117 category='api-schema') 118 119 @memoize 120 def ForTemplates(self, file_system): 121 '''Creates a CompiledFileSystem for parsing templates. 122 ''' 123 return self.Create( 124 file_system, 125 SingleFile(lambda path, text: Handlebar(ToUnicode(text), name=path)), 126 CompiledFileSystem) 127 128 @memoize 129 def ForUnicode(self, file_system): 130 '''Creates a CompiledFileSystem for Unicode text processing. 131 ''' 132 return self.Create( 133 file_system, 134 SingleFile(lambda _, text: ToUnicode(text)), 135 CompiledFileSystem, 136 category='text') 137 138 def __init__(self, 139 file_system, 140 compilation_function, 141 file_object_store, 142 list_object_store): 143 self._file_system = file_system 144 self._compilation_function = compilation_function 145 self._file_object_store = file_object_store 146 self._list_object_store = list_object_store 147 148 def _RecursiveList(self, path): 149 '''Returns a Future containing the recursive directory listing of |path| as 150 a flat list of paths. 151 ''' 152 def split_dirs_from_files(paths): 153 '''Returns a tuple (dirs, files) where |dirs| contains the directory 154 names in |paths| and |files| contains the files. 155 ''' 156 result = [], [] 157 for path in paths: 158 result[0 if path.endswith('/') else 1].append(path) 159 return result 160 161 def add_prefix(prefix, paths): 162 return [prefix + path for path in paths] 163 164 # Read in the initial list of files. Do this eagerly (i.e. not part of the 165 # asynchronous Future contract) because there's a greater chance to 166 # parallelise fetching with the second layer (can fetch multiple paths). 167 try: 168 first_layer_dirs, first_layer_files = split_dirs_from_files( 169 self._file_system.ReadSingle(path).Get()) 170 except FileNotFoundError: 171 return Future(exc_info=sys.exc_info()) 172 173 if not first_layer_dirs: 174 return Future(value=first_layer_files) 175 176 second_layer_listing = self._file_system.Read( 177 add_prefix(path, first_layer_dirs)) 178 179 def resolve(): 180 def get_from_future_listing(futures): 181 '''Recursively lists files from directory listing |futures|. 182 ''' 183 dirs, files = [], [] 184 for dir_name, listing in futures.Get().iteritems(): 185 new_dirs, new_files = split_dirs_from_files(listing) 186 # |dirs| are paths for reading. Add the full prefix relative to 187 # |path| so that |file_system| can find the files. 188 dirs += add_prefix(dir_name, new_dirs) 189 # |files| are not for reading, they are for returning to the caller. 190 # This entire function set (i.e. GetFromFileListing) is defined to 191 # not include the fetched-path in the result, however, |dir_name| 192 # will be prefixed with |path|. Strip it. 193 assert dir_name.startswith(path) 194 files += add_prefix(dir_name[len(path):], new_files) 195 if dirs: 196 files += get_from_future_listing(self._file_system.Read(dirs)) 197 return files 198 199 return first_layer_files + get_from_future_listing(second_layer_listing) 200 201 return Future(callback=resolve) 202 203 def GetFromFile(self, path): 204 '''Calls |compilation_function| on the contents of the file at |path|. If 205 |binary| is True then the file will be read as binary - but this will only 206 apply for the first time the file is fetched; if already cached, |binary| 207 will be ignored. 208 ''' 209 AssertIsFile(path) 210 211 try: 212 version = self._file_system.Stat(path).version 213 except FileNotFoundError: 214 return Future(exc_info=sys.exc_info()) 215 216 cache_entry = self._file_object_store.Get(path).Get() 217 if (cache_entry is not None) and (version == cache_entry.version): 218 return Future(value=cache_entry._cache_data) 219 220 future_files = self._file_system.ReadSingle(path) 221 def resolve(): 222 cache_data = self._compilation_function(path, future_files.Get()) 223 self._file_object_store.Set(path, _CacheEntry(cache_data, version)) 224 return cache_data 225 return Future(callback=resolve) 226 227 def GetFromFileListing(self, path): 228 '''Calls |compilation_function| on the listing of the files at |path|. 229 Assumes that the path given is to a directory. 230 ''' 231 AssertIsDirectory(path) 232 233 try: 234 version = self._file_system.Stat(path).version 235 except FileNotFoundError: 236 return Future(exc_info=sys.exc_info()) 237 238 cache_entry = self._list_object_store.Get(path).Get() 239 if (cache_entry is not None) and (version == cache_entry.version): 240 return Future(value=cache_entry._cache_data) 241 242 recursive_list_future = self._RecursiveList(path) 243 def resolve(): 244 cache_data = self._compilation_function(path, recursive_list_future.Get()) 245 self._list_object_store.Set(path, _CacheEntry(cache_data, version)) 246 return cache_data 247 return Future(callback=resolve) 248 249 def GetFileVersion(self, path): 250 cache_entry = self._file_object_store.Get(path).Get() 251 if cache_entry is not None: 252 return cache_entry.version 253 return self._file_system.Stat(path).version 254 255 def GetFileListingVersion(self, path): 256 if not path.endswith('/'): 257 path += '/' 258 cache_entry = self._list_object_store.Get(path).Get() 259 if cache_entry is not None: 260 return cache_entry.version 261 return self._file_system.Stat(path).version 262 263 def FileExists(self, path): 264 return self._file_system.Exists(path) 265 266 def GetIdentity(self): 267 return self._file_system.GetIdentity() 268
