1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import posixpath 6 7 from api_models import GetNodeCategories 8 from api_schema_graph import APISchemaGraph 9 from branch_utility import BranchUtility, ChannelInfo 10 from compiled_file_system import CompiledFileSystem, SingleFile, Unicode 11 from extensions_paths import API_PATHS, JSON_TEMPLATES 12 from features_bundle import FeaturesBundle 13 from file_system import FileNotFoundError 14 from schema_processor import SchemaProcessor 15 from third_party.json_schema_compiler.memoize import memoize 16 from third_party.json_schema_compiler.model import UnixName 17 18 19 _DEVTOOLS_API = 'devtools_api.json' 20 _EXTENSION_API = 'extension_api.json' 21 # The version where api_features.json is first available. 22 _API_FEATURES_MIN_VERSION = 28 23 # The version where permission_ and manifest_features.json are available and 24 # presented in the current format. 25 _ORIGINAL_FEATURES_MIN_VERSION = 20 26 # API schemas are aggregated in extension_api.json up to this version. 27 _EXTENSION_API_MAX_VERSION = 17 28 # The earliest version for which we have SVN data. 29 _SVN_MIN_VERSION = 5 30 31 32 def _GetChannelFromFeatures(api_name, features): 33 '''Finds API channel information for |api_name| from |features|. 34 Returns None if channel information for the API cannot be located. 35 ''' 36 feature = features.Get().get(api_name) 37 return feature.get('channel') if feature else None 38 39 40 def _GetChannelFromAPIFeatures(api_name, features_bundle): 41 return _GetChannelFromFeatures(api_name, features_bundle.GetAPIFeatures()) 42 43 44 def _GetChannelFromManifestFeatures(api_name, features_bundle): 45 # _manifest_features.json uses unix_style API names. 46 api_name = UnixName(api_name) 47 return _GetChannelFromFeatures(api_name, 48 features_bundle.GetManifestFeatures()) 49 50 51 def _GetChannelFromPermissionFeatures(api_name, features_bundle): 52 return _GetChannelFromFeatures(api_name, 53 features_bundle.GetPermissionFeatures()) 54 55 56 def _GetAPISchemaFilename(api_name, file_system, version): 57 '''Gets the name of the file which may contain the schema for |api_name| in 58 |file_system|, or None if the API is not found. Note that this may be the 59 single _EXTENSION_API file which all APIs share in older versions of Chrome, 60 in which case it is unknown whether the API actually exists there. 61 ''' 62 if version == 'master' or version > _ORIGINAL_FEATURES_MIN_VERSION: 63 # API schema filenames switch format to unix_hacker_style. 64 api_name = UnixName(api_name) 65 66 # Devtools API names have 'devtools.' prepended to them. 67 # The corresponding filenames do not. 68 if 'devtools_' in api_name: 69 api_name = api_name.replace('devtools_', '') 70 71 for api_path in API_PATHS: 72 try: 73 for base, _, filenames in file_system.Walk(api_path): 74 for ext in ('json', 'idl'): 75 filename = '%s.%s' % (api_name, ext) 76 if filename in filenames: 77 return posixpath.join(api_path, base, filename) 78 if _EXTENSION_API in filenames: 79 return posixpath.join(api_path, base, _EXTENSION_API) 80 except FileNotFoundError: 81 continue 82 return None 83 84 85 class AvailabilityInfo(object): 86 '''Represents availability data for an API. |scheduled| is a version number 87 specifying when dev and beta APIs will become stable, or None if that data 88 is unknown. 89 ''' 90 def __init__(self, channel_info, scheduled=None): 91 assert isinstance(channel_info, ChannelInfo) 92 assert isinstance(scheduled, int) or scheduled is None 93 self.channel_info = channel_info 94 self.scheduled = scheduled 95 96 def __eq__(self, other): 97 return self.__dict__ == other.__dict__ 98 99 def __ne__(self, other): 100 return not (self == other) 101 102 def __repr__(self): 103 return '%s%s' % (type(self).__name__, repr(self.__dict__)) 104 105 def __str__(self): 106 return repr(self) 107 108 109 class AvailabilityFinder(object): 110 '''Generates availability information for APIs by looking at API schemas and 111 _features files over multiple release versions of Chrome. 112 ''' 113 114 def __init__(self, 115 branch_utility, 116 compiled_fs_factory, 117 file_system_iterator, 118 host_file_system, 119 object_store_creator, 120 platform, 121 schema_processor_factory): 122 self._branch_utility = branch_utility 123 self._compiled_fs_factory = compiled_fs_factory 124 self._file_system_iterator = file_system_iterator 125 self._host_file_system = host_file_system 126 self._object_store_creator = object_store_creator 127 def create_object_store(category): 128 return object_store_creator.Create( 129 AvailabilityFinder, category='/'.join((platform, category))) 130 self._top_level_object_store = create_object_store('top_level') 131 self._node_level_object_store = create_object_store('node_level') 132 self._json_fs = compiled_fs_factory.ForJson(self._host_file_system) 133 self._platform = platform 134 # When processing the API schemas, we retain inlined types in the schema 135 # so that there are not missing nodes in the APISchemaGraphs when trying 136 # to lookup availability. 137 self._schema_processor = schema_processor_factory.Create(True) 138 139 def _GetPredeterminedAvailability(self, api_name): 140 '''Checks a configuration file for hardcoded (i.e. predetermined) 141 availability information for an API. 142 ''' 143 api_info = self._json_fs.GetFromFile( 144 JSON_TEMPLATES + 'api_availabilities.json').Get().get(api_name) 145 if api_info is None: 146 return None 147 if api_info['channel'] == 'stable': 148 return AvailabilityInfo( 149 self._branch_utility.GetStableChannelInfo(api_info['version'])) 150 return AvailabilityInfo( 151 self._branch_utility.GetChannelInfo(api_info['channel'])) 152 153 @memoize 154 def _CreateAPISchemaFileSystem(self, file_system): 155 '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema 156 data and formatting it so that it can be used to create APISchemaGraphs. 157 ''' 158 def process_schema(path, data): 159 return self._schema_processor.Process(path, data) 160 return self._compiled_fs_factory.Create(file_system, 161 SingleFile(Unicode(process_schema)), 162 CompiledFileSystem, 163 category='api-schema') 164 165 def _GetAPISchema(self, api_name, file_system, version): 166 '''Searches |file_system| for |api_name|'s API schema data, and processes 167 and returns it if found. 168 ''' 169 api_filename = _GetAPISchemaFilename(api_name, file_system, version) 170 if api_filename is None: 171 # No file for the API could be found in the given |file_system|. 172 return None 173 174 schema_fs = self._CreateAPISchemaFileSystem(file_system) 175 api_schemas = schema_fs.GetFromFile(api_filename).Get() 176 matching_schemas = [api for api in api_schemas 177 if api['namespace'] == api_name] 178 # There should only be a single matching schema per file, or zero in the 179 # case of no API data being found in _EXTENSION_API. 180 assert len(matching_schemas) <= 1 181 return matching_schemas or None 182 183 def _HasAPISchema(self, api_name, file_system, version): 184 '''Whether or not an API schema for |api_name| exists in the given 185 |file_system|. 186 ''' 187 filename = _GetAPISchemaFilename(api_name, file_system, version) 188 if filename is None: 189 return False 190 if filename.endswith(_EXTENSION_API) or filename.endswith(_DEVTOOLS_API): 191 return self._GetAPISchema(api_name, file_system, version) is not None 192 return True 193 194 def _CheckStableAvailability(self, 195 api_name, 196 file_system, 197 version, 198 earliest_version=None): 199 '''Checks for availability of an API, |api_name|, on the stable channel. 200 Considers several _features.json files, file system existence, and 201 extension_api.json depending on the given |version|. 202 |earliest_version| is the version of Chrome at which |api_name| first became 203 available. It should only be given when checking stable availability for 204 API nodes, so it can be used as an alternative to the check for filesystem 205 existence. 206 ''' 207 earliest_version = earliest_version or _SVN_MIN_VERSION 208 if version < earliest_version: 209 # SVN data isn't available below this version. 210 return False 211 features_bundle = self._CreateFeaturesBundle(file_system) 212 available_channel = None 213 if version >= _API_FEATURES_MIN_VERSION: 214 # The _api_features.json file first appears in version 28 and should be 215 # the most reliable for finding API availability. 216 available_channel = _GetChannelFromAPIFeatures(api_name, 217 features_bundle) 218 if version >= _ORIGINAL_FEATURES_MIN_VERSION: 219 # The _permission_features.json and _manifest_features.json files are 220 # present in Chrome 20 and onwards. Use these if no information could be 221 # found using _api_features.json. 222 available_channel = ( 223 available_channel or 224 _GetChannelFromPermissionFeatures(api_name, features_bundle) or 225 _GetChannelFromManifestFeatures(api_name, features_bundle)) 226 if available_channel is not None: 227 return available_channel == 'stable' 228 229 # |earliest_version| == _SVN_MIN_VERSION implies we're dealing with an API. 230 # Fall back to a check for file system existence if the API is not 231 # stable in any of the _features.json files, or if the _features files 232 # do not exist (version 19 and earlier). 233 if earliest_version == _SVN_MIN_VERSION: 234 return self._HasAPISchema(api_name, file_system, version) 235 # For API nodes, assume it's available if |version| is greater than the 236 # version the node became available (which it is, because of the first 237 # check). 238 return True 239 240 def _CheckChannelAvailability(self, api_name, file_system, channel_info): 241 '''Searches through the _features files in a given |file_system|, falling 242 back to checking the file system for API schema existence, to determine 243 whether or not an API is available on the given channel, |channel_info|. 244 ''' 245 features_bundle = self._CreateFeaturesBundle(file_system) 246 available_channel = ( 247 _GetChannelFromAPIFeatures(api_name, features_bundle) or 248 _GetChannelFromPermissionFeatures(api_name, features_bundle) or 249 _GetChannelFromManifestFeatures(api_name, features_bundle)) 250 if (available_channel is None and 251 self._HasAPISchema(api_name, file_system, channel_info.version)): 252 # If an API is not represented in any of the _features files, but exists 253 # in the filesystem, then assume it is available in this version. 254 # The chrome.windows API is an example of this. 255 available_channel = channel_info.channel 256 # If the channel we're checking is the same as or newer than the 257 # |available_channel| then the API is available at this channel. 258 newest = BranchUtility.NewestChannel((available_channel, 259 channel_info.channel)) 260 return available_channel is not None and newest == channel_info.channel 261 262 def _CheckChannelAvailabilityForNode(self, 263 node_name, 264 file_system, 265 channel_info, 266 earliest_channel_info): 267 '''Searches through the _features files in a given |file_system| to 268 determine whether or not an API node is available on the given channel, 269 |channel_info|. |earliest_channel_info| is the earliest channel the node 270 was introduced. 271 ''' 272 features_bundle = self._CreateFeaturesBundle(file_system) 273 available_channel = None 274 # Only API nodes can have their availability overriden on a per-node basis, 275 # so we only need to check _api_features.json. 276 if channel_info.version >= _API_FEATURES_MIN_VERSION: 277 available_channel = _GetChannelFromAPIFeatures(node_name, features_bundle) 278 if (available_channel is None and 279 channel_info.version >= earliest_channel_info.version): 280 # Most API nodes inherit their availabiltity from their parent, so don't 281 # explicitly appear in _api_features.json. For example, "tabs.create" 282 # isn't listed; it inherits from "tabs". Assume these are available at 283 # |channel_info|. 284 available_channel = channel_info.channel 285 newest = BranchUtility.NewestChannel((available_channel, 286 channel_info.channel)) 287 return available_channel is not None and newest == channel_info.channel 288 289 @memoize 290 def _CreateFeaturesBundle(self, file_system): 291 return FeaturesBundle(file_system, 292 self._compiled_fs_factory, 293 self._object_store_creator, 294 self._platform) 295 296 def _CheckAPIAvailability(self, api_name, file_system, channel_info): 297 '''Determines the availability for an API at a certain version of Chrome. 298 Two branches of logic are used depending on whether or not the API is 299 determined to be 'stable' at the given version. 300 ''' 301 if channel_info.channel == 'stable': 302 return self._CheckStableAvailability(api_name, 303 file_system, 304 channel_info.version) 305 return self._CheckChannelAvailability(api_name, 306 file_system, 307 channel_info) 308 309 def _FindScheduled(self, api_name, earliest_version=None): 310 '''Determines the earliest version of Chrome where the API is stable. 311 Unlike the code in GetAPIAvailability, this checks if the API is stable 312 even when Chrome is in dev or beta, which shows that the API is scheduled 313 to be stable in that verison of Chrome. |earliest_version| is the version 314 |api_name| became first available. Only use it when finding scheduled 315 availability for nodes. 316 ''' 317 def check_scheduled(file_system, channel_info): 318 return self._CheckStableAvailability(api_name, 319 file_system, 320 channel_info.version, 321 earliest_version=earliest_version) 322 323 stable_channel = self._file_system_iterator.Descending( 324 self._branch_utility.GetChannelInfo('dev'), check_scheduled) 325 326 return stable_channel.version if stable_channel else None 327 328 def _CheckAPINodeAvailability(self, node_name, earliest_channel_info): 329 '''Gets availability data for a node by checking _features files. 330 ''' 331 def check_node_availability(file_system, channel_info): 332 return self._CheckChannelAvailabilityForNode(node_name, 333 file_system, 334 channel_info, 335 earliest_channel_info) 336 channel_info = (self._file_system_iterator.Descending( 337 self._branch_utility.GetChannelInfo('dev'), check_node_availability) or 338 earliest_channel_info) 339 340 if channel_info.channel == 'stable': 341 scheduled = None 342 else: 343 scheduled = self._FindScheduled( 344 node_name, 345 earliest_version=earliest_channel_info.version) 346 347 return AvailabilityInfo(channel_info, scheduled=scheduled) 348 349 def GetAPIAvailability(self, api_name): 350 '''Performs a search for an API's top-level availability by using a 351 HostFileSystemIterator instance to traverse multiple version of the 352 SVN filesystem. 353 ''' 354 availability = self._top_level_object_store.Get(api_name).Get() 355 if availability is not None: 356 return availability 357 358 # Check for predetermined availability and cache this information if found. 359 availability = self._GetPredeterminedAvailability(api_name) 360 if availability is not None: 361 self._top_level_object_store.Set(api_name, availability) 362 return availability 363 364 def check_api_availability(file_system, channel_info): 365 return self._CheckAPIAvailability(api_name, file_system, channel_info) 366 367 channel_info = self._file_system_iterator.Descending( 368 self._branch_utility.GetChannelInfo('dev'), 369 check_api_availability) 370 if channel_info is None: 371 # The API wasn't available on 'dev', so it must be a 'master'-only API. 372 channel_info = self._branch_utility.GetChannelInfo('master') 373 374 # If the API is not stable, check when it will be scheduled to be stable. 375 if channel_info.channel == 'stable': 376 scheduled = None 377 else: 378 scheduled = self._FindScheduled(api_name) 379 380 availability = AvailabilityInfo(channel_info, scheduled=scheduled) 381 382 self._top_level_object_store.Set(api_name, availability) 383 return availability 384 385 def GetAPINodeAvailability(self, api_name): 386 '''Returns an APISchemaGraph annotated with each node's availability (the 387 ChannelInfo at the oldest channel it's available in). 388 ''' 389 availability_graph = self._node_level_object_store.Get(api_name).Get() 390 if availability_graph is not None: 391 return availability_graph 392 393 def assert_not_none(value): 394 assert value is not None 395 return value 396 397 availability_graph = APISchemaGraph() 398 host_fs = self._host_file_system 399 master_stat = assert_not_none(host_fs.Stat(_GetAPISchemaFilename( 400 api_name, host_fs, 'master'))) 401 402 # Weird object thing here because nonlocal is Python 3. 403 previous = type('previous', (object,), {'stat': None, 'graph': None}) 404 405 def update_availability_graph(file_system, channel_info): 406 # If we can't find a filename, skip checking at this branch. 407 # For example, something could have a predetermined availability of 23, 408 # but it doesn't show up in the file system until 26. 409 # We know that the file will become available at some point. 410 # 411 # The problem with this is that at the first version where the API file 412 # exists, we'll get a huge chunk of new objects that don't match 413 # the predetermined API availability. 414 version_filename = _GetAPISchemaFilename(api_name, 415 file_system, 416 channel_info.version) 417 if version_filename is None: 418 # Continue the loop at the next version. 419 return True 420 421 version_stat = assert_not_none(file_system.Stat(version_filename)) 422 423 # Important optimisation: only re-parse the graph if the file changed in 424 # the last revision. Parsing the same schema and forming a graph on every 425 # iteration is really expensive. 426 if version_stat == previous.stat: 427 version_graph = previous.graph 428 else: 429 # Keep track of any new schema elements from this version by adding 430 # them to |availability_graph|. 431 # 432 # Calling |availability_graph|.Lookup() on the nodes being updated 433 # will return the |annotation| object -- the current |channel_info|. 434 version_graph = APISchemaGraph( 435 api_schema=self._GetAPISchema(api_name, 436 file_system, 437 channel_info.version)) 438 def annotator(node_name): 439 return self._CheckAPINodeAvailability('%s.%s' % (api_name, node_name), 440 channel_info) 441 442 availability_graph.Update(version_graph.Subtract(availability_graph), 443 annotator) 444 445 previous.stat = version_stat 446 previous.graph = version_graph 447 448 # Continue looping until there are no longer differences between this 449 # version and master. 450 return version_stat != master_stat 451 452 self._file_system_iterator.Ascending( 453 self.GetAPIAvailability(api_name).channel_info, 454 update_availability_graph) 455 456 self._node_level_object_store.Set(api_name, availability_graph) 457 return availability_graph 458
