1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 ''' 7 Checks a policy_templates.json file for conformity to its syntax specification. 8 ''' 9 10 import json 11 import optparse 12 import os 13 import re 14 import sys 15 16 17 LEADING_WHITESPACE = re.compile('^([ \t]*)') 18 TRAILING_WHITESPACE = re.compile('.*?([ \t]+)$') 19 # Matches all non-empty strings that contain no whitespaces. 20 NO_WHITESPACE = re.compile('[^\s]+$') 21 22 # Convert a 'type' to the schema types it may be converted to. 23 # The 'dict' type represents structured JSON data, and can be converted 24 # to an 'object' or an 'array'. 25 TYPE_TO_SCHEMA = { 26 'int': [ 'integer' ], 27 'list': [ 'array' ], 28 'dict': [ 'object', 'array' ], 29 'main': [ 'boolean' ], 30 'string': [ 'string' ], 31 'int-enum': [ 'integer' ], 32 'string-enum': [ 'string' ], 33 'external': [ 'object' ], 34 } 35 36 # List of boolean policies that have been introduced with negative polarity in 37 # the past and should not trigger the negative polarity check. 38 LEGACY_INVERTED_POLARITY_WHITELIST = [ 39 'DeveloperToolsDisabled', 40 'DeviceAutoUpdateDisabled', 41 'Disable3DAPIs', 42 'DisableAuthNegotiateCnameLookup', 43 'DisablePluginFinder', 44 'DisablePrintPreview', 45 'DisableSafeBrowsingProceedAnyway', 46 'DisableScreenshots', 47 'DisableSpdy', 48 'DisableSSLRecordSplitting', 49 'DriveDisabled', 50 'DriveDisabledOverCellular', 51 'ExternalStorageDisabled', 52 'SavingBrowserHistoryDisabled', 53 'SyncDisabled', 54 ] 55 56 class PolicyTemplateChecker(object): 57 58 def __init__(self): 59 self.error_count = 0 60 self.warning_count = 0 61 self.num_policies = 0 62 self.num_groups = 0 63 self.num_policies_in_groups = 0 64 self.options = None 65 self.features = [] 66 67 def _Error(self, message, parent_element=None, identifier=None, 68 offending_snippet=None): 69 self.error_count += 1 70 error = '' 71 if identifier is not None and parent_element is not None: 72 error += 'In %s %s: ' % (parent_element, identifier) 73 print error + 'Error: ' + message 74 if offending_snippet is not None: 75 print ' Offending:', json.dumps(offending_snippet, indent=2) 76 77 def _CheckContains(self, container, key, value_type, 78 optional=False, 79 parent_element='policy', 80 container_name=None, 81 identifier=None, 82 offending='__CONTAINER__', 83 regexp_check=None): 84 ''' 85 Checks |container| for presence of |key| with value of type |value_type|. 86 If |value_type| is string and |regexp_check| is specified, then an error is 87 reported when the value does not match the regular expression object. 88 89 |value_type| can also be a list, if more than one type is supported. 90 91 The other parameters are needed to generate, if applicable, an appropriate 92 human-readable error message of the following form: 93 94 In |parent_element| |identifier|: 95 (if the key is not present): 96 Error: |container_name| must have a |value_type| named |key|. 97 Offending snippet: |offending| (if specified; defaults to |container|) 98 (if the value does not have the required type): 99 Error: Value of |key| must be a |value_type|. 100 Offending snippet: |container[key]| 101 102 Returns: |container[key]| if the key is present, None otherwise. 103 ''' 104 if identifier is None: 105 try: 106 identifier = container.get('name') 107 except: 108 self._Error('Cannot access container name of "%s".' % container_name) 109 return None 110 if container_name is None: 111 container_name = parent_element 112 if offending == '__CONTAINER__': 113 offending = container 114 if key not in container: 115 if optional: 116 return 117 else: 118 self._Error('%s must have a %s "%s".' % 119 (container_name.title(), value_type.__name__, key), 120 container_name, identifier, offending) 121 return None 122 value = container[key] 123 value_types = value_type if isinstance(value_type, list) else [ value_type ] 124 if not any(isinstance(value, type) for type in value_types): 125 self._Error('Value of "%s" must one of [ %s ].' % 126 (key, ', '.join([type.__name__ for type in value_types])), 127 container_name, identifier, value) 128 if str in value_types and regexp_check and not regexp_check.match(value): 129 self._Error('Value of "%s" must match "%s".' % 130 (key, regexp_check.pattern), 131 container_name, identifier, value) 132 return value 133 134 def _AddPolicyID(self, id, policy_ids, policy): 135 ''' 136 Adds |id| to |policy_ids|. Generates an error message if the 137 |id| exists already; |policy| is needed for this message. 138 ''' 139 if id in policy_ids: 140 self._Error('Duplicate id', 'policy', policy.get('name'), 141 id) 142 else: 143 policy_ids.add(id) 144 145 def _CheckPolicyIDs(self, policy_ids): 146 ''' 147 Checks a set of policy_ids to make sure it contains a continuous range 148 of entries (i.e. no holes). 149 Holes would not be a technical problem, but we want to ensure that nobody 150 accidentally omits IDs. 151 ''' 152 for i in range(len(policy_ids)): 153 if (i + 1) not in policy_ids: 154 self._Error('No policy with id: %s' % (i + 1)) 155 156 def _CheckPolicySchema(self, policy, policy_type): 157 '''Checks that the 'schema' field matches the 'type' field.''' 158 self._CheckContains(policy, 'schema', dict) 159 if isinstance(policy.get('schema'), dict): 160 self._CheckContains(policy['schema'], 'type', str) 161 schema_type = policy['schema'].get('type') 162 if schema_type not in TYPE_TO_SCHEMA[policy_type]: 163 self._Error('Schema type must match the existing type for policy %s' % 164 policy.get('name')) 165 166 # Checks that boolean policies are not negated (which makes them harder to 167 # reason about). 168 if (schema_type == 'boolean' and 169 'disable' in policy.get('name').lower() and 170 policy.get('name') not in LEGACY_INVERTED_POLARITY_WHITELIST): 171 self._Error(('Boolean policy %s uses negative polarity, please make ' + 172 'new boolean policies follow the XYZEnabled pattern. ' + 173 'See also http://crbug.com/85687') % policy.get('name')) 174 175 176 def _CheckPolicy(self, policy, is_in_group, policy_ids): 177 if not isinstance(policy, dict): 178 self._Error('Each policy must be a dictionary.', 'policy', None, policy) 179 return 180 181 # There should not be any unknown keys in |policy|. 182 for key in policy: 183 if key not in ('name', 'type', 'caption', 'desc', 'device_only', 184 'supported_on', 'label', 'policies', 'items', 185 'example_value', 'features', 'deprecated', 'future', 186 'id', 'schema', 'max_size'): 187 self.warning_count += 1 188 print ('In policy %s: Warning: Unknown key: %s' % 189 (policy.get('name'), key)) 190 191 # Each policy must have a name. 192 self._CheckContains(policy, 'name', str, regexp_check=NO_WHITESPACE) 193 194 # Each policy must have a type. 195 policy_types = ('group', 'main', 'string', 'int', 'list', 'int-enum', 196 'string-enum', 'dict', 'external') 197 policy_type = self._CheckContains(policy, 'type', str) 198 if policy_type not in policy_types: 199 self._Error('Policy type must be one of: ' + ', '.join(policy_types), 200 'policy', policy.get('name'), policy_type) 201 return # Can't continue for unsupported type. 202 203 # Each policy must have a caption message. 204 self._CheckContains(policy, 'caption', str) 205 206 # Each policy must have a description message. 207 self._CheckContains(policy, 'desc', str) 208 209 # If 'label' is present, it must be a string. 210 self._CheckContains(policy, 'label', str, True) 211 212 # If 'deprecated' is present, it must be a bool. 213 self._CheckContains(policy, 'deprecated', bool, True) 214 215 # If 'future' is present, it must be a bool. 216 self._CheckContains(policy, 'future', bool, True) 217 218 if policy_type == 'group': 219 # Groups must not be nested. 220 if is_in_group: 221 self._Error('Policy groups must not be nested.', 'policy', policy) 222 223 # Each policy group must have a list of policies. 224 policies = self._CheckContains(policy, 'policies', list) 225 226 # Check sub-policies. 227 if policies is not None: 228 for nested_policy in policies: 229 self._CheckPolicy(nested_policy, True, policy_ids) 230 231 # Groups must not have an |id|. 232 if 'id' in policy: 233 self._Error('Policies of type "group" must not have an "id" field.', 234 'policy', policy) 235 236 # Statistics. 237 self.num_groups += 1 238 239 else: # policy_type != group 240 # Each policy must have a protobuf ID. 241 id = self._CheckContains(policy, 'id', int) 242 self._AddPolicyID(id, policy_ids, policy) 243 244 # 'schema' is the new 'type'. 245 # TODO(joaodasilva): remove the 'type' checks once 'schema' is used 246 # everywhere. 247 self._CheckPolicySchema(policy, policy_type) 248 249 # Each policy must have a supported_on list. 250 supported_on = self._CheckContains(policy, 'supported_on', list) 251 if supported_on is not None: 252 for s in supported_on: 253 if not isinstance(s, str): 254 self._Error('Entries in "supported_on" must be strings.', 'policy', 255 policy, supported_on) 256 257 # Each policy must have a 'features' dict. 258 features = self._CheckContains(policy, 'features', dict) 259 260 # All the features must have a documenting message. 261 if features: 262 for feature in features: 263 if not feature in self.features: 264 self._Error('Unknown feature "%s". Known features must have a ' 265 'documentation string in the messages dictionary.' % 266 feature, 'policy', policy.get('name', policy)) 267 268 # All user policies must have a per_profile feature flag. 269 if (not policy.get('device_only', False) and 270 not policy.get('deprecated', False) and 271 not filter(re.compile('^chrome_frame:.*').match, supported_on)): 272 self._CheckContains(features, 'per_profile', bool, 273 container_name='features', 274 identifier=policy.get('name')) 275 276 # All policies must declare whether they allow changes at runtime. 277 self._CheckContains(features, 'dynamic_refresh', bool, 278 container_name='features', 279 identifier=policy.get('name')) 280 281 # Each policy must have an 'example_value' of appropriate type. 282 if policy_type == 'main': 283 value_type = bool 284 elif policy_type in ('string', 'string-enum'): 285 value_type = str 286 elif policy_type in ('int', 'int-enum'): 287 value_type = int 288 elif policy_type == 'list': 289 value_type = list 290 elif policy_type == 'external': 291 value_type = dict 292 elif policy_type == 'dict': 293 value_type = [ dict, list ] 294 else: 295 raise NotImplementedError('Unimplemented policy type: %s' % policy_type) 296 self._CheckContains(policy, 'example_value', value_type) 297 298 # Statistics. 299 self.num_policies += 1 300 if is_in_group: 301 self.num_policies_in_groups += 1 302 303 if policy_type in ('int-enum', 'string-enum'): 304 # Enums must contain a list of items. 305 items = self._CheckContains(policy, 'items', list) 306 if items is not None: 307 if len(items) < 1: 308 self._Error('"items" must not be empty.', 'policy', policy, items) 309 for item in items: 310 # Each item must have a name. 311 # Note: |policy.get('name')| is used instead of |policy['name']| 312 # because it returns None rather than failing when no key called 313 # 'name' exists. 314 self._CheckContains(item, 'name', str, container_name='item', 315 identifier=policy.get('name'), 316 regexp_check=NO_WHITESPACE) 317 318 # Each item must have a value of the correct type. 319 self._CheckContains(item, 'value', value_type, container_name='item', 320 identifier=policy.get('name')) 321 322 # Each item must have a caption. 323 self._CheckContains(item, 'caption', str, container_name='item', 324 identifier=policy.get('name')) 325 326 if policy_type == 'external': 327 # Each policy referencing external data must specify a maximum data size. 328 self._CheckContains(policy, 'max_size', int) 329 330 def _CheckMessage(self, key, value): 331 # |key| must be a string, |value| a dict. 332 if not isinstance(key, str): 333 self._Error('Each message key must be a string.', 'message', key, key) 334 return 335 336 if not isinstance(value, dict): 337 self._Error('Each message must be a dictionary.', 'message', key, value) 338 return 339 340 # Each message must have a desc. 341 self._CheckContains(value, 'desc', str, parent_element='message', 342 identifier=key) 343 344 # Each message must have a text. 345 self._CheckContains(value, 'text', str, parent_element='message', 346 identifier=key) 347 348 # There should not be any unknown keys in |value|. 349 for vkey in value: 350 if vkey not in ('desc', 'text'): 351 self.warning_count += 1 352 print 'In message %s: Warning: Unknown key: %s' % (key, vkey) 353 354 def _LeadingWhitespace(self, line): 355 match = LEADING_WHITESPACE.match(line) 356 if match: 357 return match.group(1) 358 return '' 359 360 def _TrailingWhitespace(self, line): 361 match = TRAILING_WHITESPACE.match(line) 362 if match: 363 return match.group(1) 364 return '' 365 366 def _LineError(self, message, line_number): 367 self.error_count += 1 368 print 'In line %d: Error: %s' % (line_number, message) 369 370 def _LineWarning(self, message, line_number): 371 self.warning_count += 1 372 print ('In line %d: Warning: Automatically fixing formatting: %s' 373 % (line_number, message)) 374 375 def _CheckFormat(self, filename): 376 if self.options.fix: 377 fixed_lines = [] 378 with open(filename) as f: 379 indent = 0 380 line_number = 0 381 for line in f: 382 line_number += 1 383 line = line.rstrip('\n') 384 # Check for trailing whitespace. 385 trailing_whitespace = self._TrailingWhitespace(line) 386 if len(trailing_whitespace) > 0: 387 if self.options.fix: 388 line = line.rstrip() 389 self._LineWarning('Trailing whitespace.', line_number) 390 else: 391 self._LineError('Trailing whitespace.', line_number) 392 if self.options.fix: 393 if len(line) == 0: 394 fixed_lines += ['\n'] 395 continue 396 else: 397 if line == trailing_whitespace: 398 # This also catches the case of an empty line. 399 continue 400 # Check for correct amount of leading whitespace. 401 leading_whitespace = self._LeadingWhitespace(line) 402 if leading_whitespace.count('\t') > 0: 403 if self.options.fix: 404 leading_whitespace = leading_whitespace.replace('\t', ' ') 405 line = leading_whitespace + line.lstrip() 406 self._LineWarning('Tab character found.', line_number) 407 else: 408 self._LineError('Tab character found.', line_number) 409 if line[len(leading_whitespace)] in (']', '}'): 410 indent -= 2 411 if line[0] != '#': # Ignore 0-indented comments. 412 if len(leading_whitespace) != indent: 413 if self.options.fix: 414 line = ' ' * indent + line.lstrip() 415 self._LineWarning('Indentation should be ' + str(indent) + 416 ' spaces.', line_number) 417 else: 418 self._LineError('Bad indentation. Should be ' + str(indent) + 419 ' spaces.', line_number) 420 if line[-1] in ('[', '{'): 421 indent += 2 422 if self.options.fix: 423 fixed_lines.append(line + '\n') 424 425 # If --fix is specified: backup the file (deleting any existing backup), 426 # then write the fixed version with the old filename. 427 if self.options.fix: 428 if self.options.backup: 429 backupfilename = filename + '.bak' 430 if os.path.exists(backupfilename): 431 os.remove(backupfilename) 432 os.rename(filename, backupfilename) 433 with open(filename, 'w') as f: 434 f.writelines(fixed_lines) 435 436 def Main(self, filename, options): 437 try: 438 with open(filename) as f: 439 data = eval(f.read()) 440 except: 441 import traceback 442 traceback.print_exc(file=sys.stdout) 443 self._Error('Invalid Python/JSON syntax.') 444 return 1 445 if data == None: 446 self._Error('Invalid Python/JSON syntax.') 447 return 1 448 self.options = options 449 450 # First part: check JSON structure. 451 452 # Check (non-policy-specific) message definitions. 453 messages = self._CheckContains(data, 'messages', dict, 454 parent_element=None, 455 container_name='The root element', 456 offending=None) 457 if messages is not None: 458 for message in messages: 459 self._CheckMessage(message, messages[message]) 460 if message.startswith('doc_feature_'): 461 self.features.append(message[12:]) 462 463 # Check policy definitions. 464 policy_definitions = self._CheckContains(data, 'policy_definitions', list, 465 parent_element=None, 466 container_name='The root element', 467 offending=None) 468 if policy_definitions is not None: 469 policy_ids = set() 470 for policy in policy_definitions: 471 self._CheckPolicy(policy, False, policy_ids) 472 self._CheckPolicyIDs(policy_ids) 473 474 # Second part: check formatting. 475 self._CheckFormat(filename) 476 477 # Third part: summary and exit. 478 print ('Finished checking %s. %d errors, %d warnings.' % 479 (filename, self.error_count, self.warning_count)) 480 if self.options.stats: 481 if self.num_groups > 0: 482 print ('%d policies, %d of those in %d groups (containing on ' 483 'average %.1f policies).' % 484 (self.num_policies, self.num_policies_in_groups, self.num_groups, 485 (1.0 * self.num_policies_in_groups / self.num_groups))) 486 else: 487 print self.num_policies, 'policies, 0 policy groups.' 488 if self.error_count > 0: 489 return 1 490 return 0 491 492 def Run(self, argv, filename=None): 493 parser = optparse.OptionParser( 494 usage='usage: %prog [options] filename', 495 description='Syntax check a policy_templates.json file.') 496 parser.add_option('--fix', action='store_true', 497 help='Automatically fix formatting.') 498 parser.add_option('--backup', action='store_true', 499 help='Create backup of original file (before fixing).') 500 parser.add_option('--stats', action='store_true', 501 help='Generate statistics.') 502 (options, args) = parser.parse_args(argv) 503 if filename is None: 504 if len(args) != 2: 505 parser.print_help() 506 sys.exit(1) 507 filename = args[1] 508 return self.Main(filename, options) 509 510 511 if __name__ == '__main__': 512 sys.exit(PolicyTemplateChecker().Run(sys.argv)) 513