1 #!/usr/bin/env python 2 # Copyright 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """The frontend for the Mojo bindings system.""" 7 8 9 import argparse 10 import cPickle 11 import hashlib 12 import importlib 13 import json 14 import os 15 import pprint 16 import re 17 import struct 18 import sys 19 20 # Disable lint check for finding modules: 21 # pylint: disable=F0401 22 23 def _GetDirAbove(dirname): 24 """Returns the directory "above" this file containing |dirname| (which must 25 also be "above" this file).""" 26 path = os.path.abspath(__file__) 27 while True: 28 path, tail = os.path.split(path) 29 assert tail 30 if tail == dirname: 31 return path 32 33 # Manually check for the command-line flag. (This isn't quite right, since it 34 # ignores, e.g., "--", but it's close enough.) 35 if "--use_bundled_pylibs" in sys.argv[1:]: 36 sys.path.insert(0, os.path.join(_GetDirAbove("mojo"), "third_party")) 37 38 sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 39 "pylib")) 40 41 from mojom.error import Error 42 import mojom.fileutil as fileutil 43 from mojom.generate import template_expander 44 from mojom.generate import translate 45 from mojom.generate.generator import AddComputedData, WriteFile 46 from mojom.parse.conditional_features import RemoveDisabledDefinitions 47 from mojom.parse.parser import Parse 48 49 50 _BUILTIN_GENERATORS = { 51 "c++": "mojom_cpp_generator", 52 "javascript": "mojom_js_generator", 53 "java": "mojom_java_generator", 54 } 55 56 57 def LoadGenerators(generators_string): 58 if not generators_string: 59 return [] # No generators. 60 61 generators = {} 62 for generator_name in [s.strip() for s in generators_string.split(",")]: 63 language = generator_name.lower() 64 if language not in _BUILTIN_GENERATORS: 65 print "Unknown generator name %s" % generator_name 66 sys.exit(1) 67 generator_module = importlib.import_module( 68 "generators.%s" % _BUILTIN_GENERATORS[language]) 69 generators[language] = generator_module 70 return generators 71 72 73 def MakeImportStackMessage(imported_filename_stack): 74 """Make a (human-readable) message listing a chain of imports. (Returned 75 string begins with a newline (if nonempty) and does not end with one.)""" 76 return ''.join( 77 reversed(["\n %s was imported by %s" % (a, b) for (a, b) in \ 78 zip(imported_filename_stack[1:], imported_filename_stack)])) 79 80 81 class RelativePath(object): 82 """Represents a path relative to the source tree.""" 83 def __init__(self, path, source_root): 84 self.path = path 85 self.source_root = source_root 86 87 def relative_path(self): 88 return os.path.relpath(os.path.abspath(self.path), 89 os.path.abspath(self.source_root)) 90 91 92 def FindImportFile(rel_dir, file_name, search_rel_dirs): 93 """Finds |file_name| in either |rel_dir| or |search_rel_dirs|. Returns a 94 RelativePath with first file found, or an arbitrary non-existent file 95 otherwise.""" 96 for rel_search_dir in [rel_dir] + search_rel_dirs: 97 path = os.path.join(rel_search_dir.path, file_name) 98 if os.path.isfile(path): 99 return RelativePath(path, rel_search_dir.source_root) 100 return RelativePath(os.path.join(rel_dir.path, file_name), 101 rel_dir.source_root) 102 103 104 def ScrambleMethodOrdinals(interfaces, salt): 105 already_generated = set() 106 for interface in interfaces: 107 i = 0 108 already_generated.clear() 109 for method in interface.methods: 110 while True: 111 i = i + 1 112 if i == 1000000: 113 raise Exception("Could not generate %d method ordinals for %s" % 114 (len(interface.methods), interface.mojom_name)) 115 # Generate a scrambled method.ordinal value. The algorithm doesn't have 116 # to be very strong, cryptographically. It just needs to be non-trivial 117 # to guess the results without the secret salt, in order to make it 118 # harder for a compromised process to send fake Mojo messages. 119 sha256 = hashlib.sha256(salt) 120 sha256.update(interface.mojom_name) 121 sha256.update(str(i)) 122 # Take the first 4 bytes as a little-endian uint32. 123 ordinal = struct.unpack('<L', sha256.digest()[:4])[0] 124 # Trim to 31 bits, so it always fits into a Java (signed) int. 125 ordinal = ordinal & 0x7fffffff 126 if ordinal in already_generated: 127 continue 128 already_generated.add(ordinal) 129 method.ordinal = ordinal 130 method.ordinal_comment = ( 131 'The %s value is based on sha256(salt + "%s%d").' % 132 (ordinal, interface.mojom_name, i)) 133 break 134 135 136 def ReadFileContents(filename): 137 with open(filename, 'rb') as f: 138 return f.read() 139 140 141 class MojomProcessor(object): 142 """Parses mojom files and creates ASTs for them. 143 144 Attributes: 145 _processed_files: {Dict[str, mojom.generate.module.Module]} Mapping from 146 relative mojom filename paths to the module AST for that mojom file. 147 """ 148 def __init__(self, should_generate): 149 self._should_generate = should_generate 150 self._processed_files = {} 151 self._typemap = {} 152 153 def LoadTypemaps(self, typemaps): 154 # Support some very simple single-line comments in typemap JSON. 155 comment_expr = r"^\s*//.*$" 156 def no_comments(line): 157 return not re.match(comment_expr, line) 158 for filename in typemaps: 159 with open(filename) as f: 160 typemaps = json.loads("".join(filter(no_comments, f.readlines()))) 161 for language, typemap in typemaps.iteritems(): 162 language_map = self._typemap.get(language, {}) 163 language_map.update(typemap) 164 self._typemap[language] = language_map 165 166 def _GenerateModule(self, args, remaining_args, generator_modules, 167 rel_filename, imported_filename_stack): 168 # Return the already-generated module. 169 if rel_filename.path in self._processed_files: 170 return self._processed_files[rel_filename.path] 171 172 if rel_filename.path in imported_filename_stack: 173 print "%s: Error: Circular dependency" % rel_filename.path + \ 174 MakeImportStackMessage(imported_filename_stack + [rel_filename.path]) 175 sys.exit(1) 176 177 tree = _UnpickleAST(_FindPicklePath(rel_filename, args.gen_directories + 178 [args.output_dir])) 179 dirname = os.path.dirname(rel_filename.path) 180 181 # Process all our imports first and collect the module object for each. 182 # We use these to generate proper type info. 183 imports = {} 184 for parsed_imp in tree.import_list: 185 rel_import_file = FindImportFile( 186 RelativePath(dirname, rel_filename.source_root), 187 parsed_imp.import_filename, args.import_directories) 188 imports[parsed_imp.import_filename] = self._GenerateModule( 189 args, remaining_args, generator_modules, rel_import_file, 190 imported_filename_stack + [rel_filename.path]) 191 192 # Set the module path as relative to the source root. 193 # Normalize to unix-style path here to keep the generators simpler. 194 module_path = rel_filename.relative_path().replace('\\', '/') 195 196 module = translate.OrderedModule(tree, module_path, imports) 197 198 if args.scrambled_message_id_salt_paths: 199 salt = ''.join( 200 map(ReadFileContents, args.scrambled_message_id_salt_paths)) 201 ScrambleMethodOrdinals(module.interfaces, salt) 202 203 if self._should_generate(rel_filename.path): 204 AddComputedData(module) 205 for language, generator_module in generator_modules.iteritems(): 206 generator = generator_module.Generator( 207 module, args.output_dir, typemap=self._typemap.get(language, {}), 208 variant=args.variant, bytecode_path=args.bytecode_path, 209 for_blink=args.for_blink, 210 use_once_callback=args.use_once_callback, 211 js_bindings_mode=args.js_bindings_mode, 212 export_attribute=args.export_attribute, 213 export_header=args.export_header, 214 generate_non_variant_code=args.generate_non_variant_code, 215 support_lazy_serialization=args.support_lazy_serialization, 216 disallow_native_types=args.disallow_native_types, 217 disallow_interfaces=args.disallow_interfaces, 218 generate_message_ids=args.generate_message_ids, 219 generate_fuzzing=args.generate_fuzzing) 220 filtered_args = [] 221 if hasattr(generator_module, 'GENERATOR_PREFIX'): 222 prefix = '--' + generator_module.GENERATOR_PREFIX + '_' 223 filtered_args = [arg for arg in remaining_args 224 if arg.startswith(prefix)] 225 generator.GenerateFiles(filtered_args) 226 227 # Save result. 228 self._processed_files[rel_filename.path] = module 229 return module 230 231 232 def _Generate(args, remaining_args): 233 if args.variant == "none": 234 args.variant = None 235 236 for idx, import_dir in enumerate(args.import_directories): 237 tokens = import_dir.split(":") 238 if len(tokens) >= 2: 239 args.import_directories[idx] = RelativePath(tokens[0], tokens[1]) 240 else: 241 args.import_directories[idx] = RelativePath(tokens[0], args.depth) 242 generator_modules = LoadGenerators(args.generators_string) 243 244 fileutil.EnsureDirectoryExists(args.output_dir) 245 246 processor = MojomProcessor(lambda filename: filename in args.filename) 247 processor.LoadTypemaps(set(args.typemaps)) 248 249 if args.filelist: 250 with open(args.filelist) as f: 251 args.filename.extend(f.read().split()) 252 253 for filename in args.filename: 254 processor._GenerateModule(args, remaining_args, generator_modules, 255 RelativePath(filename, args.depth), []) 256 257 return 0 258 259 260 def _FindPicklePath(rel_filename, search_dirs): 261 filename, _ = os.path.splitext(rel_filename.relative_path()) 262 pickle_path = filename + '.p' 263 for search_dir in search_dirs: 264 path = os.path.join(search_dir, pickle_path) 265 if os.path.isfile(path): 266 return path 267 raise Exception("%s: Error: Could not find file in %r" % (pickle_path, search_dirs)) 268 269 270 def _GetPicklePath(rel_filename, output_dir): 271 filename, _ = os.path.splitext(rel_filename.relative_path()) 272 pickle_path = filename + '.p' 273 return os.path.join(output_dir, pickle_path) 274 275 276 def _PickleAST(ast, output_file): 277 full_dir = os.path.dirname(output_file) 278 fileutil.EnsureDirectoryExists(full_dir) 279 280 try: 281 WriteFile(cPickle.dumps(ast), output_file) 282 except (IOError, cPickle.PicklingError) as e: 283 print "%s: Error: %s" % (output_file, str(e)) 284 sys.exit(1) 285 286 def _UnpickleAST(input_file): 287 try: 288 with open(input_file, "rb") as f: 289 return cPickle.load(f) 290 except (IOError, cPickle.UnpicklingError) as e: 291 print "%s: Error: %s" % (input_file, str(e)) 292 sys.exit(1) 293 294 def _ParseFile(args, rel_filename): 295 try: 296 with open(rel_filename.path) as f: 297 source = f.read() 298 except IOError as e: 299 print "%s: Error: %s" % (rel_filename.path, e.strerror) 300 sys.exit(1) 301 302 try: 303 tree = Parse(source, rel_filename.path) 304 RemoveDisabledDefinitions(tree, args.enabled_features) 305 except Error as e: 306 print "%s: Error: %s" % (rel_filename.path, str(e)) 307 sys.exit(1) 308 _PickleAST(tree, _GetPicklePath(rel_filename, args.output_dir)) 309 310 311 def _Parse(args, _): 312 fileutil.EnsureDirectoryExists(args.output_dir) 313 314 if args.filelist: 315 with open(args.filelist) as f: 316 args.filename.extend(f.read().split()) 317 318 for filename in args.filename: 319 _ParseFile(args, RelativePath(filename, args.depth)) 320 return 0 321 322 323 def _Precompile(args, _): 324 generator_modules = LoadGenerators(",".join(_BUILTIN_GENERATORS.keys())) 325 326 template_expander.PrecompileTemplates(generator_modules, args.output_dir) 327 return 0 328 329 def _VerifyImportDeps(args, __): 330 fileutil.EnsureDirectoryExists(args.gen_dir) 331 332 if args.filelist: 333 with open(args.filelist) as f: 334 args.filename.extend(f.read().split()) 335 336 for filename in args.filename: 337 rel_path = RelativePath(filename, args.depth) 338 tree = _UnpickleAST(_GetPicklePath(rel_path, args.gen_dir)) 339 340 mojom_imports = set( 341 parsed_imp.import_filename for parsed_imp in tree.import_list 342 ) 343 344 # read the paths from the file 345 f_deps = open(args.deps_file, 'r') 346 deps_sources = set() 347 for deps_path in f_deps: 348 deps_path = deps_path.rstrip('\n') 349 f_sources = open(deps_path, 'r') 350 351 for source_file in f_sources: 352 source_dir = deps_path.split(args.gen_dir + "/", 1)[1] 353 full_source_path = os.path.dirname(source_dir) + "/" + \ 354 source_file 355 deps_sources.add(full_source_path.rstrip('\n')) 356 357 if (not deps_sources.issuperset(mojom_imports)): 358 print ">>> [%s] Missing dependencies for the following imports: %s" % ( \ 359 args.filename[0], \ 360 list(mojom_imports.difference(deps_sources))) 361 sys.exit(1) 362 363 source_filename, _ = os.path.splitext(rel_path.relative_path()) 364 output_file = source_filename + '.v' 365 output_file_path = os.path.join(args.gen_dir, output_file) 366 WriteFile("", output_file_path) 367 368 return 0 369 370 def main(): 371 parser = argparse.ArgumentParser( 372 description="Generate bindings from mojom files.") 373 parser.add_argument("--use_bundled_pylibs", action="store_true", 374 help="use Python modules bundled in the SDK") 375 376 subparsers = parser.add_subparsers() 377 378 parse_parser = subparsers.add_parser( 379 "parse", description="Parse mojom to AST and remove disabled definitions." 380 " Pickle pruned AST into output_dir.") 381 parse_parser.add_argument("filename", nargs="*", help="mojom input file") 382 parse_parser.add_argument("--filelist", help="mojom input file list") 383 parse_parser.add_argument( 384 "-o", 385 "--output_dir", 386 dest="output_dir", 387 default=".", 388 help="output directory for generated files") 389 parse_parser.add_argument( 390 "-d", "--depth", dest="depth", default=".", help="depth from source root") 391 parse_parser.add_argument( 392 "--enable_feature", 393 dest = "enabled_features", 394 default=[], 395 action="append", 396 help="Controls which definitions guarded by an EnabledIf attribute " 397 "will be enabled. If an EnabledIf attribute does not specify a value " 398 "that matches one of the enabled features, it will be disabled.") 399 parse_parser.set_defaults(func=_Parse) 400 401 generate_parser = subparsers.add_parser( 402 "generate", description="Generate bindings from mojom files.") 403 generate_parser.add_argument("filename", nargs="*", 404 help="mojom input file") 405 generate_parser.add_argument("--filelist", help="mojom input file list") 406 generate_parser.add_argument("-d", "--depth", dest="depth", default=".", 407 help="depth from source root") 408 generate_parser.add_argument("-o", "--output_dir", dest="output_dir", 409 default=".", 410 help="output directory for generated files") 411 generate_parser.add_argument("-g", "--generators", 412 dest="generators_string", 413 metavar="GENERATORS", 414 default="c++,javascript,java", 415 help="comma-separated list of generators") 416 generate_parser.add_argument( 417 "--gen_dir", dest="gen_directories", action="append", metavar="directory", 418 default=[], help="add a directory to be searched for the syntax trees.") 419 generate_parser.add_argument( 420 "-I", dest="import_directories", action="append", metavar="directory", 421 default=[], 422 help="add a directory to be searched for import files. The depth from " 423 "source root can be specified for each import by appending it after " 424 "a colon") 425 generate_parser.add_argument("--typemap", action="append", metavar="TYPEMAP", 426 default=[], dest="typemaps", 427 help="apply TYPEMAP to generated output") 428 generate_parser.add_argument("--variant", dest="variant", default=None, 429 help="output a named variant of the bindings") 430 generate_parser.add_argument( 431 "--bytecode_path", required=True, help=( 432 "the path from which to load template bytecode; to generate template " 433 "bytecode, run %s precompile BYTECODE_PATH" % os.path.basename( 434 sys.argv[0]))) 435 generate_parser.add_argument("--for_blink", action="store_true", 436 help="Use WTF types as generated types for mojo " 437 "string/array/map.") 438 generate_parser.add_argument( 439 "--use_once_callback", action="store_true", 440 help="Use base::OnceCallback instead of base::RepeatingCallback.") 441 generate_parser.add_argument( 442 "--js_bindings_mode", choices=["new", "both", "old"], default="new", 443 help="This option only affects the JavaScript bindings. The value could " 444 "be: \"new\" - generate only the new-style JS bindings, which use the " 445 "new module loading approach and the core api exposed by Web IDL; " 446 "\"both\" - generate both the old- and new-style bindings; \"old\" - " 447 "generate only the old-style bindings.") 448 generate_parser.add_argument( 449 "--export_attribute", default="", 450 help="Optional attribute to specify on class declaration to export it " 451 "for the component build.") 452 generate_parser.add_argument( 453 "--export_header", default="", 454 help="Optional header to include in the generated headers to support the " 455 "component build.") 456 generate_parser.add_argument( 457 "--generate_non_variant_code", action="store_true", 458 help="Generate code that is shared by different variants.") 459 generate_parser.add_argument( 460 "--scrambled_message_id_salt_path", 461 dest="scrambled_message_id_salt_paths", 462 help="If non-empty, the path to a file whose contents should be used as" 463 "a salt for generating scrambled message IDs. If this switch is specified" 464 "more than once, the contents of all salt files are concatenated to form" 465 "the salt value.", default=[], action="append") 466 generate_parser.add_argument( 467 "--support_lazy_serialization", 468 help="If set, generated bindings will serialize lazily when possible.", 469 action="store_true") 470 generate_parser.add_argument( 471 "--disallow_native_types", 472 help="Disallows the [Native] attribute to be specified on structs or " 473 "enums within the mojom file.", action="store_true") 474 generate_parser.add_argument( 475 "--disallow_interfaces", 476 help="Disallows interface definitions within the mojom file. It is an " 477 "error to specify this flag when processing a mojom file which defines " 478 "any interface.", action="store_true") 479 generate_parser.add_argument( 480 "--generate_message_ids", 481 help="Generates only the message IDs header for C++ bindings. Note that " 482 "this flag only matters if --generate_non_variant_code is also " 483 "specified.", action="store_true") 484 generate_parser.add_argument( 485 "--generate_fuzzing", 486 action="store_true", 487 help="Generates additional bindings for fuzzing in JS.") 488 generate_parser.set_defaults(func=_Generate) 489 490 precompile_parser = subparsers.add_parser("precompile", 491 description="Precompile templates for the mojom bindings generator.") 492 precompile_parser.add_argument( 493 "-o", "--output_dir", dest="output_dir", default=".", 494 help="output directory for precompiled templates") 495 precompile_parser.set_defaults(func=_Precompile) 496 497 verify_parser = subparsers.add_parser("verify", description="Checks " 498 "the set of imports against the set of dependencies.") 499 verify_parser.add_argument("filename", nargs="*", 500 help="mojom input file") 501 verify_parser.add_argument("--filelist", help="mojom input file list") 502 verify_parser.add_argument("-f", "--file", dest="deps_file", 503 help="file containing paths to the sources files for " 504 "dependencies") 505 verify_parser.add_argument("-g", "--gen_dir", 506 dest="gen_dir", 507 help="directory with the syntax tree") 508 verify_parser.add_argument( 509 "-d", "--depth", dest="depth", 510 help="depth from source root") 511 512 verify_parser.set_defaults(func=_VerifyImportDeps) 513 514 args, remaining_args = parser.parse_known_args() 515 return args.func(args, remaining_args) 516 517 518 if __name__ == "__main__": 519 sys.exit(main()) 520
