1 #!/usr/bin/env python 2 # Copyright (C) 2018 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 # This tool translates a collection of BUILD.gn files into a mostly equivalent 17 # BUILD file for the Bazel build system. The input to the tool is a 18 # JSON description of the GN build definition generated with the following 19 # command: 20 # 21 # gn desc out --format=json --all-toolchains "//*" > desc.json 22 # 23 # The tool is then given a list of GN labels for which to generate Bazel 24 # build rules. 25 26 from __future__ import print_function 27 import argparse 28 import errno 29 import functools 30 import json 31 import os 32 import re 33 import shutil 34 import subprocess 35 import sys 36 import textwrap 37 38 # Copyright header for generated code. 39 header = """# Copyright (C) 2019 The Android Open Source Project 40 # 41 # Licensed under the Apache License, Version 2.0 (the "License"); 42 # you may not use this file except in compliance with the License. 43 # You may obtain a copy of the License at 44 # 45 # http://www.apache.org/licenses/LICENSE-2.0 46 # 47 # Unless required by applicable law or agreed to in writing, software 48 # distributed under the License is distributed on an "AS IS" BASIS, 49 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 50 # See the License for the specific language governing permissions and 51 # limitations under the License. 52 # 53 # This file is automatically generated by {}. Do not edit. 54 """.format(__file__) 55 56 # Arguments for the GN output directory. 57 # host_os="linux" is to generate the right build files from Mac OS. 58 gn_args = 'target_os="linux" is_debug=false host_os="linux"' 59 60 # Default targets to translate to the blueprint file. 61 default_targets = [ 62 '//src/protozero:libprotozero', 63 '//src/trace_processor:trace_processor', 64 '//src/trace_processor:trace_processor_shell_host(//gn/standalone/toolchain:gcc_like_host)', 65 '//tools/trace_to_text:trace_to_text_host(//gn/standalone/toolchain:gcc_like_host)', 66 '//protos/perfetto/config:merged_config_gen', 67 '//protos/perfetto/trace:merged_trace_gen', 68 ] 69 70 # Aliases to add to the BUILD file 71 alias_targets = { 72 '//src/protozero:libprotozero': 'libprotozero', 73 '//src/trace_processor:trace_processor': 'trace_processor', 74 '//src/trace_processor:trace_processor_shell_host': 'trace_processor_shell', 75 '//tools/trace_to_text:trace_to_text_host': 'trace_to_text', 76 } 77 78 79 def enable_sqlite(module): 80 module.deps.add(Label('//third_party/sqlite')) 81 module.deps.add(Label('//third_party/sqlite:sqlite_ext_percentile')) 82 83 84 def enable_jsoncpp(module): 85 module.deps.add(Label('//third_party/perfetto/google:jsoncpp')) 86 87 88 def enable_linenoise(module): 89 module.deps.add(Label('//third_party/perfetto/google:linenoise')) 90 91 92 def enable_gtest_prod(module): 93 module.deps.add(Label('//third_party/perfetto/google:gtest_prod')) 94 95 96 def enable_protobuf_full(module): 97 module.deps.add(Label('//third_party/protobuf:libprotoc')) 98 module.deps.add(Label('//third_party/protobuf')) 99 100 101 def enable_perfetto_version(module): 102 module.deps.add(Label('//third_party/perfetto/google:perfetto_version')) 103 104 105 def disable_module(module): 106 pass 107 108 109 # Internal equivalents for third-party libraries that the upstream project 110 # depends on. 111 builtin_deps = { 112 '//gn:jsoncpp_deps': enable_jsoncpp, 113 '//buildtools:linenoise': enable_linenoise, 114 '//buildtools:protobuf_lite': disable_module, 115 '//buildtools:protobuf_full': enable_protobuf_full, 116 '//buildtools:protoc': disable_module, 117 '//buildtools:sqlite': enable_sqlite, 118 '//gn:default_deps': disable_module, 119 '//gn:gtest_prod_config': enable_gtest_prod, 120 '//gn:protoc_lib_deps': enable_protobuf_full, 121 '//gn/standalone:gen_git_revision': enable_perfetto_version, 122 } 123 124 # ---------------------------------------------------------------------------- 125 # End of configuration. 126 # ---------------------------------------------------------------------------- 127 128 129 def check_output(cmd, cwd): 130 try: 131 output = subprocess.check_output( 132 cmd, stderr=subprocess.STDOUT, cwd=cwd) 133 except subprocess.CalledProcessError as e: 134 print('Cmd "{}" failed in {}:'.format( 135 ' '.join(cmd), cwd), file=sys.stderr) 136 print(e.output) 137 exit(1) 138 else: 139 return output 140 141 142 class Error(Exception): 143 pass 144 145 146 def repo_root(): 147 """Returns an absolute path to the repository root.""" 148 return os.path.join( 149 os.path.realpath(os.path.dirname(__file__)), os.path.pardir) 150 151 152 def create_build_description(repo_root): 153 """Creates the JSON build description by running GN.""" 154 155 out = os.path.join(repo_root, 'out', 'tmp.gen_build') 156 try: 157 try: 158 os.makedirs(out) 159 except OSError as e: 160 if e.errno != errno.EEXIST: 161 raise 162 check_output( 163 ['gn', 'gen', out, '--args=%s' % gn_args], repo_root) 164 desc = check_output( 165 ['gn', 'desc', out, '--format=json', '--all-toolchains', '//*'], 166 repo_root) 167 return json.loads(desc) 168 finally: 169 shutil.rmtree(out) 170 171 172 def label_to_path(label): 173 """Turn a GN output label (e.g., //some_dir/file.cc) into a path.""" 174 assert label.startswith('//') 175 return label[2:] 176 177 178 def label_to_target_name_with_path(label): 179 """ 180 Turn a GN label into a target name involving the full path. 181 e.g., //src/perfetto:tests -> src_perfetto_tests 182 """ 183 name = re.sub(r'^//:?', '', label) 184 name = re.sub(r'[^a-zA-Z0-9_]', '_', name) 185 return name 186 187 188 def label_without_toolchain(label): 189 """Strips the toolchain from a GN label. 190 191 Return a GN label (e.g //buildtools:protobuf(//gn/standalone/toolchain: 192 gcc_like_host) without the parenthesised toolchain part. 193 """ 194 return label.split('(')[0] 195 196 197 def is_public_header(label): 198 """ 199 Returns if this is a c++ header file that is part of the API. 200 Args: 201 label: Label to evaluate 202 """ 203 return label.endswith('.h') and label.startswith('//include/perfetto/') 204 205 206 @functools.total_ordering 207 class Label(object): 208 """Represents a label in BUILD file terminology. This class wraps a string 209 label to allow for correct comparision of labels for sorting. 210 211 Args: 212 label: The string rerepsentation of the label. 213 """ 214 215 def __init__(self, label): 216 self.label = label 217 218 def is_absolute(self): 219 return self.label.startswith('//') 220 221 def dirname(self): 222 return self.label.split(':')[0] if ':' in self.label else self.label 223 224 def basename(self): 225 return self.label.split(':')[1] if ':' in self.label else '' 226 227 def __eq__(self, other): 228 return self.label == other.label 229 230 def __lt__(self, other): 231 return ( 232 self.is_absolute(), 233 self.dirname(), 234 self.basename() 235 ) < ( 236 other.is_absolute(), 237 other.dirname(), 238 other.basename() 239 ) 240 241 def __str__(self): 242 return self.label 243 244 def __hash__(self): 245 return hash(self.label) 246 247 248 class Writer(object): 249 def __init__(self, output, width=79): 250 self.output = output 251 self.width = width 252 253 def comment(self, text): 254 for line in textwrap.wrap(text, 255 self.width - 2, 256 break_long_words=False, 257 break_on_hyphens=False): 258 self.output.write('# {}\n'.format(line)) 259 260 def newline(self): 261 self.output.write('\n') 262 263 def line(self, s, indent=0): 264 self.output.write(' ' * indent + s + '\n') 265 266 def variable(self, key, value, sort=True): 267 if value is None: 268 return 269 if isinstance(value, set) or isinstance(value, list): 270 if len(value) == 0: 271 return 272 self.line('{} = ['.format(key), indent=1) 273 for v in sorted(list(value)) if sort else value: 274 self.line('"{}",'.format(v), indent=2) 275 self.line('],', indent=1) 276 elif isinstance(value, basestring): 277 self.line('{} = "{}",'.format(key, value), indent=1) 278 else: 279 self.line('{} = {},'.format(key, value), indent=1) 280 281 def header(self): 282 self.output.write(header) 283 284 285 class Target(object): 286 """In-memory representation of a BUILD target.""" 287 288 def __init__(self, type, name, gn_name=None): 289 assert type in ('cc_binary', 'cc_library', 'cc_proto_library', 290 'proto_library', 'filegroup', 'alias', 291 'pbzero_cc_proto_library', 'genrule', ) 292 self.type = type 293 self.name = name 294 self.srcs = set() 295 self.hdrs = set() 296 self.deps = set() 297 self.visibility = set() 298 self.gn_name = gn_name 299 self.is_pbzero = False 300 self.src_proto_library = None 301 self.outs = set() 302 self.cmd = None 303 self.tools = set() 304 305 def write(self, writer): 306 if self.gn_name: 307 writer.comment('GN target: {}'.format(self.gn_name)) 308 309 writer.line('{}('.format(self.type)) 310 writer.variable('name', self.name) 311 writer.variable('srcs', self.srcs) 312 writer.variable('hdrs', self.hdrs) 313 314 if self.type == 'proto_library' and not self.is_pbzero: 315 if self.srcs: 316 writer.variable('has_services', 1) 317 writer.variable('cc_api_version', 2) 318 if self.srcs: 319 writer.variable('cc_generic_services', 1) 320 321 writer.variable('src_proto_library', self.src_proto_library) 322 323 writer.variable('outs', self.outs) 324 writer.variable('cmd', self.cmd) 325 writer.variable('tools', self.tools) 326 327 # Keep visibility and deps last. 328 writer.variable('visibility', self.visibility) 329 330 if type != 'filegroup': 331 writer.variable('deps', self.deps) 332 333 writer.line(')') 334 335 336 class Build(object): 337 """In-memory representation of a BUILD file.""" 338 339 def __init__(self, public, header_lines=[]): 340 self.targets = {} 341 self.public = public 342 self.header_lines = header_lines 343 344 def add_target(self, target): 345 self.targets[target.name] = target 346 347 def write(self, writer): 348 writer.header() 349 writer.newline() 350 for line in self.header_lines: 351 writer.line(line) 352 if self.header_lines: 353 writer.newline() 354 if self.public: 355 writer.line( 356 'package(default_visibility = ["//visibility:public"])') 357 else: 358 writer.line( 359 'package(default_visibility = ["//third_party/perfetto:__subpackages__"])') 360 writer.newline() 361 writer.line('licenses(["notice"]) # Apache 2.0') 362 writer.newline() 363 writer.line('exports_files(["LICENSE"])') 364 writer.newline() 365 366 sorted_targets = sorted( 367 self.targets.itervalues(), key=lambda m: m.name) 368 for target in sorted_targets[:-1]: 369 target.write(writer) 370 writer.newline() 371 372 # BUILD files shouldn't have a trailing new line. 373 sorted_targets[-1].write(writer) 374 375 376 class BuildGenerator(object): 377 def __init__(self, desc): 378 self.desc = desc 379 self.action_generated_files = set() 380 381 for target in self.desc.itervalues(): 382 if target['type'] == 'action': 383 self.action_generated_files.update(target['outputs']) 384 385 386 def create_build_for_targets(self, targets): 387 """Generate a BUILD for a list of GN targets and aliases.""" 388 self.build = Build(public=True) 389 390 proto_cc_import = 'load("//tools/build_defs/proto/cpp:cc_proto_library.bzl", "cc_proto_library")' 391 pbzero_cc_import = 'load("//third_party/perfetto/google:build_defs.bzl", "pbzero_cc_proto_library")' 392 self.proto_build = Build(public=False, header_lines=[ 393 proto_cc_import, pbzero_cc_import]) 394 395 for target in targets: 396 self.create_target(target) 397 398 return (self.build, self.proto_build) 399 400 401 def resolve_dependencies(self, target_name): 402 """Return the set of direct dependent-on targets for a GN target. 403 404 Args: 405 desc: JSON GN description. 406 target_name: Name of target 407 408 Returns: 409 A set of transitive dependencies in the form of GN targets. 410 """ 411 412 if label_without_toolchain(target_name) in builtin_deps: 413 return set() 414 target = self.desc[target_name] 415 resolved_deps = set() 416 for dep in target.get('deps', []): 417 resolved_deps.add(dep) 418 return resolved_deps 419 420 421 def apply_module_sources_to_target(self, target, module_desc): 422 """ 423 Args: 424 target: Module to which dependencies should be added. 425 module_desc: JSON GN description of the module. 426 visibility: Whether the module is visible with respect to the target. 427 """ 428 for src in module_desc['sources']: 429 label = Label(label_to_path(src)) 430 if target.type == 'cc_library' and is_public_header(src): 431 target.hdrs.add(label) 432 else: 433 target.srcs.add(label) 434 435 436 def apply_module_dependency(self, target, dep_name): 437 """ 438 Args: 439 build: BUILD instance which is being generated. 440 proto_build: BUILD instance which is being generated to hold protos. 441 desc: JSON GN description. 442 target: Module to which dependencies should be added. 443 dep_name: GN target of the dependency. 444 """ 445 # If the dependency refers to a library which we can replace with an internal 446 # equivalent, stop recursing and patch the dependency in. 447 dep_name_no_toolchain = label_without_toolchain(dep_name) 448 if dep_name_no_toolchain in builtin_deps: 449 builtin_deps[dep_name_no_toolchain](target) 450 return 451 452 dep_desc = self.desc[dep_name] 453 if dep_desc['type'] == 'source_set': 454 for inner_name in self.resolve_dependencies(dep_name): 455 self.apply_module_dependency(target, inner_name) 456 457 # Any source set which has a source generated by an action doesn't need 458 # to be depended on as we will depend on the action directly. 459 if any(src in self.action_generated_files for src in dep_desc['sources']): 460 return 461 462 self.apply_module_sources_to_target(target, dep_desc) 463 elif dep_desc['type'] == 'action': 464 args = dep_desc['args'] 465 if "gen_merged_sql_metrics" in dep_name: 466 dep_target = self.create_merged_sql_metrics_target(dep_name) 467 target.deps.add(Label("//third_party/perfetto:" + dep_target.name)) 468 469 if target.type == 'cc_library' or target.type == 'cc_binary': 470 target.srcs.update(dep_target.outs) 471 elif args[0].endswith('/protoc'): 472 (proto_target, cc_target) = self.create_proto_target(dep_name) 473 if target.type == 'proto_library': 474 dep_target_name = proto_target.name 475 else: 476 dep_target_name = cc_target.name 477 target.deps.add( 478 Label("//third_party/perfetto/protos:" + dep_target_name)) 479 else: 480 raise Error('Unsupported action in target %s: %s' % (dep_target_name, 481 args)) 482 elif dep_desc['type'] == 'static_library': 483 dep_target = self.create_target(dep_name) 484 target.deps.add(Label("//third_party/perfetto:" + dep_target.name)) 485 elif dep_desc['type'] == 'group': 486 for inner_name in self.resolve_dependencies(dep_name): 487 self.apply_module_dependency(target, inner_name) 488 elif dep_desc['type'] == 'executable': 489 # Just create the dep target but don't add it as a dep because it's an 490 # executable. 491 self.create_target(dep_name) 492 else: 493 raise Error('Unknown target name %s with type: %s' % 494 (dep_name, dep_desc['type'])) 495 496 def create_merged_sql_metrics_target(self, gn_target_name): 497 target_desc = self.desc[gn_target_name] 498 gn_target_name_no_toolchain = label_without_toolchain(gn_target_name) 499 target = Target( 500 'genrule', 501 'gen_merged_sql_metrics', 502 gn_name=gn_target_name_no_toolchain, 503 ) 504 target.outs.update( 505 Label(src[src.index('gen/') + len('gen/'):]) 506 for src in target_desc.get('outputs', []) 507 ) 508 target.cmd = '$(location gen_merged_sql_metrics_py) --cpp_out=$@ $(SRCS)' 509 target.tools.update([ 510 'gen_merged_sql_metrics_py', 511 ]) 512 target.srcs.update( 513 Label(label_to_path(src)) 514 for src in target_desc.get('inputs', []) 515 if src not in self.action_generated_files 516 ) 517 self.build.add_target(target) 518 return target 519 520 def create_proto_target(self, gn_target_name): 521 target_desc = self.desc[gn_target_name] 522 args = target_desc['args'] 523 524 gn_target_name_no_toolchain = label_without_toolchain(gn_target_name) 525 stripped_path = gn_target_name_no_toolchain.replace("protos/perfetto/", "") 526 pretty_target_name = label_to_target_name_with_path(stripped_path) 527 pretty_target_name = pretty_target_name.replace("_lite_gen", "") 528 pretty_target_name = pretty_target_name.replace("_zero_gen", "_zero") 529 530 proto_target = Target( 531 'proto_library', 532 pretty_target_name, 533 gn_name=gn_target_name_no_toolchain 534 ) 535 proto_target.is_pbzero = any("pbzero" in arg for arg in args) 536 proto_target.srcs.update([ 537 Label(label_to_path(src).replace('protos/', '')) 538 for src in target_desc.get('sources', []) 539 ]) 540 if not proto_target.is_pbzero: 541 proto_target.visibility.add("//visibility:public") 542 self.proto_build.add_target(proto_target) 543 544 for dep_name in self.resolve_dependencies(gn_target_name): 545 self.apply_module_dependency(proto_target, dep_name) 546 547 if proto_target.is_pbzero: 548 # Remove all the protozero srcs from the proto_library. 549 proto_target.srcs.difference_update( 550 [src for src in proto_target.srcs if not src.label.endswith('.proto')]) 551 552 # Remove all the non-proto deps from the proto_library and add to the cc 553 # library. 554 cc_deps = [ 555 dep for dep in proto_target.deps 556 if not dep.label.startswith('//third_party/perfetto/protos') 557 ] 558 proto_target.deps.difference_update(cc_deps) 559 560 cc_target_name = proto_target.name + "_cc_proto" 561 cc_target = Target('pbzero_cc_proto_library', cc_target_name, 562 gn_name=gn_target_name_no_toolchain) 563 564 cc_target.deps.add(Label('//third_party/perfetto:libprotozero')) 565 cc_target.deps.update(cc_deps) 566 567 # Add the proto_library to the cc_target. 568 cc_target.src_proto_library = \ 569 "//third_party/perfetto/protos:" + proto_target.name 570 571 self.proto_build.add_target(cc_target) 572 else: 573 cc_target_name = proto_target.name + "_cc_proto" 574 cc_target = Target('cc_proto_library', 575 cc_target_name, gn_name=gn_target_name_no_toolchain) 576 cc_target.visibility.add("//visibility:public") 577 cc_target.deps.add( 578 Label("//third_party/perfetto/protos:" + proto_target.name)) 579 self.proto_build.add_target(cc_target) 580 581 return (proto_target, cc_target) 582 583 584 def create_target(self, gn_target_name): 585 """Generate module(s) for a given GN target. 586 587 Given a GN target name, generate one or more corresponding modules into a 588 build file. 589 590 Args: 591 build: Build instance which is being generated. 592 desc: JSON GN description. 593 gn_target_name: GN target name for module generation. 594 """ 595 596 target_desc = self.desc[gn_target_name] 597 if target_desc['type'] == 'action': 598 args = target_desc['args'] 599 if args[0].endswith('/protoc'): 600 return self.create_proto_target(gn_target_name) 601 else: 602 raise Error('Unsupported action in target %s: %s' % (gn_target_name, 603 args)) 604 elif target_desc['type'] == 'executable': 605 target_type = 'cc_binary' 606 elif target_desc['type'] == 'static_library': 607 target_type = 'cc_library' 608 elif target_desc['type'] == 'source_set': 609 target_type = 'filegroup' 610 else: 611 raise Error('Unknown target type: %s' % target_desc['type']) 612 613 label_no_toolchain = label_without_toolchain(gn_target_name) 614 target_name_path = label_to_target_name_with_path(label_no_toolchain) 615 target_name = alias_targets.get(label_no_toolchain, target_name_path) 616 target = Target(target_type, target_name, gn_name=label_no_toolchain) 617 target.srcs.update( 618 Label(label_to_path(src)) 619 for src in target_desc.get('sources', []) 620 if src not in self.action_generated_files 621 ) 622 623 for dep_name in self.resolve_dependencies(gn_target_name): 624 self.apply_module_dependency(target, dep_name) 625 626 self.build.add_target(target) 627 return target 628 629 def main(): 630 parser = argparse.ArgumentParser( 631 description='Generate BUILD from a GN description.') 632 parser.add_argument( 633 '--desc', 634 help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"' 635 ) 636 parser.add_argument( 637 '--repo-root', 638 help='Standalone Perfetto repository to generate a GN description', 639 default=repo_root(), 640 ) 641 parser.add_argument( 642 '--extras', 643 help='Extra targets to include at the end of the BUILD file', 644 default=os.path.join(repo_root(), 'BUILD.extras'), 645 ) 646 parser.add_argument( 647 '--output', 648 help='BUILD file to create', 649 default=os.path.join(repo_root(), 'BUILD'), 650 ) 651 parser.add_argument( 652 '--output-proto', 653 help='Proto BUILD file to create', 654 default=os.path.join(repo_root(), 'protos', 'BUILD'), 655 ) 656 parser.add_argument( 657 'targets', 658 nargs=argparse.REMAINDER, 659 help='Targets to include in the BUILD file (e.g., "//:perfetto_tests")') 660 args = parser.parse_args() 661 662 if args.desc: 663 with open(args.desc) as f: 664 desc = json.load(f) 665 else: 666 desc = create_build_description(args.repo_root) 667 668 build_generator = BuildGenerator(desc) 669 build, proto_build = build_generator.create_build_for_targets( 670 args.targets or default_targets) 671 with open(args.output, 'w') as f: 672 writer = Writer(f) 673 build.write(writer) 674 writer.newline() 675 676 with open(args.extras, 'r') as r: 677 for line in r: 678 writer.line(line.rstrip("\n\r")) 679 680 with open(args.output_proto, 'w') as f: 681 proto_build.write(Writer(f)) 682 683 return 0 684 685 686 if __name__ == '__main__': 687 sys.exit(main()) 688
