1 #!/usr/bin/env python 2 # Copyright 2016 the V8 project authors. All rights reserved. 3 # Copyright 2015 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """MB - the Meta-Build wrapper around GYP and GN 8 9 MB is a wrapper script for GYP and GN that can be used to generate build files 10 for sets of canned configurations and analyze them. 11 """ 12 13 from __future__ import print_function 14 15 import argparse 16 import ast 17 import errno 18 import json 19 import os 20 import pipes 21 import pprint 22 import re 23 import shutil 24 import sys 25 import subprocess 26 import tempfile 27 import traceback 28 import urllib2 29 30 from collections import OrderedDict 31 32 CHROMIUM_SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname( 33 os.path.abspath(__file__)))) 34 sys.path = [os.path.join(CHROMIUM_SRC_DIR, 'build')] + sys.path 35 36 import gn_helpers 37 38 39 def main(args): 40 mbw = MetaBuildWrapper() 41 return mbw.Main(args) 42 43 44 class MetaBuildWrapper(object): 45 def __init__(self): 46 self.chromium_src_dir = CHROMIUM_SRC_DIR 47 self.default_config = os.path.join(self.chromium_src_dir, 'infra', 'mb', 48 'mb_config.pyl') 49 self.executable = sys.executable 50 self.platform = sys.platform 51 self.sep = os.sep 52 self.args = argparse.Namespace() 53 self.configs = {} 54 self.masters = {} 55 self.mixins = {} 56 57 def Main(self, args): 58 self.ParseArgs(args) 59 try: 60 ret = self.args.func() 61 if ret: 62 self.DumpInputFiles() 63 return ret 64 except KeyboardInterrupt: 65 self.Print('interrupted, exiting', stream=sys.stderr) 66 return 130 67 except Exception: 68 self.DumpInputFiles() 69 s = traceback.format_exc() 70 for l in s.splitlines(): 71 self.Print(l) 72 return 1 73 74 def ParseArgs(self, argv): 75 def AddCommonOptions(subp): 76 subp.add_argument('-b', '--builder', 77 help='builder name to look up config from') 78 subp.add_argument('-m', '--master', 79 help='master name to look up config from') 80 subp.add_argument('-c', '--config', 81 help='configuration to analyze') 82 subp.add_argument('--phase', type=int, 83 help=('build phase for a given build ' 84 '(int in [1, 2, ...))')) 85 subp.add_argument('-f', '--config-file', metavar='PATH', 86 default=self.default_config, 87 help='path to config file ' 88 '(default is //tools/mb/mb_config.pyl)') 89 subp.add_argument('-g', '--goma-dir', 90 help='path to goma directory') 91 subp.add_argument('--gyp-script', metavar='PATH', 92 default=self.PathJoin('build', 'gyp_chromium'), 93 help='path to gyp script relative to project root ' 94 '(default is %(default)s)') 95 subp.add_argument('--android-version-code', 96 help='Sets GN arg android_default_version_code and ' 97 'GYP_DEFINE app_manifest_version_code') 98 subp.add_argument('--android-version-name', 99 help='Sets GN arg android_default_version_name and ' 100 'GYP_DEFINE app_manifest_version_name') 101 subp.add_argument('-n', '--dryrun', action='store_true', 102 help='Do a dry run (i.e., do nothing, just print ' 103 'the commands that will run)') 104 subp.add_argument('-v', '--verbose', action='store_true', 105 help='verbose logging') 106 107 parser = argparse.ArgumentParser(prog='mb') 108 subps = parser.add_subparsers() 109 110 subp = subps.add_parser('analyze', 111 help='analyze whether changes to a set of files ' 112 'will cause a set of binaries to be rebuilt.') 113 AddCommonOptions(subp) 114 subp.add_argument('path', nargs=1, 115 help='path build was generated into.') 116 subp.add_argument('input_path', nargs=1, 117 help='path to a file containing the input arguments ' 118 'as a JSON object.') 119 subp.add_argument('output_path', nargs=1, 120 help='path to a file containing the output arguments ' 121 'as a JSON object.') 122 subp.set_defaults(func=self.CmdAnalyze) 123 124 subp = subps.add_parser('gen', 125 help='generate a new set of build files') 126 AddCommonOptions(subp) 127 subp.add_argument('--swarming-targets-file', 128 help='save runtime dependencies for targets listed ' 129 'in file.') 130 subp.add_argument('path', nargs=1, 131 help='path to generate build into') 132 subp.set_defaults(func=self.CmdGen) 133 134 subp = subps.add_parser('isolate', 135 help='generate the .isolate files for a given' 136 'binary') 137 AddCommonOptions(subp) 138 subp.add_argument('path', nargs=1, 139 help='path build was generated into') 140 subp.add_argument('target', nargs=1, 141 help='ninja target to generate the isolate for') 142 subp.set_defaults(func=self.CmdIsolate) 143 144 subp = subps.add_parser('lookup', 145 help='look up the command for a given config or ' 146 'builder') 147 AddCommonOptions(subp) 148 subp.set_defaults(func=self.CmdLookup) 149 150 subp = subps.add_parser( 151 'run', 152 help='build and run the isolated version of a ' 153 'binary', 154 formatter_class=argparse.RawDescriptionHelpFormatter) 155 subp.description = ( 156 'Build, isolate, and run the given binary with the command line\n' 157 'listed in the isolate. You may pass extra arguments after the\n' 158 'target; use "--" if the extra arguments need to include switches.\n' 159 '\n' 160 'Examples:\n' 161 '\n' 162 ' % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n' 163 ' //out/Default content_browsertests\n' 164 '\n' 165 ' % tools/mb/mb.py run out/Default content_browsertests\n' 166 '\n' 167 ' % tools/mb/mb.py run out/Default content_browsertests -- \\\n' 168 ' --test-launcher-retry-limit=0' 169 '\n' 170 ) 171 172 AddCommonOptions(subp) 173 subp.add_argument('-j', '--jobs', dest='jobs', type=int, 174 help='Number of jobs to pass to ninja') 175 subp.add_argument('--no-build', dest='build', default=True, 176 action='store_false', 177 help='Do not build, just isolate and run') 178 subp.add_argument('path', nargs=1, 179 help=('path to generate build into (or use).' 180 ' This can be either a regular path or a ' 181 'GN-style source-relative path like ' 182 '//out/Default.')) 183 subp.add_argument('target', nargs=1, 184 help='ninja target to build and run') 185 subp.add_argument('extra_args', nargs='*', 186 help=('extra args to pass to the isolate to run. Use ' 187 '"--" as the first arg if you need to pass ' 188 'switches')) 189 subp.set_defaults(func=self.CmdRun) 190 191 subp = subps.add_parser('validate', 192 help='validate the config file') 193 subp.add_argument('-f', '--config-file', metavar='PATH', 194 default=self.default_config, 195 help='path to config file ' 196 '(default is //infra/mb/mb_config.pyl)') 197 subp.set_defaults(func=self.CmdValidate) 198 199 subp = subps.add_parser('audit', 200 help='Audit the config file to track progress') 201 subp.add_argument('-f', '--config-file', metavar='PATH', 202 default=self.default_config, 203 help='path to config file ' 204 '(default is //infra/mb/mb_config.pyl)') 205 subp.add_argument('-i', '--internal', action='store_true', 206 help='check internal masters also') 207 subp.add_argument('-m', '--master', action='append', 208 help='master to audit (default is all non-internal ' 209 'masters in file)') 210 subp.add_argument('-u', '--url-template', action='store', 211 default='https://build.chromium.org/p/' 212 '{master}/json/builders', 213 help='URL scheme for JSON APIs to buildbot ' 214 '(default: %(default)s) ') 215 subp.add_argument('-c', '--check-compile', action='store_true', 216 help='check whether tbd and master-only bots actually' 217 ' do compiles') 218 subp.set_defaults(func=self.CmdAudit) 219 220 subp = subps.add_parser('help', 221 help='Get help on a subcommand.') 222 subp.add_argument(nargs='?', action='store', dest='subcommand', 223 help='The command to get help for.') 224 subp.set_defaults(func=self.CmdHelp) 225 226 self.args = parser.parse_args(argv) 227 228 def DumpInputFiles(self): 229 230 def DumpContentsOfFilePassedTo(arg_name, path): 231 if path and self.Exists(path): 232 self.Print("\n# To recreate the file passed to %s:" % arg_name) 233 self.Print("%% cat > %s <<EOF)" % path) 234 contents = self.ReadFile(path) 235 self.Print(contents) 236 self.Print("EOF\n%\n") 237 238 if getattr(self.args, 'input_path', None): 239 DumpContentsOfFilePassedTo( 240 'argv[0] (input_path)', self.args.input_path[0]) 241 if getattr(self.args, 'swarming_targets_file', None): 242 DumpContentsOfFilePassedTo( 243 '--swarming-targets-file', self.args.swarming_targets_file) 244 245 def CmdAnalyze(self): 246 vals = self.Lookup() 247 self.ClobberIfNeeded(vals) 248 if vals['type'] == 'gn': 249 return self.RunGNAnalyze(vals) 250 else: 251 return self.RunGYPAnalyze(vals) 252 253 def CmdGen(self): 254 vals = self.Lookup() 255 self.ClobberIfNeeded(vals) 256 if vals['type'] == 'gn': 257 return self.RunGNGen(vals) 258 else: 259 return self.RunGYPGen(vals) 260 261 def CmdHelp(self): 262 if self.args.subcommand: 263 self.ParseArgs([self.args.subcommand, '--help']) 264 else: 265 self.ParseArgs(['--help']) 266 267 def CmdIsolate(self): 268 vals = self.GetConfig() 269 if not vals: 270 return 1 271 272 if vals['type'] == 'gn': 273 return self.RunGNIsolate(vals) 274 else: 275 return self.Build('%s_run' % self.args.target[0]) 276 277 def CmdLookup(self): 278 vals = self.Lookup() 279 if vals['type'] == 'gn': 280 cmd = self.GNCmd('gen', '_path_') 281 gn_args = self.GNArgs(vals) 282 self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args) 283 env = None 284 else: 285 cmd, env = self.GYPCmd('_path_', vals) 286 287 self.PrintCmd(cmd, env) 288 return 0 289 290 def CmdRun(self): 291 vals = self.GetConfig() 292 if not vals: 293 return 1 294 295 build_dir = self.args.path[0] 296 target = self.args.target[0] 297 298 if vals['type'] == 'gn': 299 if self.args.build: 300 ret = self.Build(target) 301 if ret: 302 return ret 303 ret = self.RunGNIsolate(vals) 304 if ret: 305 return ret 306 else: 307 ret = self.Build('%s_run' % target) 308 if ret: 309 return ret 310 311 cmd = [ 312 self.executable, 313 self.PathJoin('tools', 'swarming_client', 'isolate.py'), 314 'run', 315 '-s', 316 self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)), 317 ] 318 if self.args.extra_args: 319 cmd += ['--'] + self.args.extra_args 320 321 ret, _, _ = self.Run(cmd, force_verbose=False, buffer_output=False) 322 323 return ret 324 325 def CmdValidate(self, print_ok=True): 326 errs = [] 327 328 # Read the file to make sure it parses. 329 self.ReadConfigFile() 330 331 # Build a list of all of the configs referenced by builders. 332 all_configs = {} 333 for master in self.masters: 334 for config in self.masters[master].values(): 335 if isinstance(config, list): 336 for c in config: 337 all_configs[c] = master 338 else: 339 all_configs[config] = master 340 341 # Check that every referenced args file or config actually exists. 342 for config, loc in all_configs.items(): 343 if config.startswith('//'): 344 if not self.Exists(self.ToAbsPath(config)): 345 errs.append('Unknown args file "%s" referenced from "%s".' % 346 (config, loc)) 347 elif not config in self.configs: 348 errs.append('Unknown config "%s" referenced from "%s".' % 349 (config, loc)) 350 351 # Check that every actual config is actually referenced. 352 for config in self.configs: 353 if not config in all_configs: 354 errs.append('Unused config "%s".' % config) 355 356 # Figure out the whole list of mixins, and check that every mixin 357 # listed by a config or another mixin actually exists. 358 referenced_mixins = set() 359 for config, mixins in self.configs.items(): 360 for mixin in mixins: 361 if not mixin in self.mixins: 362 errs.append('Unknown mixin "%s" referenced by config "%s".' % 363 (mixin, config)) 364 referenced_mixins.add(mixin) 365 366 for mixin in self.mixins: 367 for sub_mixin in self.mixins[mixin].get('mixins', []): 368 if not sub_mixin in self.mixins: 369 errs.append('Unknown mixin "%s" referenced by mixin "%s".' % 370 (sub_mixin, mixin)) 371 referenced_mixins.add(sub_mixin) 372 373 # Check that every mixin defined is actually referenced somewhere. 374 for mixin in self.mixins: 375 if not mixin in referenced_mixins: 376 errs.append('Unreferenced mixin "%s".' % mixin) 377 378 if errs: 379 raise MBErr(('mb config file %s has problems:' % self.args.config_file) + 380 '\n ' + '\n '.join(errs)) 381 382 if print_ok: 383 self.Print('mb config file %s looks ok.' % self.args.config_file) 384 return 0 385 386 def CmdAudit(self): 387 """Track the progress of the GYP->GN migration on the bots.""" 388 389 # First, make sure the config file is okay, but don't print anything 390 # if it is (it will throw an error if it isn't). 391 self.CmdValidate(print_ok=False) 392 393 stats = OrderedDict() 394 STAT_MASTER_ONLY = 'Master only' 395 STAT_CONFIG_ONLY = 'Config only' 396 STAT_TBD = 'Still TBD' 397 STAT_GYP = 'Still GYP' 398 STAT_DONE = 'Done (on GN)' 399 stats[STAT_MASTER_ONLY] = 0 400 stats[STAT_CONFIG_ONLY] = 0 401 stats[STAT_TBD] = 0 402 stats[STAT_GYP] = 0 403 stats[STAT_DONE] = 0 404 405 def PrintBuilders(heading, builders, notes): 406 stats.setdefault(heading, 0) 407 stats[heading] += len(builders) 408 if builders: 409 self.Print(' %s:' % heading) 410 for builder in sorted(builders): 411 self.Print(' %s%s' % (builder, notes[builder])) 412 413 self.ReadConfigFile() 414 415 masters = self.args.master or self.masters 416 for master in sorted(masters): 417 url = self.args.url_template.replace('{master}', master) 418 419 self.Print('Auditing %s' % master) 420 421 MASTERS_TO_SKIP = ( 422 'client.skia', 423 'client.v8.fyi', 424 'tryserver.v8', 425 ) 426 if master in MASTERS_TO_SKIP: 427 # Skip these bots because converting them is the responsibility of 428 # those teams and out of scope for the Chromium migration to GN. 429 self.Print(' Skipped (out of scope)') 430 self.Print('') 431 continue 432 433 INTERNAL_MASTERS = ('official.desktop', 'official.desktop.continuous', 434 'internal.client.kitchensync') 435 if master in INTERNAL_MASTERS and not self.args.internal: 436 # Skip these because the servers aren't accessible by default ... 437 self.Print(' Skipped (internal)') 438 self.Print('') 439 continue 440 441 try: 442 # Fetch the /builders contents from the buildbot master. The 443 # keys of the dict are the builder names themselves. 444 json_contents = self.Fetch(url) 445 d = json.loads(json_contents) 446 except Exception as e: 447 self.Print(str(e)) 448 return 1 449 450 config_builders = set(self.masters[master]) 451 master_builders = set(d.keys()) 452 both = master_builders & config_builders 453 master_only = master_builders - config_builders 454 config_only = config_builders - master_builders 455 tbd = set() 456 gyp = set() 457 done = set() 458 notes = {builder: '' for builder in config_builders | master_builders} 459 460 for builder in both: 461 config = self.masters[master][builder] 462 if config == 'tbd': 463 tbd.add(builder) 464 elif isinstance(config, list): 465 vals = self.FlattenConfig(config[0]) 466 if vals['type'] == 'gyp': 467 gyp.add(builder) 468 else: 469 done.add(builder) 470 elif config.startswith('//'): 471 done.add(builder) 472 else: 473 vals = self.FlattenConfig(config) 474 if vals['type'] == 'gyp': 475 gyp.add(builder) 476 else: 477 done.add(builder) 478 479 if self.args.check_compile and (tbd or master_only): 480 either = tbd | master_only 481 for builder in either: 482 notes[builder] = ' (' + self.CheckCompile(master, builder) +')' 483 484 if master_only or config_only or tbd or gyp: 485 PrintBuilders(STAT_MASTER_ONLY, master_only, notes) 486 PrintBuilders(STAT_CONFIG_ONLY, config_only, notes) 487 PrintBuilders(STAT_TBD, tbd, notes) 488 PrintBuilders(STAT_GYP, gyp, notes) 489 else: 490 self.Print(' All GN!') 491 492 stats[STAT_DONE] += len(done) 493 494 self.Print('') 495 496 fmt = '{:<27} {:>4}' 497 self.Print(fmt.format('Totals', str(sum(int(v) for v in stats.values())))) 498 self.Print(fmt.format('-' * 27, '----')) 499 for stat, count in stats.items(): 500 self.Print(fmt.format(stat, str(count))) 501 502 return 0 503 504 def GetConfig(self): 505 build_dir = self.args.path[0] 506 507 vals = {} 508 if self.args.builder or self.args.master or self.args.config: 509 vals = self.Lookup() 510 if vals['type'] == 'gn': 511 # Re-run gn gen in order to ensure the config is consistent with the 512 # build dir. 513 self.RunGNGen(vals) 514 return vals 515 516 mb_type_path = self.PathJoin(self.ToAbsPath(build_dir), 'mb_type') 517 if not self.Exists(mb_type_path): 518 toolchain_path = self.PathJoin(self.ToAbsPath(build_dir), 519 'toolchain.ninja') 520 if not self.Exists(toolchain_path): 521 self.Print('Must either specify a path to an existing GN build dir ' 522 'or pass in a -m/-b pair or a -c flag to specify the ' 523 'configuration') 524 return {} 525 else: 526 mb_type = 'gn' 527 else: 528 mb_type = self.ReadFile(mb_type_path).strip() 529 530 if mb_type == 'gn': 531 vals = self.GNValsFromDir(build_dir) 532 else: 533 vals = {} 534 vals['type'] = mb_type 535 536 return vals 537 538 def GNValsFromDir(self, build_dir): 539 args_contents = "" 540 gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn') 541 if self.Exists(gn_args_path): 542 args_contents = self.ReadFile(gn_args_path) 543 gn_args = [] 544 for l in args_contents.splitlines(): 545 fields = l.split(' ') 546 name = fields[0] 547 val = ' '.join(fields[2:]) 548 gn_args.append('%s=%s' % (name, val)) 549 550 return { 551 'gn_args': ' '.join(gn_args), 552 'type': 'gn', 553 } 554 555 def Lookup(self): 556 vals = self.ReadBotConfig() 557 if not vals: 558 self.ReadConfigFile() 559 config = self.ConfigFromArgs() 560 if config.startswith('//'): 561 if not self.Exists(self.ToAbsPath(config)): 562 raise MBErr('args file "%s" not found' % config) 563 vals = { 564 'args_file': config, 565 'cros_passthrough': False, 566 'gn_args': '', 567 'gyp_crosscompile': False, 568 'gyp_defines': '', 569 'type': 'gn', 570 } 571 else: 572 if not config in self.configs: 573 raise MBErr('Config "%s" not found in %s' % 574 (config, self.args.config_file)) 575 vals = self.FlattenConfig(config) 576 577 # Do some basic sanity checking on the config so that we 578 # don't have to do this in every caller. 579 assert 'type' in vals, 'No meta-build type specified in the config' 580 assert vals['type'] in ('gn', 'gyp'), ( 581 'Unknown meta-build type "%s"' % vals['gn_args']) 582 583 return vals 584 585 def ReadBotConfig(self): 586 if not self.args.master or not self.args.builder: 587 return {} 588 path = self.PathJoin(self.chromium_src_dir, 'ios', 'build', 'bots', 589 self.args.master, self.args.builder + '.json') 590 if not self.Exists(path): 591 return {} 592 593 contents = json.loads(self.ReadFile(path)) 594 gyp_vals = contents.get('GYP_DEFINES', {}) 595 if isinstance(gyp_vals, dict): 596 gyp_defines = ' '.join('%s=%s' % (k, v) for k, v in gyp_vals.items()) 597 else: 598 gyp_defines = ' '.join(gyp_vals) 599 gn_args = ' '.join(contents.get('gn_args', [])) 600 601 return { 602 'args_file': '', 603 'cros_passthrough': False, 604 'gn_args': gn_args, 605 'gyp_crosscompile': False, 606 'gyp_defines': gyp_defines, 607 'type': contents.get('mb_type', ''), 608 } 609 610 def ReadConfigFile(self): 611 if not self.Exists(self.args.config_file): 612 raise MBErr('config file not found at %s' % self.args.config_file) 613 614 try: 615 contents = ast.literal_eval(self.ReadFile(self.args.config_file)) 616 except SyntaxError as e: 617 raise MBErr('Failed to parse config file "%s": %s' % 618 (self.args.config_file, e)) 619 620 self.configs = contents['configs'] 621 self.masters = contents['masters'] 622 self.mixins = contents['mixins'] 623 624 def ConfigFromArgs(self): 625 if self.args.config: 626 if self.args.master or self.args.builder: 627 raise MBErr('Can not specific both -c/--config and -m/--master or ' 628 '-b/--builder') 629 630 return self.args.config 631 632 if not self.args.master or not self.args.builder: 633 raise MBErr('Must specify either -c/--config or ' 634 '(-m/--master and -b/--builder)') 635 636 if not self.args.master in self.masters: 637 raise MBErr('Master name "%s" not found in "%s"' % 638 (self.args.master, self.args.config_file)) 639 640 if not self.args.builder in self.masters[self.args.master]: 641 raise MBErr('Builder name "%s" not found under masters[%s] in "%s"' % 642 (self.args.builder, self.args.master, self.args.config_file)) 643 644 config = self.masters[self.args.master][self.args.builder] 645 if isinstance(config, list): 646 if self.args.phase is None: 647 raise MBErr('Must specify a build --phase for %s on %s' % 648 (self.args.builder, self.args.master)) 649 phase = int(self.args.phase) 650 if phase < 1 or phase > len(config): 651 raise MBErr('Phase %d out of bounds for %s on %s' % 652 (phase, self.args.builder, self.args.master)) 653 return config[phase-1] 654 655 if self.args.phase is not None: 656 raise MBErr('Must not specify a build --phase for %s on %s' % 657 (self.args.builder, self.args.master)) 658 return config 659 660 def FlattenConfig(self, config): 661 mixins = self.configs[config] 662 vals = { 663 'args_file': '', 664 'cros_passthrough': False, 665 'gn_args': [], 666 'gyp_defines': '', 667 'gyp_crosscompile': False, 668 'type': None, 669 } 670 671 visited = [] 672 self.FlattenMixins(mixins, vals, visited) 673 return vals 674 675 def FlattenMixins(self, mixins, vals, visited): 676 for m in mixins: 677 if m not in self.mixins: 678 raise MBErr('Unknown mixin "%s"' % m) 679 680 visited.append(m) 681 682 mixin_vals = self.mixins[m] 683 684 if 'cros_passthrough' in mixin_vals: 685 vals['cros_passthrough'] = mixin_vals['cros_passthrough'] 686 if 'gn_args' in mixin_vals: 687 if vals['gn_args']: 688 vals['gn_args'] += ' ' + mixin_vals['gn_args'] 689 else: 690 vals['gn_args'] = mixin_vals['gn_args'] 691 if 'gyp_crosscompile' in mixin_vals: 692 vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile'] 693 if 'gyp_defines' in mixin_vals: 694 if vals['gyp_defines']: 695 vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines'] 696 else: 697 vals['gyp_defines'] = mixin_vals['gyp_defines'] 698 if 'type' in mixin_vals: 699 vals['type'] = mixin_vals['type'] 700 701 if 'mixins' in mixin_vals: 702 self.FlattenMixins(mixin_vals['mixins'], vals, visited) 703 return vals 704 705 def ClobberIfNeeded(self, vals): 706 path = self.args.path[0] 707 build_dir = self.ToAbsPath(path) 708 mb_type_path = self.PathJoin(build_dir, 'mb_type') 709 needs_clobber = False 710 new_mb_type = vals['type'] 711 if self.Exists(build_dir): 712 if self.Exists(mb_type_path): 713 old_mb_type = self.ReadFile(mb_type_path) 714 if old_mb_type != new_mb_type: 715 self.Print("Build type mismatch: was %s, will be %s, clobbering %s" % 716 (old_mb_type, new_mb_type, path)) 717 needs_clobber = True 718 else: 719 # There is no 'mb_type' file in the build directory, so this probably 720 # means that the prior build(s) were not done through mb, and we 721 # have no idea if this was a GYP build or a GN build. Clobber it 722 # to be safe. 723 self.Print("%s/mb_type missing, clobbering to be safe" % path) 724 needs_clobber = True 725 726 if self.args.dryrun: 727 return 728 729 if needs_clobber: 730 self.RemoveDirectory(build_dir) 731 732 self.MaybeMakeDirectory(build_dir) 733 self.WriteFile(mb_type_path, new_mb_type) 734 735 def RunGNGen(self, vals): 736 build_dir = self.args.path[0] 737 738 cmd = self.GNCmd('gen', build_dir, '--check') 739 gn_args = self.GNArgs(vals) 740 741 # Since GN hasn't run yet, the build directory may not even exist. 742 self.MaybeMakeDirectory(self.ToAbsPath(build_dir)) 743 744 gn_args_path = self.ToAbsPath(build_dir, 'args.gn') 745 self.WriteFile(gn_args_path, gn_args, force_verbose=True) 746 747 swarming_targets = [] 748 if getattr(self.args, 'swarming_targets_file', None): 749 # We need GN to generate the list of runtime dependencies for 750 # the compile targets listed (one per line) in the file so 751 # we can run them via swarming. We use ninja_to_gn.pyl to convert 752 # the compile targets to the matching GN labels. 753 path = self.args.swarming_targets_file 754 if not self.Exists(path): 755 self.WriteFailureAndRaise('"%s" does not exist' % path, 756 output_path=None) 757 contents = self.ReadFile(path) 758 swarming_targets = set(contents.splitlines()) 759 gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin( 760 self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl'))) 761 gn_labels = [] 762 err = '' 763 for target in swarming_targets: 764 target_name = self.GNTargetName(target) 765 if not target_name in gn_isolate_map: 766 err += ('test target "%s" not found\n' % target_name) 767 elif gn_isolate_map[target_name]['type'] == 'unknown': 768 err += ('test target "%s" type is unknown\n' % target_name) 769 else: 770 gn_labels.append(gn_isolate_map[target_name]['label']) 771 772 if err: 773 raise MBErr('Error: Failed to match swarming targets to %s:\n%s' % 774 ('//testing/buildbot/gn_isolate_map.pyl', err)) 775 776 gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps') 777 self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n') 778 cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path) 779 780 # Override msvs infra environment variables. 781 # TODO(machenbach): Remove after GYP_MSVS_VERSION is removed on infra side. 782 env = {} 783 env.update(os.environ) 784 env['GYP_MSVS_VERSION'] = '2015' 785 786 ret, _, _ = self.Run(cmd, env=env) 787 if ret: 788 # If `gn gen` failed, we should exit early rather than trying to 789 # generate isolates. Run() will have already logged any error output. 790 self.Print('GN gen failed: %d' % ret) 791 return ret 792 793 android = 'target_os="android"' in vals['gn_args'] 794 for target in swarming_targets: 795 if android: 796 # Android targets may be either android_apk or executable. The former 797 # will result in runtime_deps associated with the stamp file, while the 798 # latter will result in runtime_deps associated with the executable. 799 target_name = self.GNTargetName(target) 800 label = gn_isolate_map[target_name]['label'] 801 runtime_deps_targets = [ 802 target_name + '.runtime_deps', 803 'obj/%s.stamp.runtime_deps' % label.replace(':', '/')] 804 elif gn_isolate_map[target]['type'] == 'gpu_browser_test': 805 if self.platform == 'win32': 806 runtime_deps_targets = ['browser_tests.exe.runtime_deps'] 807 else: 808 runtime_deps_targets = ['browser_tests.runtime_deps'] 809 elif (gn_isolate_map[target]['type'] == 'script' or 810 gn_isolate_map[target].get('label_type') == 'group'): 811 # For script targets, the build target is usually a group, 812 # for which gn generates the runtime_deps next to the stamp file 813 # for the label, which lives under the obj/ directory. 814 label = gn_isolate_map[target]['label'] 815 runtime_deps_targets = [ 816 'obj/%s.stamp.runtime_deps' % label.replace(':', '/')] 817 elif self.platform == 'win32': 818 runtime_deps_targets = [target + '.exe.runtime_deps'] 819 else: 820 runtime_deps_targets = [target + '.runtime_deps'] 821 822 for r in runtime_deps_targets: 823 runtime_deps_path = self.ToAbsPath(build_dir, r) 824 if self.Exists(runtime_deps_path): 825 break 826 else: 827 raise MBErr('did not generate any of %s' % 828 ', '.join(runtime_deps_targets)) 829 830 command, extra_files = self.GetIsolateCommand(target, vals, 831 gn_isolate_map) 832 833 runtime_deps = self.ReadFile(runtime_deps_path).splitlines() 834 835 self.WriteIsolateFiles(build_dir, command, target, runtime_deps, 836 extra_files) 837 838 return 0 839 840 def RunGNIsolate(self, vals): 841 gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin( 842 self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl'))) 843 844 build_dir = self.args.path[0] 845 target = self.args.target[0] 846 target_name = self.GNTargetName(target) 847 command, extra_files = self.GetIsolateCommand(target, vals, gn_isolate_map) 848 849 label = gn_isolate_map[target_name]['label'] 850 cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps') 851 ret, out, _ = self.Call(cmd) 852 if ret: 853 if out: 854 self.Print(out) 855 return ret 856 857 runtime_deps = out.splitlines() 858 859 self.WriteIsolateFiles(build_dir, command, target, runtime_deps, 860 extra_files) 861 862 ret, _, _ = self.Run([ 863 self.executable, 864 self.PathJoin('tools', 'swarming_client', 'isolate.py'), 865 'check', 866 '-i', 867 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), 868 '-s', 869 self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target))], 870 buffer_output=False) 871 872 return ret 873 874 def WriteIsolateFiles(self, build_dir, command, target, runtime_deps, 875 extra_files): 876 isolate_path = self.ToAbsPath(build_dir, target + '.isolate') 877 self.WriteFile(isolate_path, 878 pprint.pformat({ 879 'variables': { 880 'command': command, 881 'files': sorted(runtime_deps + extra_files), 882 } 883 }) + '\n') 884 885 self.WriteJSON( 886 { 887 'args': [ 888 '--isolated', 889 self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)), 890 '--isolate', 891 self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)), 892 ], 893 'dir': self.chromium_src_dir, 894 'version': 1, 895 }, 896 isolate_path + 'd.gen.json', 897 ) 898 899 def GNCmd(self, subcommand, path, *args): 900 if self.platform == 'linux2': 901 subdir, exe = 'linux64', 'gn' 902 elif self.platform == 'darwin': 903 subdir, exe = 'mac', 'gn' 904 else: 905 subdir, exe = 'win', 'gn.exe' 906 907 gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, exe) 908 909 return [gn_path, subcommand, path] + list(args) 910 911 def GNArgs(self, vals): 912 if vals['cros_passthrough']: 913 if not 'GN_ARGS' in os.environ: 914 raise MBErr('MB is expecting GN_ARGS to be in the environment') 915 gn_args = os.environ['GN_ARGS'] 916 if not re.search('target_os.*=.*"chromeos"', gn_args): 917 raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' % 918 gn_args) 919 else: 920 gn_args = vals['gn_args'] 921 922 if self.args.goma_dir: 923 gn_args += ' goma_dir="%s"' % self.args.goma_dir 924 925 android_version_code = self.args.android_version_code 926 if android_version_code: 927 gn_args += ' android_default_version_code="%s"' % android_version_code 928 929 android_version_name = self.args.android_version_name 930 if android_version_name: 931 gn_args += ' android_default_version_name="%s"' % android_version_name 932 933 # Canonicalize the arg string into a sorted, newline-separated list 934 # of key-value pairs, and de-dup the keys if need be so that only 935 # the last instance of each arg is listed. 936 gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args)) 937 938 args_file = vals.get('args_file', None) 939 if args_file: 940 gn_args = ('import("%s")\n' % vals['args_file']) + gn_args 941 return gn_args 942 943 def RunGYPGen(self, vals): 944 path = self.args.path[0] 945 946 output_dir = self.ParseGYPConfigPath(path) 947 cmd, env = self.GYPCmd(output_dir, vals) 948 ret, _, _ = self.Run(cmd, env=env) 949 return ret 950 951 def RunGYPAnalyze(self, vals): 952 output_dir = self.ParseGYPConfigPath(self.args.path[0]) 953 if self.args.verbose: 954 inp = self.ReadInputJSON(['files', 'test_targets', 955 'additional_compile_targets']) 956 self.Print() 957 self.Print('analyze input:') 958 self.PrintJSON(inp) 959 self.Print() 960 961 cmd, env = self.GYPCmd(output_dir, vals) 962 cmd.extend(['-f', 'analyzer', 963 '-G', 'config_path=%s' % self.args.input_path[0], 964 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]]) 965 ret, _, _ = self.Run(cmd, env=env) 966 if not ret and self.args.verbose: 967 outp = json.loads(self.ReadFile(self.args.output_path[0])) 968 self.Print() 969 self.Print('analyze output:') 970 self.PrintJSON(outp) 971 self.Print() 972 973 return ret 974 975 def GetIsolateCommand(self, target, vals, gn_isolate_map): 976 android = 'target_os="android"' in vals['gn_args'] 977 978 # This needs to mirror the settings in //build/config/ui.gni: 979 # use_x11 = is_linux && !use_ozone. 980 use_x11 = (self.platform == 'linux2' and 981 not android and 982 not 'use_ozone=true' in vals['gn_args']) 983 984 asan = 'is_asan=true' in vals['gn_args'] 985 msan = 'is_msan=true' in vals['gn_args'] 986 tsan = 'is_tsan=true' in vals['gn_args'] 987 988 target_name = self.GNTargetName(target) 989 test_type = gn_isolate_map[target_name]['type'] 990 991 executable = gn_isolate_map[target_name].get('executable', target_name) 992 executable_suffix = '.exe' if self.platform == 'win32' else '' 993 994 cmdline = [] 995 extra_files = [] 996 997 if android and test_type != "script": 998 logdog_command = [ 999 '--logdog-bin-cmd', './../../bin/logdog_butler', 1000 '--project', 'chromium', 1001 '--service-account-json', 1002 '/creds/service_accounts/service-account-luci-logdog-publisher.json', 1003 '--prefix', 'android/swarming/logcats/${SWARMING_TASK_ID}', 1004 '--source', '${ISOLATED_OUTDIR}/logcats', 1005 '--name', 'unified_logcats', 1006 ] 1007 test_cmdline = [ 1008 self.PathJoin('bin', 'run_%s' % target_name), 1009 '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats', 1010 '--target-devices-file', '${SWARMING_BOT_FILE}', 1011 '-v' 1012 ] 1013 cmdline = (['./../../build/android/test_wrapper/logdog_wrapper.py'] 1014 + logdog_command + test_cmdline) 1015 elif use_x11 and test_type == 'windowed_test_launcher': 1016 extra_files = [ 1017 '../../testing/test_env.py', 1018 '../../testing/xvfb.py', 1019 ] 1020 cmdline = [ 1021 '../../testing/xvfb.py', 1022 '.', 1023 './' + str(executable) + executable_suffix, 1024 '--brave-new-test-launcher', 1025 '--test-launcher-bot-mode', 1026 '--asan=%d' % asan, 1027 '--msan=%d' % msan, 1028 '--tsan=%d' % tsan, 1029 ] 1030 elif test_type in ('windowed_test_launcher', 'console_test_launcher'): 1031 extra_files = [ 1032 '../../testing/test_env.py' 1033 ] 1034 cmdline = [ 1035 '../../testing/test_env.py', 1036 './' + str(executable) + executable_suffix, 1037 '--brave-new-test-launcher', 1038 '--test-launcher-bot-mode', 1039 '--asan=%d' % asan, 1040 '--msan=%d' % msan, 1041 '--tsan=%d' % tsan, 1042 ] 1043 elif test_type == 'gpu_browser_test': 1044 extra_files = [ 1045 '../../testing/test_env.py' 1046 ] 1047 gtest_filter = gn_isolate_map[target]['gtest_filter'] 1048 cmdline = [ 1049 '../../testing/test_env.py', 1050 './browser_tests' + executable_suffix, 1051 '--test-launcher-bot-mode', 1052 '--enable-gpu', 1053 '--test-launcher-jobs=1', 1054 '--gtest_filter=%s' % gtest_filter, 1055 ] 1056 elif test_type == 'script': 1057 extra_files = [ 1058 '../../testing/test_env.py' 1059 ] 1060 cmdline = [ 1061 '../../testing/test_env.py', 1062 '../../' + self.ToSrcRelPath(gn_isolate_map[target]['script']) 1063 ] 1064 elif test_type in ('raw'): 1065 extra_files = [] 1066 cmdline = [ 1067 './' + str(target) + executable_suffix, 1068 ] 1069 1070 else: 1071 self.WriteFailureAndRaise('No command line for %s found (test type %s).' 1072 % (target, test_type), output_path=None) 1073 1074 cmdline += gn_isolate_map[target_name].get('args', []) 1075 1076 return cmdline, extra_files 1077 1078 def ToAbsPath(self, build_path, *comps): 1079 return self.PathJoin(self.chromium_src_dir, 1080 self.ToSrcRelPath(build_path), 1081 *comps) 1082 1083 def ToSrcRelPath(self, path): 1084 """Returns a relative path from the top of the repo.""" 1085 if path.startswith('//'): 1086 return path[2:].replace('/', self.sep) 1087 return self.RelPath(path, self.chromium_src_dir) 1088 1089 def ParseGYPConfigPath(self, path): 1090 rpath = self.ToSrcRelPath(path) 1091 output_dir, _, _ = rpath.rpartition(self.sep) 1092 return output_dir 1093 1094 def GYPCmd(self, output_dir, vals): 1095 if vals['cros_passthrough']: 1096 if not 'GYP_DEFINES' in os.environ: 1097 raise MBErr('MB is expecting GYP_DEFINES to be in the environment') 1098 gyp_defines = os.environ['GYP_DEFINES'] 1099 if not 'chromeos=1' in gyp_defines: 1100 raise MBErr('GYP_DEFINES is missing chromeos=1: (GYP_DEFINES=%s)' % 1101 gyp_defines) 1102 else: 1103 gyp_defines = vals['gyp_defines'] 1104 1105 goma_dir = self.args.goma_dir 1106 1107 # GYP uses shlex.split() to split the gyp defines into separate arguments, 1108 # so we can support backslashes and and spaces in arguments by quoting 1109 # them, even on Windows, where this normally wouldn't work. 1110 if goma_dir and ('\\' in goma_dir or ' ' in goma_dir): 1111 goma_dir = "'%s'" % goma_dir 1112 1113 if goma_dir: 1114 gyp_defines += ' gomadir=%s' % goma_dir 1115 1116 android_version_code = self.args.android_version_code 1117 if android_version_code: 1118 gyp_defines += ' app_manifest_version_code=%s' % android_version_code 1119 1120 android_version_name = self.args.android_version_name 1121 if android_version_name: 1122 gyp_defines += ' app_manifest_version_name=%s' % android_version_name 1123 1124 cmd = [ 1125 self.executable, 1126 self.args.gyp_script, 1127 '-G', 1128 'output_dir=' + output_dir, 1129 ] 1130 1131 # Ensure that we have an environment that only contains 1132 # the exact values of the GYP variables we need. 1133 env = os.environ.copy() 1134 1135 # This is a terrible hack to work around the fact that 1136 # //tools/clang/scripts/update.py is invoked by GYP and GN but 1137 # currently relies on an environment variable to figure out 1138 # what revision to embed in the command line #defines. 1139 # For GN, we've made this work via a gn arg that will cause update.py 1140 # to get an additional command line arg, but getting that to work 1141 # via GYP_DEFINES has proven difficult, so we rewrite the GYP_DEFINES 1142 # to get rid of the arg and add the old var in, instead. 1143 # See crbug.com/582737 for more on this. This can hopefully all 1144 # go away with GYP. 1145 m = re.search('llvm_force_head_revision=1\s*', gyp_defines) 1146 if m: 1147 env['LLVM_FORCE_HEAD_REVISION'] = '1' 1148 gyp_defines = gyp_defines.replace(m.group(0), '') 1149 1150 # This is another terrible hack to work around the fact that 1151 # GYP sets the link concurrency to use via the GYP_LINK_CONCURRENCY 1152 # environment variable, and not via a proper GYP_DEFINE. See 1153 # crbug.com/611491 for more on this. 1154 m = re.search('gyp_link_concurrency=(\d+)(\s*)', gyp_defines) 1155 if m: 1156 env['GYP_LINK_CONCURRENCY'] = m.group(1) 1157 gyp_defines = gyp_defines.replace(m.group(0), '') 1158 1159 env['GYP_GENERATORS'] = 'ninja' 1160 if 'GYP_CHROMIUM_NO_ACTION' in env: 1161 del env['GYP_CHROMIUM_NO_ACTION'] 1162 if 'GYP_CROSSCOMPILE' in env: 1163 del env['GYP_CROSSCOMPILE'] 1164 env['GYP_DEFINES'] = gyp_defines 1165 if vals['gyp_crosscompile']: 1166 env['GYP_CROSSCOMPILE'] = '1' 1167 return cmd, env 1168 1169 def RunGNAnalyze(self, vals): 1170 # analyze runs before 'gn gen' now, so we need to run gn gen 1171 # in order to ensure that we have a build directory. 1172 ret = self.RunGNGen(vals) 1173 if ret: 1174 return ret 1175 1176 inp = self.ReadInputJSON(['files', 'test_targets', 1177 'additional_compile_targets']) 1178 if self.args.verbose: 1179 self.Print() 1180 self.Print('analyze input:') 1181 self.PrintJSON(inp) 1182 self.Print() 1183 1184 # TODO(crbug.com/555273) - currently GN treats targets and 1185 # additional_compile_targets identically since we can't tell the 1186 # difference between a target that is a group in GN and one that isn't. 1187 # We should eventually fix this and treat the two types differently. 1188 targets = (set(inp['test_targets']) | 1189 set(inp['additional_compile_targets'])) 1190 1191 output_path = self.args.output_path[0] 1192 1193 # Bail out early if a GN file was modified, since 'gn refs' won't know 1194 # what to do about it. Also, bail out early if 'all' was asked for, 1195 # since we can't deal with it yet. 1196 if (any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']) or 1197 'all' in targets): 1198 self.WriteJSON({ 1199 'status': 'Found dependency (all)', 1200 'compile_targets': sorted(targets), 1201 'test_targets': sorted(targets & set(inp['test_targets'])), 1202 }, output_path) 1203 return 0 1204 1205 # This shouldn't normally happen, but could due to unusual race conditions, 1206 # like a try job that gets scheduled before a patch lands but runs after 1207 # the patch has landed. 1208 if not inp['files']: 1209 self.Print('Warning: No files modified in patch, bailing out early.') 1210 self.WriteJSON({ 1211 'status': 'No dependency', 1212 'compile_targets': [], 1213 'test_targets': [], 1214 }, output_path) 1215 return 0 1216 1217 ret = 0 1218 response_file = self.TempFile() 1219 response_file.write('\n'.join(inp['files']) + '\n') 1220 response_file.close() 1221 1222 matching_targets = set() 1223 try: 1224 cmd = self.GNCmd('refs', 1225 self.args.path[0], 1226 '@%s' % response_file.name, 1227 '--all', 1228 '--as=output') 1229 ret, out, _ = self.Run(cmd, force_verbose=False) 1230 if ret and not 'The input matches no targets' in out: 1231 self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out), 1232 output_path) 1233 build_dir = self.ToSrcRelPath(self.args.path[0]) + self.sep 1234 for output in out.splitlines(): 1235 build_output = output.replace(build_dir, '') 1236 if build_output in targets: 1237 matching_targets.add(build_output) 1238 1239 cmd = self.GNCmd('refs', 1240 self.args.path[0], 1241 '@%s' % response_file.name, 1242 '--all') 1243 ret, out, _ = self.Run(cmd, force_verbose=False) 1244 if ret and not 'The input matches no targets' in out: 1245 self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out), 1246 output_path) 1247 for label in out.splitlines(): 1248 build_target = label[2:] 1249 # We want to accept 'chrome/android:chrome_public_apk' and 1250 # just 'chrome_public_apk'. This may result in too many targets 1251 # getting built, but we can adjust that later if need be. 1252 for input_target in targets: 1253 if (input_target == build_target or 1254 build_target.endswith(':' + input_target)): 1255 matching_targets.add(input_target) 1256 finally: 1257 self.RemoveFile(response_file.name) 1258 1259 if matching_targets: 1260 self.WriteJSON({ 1261 'status': 'Found dependency', 1262 'compile_targets': sorted(matching_targets), 1263 'test_targets': sorted(matching_targets & 1264 set(inp['test_targets'])), 1265 }, output_path) 1266 else: 1267 self.WriteJSON({ 1268 'status': 'No dependency', 1269 'compile_targets': [], 1270 'test_targets': [], 1271 }, output_path) 1272 1273 if self.args.verbose: 1274 outp = json.loads(self.ReadFile(output_path)) 1275 self.Print() 1276 self.Print('analyze output:') 1277 self.PrintJSON(outp) 1278 self.Print() 1279 1280 return 0 1281 1282 def ReadInputJSON(self, required_keys): 1283 path = self.args.input_path[0] 1284 output_path = self.args.output_path[0] 1285 if not self.Exists(path): 1286 self.WriteFailureAndRaise('"%s" does not exist' % path, output_path) 1287 1288 try: 1289 inp = json.loads(self.ReadFile(path)) 1290 except Exception as e: 1291 self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' % 1292 (path, e), output_path) 1293 1294 for k in required_keys: 1295 if not k in inp: 1296 self.WriteFailureAndRaise('input file is missing a "%s" key' % k, 1297 output_path) 1298 1299 return inp 1300 1301 def WriteFailureAndRaise(self, msg, output_path): 1302 if output_path: 1303 self.WriteJSON({'error': msg}, output_path, force_verbose=True) 1304 raise MBErr(msg) 1305 1306 def WriteJSON(self, obj, path, force_verbose=False): 1307 try: 1308 self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n', 1309 force_verbose=force_verbose) 1310 except Exception as e: 1311 raise MBErr('Error %s writing to the output path "%s"' % 1312 (e, path)) 1313 1314 def CheckCompile(self, master, builder): 1315 url_template = self.args.url_template + '/{builder}/builds/_all?as_text=1' 1316 url = urllib2.quote(url_template.format(master=master, builder=builder), 1317 safe=':/()?=') 1318 try: 1319 builds = json.loads(self.Fetch(url)) 1320 except Exception as e: 1321 return str(e) 1322 successes = sorted( 1323 [int(x) for x in builds.keys() if "text" in builds[x] and 1324 cmp(builds[x]["text"][:2], ["build", "successful"]) == 0], 1325 reverse=True) 1326 if not successes: 1327 return "no successful builds" 1328 build = builds[str(successes[0])] 1329 step_names = set([step["name"] for step in build["steps"]]) 1330 compile_indicators = set(["compile", "compile (with patch)", "analyze"]) 1331 if compile_indicators & step_names: 1332 return "compiles" 1333 return "does not compile" 1334 1335 def PrintCmd(self, cmd, env): 1336 if self.platform == 'win32': 1337 env_prefix = 'set ' 1338 env_quoter = QuoteForSet 1339 shell_quoter = QuoteForCmd 1340 else: 1341 env_prefix = '' 1342 env_quoter = pipes.quote 1343 shell_quoter = pipes.quote 1344 1345 def print_env(var): 1346 if env and var in env: 1347 self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var]))) 1348 1349 print_env('GYP_CROSSCOMPILE') 1350 print_env('GYP_DEFINES') 1351 print_env('GYP_LINK_CONCURRENCY') 1352 print_env('LLVM_FORCE_HEAD_REVISION') 1353 1354 if cmd[0] == self.executable: 1355 cmd = ['python'] + cmd[1:] 1356 self.Print(*[shell_quoter(arg) for arg in cmd]) 1357 1358 def PrintJSON(self, obj): 1359 self.Print(json.dumps(obj, indent=2, sort_keys=True)) 1360 1361 def GNTargetName(self, target): 1362 return target 1363 1364 def Build(self, target): 1365 build_dir = self.ToSrcRelPath(self.args.path[0]) 1366 ninja_cmd = ['ninja', '-C', build_dir] 1367 if self.args.jobs: 1368 ninja_cmd.extend(['-j', '%d' % self.args.jobs]) 1369 ninja_cmd.append(target) 1370 ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False) 1371 return ret 1372 1373 def Run(self, cmd, env=None, force_verbose=True, buffer_output=True): 1374 # This function largely exists so it can be overridden for testing. 1375 if self.args.dryrun or self.args.verbose or force_verbose: 1376 self.PrintCmd(cmd, env) 1377 if self.args.dryrun: 1378 return 0, '', '' 1379 1380 ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output) 1381 if self.args.verbose or force_verbose: 1382 if ret: 1383 self.Print(' -> returned %d' % ret) 1384 if out: 1385 self.Print(out, end='') 1386 if err: 1387 self.Print(err, end='', file=sys.stderr) 1388 return ret, out, err 1389 1390 def Call(self, cmd, env=None, buffer_output=True): 1391 if buffer_output: 1392 p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir, 1393 stdout=subprocess.PIPE, stderr=subprocess.PIPE, 1394 env=env) 1395 out, err = p.communicate() 1396 else: 1397 p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir, 1398 env=env) 1399 p.wait() 1400 out = err = '' 1401 return p.returncode, out, err 1402 1403 def ExpandUser(self, path): 1404 # This function largely exists so it can be overridden for testing. 1405 return os.path.expanduser(path) 1406 1407 def Exists(self, path): 1408 # This function largely exists so it can be overridden for testing. 1409 return os.path.exists(path) 1410 1411 def Fetch(self, url): 1412 # This function largely exists so it can be overridden for testing. 1413 f = urllib2.urlopen(url) 1414 contents = f.read() 1415 f.close() 1416 return contents 1417 1418 def MaybeMakeDirectory(self, path): 1419 try: 1420 os.makedirs(path) 1421 except OSError, e: 1422 if e.errno != errno.EEXIST: 1423 raise 1424 1425 def PathJoin(self, *comps): 1426 # This function largely exists so it can be overriden for testing. 1427 return os.path.join(*comps) 1428 1429 def Print(self, *args, **kwargs): 1430 # This function largely exists so it can be overridden for testing. 1431 print(*args, **kwargs) 1432 if kwargs.get('stream', sys.stdout) == sys.stdout: 1433 sys.stdout.flush() 1434 1435 def ReadFile(self, path): 1436 # This function largely exists so it can be overriden for testing. 1437 with open(path) as fp: 1438 return fp.read() 1439 1440 def RelPath(self, path, start='.'): 1441 # This function largely exists so it can be overriden for testing. 1442 return os.path.relpath(path, start) 1443 1444 def RemoveFile(self, path): 1445 # This function largely exists so it can be overriden for testing. 1446 os.remove(path) 1447 1448 def RemoveDirectory(self, abs_path): 1449 if self.platform == 'win32': 1450 # In other places in chromium, we often have to retry this command 1451 # because we're worried about other processes still holding on to 1452 # file handles, but when MB is invoked, it will be early enough in the 1453 # build that their should be no other processes to interfere. We 1454 # can change this if need be. 1455 self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path]) 1456 else: 1457 shutil.rmtree(abs_path, ignore_errors=True) 1458 1459 def TempFile(self, mode='w'): 1460 # This function largely exists so it can be overriden for testing. 1461 return tempfile.NamedTemporaryFile(mode=mode, delete=False) 1462 1463 def WriteFile(self, path, contents, force_verbose=False): 1464 # This function largely exists so it can be overriden for testing. 1465 if self.args.dryrun or self.args.verbose or force_verbose: 1466 self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path)) 1467 with open(path, 'w') as fp: 1468 return fp.write(contents) 1469 1470 1471 class MBErr(Exception): 1472 pass 1473 1474 1475 # See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful 1476 # details of this next section, which handles escaping command lines 1477 # so that they can be copied and pasted into a cmd window. 1478 UNSAFE_FOR_SET = set('^<>&|') 1479 UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%')) 1480 ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"')) 1481 1482 1483 def QuoteForSet(arg): 1484 if any(a in UNSAFE_FOR_SET for a in arg): 1485 arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg) 1486 return arg 1487 1488 1489 def QuoteForCmd(arg): 1490 # First, escape the arg so that CommandLineToArgvW will parse it properly. 1491 # From //tools/gyp/pylib/gyp/msvs_emulation.py:23. 1492 if arg == '' or ' ' in arg or '"' in arg: 1493 quote_re = re.compile(r'(\\*)"') 1494 arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg)) 1495 1496 # Then check to see if the arg contains any metacharacters other than 1497 # double quotes; if it does, quote everything (including the double 1498 # quotes) for safety. 1499 if any(a in UNSAFE_FOR_CMD for a in arg): 1500 arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg) 1501 return arg 1502 1503 1504 if __name__ == '__main__': 1505 sys.exit(main(sys.argv[1:])) 1506
