1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 from collections import OrderedDict 32 import itertools 33 import json 34 import multiprocessing 35 import optparse 36 import os 37 from os.path import getmtime, isdir, join 38 import platform 39 import random 40 import shlex 41 import subprocess 42 import sys 43 import time 44 45 from testrunner.local import execution 46 from testrunner.local import progress 47 from testrunner.local import testsuite 48 from testrunner.local.variants import ALL_VARIANTS 49 from testrunner.local import utils 50 from testrunner.local import verbose 51 from testrunner.network import network_execution 52 from testrunner.objects import context 53 54 55 # Base dir of the v8 checkout to be used as cwd. 56 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 57 58 DEFAULT_OUT_GN = "out.gn" 59 60 ARCH_GUESS = utils.DefaultArch() 61 62 # Map of test name synonyms to lists of test suites. Should be ordered by 63 # expected runtimes (suites with slow test cases first). These groups are 64 # invoked in seperate steps on the bots. 65 TEST_MAP = { 66 # This needs to stay in sync with test/bot_default.isolate. 67 "bot_default": [ 68 "mjsunit", 69 "cctest", 70 "debugger", 71 "inspector", 72 "webkit", 73 "fuzzer", 74 "message", 75 "preparser", 76 "intl", 77 "unittests", 78 ], 79 # This needs to stay in sync with test/default.isolate. 80 "default": [ 81 "mjsunit", 82 "cctest", 83 "debugger", 84 "inspector", 85 "fuzzer", 86 "message", 87 "preparser", 88 "intl", 89 "unittests", 90 ], 91 # This needs to stay in sync with test/optimize_for_size.isolate. 92 "optimize_for_size": [ 93 "mjsunit", 94 "cctest", 95 "debugger", 96 "inspector", 97 "webkit", 98 "intl", 99 ], 100 "unittests": [ 101 "unittests", 102 ], 103 } 104 105 TIMEOUT_DEFAULT = 60 106 107 VARIANTS = ["default", "turbofan", "ignition_staging"] 108 109 MORE_VARIANTS = [ 110 "ignition", 111 "stress", 112 "turbofan_opt", 113 "asm_wasm", 114 ] 115 116 EXHAUSTIVE_VARIANTS = VARIANTS + MORE_VARIANTS 117 118 VARIANT_ALIASES = { 119 # The default for developer workstations. 120 "dev": VARIANTS, 121 # Additional variants, run on all bots. 122 "more": MORE_VARIANTS, 123 # Additional variants, run on a subset of bots. 124 "extra": ["nocrankshaft"], 125 } 126 127 DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination", 128 "--nofold-constants", "--enable-slow-asserts", 129 "--debug-code", "--verify-heap"] 130 RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination", 131 "--nofold-constants"] 132 133 MODES = { 134 "debug": { 135 "flags": DEBUG_FLAGS, 136 "timeout_scalefactor": 4, 137 "status_mode": "debug", 138 "execution_mode": "debug", 139 "output_folder": "debug", 140 }, 141 "optdebug": { 142 "flags": DEBUG_FLAGS, 143 "timeout_scalefactor": 4, 144 "status_mode": "debug", 145 "execution_mode": "debug", 146 "output_folder": "optdebug", 147 }, 148 "release": { 149 "flags": RELEASE_FLAGS, 150 "timeout_scalefactor": 1, 151 "status_mode": "release", 152 "execution_mode": "release", 153 "output_folder": "release", 154 }, 155 # Normal trybot release configuration. There, dchecks are always on which 156 # implies debug is set. Hence, the status file needs to assume debug-like 157 # behavior/timeouts. 158 "tryrelease": { 159 "flags": RELEASE_FLAGS, 160 "timeout_scalefactor": 1, 161 "status_mode": "debug", 162 "execution_mode": "release", 163 "output_folder": "release", 164 }, 165 # This mode requires v8 to be compiled with dchecks and slow dchecks. 166 "slowrelease": { 167 "flags": RELEASE_FLAGS + ["--enable-slow-asserts"], 168 "timeout_scalefactor": 2, 169 "status_mode": "debug", 170 "execution_mode": "release", 171 "output_folder": "release", 172 }, 173 } 174 175 GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", 176 "--concurrent-recompilation-queue-length=64", 177 "--concurrent-recompilation-delay=500", 178 "--concurrent-recompilation"] 179 180 SUPPORTED_ARCHS = ["android_arm", 181 "android_arm64", 182 "android_ia32", 183 "android_x64", 184 "arm", 185 "ia32", 186 "x87", 187 "mips", 188 "mipsel", 189 "mips64", 190 "mips64el", 191 "s390", 192 "s390x", 193 "ppc", 194 "ppc64", 195 "x64", 196 "x32", 197 "arm64"] 198 # Double the timeout for these: 199 SLOW_ARCHS = ["android_arm", 200 "android_arm64", 201 "android_ia32", 202 "android_x64", 203 "arm", 204 "mips", 205 "mipsel", 206 "mips64", 207 "mips64el", 208 "s390", 209 "s390x", 210 "x87", 211 "arm64"] 212 213 214 def BuildOptions(): 215 result = optparse.OptionParser() 216 result.usage = '%prog [options] [tests]' 217 result.description = """TESTS: %s""" % (TEST_MAP["default"]) 218 result.add_option("--arch", 219 help=("The architecture to run tests for, " 220 "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS), 221 default="ia32,x64,arm") 222 result.add_option("--arch-and-mode", 223 help="Architecture and mode in the format 'arch.mode'", 224 default=None) 225 result.add_option("--asan", 226 help="Regard test expectations for ASAN", 227 default=False, action="store_true") 228 result.add_option("--sancov-dir", 229 help="Directory where to collect coverage data") 230 result.add_option("--cfi-vptr", 231 help="Run tests with UBSAN cfi_vptr option.", 232 default=False, action="store_true") 233 result.add_option("--buildbot", 234 help="Adapt to path structure used on buildbots", 235 default=False, action="store_true") 236 result.add_option("--dcheck-always-on", 237 help="Indicates that V8 was compiled with DCHECKs enabled", 238 default=False, action="store_true") 239 result.add_option("--novfp3", 240 help="Indicates that V8 was compiled without VFP3 support", 241 default=False, action="store_true") 242 result.add_option("--cat", help="Print the source of the tests", 243 default=False, action="store_true") 244 result.add_option("--slow-tests", 245 help="Regard slow tests (run|skip|dontcare)", 246 default="dontcare") 247 result.add_option("--pass-fail-tests", 248 help="Regard pass|fail tests (run|skip|dontcare)", 249 default="dontcare") 250 result.add_option("--gc-stress", 251 help="Switch on GC stress mode", 252 default=False, action="store_true") 253 result.add_option("--gcov-coverage", 254 help="Uses executables instrumented for gcov coverage", 255 default=False, action="store_true") 256 result.add_option("--command-prefix", 257 help="Prepended to each shell command used to run a test", 258 default="") 259 result.add_option("--download-data", help="Download missing test suite data", 260 default=False, action="store_true") 261 result.add_option("--download-data-only", 262 help="Deprecated", 263 default=False, action="store_true") 264 result.add_option("--enable-inspector", 265 help="Indicates a build with inspector support", 266 default=False, action="store_true") 267 result.add_option("--extra-flags", 268 help="Additional flags to pass to each test command", 269 default="") 270 result.add_option("--isolates", help="Whether to test isolates", 271 default=False, action="store_true") 272 result.add_option("-j", help="The number of parallel tasks to run", 273 default=0, type="int") 274 result.add_option("-m", "--mode", 275 help="The test modes in which to run (comma-separated," 276 " uppercase for ninja and buildbot builds): %s" % MODES.keys(), 277 default="release,debug") 278 result.add_option("--no-harness", "--noharness", 279 help="Run without test harness of a given suite", 280 default=False, action="store_true") 281 result.add_option("--no-i18n", "--noi18n", 282 help="Skip internationalization tests", 283 default=False, action="store_true") 284 result.add_option("--no-network", "--nonetwork", 285 help="Don't distribute tests on the network", 286 default=(utils.GuessOS() != "linux"), 287 dest="no_network", action="store_true") 288 result.add_option("--no-presubmit", "--nopresubmit", 289 help='Skip presubmit checks (deprecated)', 290 default=False, dest="no_presubmit", action="store_true") 291 result.add_option("--no-snap", "--nosnap", 292 help='Test a build compiled without snapshot.', 293 default=False, dest="no_snap", action="store_true") 294 result.add_option("--no-sorting", "--nosorting", 295 help="Don't sort tests according to duration of last run.", 296 default=False, dest="no_sorting", action="store_true") 297 result.add_option("--no-variants", "--novariants", 298 help="Don't run any testing variants", 299 default=False, dest="no_variants", action="store_true") 300 result.add_option("--variants", 301 help="Comma-separated list of testing variants;" 302 " default: \"%s\"" % ",".join(VARIANTS)) 303 result.add_option("--exhaustive-variants", 304 default=False, action="store_true", 305 help="Use exhaustive set of default variants:" 306 " \"%s\"" % ",".join(EXHAUSTIVE_VARIANTS)) 307 result.add_option("--outdir", help="Base directory with compile output", 308 default="out") 309 result.add_option("--gn", help="Scan out.gn for the last built configuration", 310 default=False, action="store_true") 311 result.add_option("--predictable", 312 help="Compare output of several reruns of each test", 313 default=False, action="store_true") 314 result.add_option("-p", "--progress", 315 help=("The style of progress indicator" 316 " (verbose, dots, color, mono)"), 317 choices=progress.PROGRESS_INDICATORS.keys(), default="mono") 318 result.add_option("--quickcheck", default=False, action="store_true", 319 help=("Quick check mode (skip slow tests)")) 320 result.add_option("--report", help="Print a summary of the tests to be run", 321 default=False, action="store_true") 322 result.add_option("--json-test-results", 323 help="Path to a file for storing json results.") 324 result.add_option("--rerun-failures-count", 325 help=("Number of times to rerun each failing test case. " 326 "Very slow tests will be rerun only once."), 327 default=0, type="int") 328 result.add_option("--rerun-failures-max", 329 help="Maximum number of failing test cases to rerun.", 330 default=100, type="int") 331 result.add_option("--shard-count", 332 help="Split testsuites into this number of shards", 333 default=1, type="int") 334 result.add_option("--shard-run", 335 help="Run this shard from the split up tests.", 336 default=1, type="int") 337 result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") 338 result.add_option("--shell-dir", help="Directory containing executables", 339 default="") 340 result.add_option("--dont-skip-slow-simulator-tests", 341 help="Don't skip more slow tests when using a simulator.", 342 default=False, action="store_true", 343 dest="dont_skip_simulator_slow_tests") 344 result.add_option("--swarming", 345 help="Indicates running test driver on swarming.", 346 default=False, action="store_true") 347 result.add_option("--time", help="Print timing information after running", 348 default=False, action="store_true") 349 result.add_option("-t", "--timeout", help="Timeout in seconds", 350 default=TIMEOUT_DEFAULT, type="int") 351 result.add_option("--tsan", 352 help="Regard test expectations for TSAN", 353 default=False, action="store_true") 354 result.add_option("-v", "--verbose", help="Verbose output", 355 default=False, action="store_true") 356 result.add_option("--valgrind", help="Run tests through valgrind", 357 default=False, action="store_true") 358 result.add_option("--warn-unused", help="Report unused rules", 359 default=False, action="store_true") 360 result.add_option("--junitout", help="File name of the JUnit output") 361 result.add_option("--junittestsuite", 362 help="The testsuite name in the JUnit output file", 363 default="v8tests") 364 result.add_option("--random-seed", default=0, dest="random_seed", type="int", 365 help="Default seed for initializing random generator") 366 result.add_option("--random-seed-stress-count", default=1, type="int", 367 dest="random_seed_stress_count", 368 help="Number of runs with different random seeds") 369 result.add_option("--msan", 370 help="Regard test expectations for MSAN", 371 default=False, action="store_true") 372 return result 373 374 375 def RandomSeed(): 376 seed = 0 377 while not seed: 378 seed = random.SystemRandom().randint(-2147483648, 2147483647) 379 return seed 380 381 382 def BuildbotToV8Mode(config): 383 """Convert buildbot build configs to configs understood by the v8 runner. 384 385 V8 configs are always lower case and without the additional _x64 suffix for 386 64 bit builds on windows with ninja. 387 """ 388 mode = config[:-4] if config.endswith('_x64') else config 389 return mode.lower() 390 391 def SetupEnvironment(options): 392 """Setup additional environment variables.""" 393 394 # Many tests assume an English interface. 395 os.environ['LANG'] = 'en_US.UTF-8' 396 397 symbolizer = 'external_symbolizer_path=%s' % ( 398 os.path.join( 399 BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin', 400 'llvm-symbolizer', 401 ) 402 ) 403 404 if options.asan: 405 os.environ['ASAN_OPTIONS'] = symbolizer 406 407 if options.sancov_dir: 408 assert os.path.exists(options.sancov_dir) 409 os.environ['ASAN_OPTIONS'] = ":".join([ 410 'coverage=1', 411 'coverage_dir=%s' % options.sancov_dir, 412 symbolizer, 413 ]) 414 415 if options.cfi_vptr: 416 os.environ['UBSAN_OPTIONS'] = ":".join([ 417 'print_stacktrace=1', 418 'print_summary=1', 419 'symbolize=1', 420 symbolizer, 421 ]) 422 423 if options.msan: 424 os.environ['MSAN_OPTIONS'] = symbolizer 425 426 if options.tsan: 427 suppressions_file = os.path.join( 428 BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt') 429 os.environ['TSAN_OPTIONS'] = " ".join([ 430 symbolizer, 431 'suppressions=%s' % suppressions_file, 432 'exit_code=0', 433 'report_thread_leaks=0', 434 'history_size=7', 435 'report_destroy_locked=0', 436 ]) 437 438 def ProcessOptions(options): 439 global VARIANTS 440 441 # First try to auto-detect configurations based on the build if GN was 442 # used. This can't be overridden by cmd-line arguments. 443 options.auto_detect = False 444 if options.gn: 445 gn_out_dir = os.path.join(BASE_DIR, DEFAULT_OUT_GN) 446 latest_timestamp = -1 447 latest_config = None 448 for gn_config in os.listdir(gn_out_dir): 449 gn_config_dir = os.path.join(gn_out_dir, gn_config) 450 if not isdir(gn_config_dir): 451 continue 452 if os.path.getmtime(gn_config_dir) > latest_timestamp: 453 latest_timestamp = os.path.getmtime(gn_config_dir) 454 latest_config = gn_config 455 if latest_config: 456 print(">>> Latest GN build found is %s" % latest_config) 457 options.outdir = os.path.join(DEFAULT_OUT_GN, latest_config) 458 459 if options.buildbot: 460 build_config_path = os.path.join( 461 BASE_DIR, options.outdir, options.mode, "v8_build_config.json") 462 else: 463 build_config_path = os.path.join( 464 BASE_DIR, options.outdir, "v8_build_config.json") 465 466 if os.path.exists(build_config_path): 467 try: 468 with open(build_config_path) as f: 469 build_config = json.load(f) 470 except Exception: 471 print ("%s exists but contains invalid json. Is your build up-to-date?" % 472 build_config_path) 473 return False 474 options.auto_detect = True 475 476 # In auto-detect mode the outdir is always where we found the build config. 477 # This ensures that we'll also take the build products from there. 478 options.outdir = os.path.dirname(build_config_path) 479 480 options.arch_and_mode = None 481 options.arch = build_config["v8_target_cpu"] 482 if options.arch == 'x86': 483 # TODO(machenbach): Transform all to x86 eventually. 484 options.arch = 'ia32' 485 options.asan = build_config["is_asan"] 486 options.dcheck_always_on = build_config["dcheck_always_on"] 487 options.enable_inspector = build_config["v8_enable_inspector"] 488 options.mode = 'debug' if build_config["is_debug"] else 'release' 489 options.msan = build_config["is_msan"] 490 options.no_i18n = not build_config["v8_enable_i18n_support"] 491 options.no_snap = not build_config["v8_use_snapshot"] 492 options.tsan = build_config["is_tsan"] 493 494 # Architecture and mode related stuff. 495 if options.arch_and_mode: 496 options.arch_and_mode = [arch_and_mode.split(".") 497 for arch_and_mode in options.arch_and_mode.split(",")] 498 options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) 499 options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) 500 options.mode = options.mode.split(",") 501 for mode in options.mode: 502 if not BuildbotToV8Mode(mode) in MODES: 503 print "Unknown mode %s" % mode 504 return False 505 if options.arch in ["auto", "native"]: 506 options.arch = ARCH_GUESS 507 options.arch = options.arch.split(",") 508 for arch in options.arch: 509 if not arch in SUPPORTED_ARCHS: 510 print "Unknown architecture %s" % arch 511 return False 512 513 # Store the final configuration in arch_and_mode list. Don't overwrite 514 # predefined arch_and_mode since it is more expressive than arch and mode. 515 if not options.arch_and_mode: 516 options.arch_and_mode = itertools.product(options.arch, options.mode) 517 518 # Special processing of other options, sorted alphabetically. 519 520 if options.buildbot: 521 options.no_network = True 522 if options.command_prefix: 523 print("Specifying --command-prefix disables network distribution, " 524 "running tests locally.") 525 options.no_network = True 526 options.command_prefix = shlex.split(options.command_prefix) 527 options.extra_flags = shlex.split(options.extra_flags) 528 529 if options.gc_stress: 530 options.extra_flags += GC_STRESS_FLAGS 531 532 if options.asan: 533 options.extra_flags.append("--invoke-weak-callbacks") 534 options.extra_flags.append("--omit-quit") 535 536 if options.novfp3: 537 options.extra_flags.append("--noenable-vfp3") 538 539 if options.exhaustive_variants: 540 # This is used on many bots. It includes a larger set of default variants. 541 # Other options for manipulating variants still apply afterwards. 542 VARIANTS = EXHAUSTIVE_VARIANTS 543 544 # TODO(machenbach): Figure out how to test a bigger subset of variants on 545 # msan and tsan. 546 if options.msan: 547 VARIANTS = ["default"] 548 549 if options.tsan: 550 VARIANTS = ["default"] 551 552 if options.j == 0: 553 options.j = multiprocessing.cpu_count() 554 555 if options.random_seed_stress_count <= 1 and options.random_seed == 0: 556 options.random_seed = RandomSeed() 557 558 def excl(*args): 559 """Returns true if zero or one of multiple arguments are true.""" 560 return reduce(lambda x, y: x + y, args) <= 1 561 562 if not excl(options.no_variants, bool(options.variants)): 563 print("Use only one of --no-variants or --variants.") 564 return False 565 if options.quickcheck: 566 VARIANTS = ["default", "stress"] 567 options.slow_tests = "skip" 568 options.pass_fail_tests = "skip" 569 if options.no_variants: 570 VARIANTS = ["default"] 571 if options.variants: 572 VARIANTS = options.variants.split(",") 573 574 # Resolve variant aliases. 575 VARIANTS = reduce( 576 list.__add__, 577 (VARIANT_ALIASES.get(v, [v]) for v in VARIANTS), 578 [], 579 ) 580 581 if not set(VARIANTS).issubset(ALL_VARIANTS): 582 print "All variants must be in %s" % str(ALL_VARIANTS) 583 return False 584 if options.predictable: 585 VARIANTS = ["default"] 586 options.extra_flags.append("--predictable") 587 options.extra_flags.append("--verify_predictable") 588 options.extra_flags.append("--no-inline-new") 589 590 # Dedupe. 591 VARIANTS = list(set(VARIANTS)) 592 593 if not options.shell_dir: 594 if options.shell: 595 print "Warning: --shell is deprecated, use --shell-dir instead." 596 options.shell_dir = os.path.dirname(options.shell) 597 if options.valgrind: 598 run_valgrind = os.path.join("tools", "run-valgrind.py") 599 # This is OK for distributed running, so we don't need to set no_network. 600 options.command_prefix = (["python", "-u", run_valgrind] + 601 options.command_prefix) 602 def CheckTestMode(name, option): 603 if not option in ["run", "skip", "dontcare"]: 604 print "Unknown %s mode %s" % (name, option) 605 return False 606 return True 607 if not CheckTestMode("slow test", options.slow_tests): 608 return False 609 if not CheckTestMode("pass|fail test", options.pass_fail_tests): 610 return False 611 if options.no_i18n: 612 TEST_MAP["bot_default"].remove("intl") 613 TEST_MAP["default"].remove("intl") 614 if not options.enable_inspector: 615 TEST_MAP["default"].remove("inspector") 616 TEST_MAP["bot_default"].remove("inspector") 617 TEST_MAP["optimize_for_size"].remove("inspector") 618 TEST_MAP["default"].remove("debugger") 619 TEST_MAP["bot_default"].remove("debugger") 620 TEST_MAP["optimize_for_size"].remove("debugger") 621 return True 622 623 624 def ShardTests(tests, options): 625 # Read gtest shard configuration from environment (e.g. set by swarming). 626 # If none is present, use values passed on the command line. 627 shard_count = int(os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count)) 628 shard_run = os.environ.get('GTEST_SHARD_INDEX') 629 if shard_run is not None: 630 # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0. 631 shard_run = int(shard_run) + 1 632 else: 633 shard_run = options.shard_run 634 635 if options.shard_count > 1: 636 # Log if a value was passed on the cmd line and it differs from the 637 # environment variables. 638 if options.shard_count != shard_count: 639 print("shard_count from cmd line differs from environment variable " 640 "GTEST_TOTAL_SHARDS") 641 if options.shard_run > 1 and options.shard_run != shard_run: 642 print("shard_run from cmd line differs from environment variable " 643 "GTEST_SHARD_INDEX") 644 645 if shard_count < 2: 646 return tests 647 if shard_run < 1 or shard_run > shard_count: 648 print "shard-run not a valid number, should be in [1:shard-count]" 649 print "defaulting back to running all tests" 650 return tests 651 count = 0 652 shard = [] 653 for test in tests: 654 if count % shard_count == shard_run - 1: 655 shard.append(test) 656 count += 1 657 return shard 658 659 660 def Main(): 661 # Use the v8 root as cwd as some test cases use "load" with relative paths. 662 os.chdir(BASE_DIR) 663 664 parser = BuildOptions() 665 (options, args) = parser.parse_args() 666 if not ProcessOptions(options): 667 parser.print_help() 668 return 1 669 SetupEnvironment(options) 670 671 if options.swarming: 672 # Swarming doesn't print how isolated commands are called. Lets make this 673 # less cryptic by printing it ourselves. 674 print ' '.join(sys.argv) 675 676 exit_code = 0 677 678 suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) 679 680 # Use default tests if no test configuration was provided at the cmd line. 681 if len(args) == 0: 682 args = ["default"] 683 684 # Expand arguments with grouped tests. The args should reflect the list of 685 # suites as otherwise filters would break. 686 def ExpandTestGroups(name): 687 if name in TEST_MAP: 688 return [suite for suite in TEST_MAP[name]] 689 else: 690 return [name] 691 args = reduce(lambda x, y: x + y, 692 [ExpandTestGroups(arg) for arg in args], 693 []) 694 695 args_suites = OrderedDict() # Used as set 696 for arg in args: 697 args_suites[arg.split('/')[0]] = True 698 suite_paths = [ s for s in args_suites if s in suite_paths ] 699 700 suites = [] 701 for root in suite_paths: 702 suite = testsuite.TestSuite.LoadTestSuite( 703 os.path.join(BASE_DIR, "test", root)) 704 if suite: 705 suites.append(suite) 706 707 if options.download_data or options.download_data_only: 708 for s in suites: 709 s.DownloadData() 710 711 if options.download_data_only: 712 return exit_code 713 714 for s in suites: 715 s.PrepareSources() 716 717 for (arch, mode) in options.arch_and_mode: 718 try: 719 code = Execute(arch, mode, args, options, suites) 720 except KeyboardInterrupt: 721 return 2 722 exit_code = exit_code or code 723 return exit_code 724 725 726 def Execute(arch, mode, args, options, suites): 727 print(">>> Running tests for %s.%s" % (arch, mode)) 728 729 shell_dir = options.shell_dir 730 if not shell_dir: 731 if options.auto_detect: 732 # If an output dir with a build was passed, test directly in that 733 # directory. 734 shell_dir = os.path.join(BASE_DIR, options.outdir) 735 elif options.buildbot: 736 # TODO(machenbach): Get rid of different output folder location on 737 # buildbot. Currently this is capitalized Release and Debug. 738 shell_dir = os.path.join(BASE_DIR, options.outdir, mode) 739 mode = BuildbotToV8Mode(mode) 740 else: 741 shell_dir = os.path.join( 742 BASE_DIR, 743 options.outdir, 744 "%s.%s" % (arch, MODES[mode]["output_folder"]), 745 ) 746 if not os.path.exists(shell_dir): 747 raise Exception('Could not find shell_dir: "%s"' % shell_dir) 748 749 # Populate context object. 750 mode_flags = MODES[mode]["flags"] 751 752 # Simulators are slow, therefore allow a longer timeout. 753 if arch in SLOW_ARCHS: 754 options.timeout *= 2 755 756 options.timeout *= MODES[mode]["timeout_scalefactor"] 757 758 if options.predictable: 759 # Predictable mode is slower. 760 options.timeout *= 2 761 762 ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir, 763 mode_flags, options.verbose, 764 options.timeout, 765 options.isolates, 766 options.command_prefix, 767 options.extra_flags, 768 options.no_i18n, 769 options.random_seed, 770 options.no_sorting, 771 options.rerun_failures_count, 772 options.rerun_failures_max, 773 options.predictable, 774 options.no_harness, 775 use_perf_data=not options.swarming, 776 sancov_dir=options.sancov_dir) 777 778 # TODO(all): Combine "simulator" and "simulator_run". 779 # TODO(machenbach): In GN we can derive simulator run from 780 # target_arch != v8_target_arch in the dumped build config. 781 simulator_run = not options.dont_skip_simulator_slow_tests and \ 782 arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \ 783 'ppc', 'ppc64'] and \ 784 ARCH_GUESS and arch != ARCH_GUESS 785 # Find available test suites and read test cases from them. 786 variables = { 787 "arch": arch, 788 "asan": options.asan, 789 "deopt_fuzzer": False, 790 "gc_stress": options.gc_stress, 791 "gcov_coverage": options.gcov_coverage, 792 "isolates": options.isolates, 793 "mode": MODES[mode]["status_mode"], 794 "no_i18n": options.no_i18n, 795 "no_snap": options.no_snap, 796 "simulator_run": simulator_run, 797 "simulator": utils.UseSimulator(arch), 798 "system": utils.GuessOS(), 799 "tsan": options.tsan, 800 "msan": options.msan, 801 "dcheck_always_on": options.dcheck_always_on, 802 "novfp3": options.novfp3, 803 "predictable": options.predictable, 804 "byteorder": sys.byteorder, 805 } 806 all_tests = [] 807 num_tests = 0 808 for s in suites: 809 s.ReadStatusFile(variables) 810 s.ReadTestCases(ctx) 811 if len(args) > 0: 812 s.FilterTestCasesByArgs(args) 813 all_tests += s.tests 814 815 # First filtering by status applying the generic rules (independent of 816 # variants). 817 s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, 818 options.pass_fail_tests) 819 820 if options.cat: 821 verbose.PrintTestSource(s.tests) 822 continue 823 variant_gen = s.CreateVariantGenerator(VARIANTS) 824 variant_tests = [ t.CopyAddingFlags(v, flags) 825 for t in s.tests 826 for v in variant_gen.FilterVariantsByTest(t) 827 for flags in variant_gen.GetFlagSets(t, v) ] 828 829 if options.random_seed_stress_count > 1: 830 # Duplicate test for random seed stress mode. 831 def iter_seed_flags(): 832 for i in range(0, options.random_seed_stress_count): 833 # Use given random seed for all runs (set by default in execution.py) 834 # or a new random seed if none is specified. 835 if options.random_seed: 836 yield [] 837 else: 838 yield ["--random-seed=%d" % RandomSeed()] 839 s.tests = [ 840 t.CopyAddingFlags(t.variant, flags) 841 for t in variant_tests 842 for flags in iter_seed_flags() 843 ] 844 else: 845 s.tests = variant_tests 846 847 # Second filtering by status applying the variant-dependent rules. 848 s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, 849 options.pass_fail_tests, variants=True) 850 851 s.tests = ShardTests(s.tests, options) 852 num_tests += len(s.tests) 853 854 if options.cat: 855 return 0 # We're done here. 856 857 if options.report: 858 verbose.PrintReport(all_tests) 859 860 # Run the tests, either locally or distributed on the network. 861 start_time = time.time() 862 progress_indicator = progress.IndicatorNotifier() 863 progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]()) 864 if options.junitout: 865 progress_indicator.Register(progress.JUnitTestProgressIndicator( 866 options.junitout, options.junittestsuite)) 867 if options.json_test_results: 868 progress_indicator.Register(progress.JsonTestProgressIndicator( 869 options.json_test_results, arch, MODES[mode]["execution_mode"], 870 ctx.random_seed)) 871 872 run_networked = not options.no_network 873 if not run_networked: 874 if options.verbose: 875 print("Network distribution disabled, running tests locally.") 876 elif utils.GuessOS() != "linux": 877 print("Network distribution is only supported on Linux, sorry!") 878 run_networked = False 879 peers = [] 880 if run_networked: 881 peers = network_execution.GetPeers() 882 if not peers: 883 print("No connection to distribution server; running tests locally.") 884 run_networked = False 885 elif len(peers) == 1: 886 print("No other peers on the network; running tests locally.") 887 run_networked = False 888 elif num_tests <= 100: 889 print("Less than 100 tests, running them locally.") 890 run_networked = False 891 892 if run_networked: 893 runner = network_execution.NetworkedRunner(suites, progress_indicator, 894 ctx, peers, BASE_DIR) 895 else: 896 runner = execution.Runner(suites, progress_indicator, ctx) 897 898 exit_code = runner.Run(options.j) 899 overall_duration = time.time() - start_time 900 901 if options.time: 902 verbose.PrintTestDurations(suites, overall_duration) 903 904 if num_tests == 0: 905 print("Warning: no tests were run!") 906 907 if exit_code == 1 and options.json_test_results: 908 print("Force exit code 0 after failures. Json test results file generated " 909 "with failure information.") 910 exit_code = 0 911 912 if options.sancov_dir: 913 # If tests ran with sanitizer coverage, merge coverage files in the end. 914 try: 915 print "Merging sancov files." 916 subprocess.check_call([ 917 sys.executable, 918 join(BASE_DIR, "tools", "sanitizers", "sancov_merger.py"), 919 "--coverage-dir=%s" % options.sancov_dir]) 920 except: 921 print >> sys.stderr, "Error: Merging sancov files failed." 922 exit_code = 1 923 924 return exit_code 925 926 927 if __name__ == "__main__": 928 sys.exit(Main()) 929
