1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 from collections import OrderedDict 32 import itertools 33 import json 34 import multiprocessing 35 import optparse 36 import os 37 from os.path import getmtime, isdir, join 38 import platform 39 import random 40 import shlex 41 import subprocess 42 import sys 43 import time 44 45 from testrunner.local import execution 46 from testrunner.local import progress 47 from testrunner.local import testsuite 48 from testrunner.local.variants import ALL_VARIANTS 49 from testrunner.local import utils 50 from testrunner.local import verbose 51 from testrunner.network import network_execution 52 from testrunner.objects import context 53 54 55 # Base dir of the v8 checkout to be used as cwd. 56 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 57 58 DEFAULT_OUT_GN = "out.gn" 59 60 ARCH_GUESS = utils.DefaultArch() 61 62 # Map of test name synonyms to lists of test suites. Should be ordered by 63 # expected runtimes (suites with slow test cases first). These groups are 64 # invoked in seperate steps on the bots. 65 TEST_MAP = { 66 # This needs to stay in sync with test/bot_default.isolate. 67 "bot_default": [ 68 "debugger", 69 "mjsunit", 70 "cctest", 71 "inspector", 72 "webkit", 73 "fuzzer", 74 "message", 75 "preparser", 76 "intl", 77 "unittests", 78 ], 79 # This needs to stay in sync with test/default.isolate. 80 "default": [ 81 "debugger", 82 "mjsunit", 83 "cctest", 84 "inspector", 85 "fuzzer", 86 "message", 87 "preparser", 88 "intl", 89 "unittests", 90 ], 91 # This needs to stay in sync with test/optimize_for_size.isolate. 92 "optimize_for_size": [ 93 "debugger", 94 "mjsunit", 95 "cctest", 96 "inspector", 97 "webkit", 98 "intl", 99 ], 100 "unittests": [ 101 "unittests", 102 ], 103 } 104 105 TIMEOUT_DEFAULT = 60 106 107 # Variants ordered by expected runtime (slowest first). 108 VARIANTS = ["ignition_staging", "default", "turbofan"] 109 110 MORE_VARIANTS = [ 111 "stress", 112 "turbofan_opt", 113 "ignition", 114 "asm_wasm", 115 "wasm_traps", 116 ] 117 118 EXHAUSTIVE_VARIANTS = MORE_VARIANTS + VARIANTS 119 120 VARIANT_ALIASES = { 121 # The default for developer workstations. 122 "dev": VARIANTS, 123 # Additional variants, run on all bots. 124 "more": MORE_VARIANTS, 125 # Additional variants, run on a subset of bots. 126 "extra": ["nocrankshaft"], 127 } 128 129 DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination", 130 "--nofold-constants", "--enable-slow-asserts", 131 "--verify-heap"] 132 RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination", 133 "--nofold-constants"] 134 135 MODES = { 136 "debug": { 137 "flags": DEBUG_FLAGS, 138 "timeout_scalefactor": 4, 139 "status_mode": "debug", 140 "execution_mode": "debug", 141 "output_folder": "debug", 142 }, 143 "optdebug": { 144 "flags": DEBUG_FLAGS, 145 "timeout_scalefactor": 4, 146 "status_mode": "debug", 147 "execution_mode": "debug", 148 "output_folder": "optdebug", 149 }, 150 "release": { 151 "flags": RELEASE_FLAGS, 152 "timeout_scalefactor": 1, 153 "status_mode": "release", 154 "execution_mode": "release", 155 "output_folder": "release", 156 }, 157 # Normal trybot release configuration. There, dchecks are always on which 158 # implies debug is set. Hence, the status file needs to assume debug-like 159 # behavior/timeouts. 160 "tryrelease": { 161 "flags": RELEASE_FLAGS, 162 "timeout_scalefactor": 1, 163 "status_mode": "debug", 164 "execution_mode": "release", 165 "output_folder": "release", 166 }, 167 # This mode requires v8 to be compiled with dchecks and slow dchecks. 168 "slowrelease": { 169 "flags": RELEASE_FLAGS + ["--enable-slow-asserts"], 170 "timeout_scalefactor": 2, 171 "status_mode": "debug", 172 "execution_mode": "release", 173 "output_folder": "release", 174 }, 175 } 176 177 GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", 178 "--concurrent-recompilation-queue-length=64", 179 "--concurrent-recompilation-delay=500", 180 "--concurrent-recompilation"] 181 182 SUPPORTED_ARCHS = ["android_arm", 183 "android_arm64", 184 "android_ia32", 185 "android_x64", 186 "arm", 187 "ia32", 188 "x87", 189 "mips", 190 "mipsel", 191 "mips64", 192 "mips64el", 193 "s390", 194 "s390x", 195 "ppc", 196 "ppc64", 197 "x64", 198 "x32", 199 "arm64"] 200 # Double the timeout for these: 201 SLOW_ARCHS = ["android_arm", 202 "android_arm64", 203 "android_ia32", 204 "android_x64", 205 "arm", 206 "mips", 207 "mipsel", 208 "mips64", 209 "mips64el", 210 "s390", 211 "s390x", 212 "x87", 213 "arm64"] 214 215 216 def BuildOptions(): 217 result = optparse.OptionParser() 218 result.usage = '%prog [options] [tests]' 219 result.description = """TESTS: %s""" % (TEST_MAP["default"]) 220 result.add_option("--arch", 221 help=("The architecture to run tests for, " 222 "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS), 223 default="ia32,x64,arm") 224 result.add_option("--arch-and-mode", 225 help="Architecture and mode in the format 'arch.mode'", 226 default=None) 227 result.add_option("--asan", 228 help="Regard test expectations for ASAN", 229 default=False, action="store_true") 230 result.add_option("--sancov-dir", 231 help="Directory where to collect coverage data") 232 result.add_option("--cfi-vptr", 233 help="Run tests with UBSAN cfi_vptr option.", 234 default=False, action="store_true") 235 result.add_option("--buildbot", 236 help="Adapt to path structure used on buildbots", 237 default=False, action="store_true") 238 result.add_option("--dcheck-always-on", 239 help="Indicates that V8 was compiled with DCHECKs enabled", 240 default=False, action="store_true") 241 result.add_option("--novfp3", 242 help="Indicates that V8 was compiled without VFP3 support", 243 default=False, action="store_true") 244 result.add_option("--cat", help="Print the source of the tests", 245 default=False, action="store_true") 246 result.add_option("--slow-tests", 247 help="Regard slow tests (run|skip|dontcare)", 248 default="dontcare") 249 result.add_option("--pass-fail-tests", 250 help="Regard pass|fail tests (run|skip|dontcare)", 251 default="dontcare") 252 result.add_option("--gc-stress", 253 help="Switch on GC stress mode", 254 default=False, action="store_true") 255 result.add_option("--gcov-coverage", 256 help="Uses executables instrumented for gcov coverage", 257 default=False, action="store_true") 258 result.add_option("--command-prefix", 259 help="Prepended to each shell command used to run a test", 260 default="") 261 result.add_option("--download-data", help="Download missing test suite data", 262 default=False, action="store_true") 263 result.add_option("--download-data-only", 264 help="Deprecated", 265 default=False, action="store_true") 266 result.add_option("--enable-inspector", 267 help="Indicates a build with inspector support", 268 default=False, action="store_true") 269 result.add_option("--extra-flags", 270 help="Additional flags to pass to each test command", 271 default="") 272 result.add_option("--isolates", help="Whether to test isolates", 273 default=False, action="store_true") 274 result.add_option("-j", help="The number of parallel tasks to run", 275 default=0, type="int") 276 result.add_option("-m", "--mode", 277 help="The test modes in which to run (comma-separated," 278 " uppercase for ninja and buildbot builds): %s" % MODES.keys(), 279 default="release,debug") 280 result.add_option("--no-harness", "--noharness", 281 help="Run without test harness of a given suite", 282 default=False, action="store_true") 283 result.add_option("--no-i18n", "--noi18n", 284 help="Skip internationalization tests", 285 default=False, action="store_true") 286 result.add_option("--no-network", "--nonetwork", 287 help="Don't distribute tests on the network", 288 default=(utils.GuessOS() != "linux"), 289 dest="no_network", action="store_true") 290 result.add_option("--no-presubmit", "--nopresubmit", 291 help='Skip presubmit checks (deprecated)', 292 default=False, dest="no_presubmit", action="store_true") 293 result.add_option("--no-snap", "--nosnap", 294 help='Test a build compiled without snapshot.', 295 default=False, dest="no_snap", action="store_true") 296 result.add_option("--no-sorting", "--nosorting", 297 help="Don't sort tests according to duration of last run.", 298 default=False, dest="no_sorting", action="store_true") 299 result.add_option("--no-variants", "--novariants", 300 help="Don't run any testing variants", 301 default=False, dest="no_variants", action="store_true") 302 result.add_option("--variants", 303 help="Comma-separated list of testing variants;" 304 " default: \"%s\"" % ",".join(VARIANTS)) 305 result.add_option("--exhaustive-variants", 306 default=False, action="store_true", 307 help="Use exhaustive set of default variants:" 308 " \"%s\"" % ",".join(EXHAUSTIVE_VARIANTS)) 309 result.add_option("--outdir", help="Base directory with compile output", 310 default="out") 311 result.add_option("--gn", help="Scan out.gn for the last built configuration", 312 default=False, action="store_true") 313 result.add_option("--predictable", 314 help="Compare output of several reruns of each test", 315 default=False, action="store_true") 316 result.add_option("-p", "--progress", 317 help=("The style of progress indicator" 318 " (verbose, dots, color, mono)"), 319 choices=progress.PROGRESS_INDICATORS.keys(), default="mono") 320 result.add_option("--quickcheck", default=False, action="store_true", 321 help=("Quick check mode (skip slow tests)")) 322 result.add_option("--report", help="Print a summary of the tests to be run", 323 default=False, action="store_true") 324 result.add_option("--json-test-results", 325 help="Path to a file for storing json results.") 326 result.add_option("--rerun-failures-count", 327 help=("Number of times to rerun each failing test case. " 328 "Very slow tests will be rerun only once."), 329 default=0, type="int") 330 result.add_option("--rerun-failures-max", 331 help="Maximum number of failing test cases to rerun.", 332 default=100, type="int") 333 result.add_option("--shard-count", 334 help="Split testsuites into this number of shards", 335 default=1, type="int") 336 result.add_option("--shard-run", 337 help="Run this shard from the split up tests.", 338 default=1, type="int") 339 result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") 340 result.add_option("--shell-dir", help="Directory containing executables", 341 default="") 342 result.add_option("--dont-skip-slow-simulator-tests", 343 help="Don't skip more slow tests when using a simulator.", 344 default=False, action="store_true", 345 dest="dont_skip_simulator_slow_tests") 346 result.add_option("--swarming", 347 help="Indicates running test driver on swarming.", 348 default=False, action="store_true") 349 result.add_option("--time", help="Print timing information after running", 350 default=False, action="store_true") 351 result.add_option("-t", "--timeout", help="Timeout in seconds", 352 default=TIMEOUT_DEFAULT, type="int") 353 result.add_option("--tsan", 354 help="Regard test expectations for TSAN", 355 default=False, action="store_true") 356 result.add_option("-v", "--verbose", help="Verbose output", 357 default=False, action="store_true") 358 result.add_option("--valgrind", help="Run tests through valgrind", 359 default=False, action="store_true") 360 result.add_option("--warn-unused", help="Report unused rules", 361 default=False, action="store_true") 362 result.add_option("--junitout", help="File name of the JUnit output") 363 result.add_option("--junittestsuite", 364 help="The testsuite name in the JUnit output file", 365 default="v8tests") 366 result.add_option("--random-seed", default=0, dest="random_seed", type="int", 367 help="Default seed for initializing random generator") 368 result.add_option("--random-seed-stress-count", default=1, type="int", 369 dest="random_seed_stress_count", 370 help="Number of runs with different random seeds") 371 result.add_option("--msan", 372 help="Regard test expectations for MSAN", 373 default=False, action="store_true") 374 return result 375 376 377 def RandomSeed(): 378 seed = 0 379 while not seed: 380 seed = random.SystemRandom().randint(-2147483648, 2147483647) 381 return seed 382 383 384 def BuildbotToV8Mode(config): 385 """Convert buildbot build configs to configs understood by the v8 runner. 386 387 V8 configs are always lower case and without the additional _x64 suffix for 388 64 bit builds on windows with ninja. 389 """ 390 mode = config[:-4] if config.endswith('_x64') else config 391 return mode.lower() 392 393 def SetupEnvironment(options): 394 """Setup additional environment variables.""" 395 396 # Many tests assume an English interface. 397 os.environ['LANG'] = 'en_US.UTF-8' 398 399 symbolizer = 'external_symbolizer_path=%s' % ( 400 os.path.join( 401 BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin', 402 'llvm-symbolizer', 403 ) 404 ) 405 406 if options.asan: 407 asan_options = [symbolizer] 408 if not utils.GuessOS() == 'macos': 409 # LSAN is not available on mac. 410 asan_options.append('detect_leaks=1') 411 os.environ['LSAN_OPTIONS'] = ":".join([ 412 'suppressions=%s' % os.path.join( 413 BASE_DIR, 'tools', 'memory', 'lsan', 'suppressions.txt'), 414 ]) 415 os.environ['ASAN_OPTIONS'] = ":".join(asan_options) 416 417 if options.sancov_dir: 418 assert os.path.exists(options.sancov_dir) 419 os.environ['ASAN_OPTIONS'] = ":".join([ 420 'coverage=1', 421 'coverage_dir=%s' % options.sancov_dir, 422 symbolizer, 423 ]) 424 425 if options.cfi_vptr: 426 os.environ['UBSAN_OPTIONS'] = ":".join([ 427 'print_stacktrace=1', 428 'print_summary=1', 429 'symbolize=1', 430 symbolizer, 431 ]) 432 433 if options.msan: 434 os.environ['MSAN_OPTIONS'] = symbolizer 435 436 if options.tsan: 437 suppressions_file = os.path.join( 438 BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt') 439 os.environ['TSAN_OPTIONS'] = " ".join([ 440 symbolizer, 441 'suppressions=%s' % suppressions_file, 442 'exit_code=0', 443 'report_thread_leaks=0', 444 'history_size=7', 445 'report_destroy_locked=0', 446 ]) 447 448 def ProcessOptions(options): 449 global VARIANTS 450 451 # First try to auto-detect configurations based on the build if GN was 452 # used. This can't be overridden by cmd-line arguments. 453 options.auto_detect = False 454 if options.gn: 455 gn_out_dir = os.path.join(BASE_DIR, DEFAULT_OUT_GN) 456 latest_timestamp = -1 457 latest_config = None 458 for gn_config in os.listdir(gn_out_dir): 459 gn_config_dir = os.path.join(gn_out_dir, gn_config) 460 if not isdir(gn_config_dir): 461 continue 462 if os.path.getmtime(gn_config_dir) > latest_timestamp: 463 latest_timestamp = os.path.getmtime(gn_config_dir) 464 latest_config = gn_config 465 if latest_config: 466 print(">>> Latest GN build found is %s" % latest_config) 467 options.outdir = os.path.join(DEFAULT_OUT_GN, latest_config) 468 469 if options.buildbot: 470 build_config_path = os.path.join( 471 BASE_DIR, options.outdir, options.mode, "v8_build_config.json") 472 else: 473 build_config_path = os.path.join( 474 BASE_DIR, options.outdir, "v8_build_config.json") 475 476 if os.path.exists(build_config_path): 477 try: 478 with open(build_config_path) as f: 479 build_config = json.load(f) 480 except Exception: 481 print ("%s exists but contains invalid json. Is your build up-to-date?" % 482 build_config_path) 483 return False 484 options.auto_detect = True 485 486 # In auto-detect mode the outdir is always where we found the build config. 487 # This ensures that we'll also take the build products from there. 488 options.outdir = os.path.dirname(build_config_path) 489 490 options.arch_and_mode = None 491 options.arch = build_config["v8_target_cpu"] 492 if options.arch == 'x86': 493 # TODO(machenbach): Transform all to x86 eventually. 494 options.arch = 'ia32' 495 options.asan = build_config["is_asan"] 496 options.dcheck_always_on = build_config["dcheck_always_on"] 497 options.enable_inspector = build_config["v8_enable_inspector"] 498 options.mode = 'debug' if build_config["is_debug"] else 'release' 499 options.msan = build_config["is_msan"] 500 options.no_i18n = not build_config["v8_enable_i18n_support"] 501 options.no_snap = not build_config["v8_use_snapshot"] 502 options.tsan = build_config["is_tsan"] 503 504 # Architecture and mode related stuff. 505 if options.arch_and_mode: 506 options.arch_and_mode = [arch_and_mode.split(".") 507 for arch_and_mode in options.arch_and_mode.split(",")] 508 options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) 509 options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) 510 options.mode = options.mode.split(",") 511 for mode in options.mode: 512 if not BuildbotToV8Mode(mode) in MODES: 513 print "Unknown mode %s" % mode 514 return False 515 if options.arch in ["auto", "native"]: 516 options.arch = ARCH_GUESS 517 options.arch = options.arch.split(",") 518 for arch in options.arch: 519 if not arch in SUPPORTED_ARCHS: 520 print "Unknown architecture %s" % arch 521 return False 522 523 # Store the final configuration in arch_and_mode list. Don't overwrite 524 # predefined arch_and_mode since it is more expressive than arch and mode. 525 if not options.arch_and_mode: 526 options.arch_and_mode = itertools.product(options.arch, options.mode) 527 528 # Special processing of other options, sorted alphabetically. 529 530 if options.buildbot: 531 options.no_network = True 532 if options.command_prefix: 533 print("Specifying --command-prefix disables network distribution, " 534 "running tests locally.") 535 options.no_network = True 536 options.command_prefix = shlex.split(options.command_prefix) 537 options.extra_flags = shlex.split(options.extra_flags) 538 539 if options.gc_stress: 540 options.extra_flags += GC_STRESS_FLAGS 541 542 if options.asan: 543 options.extra_flags.append("--invoke-weak-callbacks") 544 options.extra_flags.append("--omit-quit") 545 546 if options.novfp3: 547 options.extra_flags.append("--noenable-vfp3") 548 549 if options.exhaustive_variants: 550 # This is used on many bots. It includes a larger set of default variants. 551 # Other options for manipulating variants still apply afterwards. 552 VARIANTS = EXHAUSTIVE_VARIANTS 553 554 # TODO(machenbach): Figure out how to test a bigger subset of variants on 555 # msan and tsan. 556 if options.msan: 557 VARIANTS = ["default"] 558 559 if options.tsan: 560 VARIANTS = ["default"] 561 562 if options.j == 0: 563 options.j = multiprocessing.cpu_count() 564 565 if options.random_seed_stress_count <= 1 and options.random_seed == 0: 566 options.random_seed = RandomSeed() 567 568 def excl(*args): 569 """Returns true if zero or one of multiple arguments are true.""" 570 return reduce(lambda x, y: x + y, args) <= 1 571 572 if not excl(options.no_variants, bool(options.variants)): 573 print("Use only one of --no-variants or --variants.") 574 return False 575 if options.quickcheck: 576 VARIANTS = ["default", "stress"] 577 options.slow_tests = "skip" 578 options.pass_fail_tests = "skip" 579 if options.no_variants: 580 VARIANTS = ["default"] 581 if options.variants: 582 VARIANTS = options.variants.split(",") 583 584 # Resolve variant aliases. 585 VARIANTS = reduce( 586 list.__add__, 587 (VARIANT_ALIASES.get(v, [v]) for v in VARIANTS), 588 [], 589 ) 590 591 if not set(VARIANTS).issubset(ALL_VARIANTS): 592 print "All variants must be in %s" % str(ALL_VARIANTS) 593 return False 594 if options.predictable: 595 VARIANTS = ["default"] 596 options.extra_flags.append("--predictable") 597 options.extra_flags.append("--verify_predictable") 598 options.extra_flags.append("--no-inline-new") 599 600 # Dedupe. 601 VARIANTS = list(set(VARIANTS)) 602 603 if not options.shell_dir: 604 if options.shell: 605 print "Warning: --shell is deprecated, use --shell-dir instead." 606 options.shell_dir = os.path.dirname(options.shell) 607 if options.valgrind: 608 run_valgrind = os.path.join("tools", "run-valgrind.py") 609 # This is OK for distributed running, so we don't need to set no_network. 610 options.command_prefix = (["python", "-u", run_valgrind] + 611 options.command_prefix) 612 def CheckTestMode(name, option): 613 if not option in ["run", "skip", "dontcare"]: 614 print "Unknown %s mode %s" % (name, option) 615 return False 616 return True 617 if not CheckTestMode("slow test", options.slow_tests): 618 return False 619 if not CheckTestMode("pass|fail test", options.pass_fail_tests): 620 return False 621 if options.no_i18n: 622 TEST_MAP["bot_default"].remove("intl") 623 TEST_MAP["default"].remove("intl") 624 if not options.enable_inspector: 625 TEST_MAP["default"].remove("inspector") 626 TEST_MAP["bot_default"].remove("inspector") 627 TEST_MAP["optimize_for_size"].remove("inspector") 628 TEST_MAP["default"].remove("debugger") 629 TEST_MAP["bot_default"].remove("debugger") 630 TEST_MAP["optimize_for_size"].remove("debugger") 631 return True 632 633 634 def ShardTests(tests, options): 635 # Read gtest shard configuration from environment (e.g. set by swarming). 636 # If none is present, use values passed on the command line. 637 shard_count = int(os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count)) 638 shard_run = os.environ.get('GTEST_SHARD_INDEX') 639 if shard_run is not None: 640 # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0. 641 shard_run = int(shard_run) + 1 642 else: 643 shard_run = options.shard_run 644 645 if options.shard_count > 1: 646 # Log if a value was passed on the cmd line and it differs from the 647 # environment variables. 648 if options.shard_count != shard_count: 649 print("shard_count from cmd line differs from environment variable " 650 "GTEST_TOTAL_SHARDS") 651 if options.shard_run > 1 and options.shard_run != shard_run: 652 print("shard_run from cmd line differs from environment variable " 653 "GTEST_SHARD_INDEX") 654 655 if shard_count < 2: 656 return tests 657 if shard_run < 1 or shard_run > shard_count: 658 print "shard-run not a valid number, should be in [1:shard-count]" 659 print "defaulting back to running all tests" 660 return tests 661 count = 0 662 shard = [] 663 for test in tests: 664 if count % shard_count == shard_run - 1: 665 shard.append(test) 666 count += 1 667 return shard 668 669 670 def Main(): 671 # Use the v8 root as cwd as some test cases use "load" with relative paths. 672 os.chdir(BASE_DIR) 673 674 parser = BuildOptions() 675 (options, args) = parser.parse_args() 676 if not ProcessOptions(options): 677 parser.print_help() 678 return 1 679 SetupEnvironment(options) 680 681 if options.swarming: 682 # Swarming doesn't print how isolated commands are called. Lets make this 683 # less cryptic by printing it ourselves. 684 print ' '.join(sys.argv) 685 686 exit_code = 0 687 688 suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) 689 690 # Use default tests if no test configuration was provided at the cmd line. 691 if len(args) == 0: 692 args = ["default"] 693 694 # Expand arguments with grouped tests. The args should reflect the list of 695 # suites as otherwise filters would break. 696 def ExpandTestGroups(name): 697 if name in TEST_MAP: 698 return [suite for suite in TEST_MAP[name]] 699 else: 700 return [name] 701 args = reduce(lambda x, y: x + y, 702 [ExpandTestGroups(arg) for arg in args], 703 []) 704 705 args_suites = OrderedDict() # Used as set 706 for arg in args: 707 args_suites[arg.split('/')[0]] = True 708 suite_paths = [ s for s in args_suites if s in suite_paths ] 709 710 suites = [] 711 for root in suite_paths: 712 suite = testsuite.TestSuite.LoadTestSuite( 713 os.path.join(BASE_DIR, "test", root)) 714 if suite: 715 suites.append(suite) 716 717 if options.download_data or options.download_data_only: 718 for s in suites: 719 s.DownloadData() 720 721 if options.download_data_only: 722 return exit_code 723 724 for s in suites: 725 s.PrepareSources() 726 727 for (arch, mode) in options.arch_and_mode: 728 try: 729 code = Execute(arch, mode, args, options, suites) 730 except KeyboardInterrupt: 731 return 2 732 exit_code = exit_code or code 733 return exit_code 734 735 736 def Execute(arch, mode, args, options, suites): 737 print(">>> Running tests for %s.%s" % (arch, mode)) 738 739 shell_dir = options.shell_dir 740 if not shell_dir: 741 if options.auto_detect: 742 # If an output dir with a build was passed, test directly in that 743 # directory. 744 shell_dir = os.path.join(BASE_DIR, options.outdir) 745 elif options.buildbot: 746 # TODO(machenbach): Get rid of different output folder location on 747 # buildbot. Currently this is capitalized Release and Debug. 748 shell_dir = os.path.join(BASE_DIR, options.outdir, mode) 749 mode = BuildbotToV8Mode(mode) 750 else: 751 shell_dir = os.path.join( 752 BASE_DIR, 753 options.outdir, 754 "%s.%s" % (arch, MODES[mode]["output_folder"]), 755 ) 756 if not os.path.exists(shell_dir): 757 raise Exception('Could not find shell_dir: "%s"' % shell_dir) 758 759 # Populate context object. 760 mode_flags = MODES[mode]["flags"] 761 762 # Simulators are slow, therefore allow a longer timeout. 763 if arch in SLOW_ARCHS: 764 options.timeout *= 2 765 766 options.timeout *= MODES[mode]["timeout_scalefactor"] 767 768 if options.predictable: 769 # Predictable mode is slower. 770 options.timeout *= 2 771 772 ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir, 773 mode_flags, options.verbose, 774 options.timeout, 775 options.isolates, 776 options.command_prefix, 777 options.extra_flags, 778 options.no_i18n, 779 options.random_seed, 780 options.no_sorting, 781 options.rerun_failures_count, 782 options.rerun_failures_max, 783 options.predictable, 784 options.no_harness, 785 use_perf_data=not options.swarming, 786 sancov_dir=options.sancov_dir) 787 788 # TODO(all): Combine "simulator" and "simulator_run". 789 # TODO(machenbach): In GN we can derive simulator run from 790 # target_arch != v8_target_arch in the dumped build config. 791 simulator_run = not options.dont_skip_simulator_slow_tests and \ 792 arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \ 793 'ppc', 'ppc64'] and \ 794 ARCH_GUESS and arch != ARCH_GUESS 795 # Find available test suites and read test cases from them. 796 variables = { 797 "arch": arch, 798 "asan": options.asan, 799 "deopt_fuzzer": False, 800 "gc_stress": options.gc_stress, 801 "gcov_coverage": options.gcov_coverage, 802 "isolates": options.isolates, 803 "mode": MODES[mode]["status_mode"], 804 "no_i18n": options.no_i18n, 805 "no_snap": options.no_snap, 806 "simulator_run": simulator_run, 807 "simulator": utils.UseSimulator(arch), 808 "system": utils.GuessOS(), 809 "tsan": options.tsan, 810 "msan": options.msan, 811 "dcheck_always_on": options.dcheck_always_on, 812 "novfp3": options.novfp3, 813 "predictable": options.predictable, 814 "byteorder": sys.byteorder, 815 } 816 all_tests = [] 817 num_tests = 0 818 for s in suites: 819 s.ReadStatusFile(variables) 820 s.ReadTestCases(ctx) 821 if len(args) > 0: 822 s.FilterTestCasesByArgs(args) 823 all_tests += s.tests 824 825 # First filtering by status applying the generic rules (independent of 826 # variants). 827 s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, 828 options.pass_fail_tests) 829 830 if options.cat: 831 verbose.PrintTestSource(s.tests) 832 continue 833 variant_gen = s.CreateVariantGenerator(VARIANTS) 834 variant_tests = [ t.CopyAddingFlags(v, flags) 835 for t in s.tests 836 for v in variant_gen.FilterVariantsByTest(t) 837 for flags in variant_gen.GetFlagSets(t, v) ] 838 839 if options.random_seed_stress_count > 1: 840 # Duplicate test for random seed stress mode. 841 def iter_seed_flags(): 842 for i in range(0, options.random_seed_stress_count): 843 # Use given random seed for all runs (set by default in execution.py) 844 # or a new random seed if none is specified. 845 if options.random_seed: 846 yield [] 847 else: 848 yield ["--random-seed=%d" % RandomSeed()] 849 s.tests = [ 850 t.CopyAddingFlags(t.variant, flags) 851 for t in variant_tests 852 for flags in iter_seed_flags() 853 ] 854 else: 855 s.tests = variant_tests 856 857 # Second filtering by status applying the variant-dependent rules. 858 s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests, 859 options.pass_fail_tests, variants=True) 860 861 s.tests = ShardTests(s.tests, options) 862 num_tests += len(s.tests) 863 864 if options.cat: 865 return 0 # We're done here. 866 867 if options.report: 868 verbose.PrintReport(all_tests) 869 870 # Run the tests, either locally or distributed on the network. 871 start_time = time.time() 872 progress_indicator = progress.IndicatorNotifier() 873 progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]()) 874 if options.junitout: 875 progress_indicator.Register(progress.JUnitTestProgressIndicator( 876 options.junitout, options.junittestsuite)) 877 if options.json_test_results: 878 progress_indicator.Register(progress.JsonTestProgressIndicator( 879 options.json_test_results, arch, MODES[mode]["execution_mode"], 880 ctx.random_seed)) 881 882 run_networked = not options.no_network 883 if not run_networked: 884 if options.verbose: 885 print("Network distribution disabled, running tests locally.") 886 elif utils.GuessOS() != "linux": 887 print("Network distribution is only supported on Linux, sorry!") 888 run_networked = False 889 peers = [] 890 if run_networked: 891 peers = network_execution.GetPeers() 892 if not peers: 893 print("No connection to distribution server; running tests locally.") 894 run_networked = False 895 elif len(peers) == 1: 896 print("No other peers on the network; running tests locally.") 897 run_networked = False 898 elif num_tests <= 100: 899 print("Less than 100 tests, running them locally.") 900 run_networked = False 901 902 if run_networked: 903 runner = network_execution.NetworkedRunner(suites, progress_indicator, 904 ctx, peers, BASE_DIR) 905 else: 906 runner = execution.Runner(suites, progress_indicator, ctx) 907 908 exit_code = runner.Run(options.j) 909 overall_duration = time.time() - start_time 910 911 if options.time: 912 verbose.PrintTestDurations(suites, overall_duration) 913 914 if num_tests == 0: 915 print("Warning: no tests were run!") 916 917 if exit_code == 1 and options.json_test_results: 918 print("Force exit code 0 after failures. Json test results file generated " 919 "with failure information.") 920 exit_code = 0 921 922 if options.sancov_dir: 923 # If tests ran with sanitizer coverage, merge coverage files in the end. 924 try: 925 print "Merging sancov files." 926 subprocess.check_call([ 927 sys.executable, 928 join(BASE_DIR, "tools", "sanitizers", "sancov_merger.py"), 929 "--coverage-dir=%s" % options.sancov_dir]) 930 except: 931 print >> sys.stderr, "Error: Merging sancov files failed." 932 exit_code = 1 933 934 return exit_code 935 936 937 if __name__ == "__main__": 938 sys.exit(Main()) 939
