1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 from collections import OrderedDict 32 import itertools 33 import multiprocessing 34 import optparse 35 import os 36 from os.path import join 37 import platform 38 import random 39 import shlex 40 import subprocess 41 import sys 42 import time 43 44 from testrunner.local import execution 45 from testrunner.local import progress 46 from testrunner.local import testsuite 47 from testrunner.local import utils 48 from testrunner.local import verbose 49 from testrunner.network import network_execution 50 from testrunner.objects import context 51 52 53 ARCH_GUESS = utils.DefaultArch() 54 DEFAULT_TESTS = ["mjsunit", "fuzz-natives", "base-unittests", 55 "cctest", "compiler-unittests", "heap-unittests", 56 "libplatform-unittests", "message", "preparser"] 57 58 # Map of test name synonyms to lists of test suites. Should be ordered by 59 # expected runtimes (suites with slow test cases first). These groups are 60 # invoked in seperate steps on the bots. 61 TEST_MAP = { 62 "default": [ 63 "mjsunit", 64 "fuzz-natives", 65 "cctest", 66 "message", 67 "preparser", 68 ], 69 "optimize_for_size": [ 70 "mjsunit", 71 "cctest", 72 "webkit", 73 ], 74 "unittests": [ 75 "compiler-unittests", 76 "heap-unittests", 77 "base-unittests", 78 "libplatform-unittests", 79 ], 80 } 81 82 TIMEOUT_DEFAULT = 60 83 TIMEOUT_SCALEFACTOR = {"debug" : 4, 84 "release" : 1 } 85 86 # Use this to run several variants of the tests. 87 VARIANT_FLAGS = { 88 "default": [], 89 "stress": ["--stress-opt", "--always-opt"], 90 "turbofan": ["--turbo-filter=*", "--always-opt"], 91 "nocrankshaft": ["--nocrankshaft"]} 92 93 VARIANTS = ["default", "stress", "turbofan", "nocrankshaft"] 94 95 MODE_FLAGS = { 96 "debug" : ["--nohard-abort", "--nodead-code-elimination", 97 "--nofold-constants", "--enable-slow-asserts", 98 "--debug-code", "--verify-heap"], 99 "release" : ["--nohard-abort", "--nodead-code-elimination", 100 "--nofold-constants"]} 101 102 GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", 103 "--concurrent-recompilation-queue-length=64", 104 "--concurrent-recompilation-delay=500", 105 "--concurrent-recompilation"] 106 107 SUPPORTED_ARCHS = ["android_arm", 108 "android_arm64", 109 "android_ia32", 110 "arm", 111 "ia32", 112 "x87", 113 "mips", 114 "mipsel", 115 "mips64el", 116 "nacl_ia32", 117 "nacl_x64", 118 "x64", 119 "x32", 120 "arm64"] 121 # Double the timeout for these: 122 SLOW_ARCHS = ["android_arm", 123 "android_arm64", 124 "android_ia32", 125 "arm", 126 "mips", 127 "mipsel", 128 "mips64el", 129 "nacl_ia32", 130 "nacl_x64", 131 "x87", 132 "arm64"] 133 134 135 def BuildOptions(): 136 result = optparse.OptionParser() 137 result.add_option("--arch", 138 help=("The architecture to run tests for, " 139 "'auto' or 'native' for auto-detect"), 140 default="ia32,x64,arm") 141 result.add_option("--arch-and-mode", 142 help="Architecture and mode in the format 'arch.mode'", 143 default=None) 144 result.add_option("--asan", 145 help="Regard test expectations for ASAN", 146 default=False, action="store_true") 147 result.add_option("--buildbot", 148 help="Adapt to path structure used on buildbots", 149 default=False, action="store_true") 150 result.add_option("--cat", help="Print the source of the tests", 151 default=False, action="store_true") 152 result.add_option("--flaky-tests", 153 help="Regard tests marked as flaky (run|skip|dontcare)", 154 default="dontcare") 155 result.add_option("--slow-tests", 156 help="Regard slow tests (run|skip|dontcare)", 157 default="dontcare") 158 result.add_option("--pass-fail-tests", 159 help="Regard pass|fail tests (run|skip|dontcare)", 160 default="dontcare") 161 result.add_option("--gc-stress", 162 help="Switch on GC stress mode", 163 default=False, action="store_true") 164 result.add_option("--command-prefix", 165 help="Prepended to each shell command used to run a test", 166 default="") 167 result.add_option("--download-data", help="Download missing test suite data", 168 default=False, action="store_true") 169 result.add_option("--extra-flags", 170 help="Additional flags to pass to each test command", 171 default="") 172 result.add_option("--isolates", help="Whether to test isolates", 173 default=False, action="store_true") 174 result.add_option("-j", help="The number of parallel tasks to run", 175 default=0, type="int") 176 result.add_option("-m", "--mode", 177 help="The test modes in which to run (comma-separated)", 178 default="release,debug") 179 result.add_option("--no-i18n", "--noi18n", 180 help="Skip internationalization tests", 181 default=False, action="store_true") 182 result.add_option("--no-network", "--nonetwork", 183 help="Don't distribute tests on the network", 184 default=(utils.GuessOS() != "linux"), 185 dest="no_network", action="store_true") 186 result.add_option("--no-presubmit", "--nopresubmit", 187 help='Skip presubmit checks', 188 default=False, dest="no_presubmit", action="store_true") 189 result.add_option("--no-snap", "--nosnap", 190 help='Test a build compiled without snapshot.', 191 default=False, dest="no_snap", action="store_true") 192 result.add_option("--no-sorting", "--nosorting", 193 help="Don't sort tests according to duration of last run.", 194 default=False, dest="no_sorting", action="store_true") 195 result.add_option("--no-stress", "--nostress", 196 help="Don't run crankshaft --always-opt --stress-op test", 197 default=False, dest="no_stress", action="store_true") 198 result.add_option("--no-variants", "--novariants", 199 help="Don't run any testing variants", 200 default=False, dest="no_variants", action="store_true") 201 result.add_option("--variants", 202 help="Comma-separated list of testing variants") 203 result.add_option("--outdir", help="Base directory with compile output", 204 default="out") 205 result.add_option("--predictable", 206 help="Compare output of several reruns of each test", 207 default=False, action="store_true") 208 result.add_option("-p", "--progress", 209 help=("The style of progress indicator" 210 " (verbose, dots, color, mono)"), 211 choices=progress.PROGRESS_INDICATORS.keys(), default="mono") 212 result.add_option("--quickcheck", default=False, action="store_true", 213 help=("Quick check mode (skip slow/flaky tests)")) 214 result.add_option("--report", help="Print a summary of the tests to be run", 215 default=False, action="store_true") 216 result.add_option("--json-test-results", 217 help="Path to a file for storing json results.") 218 result.add_option("--rerun-failures-count", 219 help=("Number of times to rerun each failing test case. " 220 "Very slow tests will be rerun only once."), 221 default=0, type="int") 222 result.add_option("--rerun-failures-max", 223 help="Maximum number of failing test cases to rerun.", 224 default=100, type="int") 225 result.add_option("--shard-count", 226 help="Split testsuites into this number of shards", 227 default=1, type="int") 228 result.add_option("--shard-run", 229 help="Run this shard from the split up tests.", 230 default=1, type="int") 231 result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") 232 result.add_option("--shell-dir", help="Directory containing executables", 233 default="") 234 result.add_option("--dont-skip-slow-simulator-tests", 235 help="Don't skip more slow tests when using a simulator.", 236 default=False, action="store_true", 237 dest="dont_skip_simulator_slow_tests") 238 result.add_option("--stress-only", 239 help="Only run tests with --always-opt --stress-opt", 240 default=False, action="store_true") 241 result.add_option("--time", help="Print timing information after running", 242 default=False, action="store_true") 243 result.add_option("-t", "--timeout", help="Timeout in seconds", 244 default= -1, type="int") 245 result.add_option("--tsan", 246 help="Regard test expectations for TSAN", 247 default=False, action="store_true") 248 result.add_option("-v", "--verbose", help="Verbose output", 249 default=False, action="store_true") 250 result.add_option("--valgrind", help="Run tests through valgrind", 251 default=False, action="store_true") 252 result.add_option("--warn-unused", help="Report unused rules", 253 default=False, action="store_true") 254 result.add_option("--junitout", help="File name of the JUnit output") 255 result.add_option("--junittestsuite", 256 help="The testsuite name in the JUnit output file", 257 default="v8tests") 258 result.add_option("--random-seed", default=0, dest="random_seed", 259 help="Default seed for initializing random generator") 260 return result 261 262 263 def ProcessOptions(options): 264 global VARIANT_FLAGS 265 global VARIANTS 266 267 # Architecture and mode related stuff. 268 if options.arch_and_mode: 269 options.arch_and_mode = [arch_and_mode.split(".") 270 for arch_and_mode in options.arch_and_mode.split(",")] 271 options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) 272 options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) 273 options.mode = options.mode.split(",") 274 for mode in options.mode: 275 if not mode.lower() in ["debug", "release", "optdebug"]: 276 print "Unknown mode %s" % mode 277 return False 278 if options.arch in ["auto", "native"]: 279 options.arch = ARCH_GUESS 280 options.arch = options.arch.split(",") 281 for arch in options.arch: 282 if not arch in SUPPORTED_ARCHS: 283 print "Unknown architecture %s" % arch 284 return False 285 286 # Store the final configuration in arch_and_mode list. Don't overwrite 287 # predefined arch_and_mode since it is more expressive than arch and mode. 288 if not options.arch_and_mode: 289 options.arch_and_mode = itertools.product(options.arch, options.mode) 290 291 # Special processing of other options, sorted alphabetically. 292 293 if options.buildbot: 294 # Buildbots run presubmit tests as a separate step. 295 options.no_presubmit = True 296 options.no_network = True 297 if options.command_prefix: 298 print("Specifying --command-prefix disables network distribution, " 299 "running tests locally.") 300 options.no_network = True 301 options.command_prefix = shlex.split(options.command_prefix) 302 options.extra_flags = shlex.split(options.extra_flags) 303 304 if options.gc_stress: 305 options.extra_flags += GC_STRESS_FLAGS 306 307 if options.asan: 308 options.extra_flags.append("--invoke-weak-callbacks") 309 310 if options.tsan: 311 VARIANTS = ["default"] 312 313 if options.j == 0: 314 options.j = multiprocessing.cpu_count() 315 316 while options.random_seed == 0: 317 options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647) 318 319 def excl(*args): 320 """Returns true if zero or one of multiple arguments are true.""" 321 return reduce(lambda x, y: x + y, args) <= 1 322 323 if not excl(options.no_stress, options.stress_only, options.no_variants, 324 bool(options.variants)): 325 print("Use only one of --no-stress, --stress-only, --no-variants, " 326 "or --variants.") 327 return False 328 if options.quickcheck: 329 VARIANTS = ["default", "stress"] 330 options.flaky_tests = "skip" 331 options.slow_tests = "skip" 332 options.pass_fail_tests = "skip" 333 if options.no_stress: 334 VARIANTS = ["default", "nocrankshaft"] 335 if options.no_variants: 336 VARIANTS = ["default"] 337 if options.stress_only: 338 VARIANTS = ["stress"] 339 if options.variants: 340 VARIANTS = options.variants.split(",") 341 if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()): 342 print "All variants must be in %s" % str(VARIANT_FLAGS.keys()) 343 return False 344 if options.predictable: 345 VARIANTS = ["default"] 346 options.extra_flags.append("--predictable") 347 options.extra_flags.append("--verify_predictable") 348 options.extra_flags.append("--no-inline-new") 349 350 if not options.shell_dir: 351 if options.shell: 352 print "Warning: --shell is deprecated, use --shell-dir instead." 353 options.shell_dir = os.path.dirname(options.shell) 354 if options.valgrind: 355 run_valgrind = os.path.join("tools", "run-valgrind.py") 356 # This is OK for distributed running, so we don't need to set no_network. 357 options.command_prefix = (["python", "-u", run_valgrind] + 358 options.command_prefix) 359 def CheckTestMode(name, option): 360 if not option in ["run", "skip", "dontcare"]: 361 print "Unknown %s mode %s" % (name, option) 362 return False 363 return True 364 if not CheckTestMode("flaky test", options.flaky_tests): 365 return False 366 if not CheckTestMode("slow test", options.slow_tests): 367 return False 368 if not CheckTestMode("pass|fail test", options.pass_fail_tests): 369 return False 370 if not options.no_i18n: 371 DEFAULT_TESTS.append("intl") 372 return True 373 374 375 def ShardTests(tests, shard_count, shard_run): 376 if shard_count < 2: 377 return tests 378 if shard_run < 1 or shard_run > shard_count: 379 print "shard-run not a valid number, should be in [1:shard-count]" 380 print "defaulting back to running all tests" 381 return tests 382 count = 0 383 shard = [] 384 for test in tests: 385 if count % shard_count == shard_run - 1: 386 shard.append(test) 387 count += 1 388 return shard 389 390 391 def Main(): 392 parser = BuildOptions() 393 (options, args) = parser.parse_args() 394 if not ProcessOptions(options): 395 parser.print_help() 396 return 1 397 398 exit_code = 0 399 workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), "..")) 400 if not options.no_presubmit: 401 print ">>> running presubmit tests" 402 exit_code = subprocess.call( 403 [sys.executable, join(workspace, "tools", "presubmit.py")]) 404 405 suite_paths = utils.GetSuitePaths(join(workspace, "test")) 406 407 # Expand arguments with grouped tests. The args should reflect the list of 408 # suites as otherwise filters would break. 409 def ExpandTestGroups(name): 410 if name in TEST_MAP: 411 return [suite for suite in TEST_MAP[arg]] 412 else: 413 return [name] 414 args = reduce(lambda x, y: x + y, 415 [ExpandTestGroups(arg) for arg in args], 416 []) 417 418 if len(args) == 0: 419 suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ] 420 else: 421 args_suites = OrderedDict() # Used as set 422 for arg in args: 423 args_suites[arg.split(os.path.sep)[0]] = True 424 suite_paths = [ s for s in args_suites if s in suite_paths ] 425 426 suites = [] 427 for root in suite_paths: 428 suite = testsuite.TestSuite.LoadTestSuite( 429 os.path.join(workspace, "test", root)) 430 if suite: 431 suites.append(suite) 432 433 if options.download_data: 434 for s in suites: 435 s.DownloadData() 436 437 for (arch, mode) in options.arch_and_mode: 438 try: 439 code = Execute(arch, mode, args, options, suites, workspace) 440 except KeyboardInterrupt: 441 return 2 442 exit_code = exit_code or code 443 return exit_code 444 445 446 def Execute(arch, mode, args, options, suites, workspace): 447 print(">>> Running tests for %s.%s" % (arch, mode)) 448 449 shell_dir = options.shell_dir 450 if not shell_dir: 451 if options.buildbot: 452 shell_dir = os.path.join(workspace, options.outdir, mode) 453 mode = mode.lower() 454 else: 455 shell_dir = os.path.join(workspace, options.outdir, 456 "%s.%s" % (arch, mode)) 457 shell_dir = os.path.relpath(shell_dir) 458 459 if mode == "optdebug": 460 mode = "debug" # "optdebug" is just an alias. 461 462 # Populate context object. 463 mode_flags = MODE_FLAGS[mode] 464 timeout = options.timeout 465 if timeout == -1: 466 # Simulators are slow, therefore allow a longer default timeout. 467 if arch in SLOW_ARCHS: 468 timeout = 2 * TIMEOUT_DEFAULT; 469 else: 470 timeout = TIMEOUT_DEFAULT; 471 472 timeout *= TIMEOUT_SCALEFACTOR[mode] 473 474 if options.predictable: 475 # Predictable mode is slower. 476 timeout *= 2 477 478 ctx = context.Context(arch, mode, shell_dir, 479 mode_flags, options.verbose, 480 timeout, options.isolates, 481 options.command_prefix, 482 options.extra_flags, 483 options.no_i18n, 484 options.random_seed, 485 options.no_sorting, 486 options.rerun_failures_count, 487 options.rerun_failures_max, 488 options.predictable) 489 490 # TODO(all): Combine "simulator" and "simulator_run". 491 simulator_run = not options.dont_skip_simulator_slow_tests and \ 492 arch in ['arm64', 'arm', 'mips'] and ARCH_GUESS and arch != ARCH_GUESS 493 # Find available test suites and read test cases from them. 494 variables = { 495 "arch": arch, 496 "asan": options.asan, 497 "deopt_fuzzer": False, 498 "gc_stress": options.gc_stress, 499 "isolates": options.isolates, 500 "mode": mode, 501 "no_i18n": options.no_i18n, 502 "no_snap": options.no_snap, 503 "simulator_run": simulator_run, 504 "simulator": utils.UseSimulator(arch), 505 "system": utils.GuessOS(), 506 "tsan": options.tsan, 507 } 508 all_tests = [] 509 num_tests = 0 510 test_id = 0 511 for s in suites: 512 s.ReadStatusFile(variables) 513 s.ReadTestCases(ctx) 514 if len(args) > 0: 515 s.FilterTestCasesByArgs(args) 516 all_tests += s.tests 517 s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests, 518 options.slow_tests, options.pass_fail_tests) 519 if options.cat: 520 verbose.PrintTestSource(s.tests) 521 continue 522 variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS] 523 s.tests = [ t.CopyAddingFlags(v) 524 for t in s.tests 525 for v in s.VariantFlags(t, variant_flags) ] 526 s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) 527 num_tests += len(s.tests) 528 for t in s.tests: 529 t.id = test_id 530 test_id += 1 531 532 if options.cat: 533 return 0 # We're done here. 534 535 if options.report: 536 verbose.PrintReport(all_tests) 537 538 if num_tests == 0: 539 print "No tests to run." 540 return 0 541 542 # Run the tests, either locally or distributed on the network. 543 start_time = time.time() 544 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 545 if options.junitout: 546 progress_indicator = progress.JUnitTestProgressIndicator( 547 progress_indicator, options.junitout, options.junittestsuite) 548 if options.json_test_results: 549 progress_indicator = progress.JsonTestProgressIndicator( 550 progress_indicator, options.json_test_results, arch, mode) 551 552 run_networked = not options.no_network 553 if not run_networked: 554 print("Network distribution disabled, running tests locally.") 555 elif utils.GuessOS() != "linux": 556 print("Network distribution is only supported on Linux, sorry!") 557 run_networked = False 558 peers = [] 559 if run_networked: 560 peers = network_execution.GetPeers() 561 if not peers: 562 print("No connection to distribution server; running tests locally.") 563 run_networked = False 564 elif len(peers) == 1: 565 print("No other peers on the network; running tests locally.") 566 run_networked = False 567 elif num_tests <= 100: 568 print("Less than 100 tests, running them locally.") 569 run_networked = False 570 571 if run_networked: 572 runner = network_execution.NetworkedRunner(suites, progress_indicator, 573 ctx, peers, workspace) 574 else: 575 runner = execution.Runner(suites, progress_indicator, ctx) 576 577 exit_code = runner.Run(options.j) 578 overall_duration = time.time() - start_time 579 580 if options.time: 581 verbose.PrintTestDurations(suites, overall_duration) 582 return exit_code 583 584 585 if __name__ == "__main__": 586 sys.exit(Main()) 587
