1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 import itertools 32 import multiprocessing 33 import optparse 34 import os 35 from os.path import join 36 import platform 37 import random 38 import shlex 39 import subprocess 40 import sys 41 import time 42 43 from testrunner.local import execution 44 from testrunner.local import progress 45 from testrunner.local import testsuite 46 from testrunner.local import utils 47 from testrunner.local import verbose 48 from testrunner.network import network_execution 49 from testrunner.objects import context 50 51 52 ARCH_GUESS = utils.DefaultArch() 53 DEFAULT_TESTS = ["mjsunit", "fuzz-natives", "cctest", "message", "preparser"] 54 TIMEOUT_DEFAULT = 60 55 TIMEOUT_SCALEFACTOR = {"debug" : 4, 56 "release" : 1 } 57 58 # Use this to run several variants of the tests. 59 VARIANT_FLAGS = { 60 "default": [], 61 "stress": ["--stress-opt", "--always-opt"], 62 "nocrankshaft": ["--nocrankshaft"]} 63 64 VARIANTS = ["default", "stress", "nocrankshaft"] 65 66 MODE_FLAGS = { 67 "debug" : ["--nohard-abort", "--nodead-code-elimination", 68 "--nofold-constants", "--enable-slow-asserts", 69 "--debug-code", "--verify-heap"], 70 "release" : ["--nohard-abort", "--nodead-code-elimination", 71 "--nofold-constants"]} 72 73 GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", 74 "--concurrent-recompilation-queue-length=64", 75 "--concurrent-recompilation-delay=500", 76 "--concurrent-recompilation"] 77 78 SUPPORTED_ARCHS = ["android_arm", 79 "android_arm64", 80 "android_ia32", 81 "arm", 82 "ia32", 83 "x87", 84 "mips", 85 "mipsel", 86 "nacl_ia32", 87 "nacl_x64", 88 "x64", 89 "arm64"] 90 # Double the timeout for these: 91 SLOW_ARCHS = ["android_arm", 92 "android_arm64", 93 "android_ia32", 94 "arm", 95 "mips", 96 "mipsel", 97 "nacl_ia32", 98 "nacl_x64", 99 "x87", 100 "arm64"] 101 102 103 def BuildOptions(): 104 result = optparse.OptionParser() 105 result.add_option("--arch", 106 help=("The architecture to run tests for, " 107 "'auto' or 'native' for auto-detect"), 108 default="ia32,x64,arm") 109 result.add_option("--arch-and-mode", 110 help="Architecture and mode in the format 'arch.mode'", 111 default=None) 112 result.add_option("--asan", 113 help="Regard test expectations for ASAN", 114 default=False, action="store_true") 115 result.add_option("--buildbot", 116 help="Adapt to path structure used on buildbots", 117 default=False, action="store_true") 118 result.add_option("--cat", help="Print the source of the tests", 119 default=False, action="store_true") 120 result.add_option("--flaky-tests", 121 help="Regard tests marked as flaky (run|skip|dontcare)", 122 default="dontcare") 123 result.add_option("--slow-tests", 124 help="Regard slow tests (run|skip|dontcare)", 125 default="dontcare") 126 result.add_option("--pass-fail-tests", 127 help="Regard pass|fail tests (run|skip|dontcare)", 128 default="dontcare") 129 result.add_option("--gc-stress", 130 help="Switch on GC stress mode", 131 default=False, action="store_true") 132 result.add_option("--command-prefix", 133 help="Prepended to each shell command used to run a test", 134 default="") 135 result.add_option("--download-data", help="Download missing test suite data", 136 default=False, action="store_true") 137 result.add_option("--extra-flags", 138 help="Additional flags to pass to each test command", 139 default="") 140 result.add_option("--isolates", help="Whether to test isolates", 141 default=False, action="store_true") 142 result.add_option("-j", help="The number of parallel tasks to run", 143 default=0, type="int") 144 result.add_option("-m", "--mode", 145 help="The test modes in which to run (comma-separated)", 146 default="release,debug") 147 result.add_option("--no-i18n", "--noi18n", 148 help="Skip internationalization tests", 149 default=False, action="store_true") 150 result.add_option("--no-network", "--nonetwork", 151 help="Don't distribute tests on the network", 152 default=(utils.GuessOS() != "linux"), 153 dest="no_network", action="store_true") 154 result.add_option("--no-presubmit", "--nopresubmit", 155 help='Skip presubmit checks', 156 default=False, dest="no_presubmit", action="store_true") 157 result.add_option("--no-snap", "--nosnap", 158 help='Test a build compiled without snapshot.', 159 default=False, dest="no_snap", action="store_true") 160 result.add_option("--no-sorting", "--nosorting", 161 help="Don't sort tests according to duration of last run.", 162 default=False, dest="no_sorting", action="store_true") 163 result.add_option("--no-stress", "--nostress", 164 help="Don't run crankshaft --always-opt --stress-op test", 165 default=False, dest="no_stress", action="store_true") 166 result.add_option("--no-variants", "--novariants", 167 help="Don't run any testing variants", 168 default=False, dest="no_variants", action="store_true") 169 result.add_option("--variants", 170 help="Comma-separated list of testing variants") 171 result.add_option("--outdir", help="Base directory with compile output", 172 default="out") 173 result.add_option("-p", "--progress", 174 help=("The style of progress indicator" 175 " (verbose, dots, color, mono)"), 176 choices=progress.PROGRESS_INDICATORS.keys(), default="mono") 177 result.add_option("--quickcheck", default=False, action="store_true", 178 help=("Quick check mode (skip slow/flaky tests)")) 179 result.add_option("--report", help="Print a summary of the tests to be run", 180 default=False, action="store_true") 181 result.add_option("--json-test-results", 182 help="Path to a file for storing json results.") 183 result.add_option("--shard-count", 184 help="Split testsuites into this number of shards", 185 default=1, type="int") 186 result.add_option("--shard-run", 187 help="Run this shard from the split up tests.", 188 default=1, type="int") 189 result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") 190 result.add_option("--shell-dir", help="Directory containing executables", 191 default="") 192 result.add_option("--dont-skip-slow-simulator-tests", 193 help="Don't skip more slow tests when using a simulator.", 194 default=False, action="store_true", 195 dest="dont_skip_simulator_slow_tests") 196 result.add_option("--stress-only", 197 help="Only run tests with --always-opt --stress-opt", 198 default=False, action="store_true") 199 result.add_option("--time", help="Print timing information after running", 200 default=False, action="store_true") 201 result.add_option("-t", "--timeout", help="Timeout in seconds", 202 default= -1, type="int") 203 result.add_option("-v", "--verbose", help="Verbose output", 204 default=False, action="store_true") 205 result.add_option("--valgrind", help="Run tests through valgrind", 206 default=False, action="store_true") 207 result.add_option("--warn-unused", help="Report unused rules", 208 default=False, action="store_true") 209 result.add_option("--junitout", help="File name of the JUnit output") 210 result.add_option("--junittestsuite", 211 help="The testsuite name in the JUnit output file", 212 default="v8tests") 213 result.add_option("--random-seed", default=0, dest="random_seed", 214 help="Default seed for initializing random generator") 215 return result 216 217 218 def ProcessOptions(options): 219 global VARIANT_FLAGS 220 global VARIANTS 221 222 # Architecture and mode related stuff. 223 if options.arch_and_mode: 224 options.arch_and_mode = [arch_and_mode.split(".") 225 for arch_and_mode in options.arch_and_mode.split(",")] 226 options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode]) 227 options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode]) 228 options.mode = options.mode.split(",") 229 for mode in options.mode: 230 if not mode.lower() in ["debug", "release", "optdebug"]: 231 print "Unknown mode %s" % mode 232 return False 233 if options.arch in ["auto", "native"]: 234 options.arch = ARCH_GUESS 235 options.arch = options.arch.split(",") 236 for arch in options.arch: 237 if not arch in SUPPORTED_ARCHS: 238 print "Unknown architecture %s" % arch 239 return False 240 241 # Store the final configuration in arch_and_mode list. Don't overwrite 242 # predefined arch_and_mode since it is more expressive than arch and mode. 243 if not options.arch_and_mode: 244 options.arch_and_mode = itertools.product(options.arch, options.mode) 245 246 # Special processing of other options, sorted alphabetically. 247 248 if options.buildbot: 249 # Buildbots run presubmit tests as a separate step. 250 options.no_presubmit = True 251 options.no_network = True 252 if options.command_prefix: 253 print("Specifying --command-prefix disables network distribution, " 254 "running tests locally.") 255 options.no_network = True 256 options.command_prefix = shlex.split(options.command_prefix) 257 options.extra_flags = shlex.split(options.extra_flags) 258 259 if options.gc_stress: 260 options.extra_flags += GC_STRESS_FLAGS 261 262 if options.asan: 263 options.extra_flags.append("--invoke-weak-callbacks") 264 265 if options.j == 0: 266 options.j = multiprocessing.cpu_count() 267 268 while options.random_seed == 0: 269 options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647) 270 271 def excl(*args): 272 """Returns true if zero or one of multiple arguments are true.""" 273 return reduce(lambda x, y: x + y, args) <= 1 274 275 if not excl(options.no_stress, options.stress_only, options.no_variants, 276 bool(options.variants), options.quickcheck): 277 print("Use only one of --no-stress, --stress-only, --no-variants, " 278 "--variants, or --quickcheck.") 279 return False 280 if options.no_stress: 281 VARIANTS = ["default", "nocrankshaft"] 282 if options.no_variants: 283 VARIANTS = ["default"] 284 if options.stress_only: 285 VARIANTS = ["stress"] 286 if options.variants: 287 VARIANTS = options.variants.split(",") 288 if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()): 289 print "All variants must be in %s" % str(VARIANT_FLAGS.keys()) 290 return False 291 if options.quickcheck: 292 VARIANTS = ["default", "stress"] 293 options.flaky_tests = "skip" 294 options.slow_tests = "skip" 295 options.pass_fail_tests = "skip" 296 297 if not options.shell_dir: 298 if options.shell: 299 print "Warning: --shell is deprecated, use --shell-dir instead." 300 options.shell_dir = os.path.dirname(options.shell) 301 if options.valgrind: 302 run_valgrind = os.path.join("tools", "run-valgrind.py") 303 # This is OK for distributed running, so we don't need to set no_network. 304 options.command_prefix = (["python", "-u", run_valgrind] + 305 options.command_prefix) 306 def CheckTestMode(name, option): 307 if not option in ["run", "skip", "dontcare"]: 308 print "Unknown %s mode %s" % (name, option) 309 return False 310 return True 311 if not CheckTestMode("flaky test", options.flaky_tests): 312 return False 313 if not CheckTestMode("slow test", options.slow_tests): 314 return False 315 if not CheckTestMode("pass|fail test", options.pass_fail_tests): 316 return False 317 if not options.no_i18n: 318 DEFAULT_TESTS.append("intl") 319 return True 320 321 322 def ShardTests(tests, shard_count, shard_run): 323 if shard_count < 2: 324 return tests 325 if shard_run < 1 or shard_run > shard_count: 326 print "shard-run not a valid number, should be in [1:shard-count]" 327 print "defaulting back to running all tests" 328 return tests 329 count = 0 330 shard = [] 331 for test in tests: 332 if count % shard_count == shard_run - 1: 333 shard.append(test) 334 count += 1 335 return shard 336 337 338 def Main(): 339 parser = BuildOptions() 340 (options, args) = parser.parse_args() 341 if not ProcessOptions(options): 342 parser.print_help() 343 return 1 344 345 exit_code = 0 346 workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), "..")) 347 if not options.no_presubmit: 348 print ">>> running presubmit tests" 349 exit_code = subprocess.call( 350 [sys.executable, join(workspace, "tools", "presubmit.py")]) 351 352 suite_paths = utils.GetSuitePaths(join(workspace, "test")) 353 354 if len(args) == 0: 355 suite_paths = [ s for s in DEFAULT_TESTS if s in suite_paths ] 356 else: 357 args_suites = set() 358 for arg in args: 359 suite = arg.split(os.path.sep)[0] 360 if not suite in args_suites: 361 args_suites.add(suite) 362 suite_paths = [ s for s in args_suites if s in suite_paths ] 363 364 suites = [] 365 for root in suite_paths: 366 suite = testsuite.TestSuite.LoadTestSuite( 367 os.path.join(workspace, "test", root)) 368 if suite: 369 suites.append(suite) 370 371 if options.download_data: 372 for s in suites: 373 s.DownloadData() 374 375 for (arch, mode) in options.arch_and_mode: 376 try: 377 code = Execute(arch, mode, args, options, suites, workspace) 378 except KeyboardInterrupt: 379 return 2 380 exit_code = exit_code or code 381 return exit_code 382 383 384 def Execute(arch, mode, args, options, suites, workspace): 385 print(">>> Running tests for %s.%s" % (arch, mode)) 386 387 shell_dir = options.shell_dir 388 if not shell_dir: 389 if options.buildbot: 390 shell_dir = os.path.join(workspace, options.outdir, mode) 391 mode = mode.lower() 392 else: 393 shell_dir = os.path.join(workspace, options.outdir, 394 "%s.%s" % (arch, mode)) 395 shell_dir = os.path.relpath(shell_dir) 396 397 if mode == "optdebug": 398 mode = "debug" # "optdebug" is just an alias. 399 400 # Populate context object. 401 mode_flags = MODE_FLAGS[mode] 402 timeout = options.timeout 403 if timeout == -1: 404 # Simulators are slow, therefore allow a longer default timeout. 405 if arch in SLOW_ARCHS: 406 timeout = 2 * TIMEOUT_DEFAULT; 407 else: 408 timeout = TIMEOUT_DEFAULT; 409 410 timeout *= TIMEOUT_SCALEFACTOR[mode] 411 ctx = context.Context(arch, mode, shell_dir, 412 mode_flags, options.verbose, 413 timeout, options.isolates, 414 options.command_prefix, 415 options.extra_flags, 416 options.no_i18n, 417 options.random_seed, 418 options.no_sorting) 419 420 # TODO(all): Combine "simulator" and "simulator_run". 421 simulator_run = not options.dont_skip_simulator_slow_tests and \ 422 arch in ['arm64', 'arm', 'mips'] and ARCH_GUESS and arch != ARCH_GUESS 423 # Find available test suites and read test cases from them. 424 variables = { 425 "arch": arch, 426 "asan": options.asan, 427 "deopt_fuzzer": False, 428 "gc_stress": options.gc_stress, 429 "isolates": options.isolates, 430 "mode": mode, 431 "no_i18n": options.no_i18n, 432 "no_snap": options.no_snap, 433 "simulator_run": simulator_run, 434 "simulator": utils.UseSimulator(arch), 435 "system": utils.GuessOS(), 436 } 437 all_tests = [] 438 num_tests = 0 439 test_id = 0 440 for s in suites: 441 s.ReadStatusFile(variables) 442 s.ReadTestCases(ctx) 443 if len(args) > 0: 444 s.FilterTestCasesByArgs(args) 445 all_tests += s.tests 446 s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests, 447 options.slow_tests, options.pass_fail_tests) 448 if options.cat: 449 verbose.PrintTestSource(s.tests) 450 continue 451 variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS] 452 s.tests = [ t.CopyAddingFlags(v) 453 for t in s.tests 454 for v in s.VariantFlags(t, variant_flags) ] 455 s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) 456 num_tests += len(s.tests) 457 for t in s.tests: 458 t.id = test_id 459 test_id += 1 460 461 if options.cat: 462 return 0 # We're done here. 463 464 if options.report: 465 verbose.PrintReport(all_tests) 466 467 if num_tests == 0: 468 print "No tests to run." 469 return 0 470 471 # Run the tests, either locally or distributed on the network. 472 start_time = time.time() 473 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 474 if options.junitout: 475 progress_indicator = progress.JUnitTestProgressIndicator( 476 progress_indicator, options.junitout, options.junittestsuite) 477 if options.json_test_results: 478 progress_indicator = progress.JsonTestProgressIndicator( 479 progress_indicator, options.json_test_results, arch, mode) 480 481 run_networked = not options.no_network 482 if not run_networked: 483 print("Network distribution disabled, running tests locally.") 484 elif utils.GuessOS() != "linux": 485 print("Network distribution is only supported on Linux, sorry!") 486 run_networked = False 487 peers = [] 488 if run_networked: 489 peers = network_execution.GetPeers() 490 if not peers: 491 print("No connection to distribution server; running tests locally.") 492 run_networked = False 493 elif len(peers) == 1: 494 print("No other peers on the network; running tests locally.") 495 run_networked = False 496 elif num_tests <= 100: 497 print("Less than 100 tests, running them locally.") 498 run_networked = False 499 500 if run_networked: 501 runner = network_execution.NetworkedRunner(suites, progress_indicator, 502 ctx, peers, workspace) 503 else: 504 runner = execution.Runner(suites, progress_indicator, ctx) 505 506 exit_code = runner.Run(options.j) 507 overall_duration = time.time() - start_time 508 509 if options.time: 510 verbose.PrintTestDurations(suites, overall_duration) 511 return exit_code 512 513 514 if __name__ == "__main__": 515 sys.exit(Main()) 516
