1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 import multiprocessing 32 import optparse 33 import os 34 from os.path import join 35 import shlex 36 import subprocess 37 import sys 38 import time 39 40 from testrunner.local import execution 41 from testrunner.local import progress 42 from testrunner.local import testsuite 43 from testrunner.local import utils 44 from testrunner.local import verbose 45 from testrunner.network import network_execution 46 from testrunner.objects import context 47 48 49 ARCH_GUESS = utils.DefaultArch() 50 DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"] 51 TIMEOUT_DEFAULT = 60 52 TIMEOUT_SCALEFACTOR = {"debug" : 4, 53 "release" : 1 } 54 55 # Use this to run several variants of the tests. 56 VARIANT_FLAGS = [[], 57 ["--stress-opt", "--always-opt"], 58 ["--nocrankshaft"]] 59 MODE_FLAGS = { 60 "debug" : ["--nobreak-on-abort", "--nodead-code-elimination", 61 "--nofold-constants", "--enable-slow-asserts", 62 "--debug-code", "--verify-heap"], 63 "release" : ["--nobreak-on-abort", "--nodead-code-elimination", 64 "--nofold-constants"]} 65 66 SUPPORTED_ARCHS = ["android_arm", 67 "android_ia32", 68 "arm", 69 "ia32", 70 "mipsel", 71 "nacl_ia32", 72 "nacl_x64", 73 "x64"] 74 # Double the timeout for these: 75 SLOW_ARCHS = ["android_arm", 76 "android_ia32", 77 "arm", 78 "mipsel", 79 "nacl_ia32", 80 "nacl_x64"] 81 82 83 def BuildOptions(): 84 result = optparse.OptionParser() 85 result.add_option("--arch", 86 help=("The architecture to run tests for, " 87 "'auto' or 'native' for auto-detect"), 88 default="ia32,x64,arm") 89 result.add_option("--arch-and-mode", 90 help="Architecture and mode in the format 'arch.mode'", 91 default=None) 92 result.add_option("--buildbot", 93 help="Adapt to path structure used on buildbots", 94 default=False, action="store_true") 95 result.add_option("--cat", help="Print the source of the tests", 96 default=False, action="store_true") 97 result.add_option("--flaky-tests", 98 help="Regard tests marked as flaky (run|skip|dontcare)", 99 default="dontcare") 100 result.add_option("--command-prefix", 101 help="Prepended to each shell command used to run a test", 102 default="") 103 result.add_option("--download-data", help="Download missing test suite data", 104 default=False, action="store_true") 105 result.add_option("--extra-flags", 106 help="Additional flags to pass to each test command", 107 default="") 108 result.add_option("--isolates", help="Whether to test isolates", 109 default=False, action="store_true") 110 result.add_option("-j", help="The number of parallel tasks to run", 111 default=0, type="int") 112 result.add_option("-m", "--mode", 113 help="The test modes in which to run (comma-separated)", 114 default="release,debug") 115 result.add_option("--no-network", "--nonetwork", 116 help="Don't distribute tests on the network", 117 default=(utils.GuessOS() != "linux"), 118 dest="no_network", action="store_true") 119 result.add_option("--no-presubmit", "--nopresubmit", 120 help='Skip presubmit checks', 121 default=False, dest="no_presubmit", action="store_true") 122 result.add_option("--no-stress", "--nostress", 123 help="Don't run crankshaft --always-opt --stress-op test", 124 default=False, dest="no_stress", action="store_true") 125 result.add_option("--outdir", help="Base directory with compile output", 126 default="out") 127 result.add_option("-p", "--progress", 128 help=("The style of progress indicator" 129 " (verbose, dots, color, mono)"), 130 choices=progress.PROGRESS_INDICATORS.keys(), default="mono") 131 result.add_option("--report", help="Print a summary of the tests to be run", 132 default=False, action="store_true") 133 result.add_option("--shard-count", 134 help="Split testsuites into this number of shards", 135 default=1, type="int") 136 result.add_option("--shard-run", 137 help="Run this shard from the split up tests.", 138 default=1, type="int") 139 result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") 140 result.add_option("--shell-dir", help="Directory containing executables", 141 default="") 142 result.add_option("--stress-only", 143 help="Only run tests with --always-opt --stress-opt", 144 default=False, action="store_true") 145 result.add_option("--time", help="Print timing information after running", 146 default=False, action="store_true") 147 result.add_option("-t", "--timeout", help="Timeout in seconds", 148 default= -1, type="int") 149 result.add_option("-v", "--verbose", help="Verbose output", 150 default=False, action="store_true") 151 result.add_option("--valgrind", help="Run tests through valgrind", 152 default=False, action="store_true") 153 result.add_option("--warn-unused", help="Report unused rules", 154 default=False, action="store_true") 155 result.add_option("--junitout", help="File name of the JUnit output") 156 result.add_option("--junittestsuite", 157 help="The testsuite name in the JUnit output file", 158 default="v8tests") 159 return result 160 161 162 def ProcessOptions(options): 163 global VARIANT_FLAGS 164 165 # Architecture and mode related stuff. 166 if options.arch_and_mode: 167 tokens = options.arch_and_mode.split(".") 168 options.arch = tokens[0] 169 options.mode = tokens[1] 170 options.mode = options.mode.split(",") 171 for mode in options.mode: 172 if not mode.lower() in ["debug", "release"]: 173 print "Unknown mode %s" % mode 174 return False 175 if options.arch in ["auto", "native"]: 176 options.arch = ARCH_GUESS 177 options.arch = options.arch.split(",") 178 for arch in options.arch: 179 if not arch in SUPPORTED_ARCHS: 180 print "Unknown architecture %s" % arch 181 return False 182 183 # Special processing of other options, sorted alphabetically. 184 185 if options.buildbot: 186 # Buildbots run presubmit tests as a separate step. 187 options.no_presubmit = True 188 options.no_network = True 189 if options.command_prefix: 190 print("Specifying --command-prefix disables network distribution, " 191 "running tests locally.") 192 options.no_network = True 193 options.command_prefix = shlex.split(options.command_prefix) 194 options.extra_flags = shlex.split(options.extra_flags) 195 if options.j == 0: 196 options.j = multiprocessing.cpu_count() 197 if options.no_stress: 198 VARIANT_FLAGS = [[], ["--nocrankshaft"]] 199 if not options.shell_dir: 200 if options.shell: 201 print "Warning: --shell is deprecated, use --shell-dir instead." 202 options.shell_dir = os.path.dirname(options.shell) 203 if options.stress_only: 204 VARIANT_FLAGS = [["--stress-opt", "--always-opt"]] 205 if options.valgrind: 206 run_valgrind = os.path.join("tools", "run-valgrind.py") 207 # This is OK for distributed running, so we don't need to set no_network. 208 options.command_prefix = (["python", "-u", run_valgrind] + 209 options.command_prefix) 210 if not options.flaky_tests in ["run", "skip", "dontcare"]: 211 print "Unknown flaky test mode %s" % options.flaky_tests 212 return False 213 return True 214 215 216 def ShardTests(tests, shard_count, shard_run): 217 if shard_count < 2: 218 return tests 219 if shard_run < 1 or shard_run > shard_count: 220 print "shard-run not a valid number, should be in [1:shard-count]" 221 print "defaulting back to running all tests" 222 return tests 223 count = 0 224 shard = [] 225 for test in tests: 226 if count % shard_count == shard_run - 1: 227 shard.append(test) 228 count += 1 229 return shard 230 231 232 def Main(): 233 parser = BuildOptions() 234 (options, args) = parser.parse_args() 235 if not ProcessOptions(options): 236 parser.print_help() 237 return 1 238 239 exit_code = 0 240 workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), "..")) 241 if not options.no_presubmit: 242 print ">>> running presubmit tests" 243 code = subprocess.call( 244 [sys.executable, join(workspace, "tools", "presubmit.py")]) 245 exit_code = code 246 247 suite_paths = utils.GetSuitePaths(join(workspace, "test")) 248 249 if len(args) == 0: 250 suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ] 251 else: 252 args_suites = set() 253 for arg in args: 254 suite = arg.split(os.path.sep)[0] 255 if not suite in args_suites: 256 args_suites.add(suite) 257 suite_paths = [ s for s in suite_paths if s in args_suites ] 258 259 suites = [] 260 for root in suite_paths: 261 suite = testsuite.TestSuite.LoadTestSuite( 262 os.path.join(workspace, "test", root)) 263 if suite: 264 suites.append(suite) 265 266 if options.download_data: 267 for s in suites: 268 s.DownloadData() 269 270 for mode in options.mode: 271 for arch in options.arch: 272 code = Execute(arch, mode, args, options, suites, workspace) 273 exit_code = exit_code or code 274 return exit_code 275 276 277 def Execute(arch, mode, args, options, suites, workspace): 278 print(">>> Running tests for %s.%s" % (arch, mode)) 279 280 shell_dir = options.shell_dir 281 if not shell_dir: 282 if options.buildbot: 283 shell_dir = os.path.join(workspace, options.outdir, mode) 284 mode = mode.lower() 285 else: 286 shell_dir = os.path.join(workspace, options.outdir, 287 "%s.%s" % (arch, mode)) 288 shell_dir = os.path.relpath(shell_dir) 289 290 # Populate context object. 291 mode_flags = MODE_FLAGS[mode] 292 timeout = options.timeout 293 if timeout == -1: 294 # Simulators are slow, therefore allow a longer default timeout. 295 if arch in SLOW_ARCHS: 296 timeout = 2 * TIMEOUT_DEFAULT; 297 else: 298 timeout = TIMEOUT_DEFAULT; 299 300 timeout *= TIMEOUT_SCALEFACTOR[mode] 301 ctx = context.Context(arch, mode, shell_dir, 302 mode_flags, options.verbose, 303 timeout, options.isolates, 304 options.command_prefix, 305 options.extra_flags) 306 307 # Find available test suites and read test cases from them. 308 variables = { 309 "mode": mode, 310 "arch": arch, 311 "system": utils.GuessOS(), 312 "isolates": options.isolates, 313 "deopt_fuzzer": False, 314 } 315 all_tests = [] 316 num_tests = 0 317 test_id = 0 318 for s in suites: 319 s.ReadStatusFile(variables) 320 s.ReadTestCases(ctx) 321 if len(args) > 0: 322 s.FilterTestCasesByArgs(args) 323 all_tests += s.tests 324 s.FilterTestCasesByStatus(options.warn_unused, options.flaky_tests) 325 if options.cat: 326 verbose.PrintTestSource(s.tests) 327 continue 328 variant_flags = s.VariantFlags() or VARIANT_FLAGS 329 s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in variant_flags ] 330 s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) 331 num_tests += len(s.tests) 332 for t in s.tests: 333 t.id = test_id 334 test_id += 1 335 336 if options.cat: 337 return 0 # We're done here. 338 339 if options.report: 340 verbose.PrintReport(all_tests) 341 342 if num_tests == 0: 343 print "No tests to run." 344 return 0 345 346 # Run the tests, either locally or distributed on the network. 347 try: 348 start_time = time.time() 349 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 350 if options.junitout: 351 progress_indicator = progress.JUnitTestProgressIndicator( 352 progress_indicator, options.junitout, options.junittestsuite) 353 354 run_networked = not options.no_network 355 if not run_networked: 356 print("Network distribution disabled, running tests locally.") 357 elif utils.GuessOS() != "linux": 358 print("Network distribution is only supported on Linux, sorry!") 359 run_networked = False 360 peers = [] 361 if run_networked: 362 peers = network_execution.GetPeers() 363 if not peers: 364 print("No connection to distribution server; running tests locally.") 365 run_networked = False 366 elif len(peers) == 1: 367 print("No other peers on the network; running tests locally.") 368 run_networked = False 369 elif num_tests <= 100: 370 print("Less than 100 tests, running them locally.") 371 run_networked = False 372 373 if run_networked: 374 runner = network_execution.NetworkedRunner(suites, progress_indicator, 375 ctx, peers, workspace) 376 else: 377 runner = execution.Runner(suites, progress_indicator, ctx) 378 379 exit_code = runner.Run(options.j) 380 if runner.terminate: 381 return exit_code 382 overall_duration = time.time() - start_time 383 except KeyboardInterrupt: 384 return 1 385 386 if options.time: 387 verbose.PrintTestDurations(suites, overall_duration) 388 return exit_code 389 390 391 if __name__ == "__main__": 392 sys.exit(Main()) 393
