1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 import json 32 import math 33 import multiprocessing 34 import optparse 35 import os 36 from os.path import join 37 import random 38 import shlex 39 import subprocess 40 import sys 41 import time 42 43 from testrunner.local import execution 44 from testrunner.local import progress 45 from testrunner.local import testsuite 46 from testrunner.local import utils 47 from testrunner.local import verbose 48 from testrunner.objects import context 49 50 51 # Base dir of the v8 checkout to be used as cwd. 52 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 53 54 ARCH_GUESS = utils.DefaultArch() 55 DEFAULT_TESTS = ["mjsunit", "webkit"] 56 TIMEOUT_DEFAULT = 60 57 TIMEOUT_SCALEFACTOR = {"debug" : 4, 58 "release" : 1 } 59 60 MODE_FLAGS = { 61 "debug" : ["--nohard-abort", "--nodead-code-elimination", 62 "--nofold-constants", "--enable-slow-asserts", 63 "--debug-code", "--verify-heap", 64 "--noconcurrent-recompilation"], 65 "release" : ["--nohard-abort", "--nodead-code-elimination", 66 "--nofold-constants", "--noconcurrent-recompilation"]} 67 68 SUPPORTED_ARCHS = ["android_arm", 69 "android_ia32", 70 "arm", 71 "ia32", 72 "ppc", 73 "ppc64", 74 "s390", 75 "s390x", 76 "mipsel", 77 "x64"] 78 # Double the timeout for these: 79 SLOW_ARCHS = ["android_arm", 80 "android_ia32", 81 "arm", 82 "mipsel"] 83 MAX_DEOPT = 1000000000 84 DISTRIBUTION_MODES = ["smooth", "random"] 85 86 87 class RandomDistribution: 88 def __init__(self, seed=None): 89 seed = seed or random.randint(1, sys.maxint) 90 print "Using random distribution with seed %d" % seed 91 self._random = random.Random(seed) 92 93 def Distribute(self, n, m): 94 if n > m: 95 n = m 96 return self._random.sample(xrange(1, m + 1), n) 97 98 99 class SmoothDistribution: 100 """Distribute n numbers into the interval [1:m]. 101 F1: Factor of the first derivation of the distribution function. 102 F2: Factor of the second derivation of the distribution function. 103 With F1 and F2 set to 0, the distribution will be equal. 104 """ 105 def __init__(self, factor1=2.0, factor2=0.2): 106 self._factor1 = factor1 107 self._factor2 = factor2 108 109 def Distribute(self, n, m): 110 if n > m: 111 n = m 112 if n <= 1: 113 return [ 1 ] 114 115 result = [] 116 x = 0.0 117 dx = 1.0 118 ddx = self._factor1 119 dddx = self._factor2 120 for i in range(0, n): 121 result += [ x ] 122 x += dx 123 dx += ddx 124 ddx += dddx 125 126 # Project the distribution into the interval [0:M]. 127 result = [ x * m / result[-1] for x in result ] 128 129 # Equalize by n. The closer n is to m, the more equal will be the 130 # distribution. 131 for (i, x) in enumerate(result): 132 # The value of x if it was equally distributed. 133 equal_x = i / float(n - 1) * float(m - 1) + 1 134 135 # Difference factor between actual and equal distribution. 136 diff = 1 - (x / equal_x) 137 138 # Equalize x dependent on the number of values to distribute. 139 result[i] = int(x + (i + 1) * diff) 140 return result 141 142 143 def Distribution(options): 144 if options.distribution_mode == "random": 145 return RandomDistribution(options.seed) 146 if options.distribution_mode == "smooth": 147 return SmoothDistribution(options.distribution_factor1, 148 options.distribution_factor2) 149 150 151 def BuildOptions(): 152 result = optparse.OptionParser() 153 result.add_option("--arch", 154 help=("The architecture to run tests for, " 155 "'auto' or 'native' for auto-detect"), 156 default="ia32,x64,arm") 157 result.add_option("--arch-and-mode", 158 help="Architecture and mode in the format 'arch.mode'", 159 default=None) 160 result.add_option("--asan", 161 help="Regard test expectations for ASAN", 162 default=False, action="store_true") 163 result.add_option("--buildbot", 164 help="Adapt to path structure used on buildbots", 165 default=False, action="store_true") 166 result.add_option("--dcheck-always-on", 167 help="Indicates that V8 was compiled with DCHECKs enabled", 168 default=False, action="store_true") 169 result.add_option("--command-prefix", 170 help="Prepended to each shell command used to run a test", 171 default="") 172 result.add_option("--coverage", help=("Exponential test coverage " 173 "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"), 174 default=0.4, type="float") 175 result.add_option("--coverage-lift", help=("Lifts test coverage for tests " 176 "with a small number of deopt points (range 0, inf)"), 177 default=20, type="int") 178 result.add_option("--download-data", help="Download missing test suite data", 179 default=False, action="store_true") 180 result.add_option("--distribution-factor1", help=("Factor of the first " 181 "derivation of the distribution function"), default=2.0, 182 type="float") 183 result.add_option("--distribution-factor2", help=("Factor of the second " 184 "derivation of the distribution function"), default=0.7, 185 type="float") 186 result.add_option("--distribution-mode", help=("How to select deopt points " 187 "for a given test (smooth|random)"), 188 default="smooth") 189 result.add_option("--dump-results-file", help=("Dump maximum number of " 190 "deopt points per test to a file")) 191 result.add_option("--extra-flags", 192 help="Additional flags to pass to each test command", 193 default="") 194 result.add_option("--isolates", help="Whether to test isolates", 195 default=False, action="store_true") 196 result.add_option("-j", help="The number of parallel tasks to run", 197 default=0, type="int") 198 result.add_option("-m", "--mode", 199 help="The test modes in which to run (comma-separated)", 200 default="release,debug") 201 result.add_option("--outdir", help="Base directory with compile output", 202 default="out") 203 result.add_option("-p", "--progress", 204 help=("The style of progress indicator" 205 " (verbose, dots, color, mono)"), 206 choices=progress.PROGRESS_INDICATORS.keys(), 207 default="mono") 208 result.add_option("--shard-count", 209 help="Split testsuites into this number of shards", 210 default=1, type="int") 211 result.add_option("--shard-run", 212 help="Run this shard from the split up tests.", 213 default=1, type="int") 214 result.add_option("--shell-dir", help="Directory containing executables", 215 default="") 216 result.add_option("--seed", help="The seed for the random distribution", 217 type="int") 218 result.add_option("-t", "--timeout", help="Timeout in seconds", 219 default= -1, type="int") 220 result.add_option("-v", "--verbose", help="Verbose output", 221 default=False, action="store_true") 222 result.add_option("--random-seed", default=0, dest="random_seed", 223 help="Default seed for initializing random generator") 224 return result 225 226 227 def ProcessOptions(options): 228 global VARIANT_FLAGS 229 230 # Architecture and mode related stuff. 231 if options.arch_and_mode: 232 tokens = options.arch_and_mode.split(".") 233 options.arch = tokens[0] 234 options.mode = tokens[1] 235 options.mode = options.mode.split(",") 236 for mode in options.mode: 237 if not mode.lower() in ["debug", "release"]: 238 print "Unknown mode %s" % mode 239 return False 240 if options.arch in ["auto", "native"]: 241 options.arch = ARCH_GUESS 242 options.arch = options.arch.split(",") 243 for arch in options.arch: 244 if not arch in SUPPORTED_ARCHS: 245 print "Unknown architecture %s" % arch 246 return False 247 248 # Special processing of other options, sorted alphabetically. 249 options.command_prefix = shlex.split(options.command_prefix) 250 options.extra_flags = shlex.split(options.extra_flags) 251 if options.j == 0: 252 options.j = multiprocessing.cpu_count() 253 while options.random_seed == 0: 254 options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647) 255 if not options.distribution_mode in DISTRIBUTION_MODES: 256 print "Unknown distribution mode %s" % options.distribution_mode 257 return False 258 if options.distribution_factor1 < 0.0: 259 print ("Distribution factor1 %s is out of range. Defaulting to 0.0" 260 % options.distribution_factor1) 261 options.distribution_factor1 = 0.0 262 if options.distribution_factor2 < 0.0: 263 print ("Distribution factor2 %s is out of range. Defaulting to 0.0" 264 % options.distribution_factor2) 265 options.distribution_factor2 = 0.0 266 if options.coverage < 0.0 or options.coverage > 1.0: 267 print ("Coverage %s is out of range. Defaulting to 0.4" 268 % options.coverage) 269 options.coverage = 0.4 270 if options.coverage_lift < 0: 271 print ("Coverage lift %s is out of range. Defaulting to 0" 272 % options.coverage_lift) 273 options.coverage_lift = 0 274 return True 275 276 277 def ShardTests(tests, shard_count, shard_run): 278 if shard_count < 2: 279 return tests 280 if shard_run < 1 or shard_run > shard_count: 281 print "shard-run not a valid number, should be in [1:shard-count]" 282 print "defaulting back to running all tests" 283 return tests 284 count = 0 285 shard = [] 286 for test in tests: 287 if count % shard_count == shard_run - 1: 288 shard.append(test) 289 count += 1 290 return shard 291 292 293 def Main(): 294 # Use the v8 root as cwd as some test cases use "load" with relative paths. 295 os.chdir(BASE_DIR) 296 297 parser = BuildOptions() 298 (options, args) = parser.parse_args() 299 if not ProcessOptions(options): 300 parser.print_help() 301 return 1 302 303 exit_code = 0 304 305 suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) 306 307 if len(args) == 0: 308 suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ] 309 else: 310 args_suites = set() 311 for arg in args: 312 suite = arg.split(os.path.sep)[0] 313 if not suite in args_suites: 314 args_suites.add(suite) 315 suite_paths = [ s for s in suite_paths if s in args_suites ] 316 317 suites = [] 318 for root in suite_paths: 319 suite = testsuite.TestSuite.LoadTestSuite( 320 os.path.join(BASE_DIR, "test", root)) 321 if suite: 322 suites.append(suite) 323 324 if options.download_data: 325 for s in suites: 326 s.DownloadData() 327 328 for mode in options.mode: 329 for arch in options.arch: 330 try: 331 code = Execute(arch, mode, args, options, suites, BASE_DIR) 332 exit_code = exit_code or code 333 except KeyboardInterrupt: 334 return 2 335 return exit_code 336 337 338 def CalculateNTests(m, options): 339 """Calculates the number of tests from m deopt points with exponential 340 coverage. 341 The coverage is expected to be between 0.0 and 1.0. 342 The 'coverage lift' lifts the coverage for tests with smaller m values. 343 """ 344 c = float(options.coverage) 345 l = float(options.coverage_lift) 346 return int(math.pow(m, (m * c + l) / (m + l))) 347 348 349 def Execute(arch, mode, args, options, suites, workspace): 350 print(">>> Running tests for %s.%s" % (arch, mode)) 351 352 dist = Distribution(options) 353 354 shell_dir = options.shell_dir 355 if not shell_dir: 356 if options.buildbot: 357 shell_dir = os.path.join(workspace, options.outdir, mode) 358 mode = mode.lower() 359 else: 360 shell_dir = os.path.join(workspace, options.outdir, 361 "%s.%s" % (arch, mode)) 362 shell_dir = os.path.relpath(shell_dir) 363 364 # Populate context object. 365 mode_flags = MODE_FLAGS[mode] 366 timeout = options.timeout 367 if timeout == -1: 368 # Simulators are slow, therefore allow a longer default timeout. 369 if arch in SLOW_ARCHS: 370 timeout = 2 * TIMEOUT_DEFAULT; 371 else: 372 timeout = TIMEOUT_DEFAULT; 373 374 timeout *= TIMEOUT_SCALEFACTOR[mode] 375 ctx = context.Context(arch, mode, shell_dir, 376 mode_flags, options.verbose, 377 timeout, options.isolates, 378 options.command_prefix, 379 options.extra_flags, 380 False, # Keep i18n on by default. 381 options.random_seed, 382 True, # No sorting of test cases. 383 0, # Don't rerun failing tests. 384 0, # No use of a rerun-failing-tests maximum. 385 False, # No predictable mode. 386 False, # No no_harness mode. 387 False, # Don't use perf data. 388 False) # Coverage not supported. 389 390 # Find available test suites and read test cases from them. 391 variables = { 392 "arch": arch, 393 "asan": options.asan, 394 "deopt_fuzzer": True, 395 "gc_stress": False, 396 "gcov_coverage": False, 397 "isolates": options.isolates, 398 "mode": mode, 399 "no_i18n": False, 400 "no_snap": False, 401 "simulator": utils.UseSimulator(arch), 402 "system": utils.GuessOS(), 403 "tsan": False, 404 "msan": False, 405 "dcheck_always_on": options.dcheck_always_on, 406 "novfp3": False, 407 "predictable": False, 408 "byteorder": sys.byteorder, 409 } 410 all_tests = [] 411 num_tests = 0 412 test_id = 0 413 414 # Remember test case prototypes for the fuzzing phase. 415 test_backup = dict((s, []) for s in suites) 416 417 for s in suites: 418 s.ReadStatusFile(variables) 419 s.ReadTestCases(ctx) 420 if len(args) > 0: 421 s.FilterTestCasesByArgs(args) 422 all_tests += s.tests 423 s.FilterTestCasesByStatus(False) 424 test_backup[s] = s.tests 425 analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT, 426 "--print-deopt-stress"] 427 s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ] 428 num_tests += len(s.tests) 429 for t in s.tests: 430 t.id = test_id 431 test_id += 1 432 433 if num_tests == 0: 434 print "No tests to run." 435 return 0 436 437 print(">>> Collection phase") 438 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 439 runner = execution.Runner(suites, progress_indicator, ctx) 440 441 exit_code = runner.Run(options.j) 442 443 print(">>> Analysis phase") 444 num_tests = 0 445 test_id = 0 446 for s in suites: 447 test_results = {} 448 for t in s.tests: 449 for line in t.output.stdout.splitlines(): 450 if line.startswith("=== Stress deopt counter: "): 451 test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1]) 452 for t in s.tests: 453 if t.path not in test_results: 454 print "Missing results for %s" % t.path 455 if options.dump_results_file: 456 results_dict = dict((t.path, n) for (t, n) in test_results.iteritems()) 457 with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f: 458 f.write(json.dumps(results_dict)) 459 460 # Reset tests and redistribute the prototypes from the collection phase. 461 s.tests = [] 462 if options.verbose: 463 print "Test distributions:" 464 for t in test_backup[s]: 465 max_deopt = test_results.get(t.path, 0) 466 if max_deopt == 0: 467 continue 468 n_deopt = CalculateNTests(max_deopt, options) 469 distribution = dist.Distribute(n_deopt, max_deopt) 470 if options.verbose: 471 print "%s %s" % (t.path, distribution) 472 for i in distribution: 473 fuzzing_flags = ["--deopt-every-n-times", "%d" % i] 474 s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags)) 475 num_tests += len(s.tests) 476 for t in s.tests: 477 t.id = test_id 478 test_id += 1 479 480 if num_tests == 0: 481 print "No tests to run." 482 return 0 483 484 print(">>> Deopt fuzzing phase (%d test cases)" % num_tests) 485 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 486 runner = execution.Runner(suites, progress_indicator, ctx) 487 488 code = runner.Run(options.j) 489 return exit_code or code 490 491 492 if __name__ == "__main__": 493 sys.exit(Main()) 494
