1 #!/usr/bin/env python 2 # 3 # Copyright 2012 the V8 project authors. All rights reserved. 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following 12 # disclaimer in the documentation and/or other materials provided 13 # with the distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived 16 # from this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 import json 32 import math 33 import multiprocessing 34 import optparse 35 import os 36 from os.path import join 37 import random 38 import shlex 39 import subprocess 40 import sys 41 import time 42 43 from testrunner.local import execution 44 from testrunner.local import progress 45 from testrunner.local import testsuite 46 from testrunner.local import utils 47 from testrunner.local import verbose 48 from testrunner.objects import context 49 50 51 # Base dir of the v8 checkout to be used as cwd. 52 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 53 54 ARCH_GUESS = utils.DefaultArch() 55 DEFAULT_TESTS = ["mjsunit", "webkit"] 56 TIMEOUT_DEFAULT = 60 57 TIMEOUT_SCALEFACTOR = {"debug" : 4, 58 "release" : 1 } 59 60 MODE_FLAGS = { 61 "debug" : ["--nohard-abort", "--nodead-code-elimination", 62 "--nofold-constants", "--enable-slow-asserts", 63 "--verify-heap", "--noconcurrent-recompilation"], 64 "release" : ["--nohard-abort", "--nodead-code-elimination", 65 "--nofold-constants", "--noconcurrent-recompilation"]} 66 67 SUPPORTED_ARCHS = ["android_arm", 68 "android_ia32", 69 "arm", 70 "ia32", 71 "ppc", 72 "ppc64", 73 "s390", 74 "s390x", 75 "mipsel", 76 "x64"] 77 # Double the timeout for these: 78 SLOW_ARCHS = ["android_arm", 79 "android_ia32", 80 "arm", 81 "mipsel"] 82 MAX_DEOPT = 1000000000 83 DISTRIBUTION_MODES = ["smooth", "random"] 84 85 86 class RandomDistribution: 87 def __init__(self, seed=None): 88 seed = seed or random.randint(1, sys.maxint) 89 print "Using random distribution with seed %d" % seed 90 self._random = random.Random(seed) 91 92 def Distribute(self, n, m): 93 if n > m: 94 n = m 95 return self._random.sample(xrange(1, m + 1), n) 96 97 98 class SmoothDistribution: 99 """Distribute n numbers into the interval [1:m]. 100 F1: Factor of the first derivation of the distribution function. 101 F2: Factor of the second derivation of the distribution function. 102 With F1 and F2 set to 0, the distribution will be equal. 103 """ 104 def __init__(self, factor1=2.0, factor2=0.2): 105 self._factor1 = factor1 106 self._factor2 = factor2 107 108 def Distribute(self, n, m): 109 if n > m: 110 n = m 111 if n <= 1: 112 return [ 1 ] 113 114 result = [] 115 x = 0.0 116 dx = 1.0 117 ddx = self._factor1 118 dddx = self._factor2 119 for i in range(0, n): 120 result += [ x ] 121 x += dx 122 dx += ddx 123 ddx += dddx 124 125 # Project the distribution into the interval [0:M]. 126 result = [ x * m / result[-1] for x in result ] 127 128 # Equalize by n. The closer n is to m, the more equal will be the 129 # distribution. 130 for (i, x) in enumerate(result): 131 # The value of x if it was equally distributed. 132 equal_x = i / float(n - 1) * float(m - 1) + 1 133 134 # Difference factor between actual and equal distribution. 135 diff = 1 - (x / equal_x) 136 137 # Equalize x dependent on the number of values to distribute. 138 result[i] = int(x + (i + 1) * diff) 139 return result 140 141 142 def Distribution(options): 143 if options.distribution_mode == "random": 144 return RandomDistribution(options.seed) 145 if options.distribution_mode == "smooth": 146 return SmoothDistribution(options.distribution_factor1, 147 options.distribution_factor2) 148 149 150 def BuildOptions(): 151 result = optparse.OptionParser() 152 result.add_option("--arch", 153 help=("The architecture to run tests for, " 154 "'auto' or 'native' for auto-detect"), 155 default="ia32,x64,arm") 156 result.add_option("--arch-and-mode", 157 help="Architecture and mode in the format 'arch.mode'", 158 default=None) 159 result.add_option("--asan", 160 help="Regard test expectations for ASAN", 161 default=False, action="store_true") 162 result.add_option("--buildbot", 163 help="Adapt to path structure used on buildbots", 164 default=False, action="store_true") 165 result.add_option("--dcheck-always-on", 166 help="Indicates that V8 was compiled with DCHECKs enabled", 167 default=False, action="store_true") 168 result.add_option("--command-prefix", 169 help="Prepended to each shell command used to run a test", 170 default="") 171 result.add_option("--coverage", help=("Exponential test coverage " 172 "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"), 173 default=0.4, type="float") 174 result.add_option("--coverage-lift", help=("Lifts test coverage for tests " 175 "with a small number of deopt points (range 0, inf)"), 176 default=20, type="int") 177 result.add_option("--download-data", help="Download missing test suite data", 178 default=False, action="store_true") 179 result.add_option("--distribution-factor1", help=("Factor of the first " 180 "derivation of the distribution function"), default=2.0, 181 type="float") 182 result.add_option("--distribution-factor2", help=("Factor of the second " 183 "derivation of the distribution function"), default=0.7, 184 type="float") 185 result.add_option("--distribution-mode", help=("How to select deopt points " 186 "for a given test (smooth|random)"), 187 default="smooth") 188 result.add_option("--dump-results-file", help=("Dump maximum number of " 189 "deopt points per test to a file")) 190 result.add_option("--extra-flags", 191 help="Additional flags to pass to each test command", 192 default="") 193 result.add_option("--isolates", help="Whether to test isolates", 194 default=False, action="store_true") 195 result.add_option("-j", help="The number of parallel tasks to run", 196 default=0, type="int") 197 result.add_option("-m", "--mode", 198 help="The test modes in which to run (comma-separated)", 199 default="release,debug") 200 result.add_option("--outdir", help="Base directory with compile output", 201 default="out") 202 result.add_option("-p", "--progress", 203 help=("The style of progress indicator" 204 " (verbose, dots, color, mono)"), 205 choices=progress.PROGRESS_INDICATORS.keys(), 206 default="mono") 207 result.add_option("--shard-count", 208 help="Split testsuites into this number of shards", 209 default=1, type="int") 210 result.add_option("--shard-run", 211 help="Run this shard from the split up tests.", 212 default=1, type="int") 213 result.add_option("--shell-dir", help="Directory containing executables", 214 default="") 215 result.add_option("--seed", help="The seed for the random distribution", 216 type="int") 217 result.add_option("-t", "--timeout", help="Timeout in seconds", 218 default= -1, type="int") 219 result.add_option("-v", "--verbose", help="Verbose output", 220 default=False, action="store_true") 221 result.add_option("--random-seed", default=0, dest="random_seed", 222 help="Default seed for initializing random generator") 223 return result 224 225 226 def ProcessOptions(options): 227 global VARIANT_FLAGS 228 229 # Architecture and mode related stuff. 230 if options.arch_and_mode: 231 tokens = options.arch_and_mode.split(".") 232 options.arch = tokens[0] 233 options.mode = tokens[1] 234 options.mode = options.mode.split(",") 235 for mode in options.mode: 236 if not mode.lower() in ["debug", "release"]: 237 print "Unknown mode %s" % mode 238 return False 239 if options.arch in ["auto", "native"]: 240 options.arch = ARCH_GUESS 241 options.arch = options.arch.split(",") 242 for arch in options.arch: 243 if not arch in SUPPORTED_ARCHS: 244 print "Unknown architecture %s" % arch 245 return False 246 247 # Special processing of other options, sorted alphabetically. 248 options.command_prefix = shlex.split(options.command_prefix) 249 options.extra_flags = shlex.split(options.extra_flags) 250 if options.j == 0: 251 options.j = multiprocessing.cpu_count() 252 while options.random_seed == 0: 253 options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647) 254 if not options.distribution_mode in DISTRIBUTION_MODES: 255 print "Unknown distribution mode %s" % options.distribution_mode 256 return False 257 if options.distribution_factor1 < 0.0: 258 print ("Distribution factor1 %s is out of range. Defaulting to 0.0" 259 % options.distribution_factor1) 260 options.distribution_factor1 = 0.0 261 if options.distribution_factor2 < 0.0: 262 print ("Distribution factor2 %s is out of range. Defaulting to 0.0" 263 % options.distribution_factor2) 264 options.distribution_factor2 = 0.0 265 if options.coverage < 0.0 or options.coverage > 1.0: 266 print ("Coverage %s is out of range. Defaulting to 0.4" 267 % options.coverage) 268 options.coverage = 0.4 269 if options.coverage_lift < 0: 270 print ("Coverage lift %s is out of range. Defaulting to 0" 271 % options.coverage_lift) 272 options.coverage_lift = 0 273 return True 274 275 276 def ShardTests(tests, shard_count, shard_run): 277 if shard_count < 2: 278 return tests 279 if shard_run < 1 or shard_run > shard_count: 280 print "shard-run not a valid number, should be in [1:shard-count]" 281 print "defaulting back to running all tests" 282 return tests 283 count = 0 284 shard = [] 285 for test in tests: 286 if count % shard_count == shard_run - 1: 287 shard.append(test) 288 count += 1 289 return shard 290 291 292 def Main(): 293 # Use the v8 root as cwd as some test cases use "load" with relative paths. 294 os.chdir(BASE_DIR) 295 296 parser = BuildOptions() 297 (options, args) = parser.parse_args() 298 if not ProcessOptions(options): 299 parser.print_help() 300 return 1 301 302 exit_code = 0 303 304 suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test")) 305 306 if len(args) == 0: 307 suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ] 308 else: 309 args_suites = set() 310 for arg in args: 311 suite = arg.split(os.path.sep)[0] 312 if not suite in args_suites: 313 args_suites.add(suite) 314 suite_paths = [ s for s in suite_paths if s in args_suites ] 315 316 suites = [] 317 for root in suite_paths: 318 suite = testsuite.TestSuite.LoadTestSuite( 319 os.path.join(BASE_DIR, "test", root)) 320 if suite: 321 suites.append(suite) 322 323 if options.download_data: 324 for s in suites: 325 s.DownloadData() 326 327 for mode in options.mode: 328 for arch in options.arch: 329 try: 330 code = Execute(arch, mode, args, options, suites, BASE_DIR) 331 exit_code = exit_code or code 332 except KeyboardInterrupt: 333 return 2 334 return exit_code 335 336 337 def CalculateNTests(m, options): 338 """Calculates the number of tests from m deopt points with exponential 339 coverage. 340 The coverage is expected to be between 0.0 and 1.0. 341 The 'coverage lift' lifts the coverage for tests with smaller m values. 342 """ 343 c = float(options.coverage) 344 l = float(options.coverage_lift) 345 return int(math.pow(m, (m * c + l) / (m + l))) 346 347 348 def Execute(arch, mode, args, options, suites, workspace): 349 print(">>> Running tests for %s.%s" % (arch, mode)) 350 351 dist = Distribution(options) 352 353 shell_dir = options.shell_dir 354 if not shell_dir: 355 if options.buildbot: 356 shell_dir = os.path.join(workspace, options.outdir, mode) 357 mode = mode.lower() 358 else: 359 shell_dir = os.path.join(workspace, options.outdir, 360 "%s.%s" % (arch, mode)) 361 shell_dir = os.path.relpath(shell_dir) 362 363 # Populate context object. 364 mode_flags = MODE_FLAGS[mode] 365 timeout = options.timeout 366 if timeout == -1: 367 # Simulators are slow, therefore allow a longer default timeout. 368 if arch in SLOW_ARCHS: 369 timeout = 2 * TIMEOUT_DEFAULT; 370 else: 371 timeout = TIMEOUT_DEFAULT; 372 373 timeout *= TIMEOUT_SCALEFACTOR[mode] 374 ctx = context.Context(arch, mode, shell_dir, 375 mode_flags, options.verbose, 376 timeout, options.isolates, 377 options.command_prefix, 378 options.extra_flags, 379 False, # Keep i18n on by default. 380 options.random_seed, 381 True, # No sorting of test cases. 382 0, # Don't rerun failing tests. 383 0, # No use of a rerun-failing-tests maximum. 384 False, # No predictable mode. 385 False, # No no_harness mode. 386 False, # Don't use perf data. 387 False) # Coverage not supported. 388 389 # Find available test suites and read test cases from them. 390 variables = { 391 "arch": arch, 392 "asan": options.asan, 393 "deopt_fuzzer": True, 394 "gc_stress": False, 395 "gcov_coverage": False, 396 "isolates": options.isolates, 397 "mode": mode, 398 "no_i18n": False, 399 "no_snap": False, 400 "simulator": utils.UseSimulator(arch), 401 "system": utils.GuessOS(), 402 "tsan": False, 403 "msan": False, 404 "dcheck_always_on": options.dcheck_always_on, 405 "novfp3": False, 406 "predictable": False, 407 "byteorder": sys.byteorder, 408 } 409 all_tests = [] 410 num_tests = 0 411 test_id = 0 412 413 # Remember test case prototypes for the fuzzing phase. 414 test_backup = dict((s, []) for s in suites) 415 416 for s in suites: 417 s.ReadStatusFile(variables) 418 s.ReadTestCases(ctx) 419 if len(args) > 0: 420 s.FilterTestCasesByArgs(args) 421 all_tests += s.tests 422 s.FilterTestCasesByStatus(False) 423 test_backup[s] = s.tests 424 analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT, 425 "--print-deopt-stress"] 426 s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ] 427 num_tests += len(s.tests) 428 for t in s.tests: 429 t.id = test_id 430 test_id += 1 431 432 if num_tests == 0: 433 print "No tests to run." 434 return 0 435 436 print(">>> Collection phase") 437 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 438 runner = execution.Runner(suites, progress_indicator, ctx) 439 440 exit_code = runner.Run(options.j) 441 442 print(">>> Analysis phase") 443 num_tests = 0 444 test_id = 0 445 for s in suites: 446 test_results = {} 447 for t in s.tests: 448 for line in t.output.stdout.splitlines(): 449 if line.startswith("=== Stress deopt counter: "): 450 test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1]) 451 for t in s.tests: 452 if t.path not in test_results: 453 print "Missing results for %s" % t.path 454 if options.dump_results_file: 455 results_dict = dict((t.path, n) for (t, n) in test_results.iteritems()) 456 with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f: 457 f.write(json.dumps(results_dict)) 458 459 # Reset tests and redistribute the prototypes from the collection phase. 460 s.tests = [] 461 if options.verbose: 462 print "Test distributions:" 463 for t in test_backup[s]: 464 max_deopt = test_results.get(t.path, 0) 465 if max_deopt == 0: 466 continue 467 n_deopt = CalculateNTests(max_deopt, options) 468 distribution = dist.Distribute(n_deopt, max_deopt) 469 if options.verbose: 470 print "%s %s" % (t.path, distribution) 471 for i in distribution: 472 fuzzing_flags = ["--deopt-every-n-times", "%d" % i] 473 s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags)) 474 num_tests += len(s.tests) 475 for t in s.tests: 476 t.id = test_id 477 test_id += 1 478 479 if num_tests == 0: 480 print "No tests to run." 481 return 0 482 483 print(">>> Deopt fuzzing phase (%d test cases)" % num_tests) 484 progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() 485 runner = execution.Runner(suites, progress_indicator, ctx) 486 487 code = runner.Run(options.j) 488 return exit_code or code 489 490 491 if __name__ == "__main__": 492 sys.exit(Main()) 493
