1 #!/usr/bin/env python3 2 # 3 # Copyright 2017, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """ART Run-Test TestRunner 18 19 The testrunner runs the ART run-tests by simply invoking the script. 20 It fetches the list of eligible tests from art/test directory, and list of 21 disabled tests from art/test/knownfailures.json. It runs the tests by 22 invoking art/test/run-test script and checks the exit value to decide if the 23 test passed or failed. 24 25 Before invoking the script, first build all the tests dependencies. 26 There are two major build targets for building target and host tests 27 dependencies: 28 1) test-art-host-run-test 29 2) test-art-target-run-test 30 31 There are various options to invoke the script which are: 32 -t: Either the test name as in art/test or the test name including the variant 33 information. Eg, "-t 001-HelloWorld", 34 "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32" 35 -j: Number of thread workers to be used. Eg - "-j64" 36 --dry-run: Instead of running the test name, just print its name. 37 --verbose 38 -b / --build-dependencies: to build the dependencies before running the test 39 40 To specify any specific variants for the test, use --<<variant-name>>. 41 For eg, for compiler type as optimizing, use --optimizing. 42 43 44 In the end, the script will print the failed and skipped tests if any. 45 46 """ 47 import argparse 48 import fnmatch 49 import itertools 50 import json 51 import multiprocessing 52 import os 53 import re 54 import subprocess 55 import sys 56 import tempfile 57 import threading 58 import time 59 60 import env 61 from target_config import target_config 62 63 TARGET_TYPES = set() 64 RUN_TYPES = set() 65 PREBUILD_TYPES = set() 66 COMPILER_TYPES = set() 67 RELOCATE_TYPES = set() 68 TRACE_TYPES = set() 69 GC_TYPES = set() 70 JNI_TYPES = set() 71 IMAGE_TYPES = set() 72 PICTEST_TYPES = set() 73 DEBUGGABLE_TYPES = set() 74 ADDRESS_SIZES = set() 75 OPTIMIZING_COMPILER_TYPES = set() 76 JVMTI_TYPES = set() 77 ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()} 78 # timeout for individual tests. 79 # TODO: make it adjustable per tests and for buildbots 80 timeout = 3000 # 50 minutes 81 82 # DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map 83 # that has key as the test name (like 001-HelloWorld), and value as set of 84 # variants that the test is disabled for. 85 DISABLED_TEST_CONTAINER = {} 86 87 # The Dict contains the list of all possible variants for a given type. For example, 88 # for key TARGET, the value would be target and host. The list is used to parse 89 # the test name given as the argument to run. 90 VARIANT_TYPE_DICT = {} 91 92 # The set contains all the variants of each time. 93 TOTAL_VARIANTS_SET = set() 94 95 # The colors are used in the output. When a test passes, COLOR_PASS is used, 96 # and so on. 97 COLOR_ERROR = '\033[91m' 98 COLOR_PASS = '\033[92m' 99 COLOR_SKIP = '\033[93m' 100 COLOR_NORMAL = '\033[0m' 101 102 # The mutex object is used by the threads for exclusive access of test_count 103 # to make any changes in its value. 104 test_count_mutex = threading.Lock() 105 106 # The set contains the list of all the possible run tests that are in art/test 107 # directory. 108 RUN_TEST_SET = set() 109 110 # The semaphore object is used by the testrunner to limit the number of 111 # threads to the user requested concurrency value. 112 semaphore = threading.Semaphore(1) 113 114 # The mutex object is used to provide exclusive access to a thread to print 115 # its output. 116 print_mutex = threading.Lock() 117 failed_tests = [] 118 skipped_tests = [] 119 120 # Flags 121 n_thread = -1 122 test_count = 0 123 total_test_count = 0 124 verbose = False 125 dry_run = False 126 build = False 127 gdb = False 128 gdb_arg = '' 129 stop_testrunner = False 130 131 def gather_test_info(): 132 """The method gathers test information about the test to be run which includes 133 generating the list of total tests from the art/test directory and the list 134 of disabled test. It also maps various variants to types. 135 """ 136 global TOTAL_VARIANTS_SET 137 global DISABLED_TEST_CONTAINER 138 # TODO: Avoid duplication of the variant names in different lists. 139 VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'} 140 VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'} 141 VARIANT_TYPE_DICT['target'] = {'target', 'host'} 142 VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'} 143 VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'multipicimage'} 144 VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'} 145 VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'} 146 VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'} 147 VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'} 148 VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} 149 VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} 150 VARIANT_TYPE_DICT['jvmti'] = {'no-jvmti', 'jvmti-stress', 'redefine-stress', 'trace-stress', 151 'field-stress', 'step-stress'} 152 VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', 153 'regalloc_gc', 'speed-profile'} 154 155 for v_type in VARIANT_TYPE_DICT: 156 TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) 157 158 test_dir = env.ANDROID_BUILD_TOP + '/art/test' 159 for f in os.listdir(test_dir): 160 if fnmatch.fnmatch(f, '[0-9]*'): 161 RUN_TEST_SET.add(f) 162 DISABLED_TEST_CONTAINER = get_disabled_test_info() 163 164 165 def setup_test_env(): 166 """The method sets default value for the various variants of the tests if they 167 are already not set. 168 """ 169 if env.ART_TEST_BISECTION: 170 env.ART_TEST_RUN_TEST_NO_PREBUILD = True 171 env.ART_TEST_RUN_TEST_PREBUILD = False 172 # Bisection search writes to standard output. 173 env.ART_TEST_QUIET = False 174 175 if not TARGET_TYPES: 176 TARGET_TYPES.add('host') 177 TARGET_TYPES.add('target') 178 179 if env.ART_TEST_RUN_TEST_NO_PREBUILD: 180 PREBUILD_TYPES.add('no-prebuild') 181 if env.ART_TEST_RUN_TEST_NO_DEX2OAT: 182 PREBUILD_TYPES.add('no-dex2oat') 183 if env.ART_TEST_RUN_TEST_PREBUILD or not PREBUILD_TYPES: # Default 184 PREBUILD_TYPES.add('prebuild') 185 186 if env.ART_TEST_INTERPRETER_ACCESS_CHECKS: 187 COMPILER_TYPES.add('interp-ac') 188 if env.ART_TEST_INTERPRETER: 189 COMPILER_TYPES.add('interpreter') 190 if env.ART_TEST_JIT: 191 COMPILER_TYPES.add('jit') 192 if env.ART_TEST_OPTIMIZING_GRAPH_COLOR: 193 COMPILER_TYPES.add('regalloc_gc') 194 OPTIMIZING_COMPILER_TYPES.add('regalloc_gc') 195 if env.ART_TEST_OPTIMIZING: 196 COMPILER_TYPES.add('optimizing') 197 OPTIMIZING_COMPILER_TYPES.add('optimizing') 198 if env.ART_TEST_SPEED_PROFILE: 199 COMPILER_TYPES.add('speed-profile') 200 201 # By default only run without jvmti 202 if not JVMTI_TYPES: 203 JVMTI_TYPES.add('no-jvmti') 204 205 # By default we run all 'compiler' variants. 206 if not COMPILER_TYPES: 207 COMPILER_TYPES.add('optimizing') 208 COMPILER_TYPES.add('jit') 209 COMPILER_TYPES.add('interpreter') 210 COMPILER_TYPES.add('interp-ac') 211 COMPILER_TYPES.add('speed-profile') 212 OPTIMIZING_COMPILER_TYPES.add('optimizing') 213 214 if env.ART_TEST_RUN_TEST_RELOCATE: 215 RELOCATE_TYPES.add('relocate') 216 if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT: 217 RELOCATE_TYPES.add('relocate-npatchoat') 218 if not RELOCATE_TYPES: # Default 219 RELOCATE_TYPES.add('no-relocate') 220 221 if env.ART_TEST_TRACE: 222 TRACE_TYPES.add('trace') 223 if env.ART_TEST_TRACE_STREAM: 224 TRACE_TYPES.add('stream') 225 if not TRACE_TYPES: # Default 226 TRACE_TYPES.add('ntrace') 227 228 if env.ART_TEST_GC_STRESS: 229 GC_TYPES.add('gcstress') 230 if env.ART_TEST_GC_VERIFY: 231 GC_TYPES.add('gcverify') 232 if not GC_TYPES: # Default 233 GC_TYPES.add('cms') 234 235 if env.ART_TEST_JNI_FORCECOPY: 236 JNI_TYPES.add('forcecopy') 237 if not JNI_TYPES: # Default 238 JNI_TYPES.add('checkjni') 239 240 if env.ART_TEST_RUN_TEST_NO_IMAGE: 241 IMAGE_TYPES.add('no-image') 242 if env.ART_TEST_RUN_TEST_MULTI_IMAGE: 243 IMAGE_TYPES.add('multipicimage') 244 if env.ART_TEST_RUN_TEST_IMAGE or not IMAGE_TYPES: # Default 245 IMAGE_TYPES.add('picimage') 246 247 if env.ART_TEST_PIC_TEST: 248 PICTEST_TYPES.add('pictest') 249 if not PICTEST_TYPES: # Default 250 PICTEST_TYPES.add('npictest') 251 252 if env.ART_TEST_RUN_TEST_NDEBUG: 253 RUN_TYPES.add('ndebug') 254 if env.ART_TEST_RUN_TEST_DEBUG or not RUN_TYPES: # Default 255 RUN_TYPES.add('debug') 256 257 if env.ART_TEST_RUN_TEST_DEBUGGABLE: 258 DEBUGGABLE_TYPES.add('debuggable') 259 if not DEBUGGABLE_TYPES: # Default 260 DEBUGGABLE_TYPES.add('ndebuggable') 261 262 if not ADDRESS_SIZES: 263 ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX) 264 ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX) 265 if env.ART_TEST_RUN_TEST_2ND_ARCH: 266 ADDRESS_SIZES_TARGET['host'].add(env.ART_2ND_PHONY_TEST_HOST_SUFFIX) 267 ADDRESS_SIZES_TARGET['target'].add(env.ART_2ND_PHONY_TEST_TARGET_SUFFIX) 268 else: 269 ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES) 270 ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES) 271 272 global n_thread 273 if n_thread is -1: 274 if 'target' in TARGET_TYPES: 275 n_thread = get_default_threads('target') 276 else: 277 n_thread = get_default_threads('host') 278 279 global semaphore 280 semaphore = threading.Semaphore(n_thread) 281 282 if not sys.stdout.isatty(): 283 global COLOR_ERROR 284 global COLOR_PASS 285 global COLOR_SKIP 286 global COLOR_NORMAL 287 COLOR_ERROR = '' 288 COLOR_PASS = '' 289 COLOR_SKIP = '' 290 COLOR_NORMAL = '' 291 292 293 def run_tests(tests): 294 """Creates thread workers to run the tests. 295 296 The method generates command and thread worker to run the tests. Depending on 297 the user input for the number of threads to be used, the method uses a 298 semaphore object to keep a count in control for the thread workers. When a new 299 worker is created, it acquires the semaphore object, and when the number of 300 workers reaches the maximum allowed concurrency, the method wait for an 301 existing thread worker to release the semaphore object. Worker releases the 302 semaphore object when they finish printing the output. 303 304 Args: 305 tests: The set of tests to be run. 306 """ 307 options_all = '' 308 global total_test_count 309 total_test_count = len(tests) 310 total_test_count *= len(RUN_TYPES) 311 total_test_count *= len(PREBUILD_TYPES) 312 total_test_count *= len(RELOCATE_TYPES) 313 total_test_count *= len(TRACE_TYPES) 314 total_test_count *= len(GC_TYPES) 315 total_test_count *= len(JNI_TYPES) 316 total_test_count *= len(IMAGE_TYPES) 317 total_test_count *= len(PICTEST_TYPES) 318 total_test_count *= len(DEBUGGABLE_TYPES) 319 total_test_count *= len(COMPILER_TYPES) 320 total_test_count *= len(JVMTI_TYPES) 321 target_address_combinations = 0 322 for target in TARGET_TYPES: 323 for address_size in ADDRESS_SIZES_TARGET[target]: 324 target_address_combinations += 1 325 total_test_count *= target_address_combinations 326 327 if env.ART_TEST_WITH_STRACE: 328 options_all += ' --strace' 329 330 if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN: 331 options_all += ' --always-clean' 332 333 if env.ART_TEST_BISECTION: 334 options_all += ' --bisection-search' 335 336 if env.ART_TEST_ANDROID_ROOT: 337 options_all += ' --android-root ' + env.ART_TEST_ANDROID_ROOT 338 339 if gdb: 340 options_all += ' --gdb' 341 if gdb_arg: 342 options_all += ' --gdb-arg ' + gdb_arg 343 344 config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES, 345 COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES, 346 GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES, 347 DEBUGGABLE_TYPES, JVMTI_TYPES) 348 349 for test, target, run, prebuild, compiler, relocate, trace, gc, \ 350 jni, image, pictest, debuggable, jvmti in config: 351 for address_size in ADDRESS_SIZES_TARGET[target]: 352 if stop_testrunner: 353 # When ART_TEST_KEEP_GOING is set to false, then as soon as a test 354 # fails, stop_testrunner is set to True. When this happens, the method 355 # stops creating any any thread and wait for all the exising threads 356 # to end. 357 while threading.active_count() > 2: 358 time.sleep(0.1) 359 return 360 test_name = 'test-art-' 361 test_name += target + '-run-test-' 362 test_name += run + '-' 363 test_name += prebuild + '-' 364 test_name += compiler + '-' 365 test_name += relocate + '-' 366 test_name += trace + '-' 367 test_name += gc + '-' 368 test_name += jni + '-' 369 test_name += image + '-' 370 test_name += pictest + '-' 371 test_name += debuggable + '-' 372 test_name += jvmti + '-' 373 test_name += test 374 test_name += address_size 375 376 variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni, 377 image, pictest, debuggable, jvmti, address_size} 378 379 options_test = options_all 380 381 if target == 'host': 382 options_test += ' --host' 383 384 if run == 'ndebug': 385 options_test += ' -O' 386 387 if prebuild == 'prebuild': 388 options_test += ' --prebuild' 389 elif prebuild == 'no-prebuild': 390 options_test += ' --no-prebuild' 391 elif prebuild == 'no-dex2oat': 392 options_test += ' --no-prebuild --no-dex2oat' 393 394 if compiler == 'optimizing': 395 options_test += ' --optimizing' 396 elif compiler == 'regalloc_gc': 397 options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color' 398 elif compiler == 'interpreter': 399 options_test += ' --interpreter' 400 elif compiler == 'interp-ac': 401 options_test += ' --interpreter --verify-soft-fail' 402 elif compiler == 'jit': 403 options_test += ' --jit' 404 elif compiler == 'speed-profile': 405 options_test += ' --random-profile' 406 407 if relocate == 'relocate': 408 options_test += ' --relocate' 409 elif relocate == 'no-relocate': 410 options_test += ' --no-relocate' 411 elif relocate == 'relocate-npatchoat': 412 options_test += ' --relocate --no-patchoat' 413 414 if trace == 'trace': 415 options_test += ' --trace' 416 elif trace == 'stream': 417 options_test += ' --trace --stream' 418 419 if gc == 'gcverify': 420 options_test += ' --gcverify' 421 elif gc == 'gcstress': 422 options_test += ' --gcstress' 423 424 if jni == 'forcecopy': 425 options_test += ' --runtime-option -Xjniopts:forcecopy' 426 elif jni == 'checkjni': 427 options_test += ' --runtime-option -Xcheck:jni' 428 429 if image == 'no-image': 430 options_test += ' --no-image' 431 elif image == 'multipicimage': 432 options_test += ' --multi-image' 433 434 if pictest == 'pictest': 435 options_test += ' --pic-test' 436 437 if debuggable == 'debuggable': 438 options_test += ' --debuggable' 439 440 if jvmti == 'jvmti-stress': 441 options_test += ' --jvmti-trace-stress --jvmti-redefine-stress --jvmti-field-stress' 442 elif jvmti == 'field-stress': 443 options_test += ' --jvmti-field-stress' 444 elif jvmti == 'trace-stress': 445 options_test += ' --jvmti-trace-stress' 446 elif jvmti == 'redefine-stress': 447 options_test += ' --jvmti-redefine-stress' 448 elif jvmti == 'step-stress': 449 options_test += ' --jvmti-step-stress' 450 451 if address_size == '64': 452 options_test += ' --64' 453 454 if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: 455 options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES 456 457 elif address_size == '32': 458 if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: 459 options_test += ' --instruction-set-features ' + \ 460 env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES 461 462 # Use the default run-test behavior unless ANDROID_COMPILE_WITH_JACK is explicitly set. 463 if env.ANDROID_COMPILE_WITH_JACK == True: 464 options_test += ' --build-with-jack' 465 elif env.ANDROID_COMPILE_WITH_JACK == False: 466 options_test += ' --build-with-javac-dx' 467 468 # TODO(http://36039166): This is a temporary solution to 469 # fix build breakages. 470 options_test = (' --output-path %s') % ( 471 tempfile.mkdtemp(dir=env.ART_HOST_TEST_DIR)) + options_test 472 473 run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' 474 command = run_test_sh + ' ' + options_test + ' ' + test 475 476 semaphore.acquire() 477 worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name)) 478 worker.daemon = True 479 worker.start() 480 481 while threading.active_count() > 2: 482 time.sleep(0.1) 483 484 485 def run_test(command, test, test_variant, test_name): 486 """Runs the test. 487 488 It invokes art/test/run-test script to run the test. The output of the script 489 is checked, and if it ends with "Succeeded!", it assumes that the tests 490 passed, otherwise, put it in the list of failed test. Before actually running 491 the test, it also checks if the test is placed in the list of disabled tests, 492 and if yes, it skips running it, and adds the test in the list of skipped 493 tests. The method uses print_text method to actually print the output. After 494 successfully running and capturing the output for the test, it releases the 495 semaphore object. 496 497 Args: 498 command: The command to be used to invoke the script 499 test: The name of the test without the variant information. 500 test_variant: The set of variant for the test. 501 test_name: The name of the test along with the variants. 502 """ 503 global stop_testrunner 504 try: 505 if is_test_disabled(test, test_variant): 506 test_skipped = True 507 else: 508 test_skipped = False 509 if gdb: 510 proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, universal_newlines=True) 511 else: 512 proc = subprocess.Popen(command.split(), stderr=subprocess.STDOUT, stdout = subprocess.PIPE, 513 universal_newlines=True) 514 script_output = proc.communicate(timeout=timeout)[0] 515 test_passed = not proc.wait() 516 517 if not test_skipped: 518 if test_passed: 519 print_test_info(test_name, 'PASS') 520 else: 521 failed_tests.append((test_name, script_output)) 522 if not env.ART_TEST_KEEP_GOING: 523 stop_testrunner = True 524 print_test_info(test_name, 'FAIL', ('%s\n%s') % ( 525 command, script_output)) 526 elif not dry_run: 527 print_test_info(test_name, 'SKIP') 528 skipped_tests.append(test_name) 529 else: 530 print_test_info(test_name, '') 531 except subprocess.TimeoutExpired as e: 532 failed_tests.append((test_name, 'Timed out in %d seconds' % timeout)) 533 print_test_info(test_name, 'TIMEOUT', 'Timed out in %d seconds\n%s' % ( 534 timeout, command)) 535 except Exception as e: 536 failed_tests.append((test_name, str(e))) 537 print_test_info(test_name, 'FAIL', 538 ('%s\n%s\n\n') % (command, str(e))) 539 finally: 540 semaphore.release() 541 542 543 def print_test_info(test_name, result, failed_test_info=""): 544 """Print the continous test information 545 546 If verbose is set to True, it continuously prints test status information 547 on a new line. 548 If verbose is set to False, it keeps on erasing test 549 information by overriding it with the latest test information. Also, 550 in this case it stictly makes sure that the information length doesn't 551 exceed the console width. It does so by shortening the test_name. 552 553 When a test fails, it prints the output of the run-test script and 554 command used to invoke the script. It doesn't override the failing 555 test information in either of the cases. 556 """ 557 558 global test_count 559 info = '' 560 if not verbose: 561 # Without --verbose, the testrunner erases passing test info. It 562 # does that by overriding the printed text with white spaces all across 563 # the console width. 564 console_width = int(os.popen('stty size', 'r').read().split()[1]) 565 info = '\r' + ' ' * console_width + '\r' 566 try: 567 print_mutex.acquire() 568 test_count += 1 569 percent = (test_count * 100) / total_test_count 570 progress_info = ('[ %d%% %d/%d ]') % ( 571 percent, 572 test_count, 573 total_test_count) 574 575 if result == 'FAIL' or result == 'TIMEOUT': 576 info += ('%s %s %s\n%s\n') % ( 577 progress_info, 578 test_name, 579 COLOR_ERROR + result + COLOR_NORMAL, 580 failed_test_info) 581 else: 582 result_text = '' 583 if result == 'PASS': 584 result_text += COLOR_PASS + 'PASS' + COLOR_NORMAL 585 elif result == 'SKIP': 586 result_text += COLOR_SKIP + 'SKIP' + COLOR_NORMAL 587 588 if verbose: 589 info += ('%s %s %s\n') % ( 590 progress_info, 591 test_name, 592 result_text) 593 else: 594 total_output_length = 2 # Two spaces 595 total_output_length += len(progress_info) 596 total_output_length += len(result) 597 allowed_test_length = console_width - total_output_length 598 test_name_len = len(test_name) 599 if allowed_test_length < test_name_len: 600 test_name = ('...%s') % ( 601 test_name[-(allowed_test_length - 3):]) 602 info += ('%s %s %s') % ( 603 progress_info, 604 test_name, 605 result_text) 606 print_text(info) 607 except Exception as e: 608 print_text(('%s\n%s\n') % (test_name, str(e))) 609 failed_tests.append(test_name) 610 finally: 611 print_mutex.release() 612 613 def verify_knownfailure_entry(entry): 614 supported_field = { 615 'tests' : (list, str), 616 'description' : (list, str), 617 'bug' : (str,), 618 'variant' : (str,), 619 'env_vars' : (dict,), 620 } 621 for field in entry: 622 field_type = type(entry[field]) 623 if field_type not in supported_field[field]: 624 raise ValueError('%s is not supported type for %s\n%s' % ( 625 str(field_type), 626 field, 627 str(entry))) 628 629 def get_disabled_test_info(): 630 """Generate set of known failures. 631 632 It parses the art/test/knownfailures.json file to generate the list of 633 disabled tests. 634 635 Returns: 636 The method returns a dict of tests mapped to the variants list 637 for which the test should not be run. 638 """ 639 known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json' 640 with open(known_failures_file) as known_failures_json: 641 known_failures_info = json.loads(known_failures_json.read()) 642 643 disabled_test_info = {} 644 for failure in known_failures_info: 645 verify_knownfailure_entry(failure) 646 tests = failure.get('tests', []) 647 if isinstance(tests, str): 648 tests = [tests] 649 variants = parse_variants(failure.get('variant')) 650 env_vars = failure.get('env_vars') 651 652 if check_env_vars(env_vars): 653 for test in tests: 654 if test not in RUN_TEST_SET: 655 raise ValueError('%s is not a valid run-test' % ( 656 test)) 657 if test in disabled_test_info: 658 disabled_test_info[test] = disabled_test_info[test].union(variants) 659 else: 660 disabled_test_info[test] = variants 661 return disabled_test_info 662 663 664 def check_env_vars(env_vars): 665 """Checks if the env variables are set as required to run the test. 666 667 Returns: 668 True if all the env variables are set as required, otherwise False. 669 """ 670 671 if not env_vars: 672 return True 673 for key in env_vars: 674 if env.get_env(key) != env_vars.get(key): 675 return False 676 return True 677 678 679 def is_test_disabled(test, variant_set): 680 """Checks if the test along with the variant_set is disabled. 681 682 Args: 683 test: The name of the test as in art/test directory. 684 variant_set: Variants to be used for the test. 685 Returns: 686 True, if the test is disabled. 687 """ 688 if dry_run: 689 return True 690 if test in env.EXTRA_DISABLED_TESTS: 691 return True 692 variants_list = DISABLED_TEST_CONTAINER.get(test, {}) 693 for variants in variants_list: 694 variants_present = True 695 for variant in variants: 696 if variant not in variant_set: 697 variants_present = False 698 break 699 if variants_present: 700 return True 701 return False 702 703 704 def parse_variants(variants): 705 """Parse variants fetched from art/test/knownfailures.json. 706 """ 707 if not variants: 708 variants = '' 709 for variant in TOTAL_VARIANTS_SET: 710 variants += variant 711 variants += '|' 712 variants = variants[:-1] 713 variant_list = set() 714 or_variants = variants.split('|') 715 for or_variant in or_variants: 716 and_variants = or_variant.split('&') 717 variant = set() 718 for and_variant in and_variants: 719 and_variant = and_variant.strip() 720 if and_variant not in TOTAL_VARIANTS_SET: 721 raise ValueError('%s is not a valid variant' % ( 722 and_variant)) 723 variant.add(and_variant) 724 variant_list.add(frozenset(variant)) 725 return variant_list 726 727 def print_text(output): 728 sys.stdout.write(output) 729 sys.stdout.flush() 730 731 def print_analysis(): 732 if not verbose: 733 # Without --verbose, the testrunner erases passing test info. It 734 # does that by overriding the printed text with white spaces all across 735 # the console width. 736 console_width = int(os.popen('stty size', 'r').read().split()[1]) 737 eraser_text = '\r' + ' ' * console_width + '\r' 738 print_text(eraser_text) 739 740 # Prints information about the total tests run. 741 # E.g., "2/38 (5%) tests passed". 742 passed_test_count = total_test_count - len(skipped_tests) - len(failed_tests) 743 passed_test_information = ('%d/%d (%d%%) %s passed.\n') % ( 744 passed_test_count, 745 total_test_count, 746 (passed_test_count*100)/total_test_count, 747 'tests' if passed_test_count > 1 else 'test') 748 print_text(passed_test_information) 749 750 # Prints the list of skipped tests, if any. 751 if skipped_tests: 752 print_text(COLOR_SKIP + 'SKIPPED TESTS: ' + COLOR_NORMAL + '\n') 753 for test in skipped_tests: 754 print_text(test + '\n') 755 print_text('\n') 756 757 # Prints the list of failed tests, if any. 758 if failed_tests: 759 print_text(COLOR_ERROR + 'FAILED: ' + COLOR_NORMAL + '\n') 760 for test_info in failed_tests: 761 print_text(('%s\n%s\n' % (test_info[0], test_info[1]))) 762 print_text(COLOR_ERROR + '----------' + COLOR_NORMAL + '\n') 763 for failed_test in sorted([test_info[0] for test_info in failed_tests]): 764 print_text(('%s\n' % (failed_test))) 765 766 767 def parse_test_name(test_name): 768 """Parses the testname provided by the user. 769 It supports two types of test_name: 770 1) Like 001-HelloWorld. In this case, it will just verify if the test actually 771 exists and if it does, it returns the testname. 772 2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32 773 In this case, it will parse all the variants and check if they are placed 774 correctly. If yes, it will set the various VARIANT_TYPES to use the 775 variants required to run the test. Again, it returns the test_name 776 without the variant information like 001-HelloWorld. 777 """ 778 test_set = set() 779 for test in RUN_TEST_SET: 780 if test.startswith(test_name): 781 test_set.add(test) 782 if test_set: 783 return test_set 784 785 regex = '^test-art-' 786 regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-' 787 regex += 'run-test-' 788 regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-' 789 regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-' 790 regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-' 791 regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-' 792 regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-' 793 regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-' 794 regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-' 795 regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-' 796 regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-' 797 regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-' 798 regex += '(' + '|'.join(VARIANT_TYPE_DICT['jvmti']) + ')-' 799 regex += '(' + '|'.join(RUN_TEST_SET) + ')' 800 regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$' 801 match = re.match(regex, test_name) 802 if match: 803 TARGET_TYPES.add(match.group(1)) 804 RUN_TYPES.add(match.group(2)) 805 PREBUILD_TYPES.add(match.group(3)) 806 COMPILER_TYPES.add(match.group(4)) 807 RELOCATE_TYPES.add(match.group(5)) 808 TRACE_TYPES.add(match.group(6)) 809 GC_TYPES.add(match.group(7)) 810 JNI_TYPES.add(match.group(8)) 811 IMAGE_TYPES.add(match.group(9)) 812 PICTEST_TYPES.add(match.group(10)) 813 DEBUGGABLE_TYPES.add(match.group(11)) 814 JVMTI_TYPES.add(match.group(12)) 815 ADDRESS_SIZES.add(match.group(14)) 816 return {match.group(13)} 817 raise ValueError(test_name + " is not a valid test") 818 819 820 def setup_env_for_build_target(build_target, parser, options): 821 """Setup environment for the build target 822 823 The method setup environment for the master-art-host targets. 824 """ 825 os.environ.update(build_target['env']) 826 os.environ['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true' 827 print_text('%s\n' % (str(os.environ))) 828 829 target_options = vars(parser.parse_args(build_target['flags'])) 830 target_options['host'] = True 831 target_options['verbose'] = True 832 target_options['build'] = True 833 target_options['n_thread'] = options['n_thread'] 834 target_options['dry_run'] = options['dry_run'] 835 836 return target_options 837 838 def get_default_threads(target): 839 if target is 'target': 840 adb_command = 'adb shell cat /sys/devices/system/cpu/present' 841 cpu_info_proc = subprocess.Popen(adb_command.split(), stdout=subprocess.PIPE) 842 cpu_info = cpu_info_proc.stdout.read() 843 if type(cpu_info) is bytes: 844 cpu_info = cpu_info.decode('utf-8') 845 cpu_info_regex = '\d*-(\d*)' 846 match = re.match(cpu_info_regex, cpu_info) 847 if match: 848 return int(match.group(1)) 849 else: 850 raise ValueError('Unable to predict the concurrency for the target. ' 851 'Is device connected?') 852 else: 853 return multiprocessing.cpu_count() 854 855 def parse_option(): 856 global verbose 857 global dry_run 858 global n_thread 859 global build 860 global gdb 861 global gdb_arg 862 global timeout 863 864 parser = argparse.ArgumentParser(description="Runs all or a subset of the ART test suite.") 865 parser.add_argument('-t', '--test', dest='test', help='name of the test') 866 parser.add_argument('-j', type=int, dest='n_thread') 867 parser.add_argument('--timeout', default=timeout, type=int, dest='timeout') 868 for variant in TOTAL_VARIANTS_SET: 869 flag = '--' + variant 870 flag_dest = variant.replace('-', '_') 871 if variant == '32' or variant == '64': 872 flag_dest = 'n' + flag_dest 873 parser.add_argument(flag, action='store_true', dest=flag_dest) 874 parser.add_argument('--verbose', '-v', action='store_true', dest='verbose') 875 parser.add_argument('--dry-run', action='store_true', dest='dry_run') 876 parser.add_argument("--skip", action="append", dest="skips", default=[], 877 help="Skip the given test in all circumstances.") 878 parser.add_argument('--no-build-dependencies', 879 action='store_false', dest='build', 880 help="Don't build dependencies under any circumstances. This is the " + 881 "behavior if ART_TEST_RUN_TEST_ALWAYS_BUILD is not set to 'true'.") 882 parser.add_argument('-b', '--build-dependencies', 883 action='store_true', dest='build', 884 help="Build dependencies under all circumstances. By default we will " + 885 "not build dependencies unless ART_TEST_RUN_TEST_BUILD=true.") 886 parser.add_argument('--build-target', dest='build_target', help='master-art-host targets') 887 parser.set_defaults(build = env.ART_TEST_RUN_TEST_BUILD) 888 parser.add_argument('--gdb', action='store_true', dest='gdb') 889 parser.add_argument('--gdb-arg', dest='gdb_arg') 890 891 options = vars(parser.parse_args()) 892 if options['build_target']: 893 options = setup_env_for_build_target(target_config[options['build_target']], 894 parser, options) 895 896 test = '' 897 env.EXTRA_DISABLED_TESTS.update(set(options['skips'])) 898 if options['test']: 899 test = parse_test_name(options['test']) 900 if options['pictest']: 901 PICTEST_TYPES.add('pictest') 902 if options['ndebug']: 903 RUN_TYPES.add('ndebug') 904 if options['interp_ac']: 905 COMPILER_TYPES.add('interp-ac') 906 if options['picimage']: 907 IMAGE_TYPES.add('picimage') 908 if options['n64']: 909 ADDRESS_SIZES.add('64') 910 if options['interpreter']: 911 COMPILER_TYPES.add('interpreter') 912 if options['jni']: 913 JNI_TYPES.add('jni') 914 if options['relocate_npatchoat']: 915 RELOCATE_TYPES.add('relocate-npatchoat') 916 if options['no_prebuild']: 917 PREBUILD_TYPES.add('no-prebuild') 918 if options['npictest']: 919 PICTEST_TYPES.add('npictest') 920 if options['no_dex2oat']: 921 PREBUILD_TYPES.add('no-dex2oat') 922 if options['jit']: 923 COMPILER_TYPES.add('jit') 924 if options['relocate']: 925 RELOCATE_TYPES.add('relocate') 926 if options['ndebuggable']: 927 DEBUGGABLE_TYPES.add('ndebuggable') 928 if options['no_image']: 929 IMAGE_TYPES.add('no-image') 930 if options['optimizing']: 931 COMPILER_TYPES.add('optimizing') 932 if options['speed_profile']: 933 COMPILER_TYPES.add('speed-profile') 934 if options['trace']: 935 TRACE_TYPES.add('trace') 936 if options['gcstress']: 937 GC_TYPES.add('gcstress') 938 if options['no_relocate']: 939 RELOCATE_TYPES.add('no-relocate') 940 if options['target']: 941 TARGET_TYPES.add('target') 942 if options['forcecopy']: 943 JNI_TYPES.add('forcecopy') 944 if options['n32']: 945 ADDRESS_SIZES.add('32') 946 if options['host']: 947 TARGET_TYPES.add('host') 948 if options['gcverify']: 949 GC_TYPES.add('gcverify') 950 if options['debuggable']: 951 DEBUGGABLE_TYPES.add('debuggable') 952 if options['prebuild']: 953 PREBUILD_TYPES.add('prebuild') 954 if options['debug']: 955 RUN_TYPES.add('debug') 956 if options['checkjni']: 957 JNI_TYPES.add('checkjni') 958 if options['ntrace']: 959 TRACE_TYPES.add('ntrace') 960 if options['cms']: 961 GC_TYPES.add('cms') 962 if options['multipicimage']: 963 IMAGE_TYPES.add('multipicimage') 964 if options['jvmti_stress']: 965 JVMTI_TYPES.add('jvmti-stress') 966 if options['redefine_stress']: 967 JVMTI_TYPES.add('redefine-stress') 968 if options['field_stress']: 969 JVMTI_TYPES.add('field-stress') 970 if options['step_stress']: 971 JVMTI_TYPES.add('step-stress') 972 if options['trace_stress']: 973 JVMTI_TYPES.add('trace-stress') 974 if options['no_jvmti']: 975 JVMTI_TYPES.add('no-jvmti') 976 if options['verbose']: 977 verbose = True 978 if options['n_thread']: 979 n_thread = max(1, options['n_thread']) 980 if options['dry_run']: 981 dry_run = True 982 verbose = True 983 build = options['build'] 984 if options['gdb']: 985 n_thread = 1 986 gdb = True 987 if options['gdb_arg']: 988 gdb_arg = options['gdb_arg'] 989 timeout = options['timeout'] 990 991 return test 992 993 def main(): 994 gather_test_info() 995 user_requested_test = parse_option() 996 setup_test_env() 997 if build: 998 build_targets = '' 999 if 'host' in TARGET_TYPES: 1000 build_targets += 'test-art-host-run-test-dependencies' 1001 if 'target' in TARGET_TYPES: 1002 build_targets += 'test-art-target-run-test-dependencies' 1003 build_command = 'make' 1004 build_command += ' -j' 1005 build_command += ' -C ' + env.ANDROID_BUILD_TOP 1006 build_command += ' ' + build_targets 1007 # Add 'dist' to avoid Jack issues b/36169180. 1008 build_command += ' dist' 1009 if subprocess.call(build_command.split()): 1010 sys.exit(1) 1011 if user_requested_test: 1012 test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,)) 1013 else: 1014 test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,)) 1015 test_runner_thread.daemon = True 1016 try: 1017 test_runner_thread.start() 1018 while threading.active_count() > 1: 1019 time.sleep(0.1) 1020 print_analysis() 1021 except Exception as e: 1022 print_analysis() 1023 print_text(str(e)) 1024 sys.exit(1) 1025 if failed_tests: 1026 sys.exit(1) 1027 sys.exit(0) 1028 1029 if __name__ == '__main__': 1030 main() 1031
