1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import collections 7 import glob 8 import hashlib 9 import json 10 import os 11 import random 12 import re 13 import shutil 14 import sys 15 16 import bb_utils 17 import bb_annotations 18 19 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 20 import provision_devices 21 from pylib import android_commands 22 from pylib import constants 23 from pylib.device import device_utils 24 from pylib.gtest import gtest_config 25 26 CHROME_SRC_DIR = bb_utils.CHROME_SRC 27 DIR_BUILD_ROOT = os.path.dirname(CHROME_SRC_DIR) 28 CHROME_OUT_DIR = bb_utils.CHROME_OUT_DIR 29 BLINK_SCRIPTS_DIR = 'third_party/WebKit/Tools/Scripts' 30 31 SLAVE_SCRIPTS_DIR = os.path.join(bb_utils.BB_BUILD_DIR, 'scripts', 'slave') 32 LOGCAT_DIR = os.path.join(bb_utils.CHROME_OUT_DIR, 'logcat') 33 GS_URL = 'https://storage.googleapis.com' 34 GS_AUTH_URL = 'https://storage.cloud.google.com' 35 36 # Describes an instrumation test suite: 37 # test: Name of test we're running. 38 # apk: apk to be installed. 39 # apk_package: package for the apk to be installed. 40 # test_apk: apk to run tests on. 41 # test_data: data folder in format destination:source. 42 # host_driven_root: The host-driven test root directory. 43 # annotation: Annotation of the tests to include. 44 # exclude_annotation: The annotation of the tests to exclude. 45 I_TEST = collections.namedtuple('InstrumentationTest', [ 46 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root', 47 'annotation', 'exclude_annotation', 'extra_flags']) 48 49 50 def SrcPath(*path): 51 return os.path.join(CHROME_SRC_DIR, *path) 52 53 54 def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None, 55 annotation=None, exclude_annotation=None, extra_flags=None): 56 return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root, 57 annotation, exclude_annotation, extra_flags) 58 59 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ 60 I('ContentShell', 61 'ContentShell.apk', 62 'org.chromium.content_shell_apk', 63 'ContentShellTest', 64 'content:content/test/data/android/device_files'), 65 I('ChromeShell', 66 'ChromeShell.apk', 67 'org.chromium.chrome.shell', 68 'ChromeShellTest', 69 'chrome:chrome/test/data/android/device_files', 70 constants.CHROME_SHELL_HOST_DRIVEN_DIR), 71 I('AndroidWebView', 72 'AndroidWebView.apk', 73 'org.chromium.android_webview.shell', 74 'AndroidWebViewTest', 75 'webview:android_webview/test/data/device_files'), 76 ]) 77 78 VALID_TESTS = set(['chromedriver', 'chrome_proxy', 'gpu', 'mojo', 'sync', 79 'telemetry_perf_unittests', 'ui', 'unit', 'webkit', 80 'webkit_layout']) 81 82 RunCmd = bb_utils.RunCmd 83 84 85 def _GetRevision(options): 86 """Get the SVN revision number. 87 88 Args: 89 options: options object. 90 91 Returns: 92 The revision number. 93 """ 94 revision = options.build_properties.get('got_revision') 95 if not revision: 96 revision = options.build_properties.get('revision', 'testing') 97 return revision 98 99 100 def _RunTest(options, cmd, suite): 101 """Run test command with runtest.py. 102 103 Args: 104 options: options object. 105 cmd: the command to run. 106 suite: test name. 107 """ 108 property_args = bb_utils.EncodeProperties(options) 109 args = [os.path.join(SLAVE_SCRIPTS_DIR, 'runtest.py')] + property_args 110 args += ['--test-platform', 'android'] 111 if options.factory_properties.get('generate_gtest_json'): 112 args.append('--generate-json-file') 113 args += ['-o', 'gtest-results/%s' % suite, 114 '--annotate', 'gtest', 115 '--build-number', str(options.build_properties.get('buildnumber', 116 '')), 117 '--builder-name', options.build_properties.get('buildername', '')] 118 if options.target == 'Release': 119 args += ['--target', 'Release'] 120 else: 121 args += ['--target', 'Debug'] 122 args += cmd 123 RunCmd(args, cwd=DIR_BUILD_ROOT) 124 125 126 def RunTestSuites(options, suites, suites_options=None): 127 """Manages an invocation of test_runner.py for gtests. 128 129 Args: 130 options: options object. 131 suites: List of suite names to run. 132 suites_options: Command line options dictionary for particular suites. 133 For example, 134 {'content_browsertests', ['--num_retries=1', '--release']} 135 will add the options only to content_browsertests. 136 """ 137 138 if not suites_options: 139 suites_options = {} 140 141 args = ['--verbose'] 142 if options.target == 'Release': 143 args.append('--release') 144 if options.asan: 145 args.append('--tool=asan') 146 if options.gtest_filter: 147 args.append('--gtest-filter=%s' % options.gtest_filter) 148 149 for suite in suites: 150 bb_annotations.PrintNamedStep(suite) 151 cmd = [suite] + args 152 cmd += suites_options.get(suite, []) 153 if suite == 'content_browsertests': 154 cmd.append('--num_retries=1') 155 _RunTest(options, cmd, suite) 156 157 158 def RunChromeDriverTests(options): 159 """Run all the steps for running chromedriver tests.""" 160 bb_annotations.PrintNamedStep('chromedriver_annotation') 161 RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py', 162 '--android-packages=%s,%s,%s,%s' % 163 ('chrome_shell', 164 'chrome_stable', 165 'chrome_beta', 166 'chromedriver_webview_shell'), 167 '--revision=%s' % _GetRevision(options), 168 '--update-log']) 169 170 def RunChromeProxyTests(options): 171 """Run the chrome_proxy tests. 172 173 Args: 174 options: options object. 175 """ 176 InstallApk(options, INSTRUMENTATION_TESTS['ChromeShell'], False) 177 args = ['--browser', 'android-chrome-shell'] 178 devices = android_commands.GetAttachedDevices() 179 if devices: 180 args = args + ['--device', devices[0]] 181 bb_annotations.PrintNamedStep('chrome_proxy') 182 RunCmd(['tools/chrome_proxy/run_tests'] + args) 183 184 def RunChromeSyncShellTests(options): 185 """Run the chrome sync shell tests""" 186 test = I('ChromeSyncShell', 187 'ChromeSyncShell.apk', 188 'org.chromium.chrome.browser.sync', 189 'ChromeSyncShellTest.apk', 190 'chrome:chrome/test/data/android/device_files') 191 RunInstrumentationSuite(options, test) 192 193 def RunTelemetryPerfUnitTests(options): 194 """Runs the telemetry perf unit tests. 195 196 Args: 197 options: options object. 198 """ 199 InstallApk(options, INSTRUMENTATION_TESTS['ChromeShell'], False) 200 args = ['--browser', 'android-chrome-shell'] 201 devices = android_commands.GetAttachedDevices() 202 if devices: 203 args = args + ['--device', devices[0]] 204 bb_annotations.PrintNamedStep('telemetry_perf_unittests') 205 RunCmd(['tools/perf/run_tests'] + args) 206 207 208 def RunMojoTests(options): 209 """Runs the mojo unit tests. 210 211 Args: 212 options: options object. 213 """ 214 test = I('MojoTest', 215 None, 216 'org.chromium.mojo.tests', 217 'MojoTest', 218 'bindings:mojo/public/interfaces/bindings/tests/data') 219 RunInstrumentationSuite(options, test) 220 221 222 def InstallApk(options, test, print_step=False): 223 """Install an apk to all phones. 224 225 Args: 226 options: options object 227 test: An I_TEST namedtuple 228 print_step: Print a buildbot step 229 """ 230 if print_step: 231 bb_annotations.PrintNamedStep('install_%s' % test.name.lower()) 232 233 args = ['--apk_package', test.apk_package] 234 if options.target == 'Release': 235 args.append('--release') 236 args.append(test.apk) 237 238 RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True) 239 240 241 def RunInstrumentationSuite(options, test, flunk_on_failure=True, 242 python_only=False, official_build=False): 243 """Manages an invocation of test_runner.py for instrumentation tests. 244 245 Args: 246 options: options object 247 test: An I_TEST namedtuple 248 flunk_on_failure: Flunk the step if tests fail. 249 Python: Run only host driven Python tests. 250 official_build: Run official-build tests. 251 """ 252 bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower()) 253 254 if test.apk: 255 InstallApk(options, test) 256 args = ['--test-apk', test.test_apk, '--verbose'] 257 if test.test_data: 258 args.extend(['--test_data', test.test_data]) 259 if options.target == 'Release': 260 args.append('--release') 261 if options.asan: 262 args.append('--tool=asan') 263 if options.flakiness_server: 264 args.append('--flakiness-dashboard-server=%s' % 265 options.flakiness_server) 266 if options.coverage_bucket: 267 args.append('--coverage-dir=%s' % options.coverage_dir) 268 if test.host_driven_root: 269 args.append('--host-driven-root=%s' % test.host_driven_root) 270 if test.annotation: 271 args.extend(['-A', test.annotation]) 272 if test.exclude_annotation: 273 args.extend(['-E', test.exclude_annotation]) 274 if test.extra_flags: 275 args.extend(test.extra_flags) 276 if python_only: 277 args.append('-p') 278 if official_build: 279 # The option needs to be assigned 'True' as it does not have an action 280 # associated with it. 281 args.append('--official-build') 282 283 RunCmd(['build/android/test_runner.py', 'instrumentation'] + args, 284 flunk_on_failure=flunk_on_failure) 285 286 287 def RunWebkitLint(): 288 """Lint WebKit's TestExpectation files.""" 289 bb_annotations.PrintNamedStep('webkit_lint') 290 RunCmd([SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'lint-test-expectations'))]) 291 292 293 def RunWebkitLayoutTests(options): 294 """Run layout tests on an actual device.""" 295 bb_annotations.PrintNamedStep('webkit_tests') 296 cmd_args = [ 297 '--no-show-results', 298 '--no-new-test-results', 299 '--full-results-html', 300 '--clobber-old-results', 301 '--exit-after-n-failures', '5000', 302 '--exit-after-n-crashes-or-timeouts', '100', 303 '--debug-rwt-logging', 304 '--results-directory', '../layout-test-results', 305 '--target', options.target, 306 '--builder-name', options.build_properties.get('buildername', ''), 307 '--build-number', str(options.build_properties.get('buildnumber', '')), 308 '--master-name', 'ChromiumWebkit', # TODO: Get this from the cfg. 309 '--build-name', options.build_properties.get('buildername', ''), 310 '--platform=android'] 311 312 for flag in 'test_results_server', 'driver_name', 'additional_drt_flag': 313 if flag in options.factory_properties: 314 cmd_args.extend(['--%s' % flag.replace('_', '-'), 315 options.factory_properties.get(flag)]) 316 317 for f in options.factory_properties.get('additional_expectations', []): 318 cmd_args.extend( 319 ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)]) 320 321 # TODO(dpranke): Remove this block after 322 # https://codereview.chromium.org/12927002/ lands. 323 for f in options.factory_properties.get('additional_expectations_files', []): 324 cmd_args.extend( 325 ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)]) 326 327 exit_code = RunCmd( 328 [SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'run-webkit-tests'))] + cmd_args) 329 if exit_code == 255: # test_run_results.UNEXPECTED_ERROR_EXIT_STATUS 330 bb_annotations.PrintMsg('?? (crashed or hung)') 331 elif exit_code == 254: # test_run_results.NO_DEVICES_EXIT_STATUS 332 bb_annotations.PrintMsg('?? (no devices found)') 333 elif exit_code == 253: # test_run_results.NO_TESTS_EXIT_STATUS 334 bb_annotations.PrintMsg('?? (no tests found)') 335 else: 336 full_results_path = os.path.join('..', 'layout-test-results', 337 'full_results.json') 338 if os.path.exists(full_results_path): 339 full_results = json.load(open(full_results_path)) 340 unexpected_passes, unexpected_failures, unexpected_flakes = ( 341 _ParseLayoutTestResults(full_results)) 342 if unexpected_failures: 343 _PrintDashboardLink('failed', unexpected_failures, 344 max_tests=25) 345 elif unexpected_passes: 346 _PrintDashboardLink('unexpected passes', unexpected_passes, 347 max_tests=10) 348 if unexpected_flakes: 349 _PrintDashboardLink('unexpected flakes', unexpected_flakes, 350 max_tests=10) 351 352 if exit_code == 0 and (unexpected_passes or unexpected_flakes): 353 # If exit_code != 0, RunCmd() will have already printed an error. 354 bb_annotations.PrintWarning() 355 else: 356 bb_annotations.PrintError() 357 bb_annotations.PrintMsg('?? (results missing)') 358 359 if options.factory_properties.get('archive_webkit_results', False): 360 bb_annotations.PrintNamedStep('archive_webkit_results') 361 base = 'https://storage.googleapis.com/chromium-layout-test-archives' 362 builder_name = options.build_properties.get('buildername', '') 363 build_number = str(options.build_properties.get('buildnumber', '')) 364 results_link = '%s/%s/%s/layout-test-results/results.html' % ( 365 base, EscapeBuilderName(builder_name), build_number) 366 bb_annotations.PrintLink('results', results_link) 367 bb_annotations.PrintLink('(zip)', '%s/%s/%s/layout-test-results.zip' % ( 368 base, EscapeBuilderName(builder_name), build_number)) 369 gs_bucket = 'gs://chromium-layout-test-archives' 370 RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'chromium', 371 'archive_layout_test_results.py'), 372 '--results-dir', '../../layout-test-results', 373 '--build-number', build_number, 374 '--builder-name', builder_name, 375 '--gs-bucket', gs_bucket], 376 cwd=DIR_BUILD_ROOT) 377 378 379 def _ParseLayoutTestResults(results): 380 """Extract the failures from the test run.""" 381 # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results 382 tests = _ConvertTrieToFlatPaths(results['tests']) 383 failures = {} 384 flakes = {} 385 passes = {} 386 for (test, result) in tests.iteritems(): 387 if result.get('is_unexpected'): 388 actual_results = result['actual'].split() 389 expected_results = result['expected'].split() 390 if len(actual_results) > 1: 391 # We report the first failure type back, even if the second 392 # was more severe. 393 if actual_results[1] in expected_results: 394 flakes[test] = actual_results[0] 395 else: 396 failures[test] = actual_results[0] 397 elif actual_results[0] == 'PASS': 398 passes[test] = result 399 else: 400 failures[test] = actual_results[0] 401 402 return (passes, failures, flakes) 403 404 405 def _ConvertTrieToFlatPaths(trie, prefix=None): 406 """Flatten the trie of failures into a list.""" 407 # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results 408 result = {} 409 for name, data in trie.iteritems(): 410 if prefix: 411 name = prefix + '/' + name 412 413 if len(data) and 'actual' not in data and 'expected' not in data: 414 result.update(_ConvertTrieToFlatPaths(data, name)) 415 else: 416 result[name] = data 417 418 return result 419 420 421 def _PrintDashboardLink(link_text, tests, max_tests): 422 """Add a link to the flakiness dashboard in the step annotations.""" 423 if len(tests) > max_tests: 424 test_list_text = ' '.join(tests[:max_tests]) + ' and more' 425 else: 426 test_list_text = ' '.join(tests) 427 428 dashboard_base = ('http://test-results.appspot.com' 429 '/dashboards/flakiness_dashboard.html#' 430 'master=ChromiumWebkit&tests=') 431 432 bb_annotations.PrintLink('%d %s: %s' % 433 (len(tests), link_text, test_list_text), 434 dashboard_base + ','.join(tests)) 435 436 437 def EscapeBuilderName(builder_name): 438 return re.sub('[ ()]', '_', builder_name) 439 440 441 def SpawnLogcatMonitor(): 442 shutil.rmtree(LOGCAT_DIR, ignore_errors=True) 443 bb_utils.SpawnCmd([ 444 os.path.join(CHROME_SRC_DIR, 'build', 'android', 'adb_logcat_monitor.py'), 445 LOGCAT_DIR]) 446 447 # Wait for logcat_monitor to pull existing logcat 448 RunCmd(['sleep', '5']) 449 450 451 def ProvisionDevices(options): 452 bb_annotations.PrintNamedStep('provision_devices') 453 454 if not bb_utils.TESTING: 455 # Restart adb to work around bugs, sleep to wait for usb discovery. 456 device_utils.RestartServer() 457 RunCmd(['sleep', '1']) 458 provision_cmd = ['build/android/provision_devices.py', '-t', options.target] 459 if options.auto_reconnect: 460 provision_cmd.append('--auto-reconnect') 461 if options.skip_wipe: 462 provision_cmd.append('--skip-wipe') 463 RunCmd(provision_cmd, halt_on_failure=True) 464 465 466 def DeviceStatusCheck(options): 467 bb_annotations.PrintNamedStep('device_status_check') 468 cmd = ['build/android/buildbot/bb_device_status_check.py'] 469 if options.restart_usb: 470 cmd.append('--restart-usb') 471 RunCmd(cmd, halt_on_failure=True) 472 473 474 def GetDeviceSetupStepCmds(): 475 return [ 476 ('device_status_check', DeviceStatusCheck), 477 ('provision_devices', ProvisionDevices), 478 ] 479 480 481 def RunUnitTests(options): 482 suites = gtest_config.STABLE_TEST_SUITES 483 if options.asan: 484 suites = [s for s in suites 485 if s not in gtest_config.ASAN_EXCLUDED_TEST_SUITES] 486 RunTestSuites(options, suites) 487 488 489 def RunInstrumentationTests(options): 490 for test in INSTRUMENTATION_TESTS.itervalues(): 491 RunInstrumentationSuite(options, test) 492 493 494 def RunWebkitTests(options): 495 RunTestSuites(options, ['webkit_unit_tests', 'blink_heap_unittests']) 496 RunWebkitLint() 497 498 499 def RunGPUTests(options): 500 revision = _GetRevision(options) 501 builder_name = options.build_properties.get('buildername', 'noname') 502 503 bb_annotations.PrintNamedStep('pixel_tests') 504 RunCmd(['content/test/gpu/run_gpu_test.py', 505 'pixel', 506 '--browser', 507 'android-content-shell', 508 '--build-revision', 509 str(revision), 510 '--upload-refimg-to-cloud-storage', 511 '--refimg-cloud-storage-bucket', 512 'chromium-gpu-archive/reference-images', 513 '--os-type', 514 'android', 515 '--test-machine-name', 516 EscapeBuilderName(builder_name)]) 517 518 bb_annotations.PrintNamedStep('webgl_conformance_tests') 519 RunCmd(['content/test/gpu/run_gpu_test.py', 520 '--browser=android-content-shell', 'webgl_conformance', 521 '--webgl-conformance-version=1.0.1']) 522 523 bb_annotations.PrintNamedStep('gpu_rasterization_tests') 524 RunCmd(['content/test/gpu/run_gpu_test.py', 525 'gpu_rasterization', 526 '--browser', 527 'android-content-shell', 528 '--build-revision', 529 str(revision), 530 '--test-machine-name', 531 EscapeBuilderName(builder_name)]) 532 533 534 def GetTestStepCmds(): 535 return [ 536 ('chromedriver', RunChromeDriverTests), 537 ('chrome_proxy', RunChromeProxyTests), 538 ('gpu', RunGPUTests), 539 ('mojo', RunMojoTests), 540 ('sync', RunChromeSyncShellTests), 541 ('telemetry_perf_unittests', RunTelemetryPerfUnitTests), 542 ('ui', RunInstrumentationTests), 543 ('unit', RunUnitTests), 544 ('webkit', RunWebkitTests), 545 ('webkit_layout', RunWebkitLayoutTests), 546 ] 547 548 549 def MakeGSPath(options, gs_base_dir): 550 revision = _GetRevision(options) 551 bot_id = options.build_properties.get('buildername', 'testing') 552 randhash = hashlib.sha1(str(random.random())).hexdigest() 553 gs_path = '%s/%s/%s/%s' % (gs_base_dir, bot_id, revision, randhash) 554 # remove double slashes, happens with blank revisions and confuses gsutil 555 gs_path = re.sub('/+', '/', gs_path) 556 return gs_path 557 558 def UploadHTML(options, gs_base_dir, dir_to_upload, link_text, 559 link_rel_path='index.html', gs_url=GS_URL): 560 """Uploads directory at |dir_to_upload| to Google Storage and output a link. 561 562 Args: 563 options: Command line options. 564 gs_base_dir: The Google Storage base directory (e.g. 565 'chromium-code-coverage/java') 566 dir_to_upload: Absolute path to the directory to be uploaded. 567 link_text: Link text to be displayed on the step. 568 link_rel_path: Link path relative to |dir_to_upload|. 569 gs_url: Google storage URL. 570 """ 571 gs_path = MakeGSPath(options, gs_base_dir) 572 RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-R', dir_to_upload, 'gs://%s' % gs_path]) 573 bb_annotations.PrintLink(link_text, 574 '%s/%s/%s' % (gs_url, gs_path, link_rel_path)) 575 576 577 def GenerateJavaCoverageReport(options): 578 """Generates an HTML coverage report using EMMA and uploads it.""" 579 bb_annotations.PrintNamedStep('java_coverage_report') 580 581 coverage_html = os.path.join(options.coverage_dir, 'coverage_html') 582 RunCmd(['build/android/generate_emma_html.py', 583 '--coverage-dir', options.coverage_dir, 584 '--metadata-dir', os.path.join(CHROME_OUT_DIR, options.target), 585 '--cleanup', 586 '--output', os.path.join(coverage_html, 'index.html')]) 587 return coverage_html 588 589 590 def LogcatDump(options): 591 # Print logcat, kill logcat monitor 592 bb_annotations.PrintNamedStep('logcat_dump') 593 logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log.txt') 594 RunCmd([SrcPath('build' , 'android', 'adb_logcat_printer.py'), 595 '--output-path', logcat_file, LOGCAT_DIR]) 596 gs_path = MakeGSPath(options, 'chromium-android/logcat_dumps') 597 RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-z', 'txt', logcat_file, 598 'gs://%s' % gs_path]) 599 bb_annotations.PrintLink('logcat dump', '%s/%s' % (GS_AUTH_URL, gs_path)) 600 601 602 def RunStackToolSteps(options): 603 """Run stack tool steps. 604 605 Stack tool is run for logcat dump, optionally for ASAN. 606 """ 607 bb_annotations.PrintNamedStep('Run stack tool with logcat dump') 608 logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log.txt') 609 RunCmd([os.path.join(CHROME_SRC_DIR, 'third_party', 'android_platform', 610 'development', 'scripts', 'stack'), 611 '--more-info', logcat_file]) 612 if options.asan_symbolize: 613 bb_annotations.PrintNamedStep('Run stack tool for ASAN') 614 RunCmd([ 615 os.path.join(CHROME_SRC_DIR, 'build', 'android', 'asan_symbolize.py'), 616 '-l', logcat_file]) 617 618 619 def GenerateTestReport(options): 620 bb_annotations.PrintNamedStep('test_report') 621 for report in glob.glob( 622 os.path.join(CHROME_OUT_DIR, options.target, 'test_logs', '*.log')): 623 RunCmd(['cat', report]) 624 os.remove(report) 625 626 627 def MainTestWrapper(options): 628 try: 629 # Spawn logcat monitor 630 SpawnLogcatMonitor() 631 632 # Run all device setup steps 633 for _, cmd in GetDeviceSetupStepCmds(): 634 cmd(options) 635 636 if options.install: 637 test_obj = INSTRUMENTATION_TESTS[options.install] 638 InstallApk(options, test_obj, print_step=True) 639 640 if options.test_filter: 641 bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options) 642 643 if options.coverage_bucket: 644 coverage_html = GenerateJavaCoverageReport(options) 645 UploadHTML(options, '%s/java' % options.coverage_bucket, coverage_html, 646 'Coverage Report') 647 shutil.rmtree(coverage_html, ignore_errors=True) 648 649 if options.experimental: 650 RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) 651 652 finally: 653 # Run all post test steps 654 LogcatDump(options) 655 if not options.disable_stack_tool: 656 RunStackToolSteps(options) 657 GenerateTestReport(options) 658 # KillHostHeartbeat() has logic to check if heartbeat process is running, 659 # and kills only if it finds the process is running on the host. 660 provision_devices.KillHostHeartbeat() 661 if options.cleanup: 662 shutil.rmtree(os.path.join(CHROME_OUT_DIR, options.target), 663 ignore_errors=True) 664 665 666 def GetDeviceStepsOptParser(): 667 parser = bb_utils.GetParser() 668 parser.add_option('--experimental', action='store_true', 669 help='Run experiemental tests') 670 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], 671 action='append', 672 help=('Run a test suite. Test suites: "%s"' % 673 '", "'.join(VALID_TESTS))) 674 parser.add_option('--gtest-filter', 675 help='Filter for running a subset of tests of a gtest test') 676 parser.add_option('--asan', action='store_true', help='Run tests with asan.') 677 parser.add_option('--install', metavar='<apk name>', 678 help='Install an apk by name') 679 parser.add_option('--no-reboot', action='store_true', 680 help='Do not reboot devices during provisioning.') 681 parser.add_option('--coverage-bucket', 682 help=('Bucket name to store coverage results. Coverage is ' 683 'only run if this is set.')) 684 parser.add_option('--restart-usb', action='store_true', 685 help='Restart usb ports before device status check.') 686 parser.add_option( 687 '--flakiness-server', 688 help=('The flakiness dashboard server to which the results should be ' 689 'uploaded.')) 690 parser.add_option( 691 '--auto-reconnect', action='store_true', 692 help='Push script to device which restarts adbd on disconnections.') 693 parser.add_option('--skip-wipe', action='store_true', 694 help='Do not wipe devices during provisioning.') 695 parser.add_option( 696 '--logcat-dump-output', 697 help='The logcat dump output will be "tee"-ed into this file') 698 # During processing perf bisects, a seperate working directory created under 699 # which builds are produced. Therefore we should look for relevent output 700 # file under this directory.(/b/build/slave/<slave_name>/build/bisect/src/out) 701 parser.add_option( 702 '--chrome-output-dir', 703 help='Chrome output directory to be used while bisecting.') 704 705 parser.add_option('--disable-stack-tool', action='store_true', 706 help='Do not run stack tool.') 707 parser.add_option('--asan-symbolize', action='store_true', 708 help='Run stack tool for ASAN') 709 parser.add_option('--cleanup', action='store_true', 710 help='Delete out/<target> directory at the end of the run.') 711 return parser 712 713 714 def main(argv): 715 parser = GetDeviceStepsOptParser() 716 options, args = parser.parse_args(argv[1:]) 717 718 if args: 719 return sys.exit('Unused args %s' % args) 720 721 unknown_tests = set(options.test_filter) - VALID_TESTS 722 if unknown_tests: 723 return sys.exit('Unknown tests %s' % list(unknown_tests)) 724 725 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) 726 727 if options.chrome_output_dir: 728 global CHROME_OUT_DIR 729 global LOGCAT_DIR 730 CHROME_OUT_DIR = options.chrome_output_dir 731 LOGCAT_DIR = os.path.join(CHROME_OUT_DIR, 'logcat') 732 733 if options.coverage_bucket: 734 setattr(options, 'coverage_dir', 735 os.path.join(CHROME_OUT_DIR, options.target, 'coverage')) 736 737 MainTestWrapper(options) 738 739 740 if __name__ == '__main__': 741 sys.exit(main(sys.argv)) 742
