1 #!/usr/bin/env python 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import collections 7 import glob 8 import multiprocessing 9 import os 10 import shutil 11 import sys 12 13 import bb_utils 14 import bb_annotations 15 16 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 17 import provision_devices 18 from pylib import android_commands 19 from pylib import constants 20 from pylib.gtest import gtest_config 21 22 sys.path.append(os.path.join( 23 constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner')) 24 import errors 25 26 27 CHROME_SRC = constants.DIR_SOURCE_ROOT 28 LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat') 29 30 # Describes an instrumation test suite: 31 # test: Name of test we're running. 32 # apk: apk to be installed. 33 # apk_package: package for the apk to be installed. 34 # test_apk: apk to run tests on. 35 # test_data: data folder in format destination:source. 36 # host_driven_root: The host-driven test root directory. 37 # annotation: Annotation of the tests to include. 38 # exclude_annotation: The annotation of the tests to exclude. 39 I_TEST = collections.namedtuple('InstrumentationTest', [ 40 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root', 41 'annotation', 'exclude_annotation', 'extra_flags']) 42 43 def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None, 44 annotation=None, exclude_annotation=None, extra_flags=None): 45 return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root, 46 annotation, exclude_annotation, extra_flags) 47 48 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ 49 I('ContentShell', 50 'ContentShell.apk', 51 'org.chromium.content_shell_apk', 52 'ContentShellTest', 53 'content:content/test/data/android/device_files'), 54 I('ChromiumTestShell', 55 'ChromiumTestShell.apk', 56 'org.chromium.chrome.testshell', 57 'ChromiumTestShellTest', 58 'chrome:chrome/test/data/android/device_files', 59 constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR), 60 I('AndroidWebView', 61 'AndroidWebView.apk', 62 'org.chromium.android_webview.shell', 63 'AndroidWebViewTest', 64 'webview:android_webview/test/data/device_files'), 65 ]) 66 67 VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout', 68 'webrtc']) 69 70 RunCmd = bb_utils.RunCmd 71 72 73 # multiprocessing map_async requires a top-level function for pickle library. 74 def RebootDeviceSafe(device): 75 """Reboot a device, wait for it to start, and squelch timeout exceptions.""" 76 try: 77 android_commands.AndroidCommands(device).Reboot(True) 78 except errors.DeviceUnresponsiveError as e: 79 return e 80 81 82 def RebootDevices(): 83 """Reboot all attached and online devices.""" 84 # Early return here to avoid presubmit dependence on adb, 85 # which might not exist in this checkout. 86 if bb_utils.TESTING: 87 return 88 devices = android_commands.GetAttachedDevices(emulator=False) 89 print 'Rebooting: %s' % devices 90 if devices: 91 pool = multiprocessing.Pool(len(devices)) 92 results = pool.map_async(RebootDeviceSafe, devices).get(99999) 93 94 for device, result in zip(devices, results): 95 if result: 96 print '%s failed to startup.' % device 97 98 if any(results): 99 bb_annotations.PrintWarning() 100 else: 101 print 'Reboots complete.' 102 103 104 def RunTestSuites(options, suites): 105 """Manages an invocation of test_runner.py for gtests. 106 107 Args: 108 options: options object. 109 suites: List of suite names to run. 110 """ 111 args = ['--verbose'] 112 if options.target == 'Release': 113 args.append('--release') 114 if options.asan: 115 args.append('--tool=asan') 116 for suite in suites: 117 bb_annotations.PrintNamedStep(suite) 118 cmd = ['build/android/test_runner.py', 'gtest', '-s', suite] + args 119 if suite == 'content_browsertests': 120 cmd.append('--num_retries=1') 121 RunCmd(cmd) 122 123 def RunChromeDriverTests(_): 124 """Run all the steps for running chromedriver tests.""" 125 bb_annotations.PrintNamedStep('chromedriver_annotation') 126 RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py', 127 '--android-package=%s' % constants.CHROMIUM_TEST_SHELL_PACKAGE]) 128 129 def InstallApk(options, test, print_step=False): 130 """Install an apk to all phones. 131 132 Args: 133 options: options object 134 test: An I_TEST namedtuple 135 print_step: Print a buildbot step 136 """ 137 if print_step: 138 bb_annotations.PrintNamedStep('install_%s' % test.name.lower()) 139 args = ['--apk', test.apk, '--apk_package', test.apk_package] 140 if options.target == 'Release': 141 args.append('--release') 142 143 RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True) 144 145 146 def RunInstrumentationSuite(options, test, flunk_on_failure=True, 147 python_only=False): 148 """Manages an invocation of test_runner.py for instrumentation tests. 149 150 Args: 151 options: options object 152 test: An I_TEST namedtuple 153 flunk_on_failure: Flunk the step if tests fail. 154 Python: Run only host driven Python tests. 155 """ 156 bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower()) 157 158 InstallApk(options, test) 159 args = ['--test-apk', test.test_apk, '--test_data', test.test_data, 160 '--verbose'] 161 if options.target == 'Release': 162 args.append('--release') 163 if options.asan: 164 args.append('--tool=asan') 165 if options.flakiness_server: 166 args.append('--flakiness-dashboard-server=%s' % 167 options.flakiness_server) 168 if test.host_driven_root: 169 args.append('--python_test_root=%s' % test.host_driven_root) 170 if test.annotation: 171 args.extend(['-A', test.annotation]) 172 if test.exclude_annotation: 173 args.extend(['-E', test.exclude_annotation]) 174 if test.extra_flags: 175 args.extend(test.extra_flags) 176 if python_only: 177 args.append('-p') 178 179 RunCmd(['build/android/test_runner.py', 'instrumentation'] + args, 180 flunk_on_failure=flunk_on_failure) 181 182 183 def RunWebkitLint(target): 184 """Lint WebKit's TestExpectation files.""" 185 bb_annotations.PrintNamedStep('webkit_lint') 186 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py', 187 '--lint-test-files', 188 '--chromium', 189 '--target', target]) 190 191 192 def RunWebkitLayoutTests(options): 193 """Run layout tests on an actual device.""" 194 bb_annotations.PrintNamedStep('webkit_tests') 195 cmd_args = [ 196 '--no-show-results', 197 '--no-new-test-results', 198 '--full-results-html', 199 '--clobber-old-results', 200 '--exit-after-n-failures', '5000', 201 '--exit-after-n-crashes-or-timeouts', '100', 202 '--debug-rwt-logging', 203 '--results-directory', '..layout-test-results', 204 '--target', options.target, 205 '--builder-name', options.build_properties.get('buildername', ''), 206 '--build-number', str(options.build_properties.get('buildnumber', '')), 207 '--master-name', options.build_properties.get('mastername', ''), 208 '--build-name', options.build_properties.get('buildername', ''), 209 '--platform=android'] 210 211 for flag in 'test_results_server', 'driver_name', 'additional_drt_flag': 212 if flag in options.factory_properties: 213 cmd_args.extend(['--%s' % flag.replace('_', '-'), 214 options.factory_properties.get(flag)]) 215 216 for f in options.factory_properties.get('additional_expectations', []): 217 cmd_args.extend( 218 ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)]) 219 220 # TODO(dpranke): Remove this block after 221 # https://codereview.chromium.org/12927002/ lands. 222 for f in options.factory_properties.get('additional_expectations_files', []): 223 cmd_args.extend( 224 ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)]) 225 226 RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args, 227 flunk_on_failure=False) 228 229 230 def SpawnLogcatMonitor(): 231 shutil.rmtree(LOGCAT_DIR, ignore_errors=True) 232 bb_utils.SpawnCmd([ 233 os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_monitor.py'), 234 LOGCAT_DIR]) 235 236 # Wait for logcat_monitor to pull existing logcat 237 RunCmd(['sleep', '5']) 238 239 def ProvisionDevices(options): 240 # Restart adb to work around bugs, sleep to wait for usb discovery. 241 RunCmd(['adb', 'kill-server']) 242 RunCmd(['adb', 'start-server']) 243 RunCmd(['sleep', '1']) 244 245 bb_annotations.PrintNamedStep('provision_devices') 246 if options.reboot: 247 RebootDevices() 248 provision_cmd = ['build/android/provision_devices.py', '-t', options.target] 249 if options.auto_reconnect: 250 provision_cmd.append('--auto-reconnect') 251 RunCmd(provision_cmd) 252 253 254 def DeviceStatusCheck(_): 255 bb_annotations.PrintNamedStep('device_status_check') 256 RunCmd(['build/android/buildbot/bb_device_status_check.py'], 257 halt_on_failure=True) 258 259 260 def GetDeviceSetupStepCmds(): 261 return [ 262 ('provision_devices', ProvisionDevices), 263 ('device_status_check', DeviceStatusCheck) 264 ] 265 266 267 def RunUnitTests(options): 268 RunTestSuites(options, gtest_config.STABLE_TEST_SUITES) 269 270 271 def RunInstrumentationTests(options): 272 for test in INSTRUMENTATION_TESTS.itervalues(): 273 RunInstrumentationSuite(options, test) 274 275 276 def RunWebkitTests(options): 277 RunTestSuites(options, ['webkit_unit_tests']) 278 RunWebkitLint(options.target) 279 280 281 def RunWebRTCTests(options): 282 RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES) 283 284 285 def GetTestStepCmds(): 286 return [ 287 ('chromedriver', RunChromeDriverTests), 288 ('unit', RunUnitTests), 289 ('ui', RunInstrumentationTests), 290 ('webkit', RunWebkitTests), 291 ('webkit_layout', RunWebkitLayoutTests), 292 ('webrtc', RunWebRTCTests), 293 ] 294 295 296 def LogcatDump(options): 297 # Print logcat, kill logcat monitor 298 bb_annotations.PrintNamedStep('logcat_dump') 299 logcat_file = os.path.join(CHROME_SRC, 'out', options.target, 'full_log') 300 with open(logcat_file, 'w') as f: 301 RunCmd([ 302 os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_printer.py'), 303 LOGCAT_DIR], stdout=f) 304 RunCmd(['cat', logcat_file]) 305 306 307 def GenerateTestReport(options): 308 bb_annotations.PrintNamedStep('test_report') 309 for report in glob.glob( 310 os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')): 311 RunCmd(['cat', report]) 312 os.remove(report) 313 314 315 def MainTestWrapper(options): 316 try: 317 # Spawn logcat monitor 318 SpawnLogcatMonitor() 319 320 # Run all device setup steps 321 for _, cmd in GetDeviceSetupStepCmds(): 322 cmd(options) 323 324 if options.install: 325 test_obj = INSTRUMENTATION_TESTS[options.install] 326 InstallApk(options, test_obj, print_step=True) 327 328 if options.test_filter: 329 bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options) 330 331 if options.experimental: 332 RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) 333 334 finally: 335 # Run all post test steps 336 LogcatDump(options) 337 GenerateTestReport(options) 338 # KillHostHeartbeat() has logic to check if heartbeat process is running, 339 # and kills only if it finds the process is running on the host. 340 provision_devices.KillHostHeartbeat() 341 342 343 def GetDeviceStepsOptParser(): 344 parser = bb_utils.GetParser() 345 parser.add_option('--experimental', action='store_true', 346 help='Run experiemental tests') 347 parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], 348 action='append', 349 help=('Run a test suite. Test suites: "%s"' % 350 '", "'.join(VALID_TESTS))) 351 parser.add_option('--asan', action='store_true', help='Run tests with asan.') 352 parser.add_option('--install', metavar='<apk name>', 353 help='Install an apk by name') 354 parser.add_option('--reboot', action='store_true', 355 help='Reboot devices before running tests') 356 parser.add_option( 357 '--flakiness-server', 358 help='The flakiness dashboard server to which the results should be ' 359 'uploaded.') 360 parser.add_option( 361 '--auto-reconnect', action='store_true', 362 help='Push script to device which restarts adbd on disconnections.') 363 parser.add_option( 364 '--logcat-dump-output', 365 help='The logcat dump output will be "tee"-ed into this file') 366 367 return parser 368 369 370 def main(argv): 371 parser = GetDeviceStepsOptParser() 372 options, args = parser.parse_args(argv[1:]) 373 374 if args: 375 return sys.exit('Unused args %s' % args) 376 377 unknown_tests = set(options.test_filter) - VALID_TESTS 378 if unknown_tests: 379 return sys.exit('Unknown tests %s' % list(unknown_tests)) 380 381 setattr(options, 'target', options.factory_properties.get('target', 'Debug')) 382 383 MainTestWrapper(options) 384 385 386 if __name__ == '__main__': 387 sys.exit(main(sys.argv)) 388
