1 #!/usr/bin/env python 2 # 3 # Copyright 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Runs all types of tests from one unified interface. 8 9 TODO(gkanwar): 10 * Add options to run Monkey tests. 11 """ 12 13 import collections 14 import optparse 15 import os 16 import shutil 17 import sys 18 19 from pylib import constants 20 from pylib import ports 21 from pylib.base import base_test_result 22 from pylib.base import test_dispatcher 23 from pylib.gtest import gtest_config 24 from pylib.gtest import setup as gtest_setup 25 from pylib.gtest import test_options as gtest_test_options 26 from pylib.host_driven import setup as host_driven_setup 27 from pylib.instrumentation import setup as instrumentation_setup 28 from pylib.instrumentation import test_options as instrumentation_test_options 29 from pylib.monkey import setup as monkey_setup 30 from pylib.monkey import test_options as monkey_test_options 31 from pylib.uiautomator import setup as uiautomator_setup 32 from pylib.uiautomator import test_options as uiautomator_test_options 33 from pylib.utils import report_results 34 from pylib.utils import run_tests_helper 35 36 37 _SDK_OUT_DIR = os.path.join(constants.DIR_SOURCE_ROOT, 'out') 38 39 40 def AddBuildTypeOption(option_parser): 41 """Adds the build type option to |option_parser|.""" 42 default_build_type = 'Debug' 43 if 'BUILDTYPE' in os.environ: 44 default_build_type = os.environ['BUILDTYPE'] 45 option_parser.add_option('--debug', action='store_const', const='Debug', 46 dest='build_type', default=default_build_type, 47 help=('If set, run test suites under out/Debug. ' 48 'Default is env var BUILDTYPE or Debug.')) 49 option_parser.add_option('--release', action='store_const', 50 const='Release', dest='build_type', 51 help=('If set, run test suites under out/Release.' 52 ' Default is env var BUILDTYPE or Debug.')) 53 54 55 def AddCommonOptions(option_parser): 56 """Adds all common options to |option_parser|.""" 57 58 AddBuildTypeOption(option_parser) 59 60 option_parser.add_option('-c', dest='cleanup_test_files', 61 help='Cleanup test files on the device after run', 62 action='store_true') 63 option_parser.add_option('--num_retries', dest='num_retries', type='int', 64 default=2, 65 help=('Number of retries for a test before ' 66 'giving up.')) 67 option_parser.add_option('-v', 68 '--verbose', 69 dest='verbose_count', 70 default=0, 71 action='count', 72 help='Verbose level (multiple times for more)') 73 option_parser.add_option('--tool', 74 dest='tool', 75 help=('Run the test under a tool ' 76 '(use --tool help to list them)')) 77 option_parser.add_option('--flakiness-dashboard-server', 78 dest='flakiness_dashboard_server', 79 help=('Address of the server that is hosting the ' 80 'Chrome for Android flakiness dashboard.')) 81 option_parser.add_option('--skip-deps-push', dest='push_deps', 82 action='store_false', default=True, 83 help=('Do not push dependencies to the device. ' 84 'Use this at own risk for speeding up test ' 85 'execution on local machine.')) 86 option_parser.add_option('-d', '--device', dest='test_device', 87 help=('Target device for the test suite ' 88 'to run on.')) 89 90 91 def ProcessCommonOptions(options): 92 """Processes and handles all common options.""" 93 run_tests_helper.SetLogLevel(options.verbose_count) 94 95 96 def AddGTestOptions(option_parser): 97 """Adds gtest options to |option_parser|.""" 98 99 option_parser.usage = '%prog gtest [options]' 100 option_parser.command_list = [] 101 option_parser.example = '%prog gtest -s base_unittests' 102 103 # TODO(gkanwar): Make this option required 104 option_parser.add_option('-s', '--suite', dest='suite_name', 105 help=('Executable name of the test suite to run ' 106 '(use -s help to list them).')) 107 option_parser.add_option('-f', '--gtest_filter', dest='test_filter', 108 help='googletest-style filter string.') 109 option_parser.add_option('-a', '--test_arguments', dest='test_arguments', 110 help='Additional arguments to pass to the test.') 111 option_parser.add_option('-t', dest='timeout', 112 help='Timeout to wait for each test', 113 type='int', 114 default=60) 115 # TODO(gkanwar): Move these to Common Options once we have the plumbing 116 # in our other test types to handle these commands 117 AddCommonOptions(option_parser) 118 119 120 def ProcessGTestOptions(options): 121 """Intercept test suite help to list test suites. 122 123 Args: 124 options: Command line options. 125 """ 126 if options.suite_name == 'help': 127 print 'Available test suites are:' 128 for test_suite in (gtest_config.STABLE_TEST_SUITES + 129 gtest_config.EXPERIMENTAL_TEST_SUITES): 130 print test_suite 131 sys.exit(0) 132 133 # Convert to a list, assuming all test suites if nothing was specified. 134 # TODO(gkanwar): Require having a test suite 135 if options.suite_name: 136 options.suite_name = [options.suite_name] 137 else: 138 options.suite_name = [s for s in gtest_config.STABLE_TEST_SUITES] 139 140 141 def AddJavaTestOptions(option_parser): 142 """Adds the Java test options to |option_parser|.""" 143 144 option_parser.add_option('-f', '--test_filter', dest='test_filter', 145 help=('Test filter (if not fully qualified, ' 146 'will run all matches).')) 147 option_parser.add_option( 148 '-A', '--annotation', dest='annotation_str', 149 help=('Comma-separated list of annotations. Run only tests with any of ' 150 'the given annotations. An annotation can be either a key or a ' 151 'key-values pair. A test that has no annotation is considered ' 152 '"SmallTest".')) 153 option_parser.add_option( 154 '-E', '--exclude-annotation', dest='exclude_annotation_str', 155 help=('Comma-separated list of annotations. Exclude tests with these ' 156 'annotations.')) 157 option_parser.add_option('--screenshot', dest='screenshot_failures', 158 action='store_true', 159 help='Capture screenshots of test failures') 160 option_parser.add_option('--save-perf-json', action='store_true', 161 help='Saves the JSON file for each UI Perf test.') 162 option_parser.add_option('--official-build', action='store_true', 163 help='Run official build tests.') 164 option_parser.add_option('--keep_test_server_ports', 165 action='store_true', 166 help=('Indicates the test server ports must be ' 167 'kept. When this is run via a sharder ' 168 'the test server ports should be kept and ' 169 'should not be reset.')) 170 option_parser.add_option('--test_data', action='append', default=[], 171 help=('Each instance defines a directory of test ' 172 'data that should be copied to the target(s) ' 173 'before running the tests. The argument ' 174 'should be of the form <target>:<source>, ' 175 '<target> is relative to the device data' 176 'directory, and <source> is relative to the ' 177 'chromium build directory.')) 178 179 180 def ProcessJavaTestOptions(options, error_func): 181 """Processes options/arguments and populates |options| with defaults.""" 182 183 if options.annotation_str: 184 options.annotations = options.annotation_str.split(',') 185 elif options.test_filter: 186 options.annotations = [] 187 else: 188 options.annotations = ['Smoke', 'SmallTest', 'MediumTest', 'LargeTest', 189 'EnormousTest'] 190 191 if options.exclude_annotation_str: 192 options.exclude_annotations = options.exclude_annotation_str.split(',') 193 else: 194 options.exclude_annotations = [] 195 196 if not options.keep_test_server_ports: 197 if not ports.ResetTestServerPortAllocation(): 198 raise Exception('Failed to reset test server port.') 199 200 201 def AddInstrumentationTestOptions(option_parser): 202 """Adds Instrumentation test options to |option_parser|.""" 203 204 option_parser.usage = '%prog instrumentation [options]' 205 option_parser.command_list = [] 206 option_parser.example = ('%prog instrumentation ' 207 '--test-apk=ChromiumTestShellTest') 208 209 AddJavaTestOptions(option_parser) 210 AddCommonOptions(option_parser) 211 212 option_parser.add_option('-j', '--java_only', action='store_true', 213 default=False, help='Run only the Java tests.') 214 option_parser.add_option('-p', '--python_only', action='store_true', 215 default=False, 216 help='Run only the host-driven tests.') 217 option_parser.add_option('--python_test_root', 218 help='Root of the host-driven tests.') 219 option_parser.add_option('-w', '--wait_debugger', dest='wait_for_debugger', 220 action='store_true', 221 help='Wait for debugger.') 222 option_parser.add_option( 223 '--test-apk', dest='test_apk', 224 help=('The name of the apk containing the tests ' 225 '(without the .apk extension; e.g. "ContentShellTest"). ' 226 'Alternatively, this can be a full path to the apk.')) 227 228 229 def ProcessInstrumentationOptions(options, error_func): 230 """Processes options/arguments and populate |options| with defaults. 231 232 Args: 233 options: optparse.Options object. 234 error_func: Function to call with the error message in case of an error. 235 236 Returns: 237 An InstrumentationOptions named tuple which contains all options relevant to 238 instrumentation tests. 239 """ 240 241 ProcessJavaTestOptions(options, error_func) 242 243 if options.java_only and options.python_only: 244 error_func('Options java_only (-j) and python_only (-p) ' 245 'are mutually exclusive.') 246 options.run_java_tests = True 247 options.run_python_tests = True 248 if options.java_only: 249 options.run_python_tests = False 250 elif options.python_only: 251 options.run_java_tests = False 252 253 if not options.python_test_root: 254 options.run_python_tests = False 255 256 if not options.test_apk: 257 error_func('--test-apk must be specified.') 258 259 if os.path.exists(options.test_apk): 260 # The APK is fully qualified, assume the JAR lives along side. 261 options.test_apk_path = options.test_apk 262 options.test_apk_jar_path = (os.path.splitext(options.test_apk_path)[0] + 263 '.jar') 264 else: 265 options.test_apk_path = os.path.join(_SDK_OUT_DIR, 266 options.build_type, 267 constants.SDK_BUILD_APKS_DIR, 268 '%s.apk' % options.test_apk) 269 options.test_apk_jar_path = os.path.join( 270 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_TEST_JAVALIB_DIR, 271 '%s.jar' % options.test_apk) 272 273 return instrumentation_test_options.InstrumentationOptions( 274 options.build_type, 275 options.tool, 276 options.cleanup_test_files, 277 options.push_deps, 278 options.annotations, 279 options.exclude_annotations, 280 options.test_filter, 281 options.test_data, 282 options.save_perf_json, 283 options.screenshot_failures, 284 options.wait_for_debugger, 285 options.test_apk, 286 options.test_apk_path, 287 options.test_apk_jar_path) 288 289 290 def AddUIAutomatorTestOptions(option_parser): 291 """Adds UI Automator test options to |option_parser|.""" 292 293 option_parser.usage = '%prog uiautomator [options]' 294 option_parser.command_list = [] 295 option_parser.example = ( 296 '%prog uiautomator --test-jar=chromium_testshell_uiautomator_tests' 297 ' --package-name=org.chromium.chrome.testshell') 298 option_parser.add_option( 299 '--package-name', 300 help='The package name used by the apk containing the application.') 301 option_parser.add_option( 302 '--test-jar', dest='test_jar', 303 help=('The name of the dexed jar containing the tests (without the ' 304 '.dex.jar extension). Alternatively, this can be a full path ' 305 'to the jar.')) 306 307 AddJavaTestOptions(option_parser) 308 AddCommonOptions(option_parser) 309 310 311 def ProcessUIAutomatorOptions(options, error_func): 312 """Processes UIAutomator options/arguments. 313 314 Args: 315 options: optparse.Options object. 316 error_func: Function to call with the error message in case of an error. 317 318 Returns: 319 A UIAutomatorOptions named tuple which contains all options relevant to 320 uiautomator tests. 321 """ 322 323 ProcessJavaTestOptions(options, error_func) 324 325 if not options.package_name: 326 error_func('--package-name must be specified.') 327 328 if not options.test_jar: 329 error_func('--test-jar must be specified.') 330 331 if os.path.exists(options.test_jar): 332 # The dexed JAR is fully qualified, assume the info JAR lives along side. 333 options.uiautomator_jar = options.test_jar 334 else: 335 options.uiautomator_jar = os.path.join( 336 _SDK_OUT_DIR, options.build_type, constants.SDK_BUILD_JAVALIB_DIR, 337 '%s.dex.jar' % options.test_jar) 338 options.uiautomator_info_jar = ( 339 options.uiautomator_jar[:options.uiautomator_jar.find('.dex.jar')] + 340 '_java.jar') 341 342 return uiautomator_test_options.UIAutomatorOptions( 343 options.build_type, 344 options.tool, 345 options.cleanup_test_files, 346 options.push_deps, 347 options.annotations, 348 options.exclude_annotations, 349 options.test_filter, 350 options.test_data, 351 options.save_perf_json, 352 options.screenshot_failures, 353 options.uiautomator_jar, 354 options.uiautomator_info_jar, 355 options.package_name) 356 357 358 def AddMonkeyTestOptions(option_parser): 359 """Adds monkey test options to |option_parser|.""" 360 361 option_parser.usage = '%prog monkey [options]' 362 option_parser.command_list = [] 363 option_parser.example = ( 364 '%prog monkey --package-name=org.chromium.content_shell_apk' 365 ' --activity-name=.ContentShellActivity') 366 367 option_parser.add_option('--package-name', help='Allowed package.') 368 option_parser.add_option( 369 '--activity-name', help='Name of the activity to start.') 370 option_parser.add_option( 371 '--event-count', default=10000, type='int', 372 help='Number of events to generate [default: %default].') 373 option_parser.add_option( 374 '--category', default='', 375 help='A list of allowed categories.') 376 option_parser.add_option( 377 '--throttle', default=100, type='int', 378 help='Delay between events (ms) [default: %default]. ') 379 option_parser.add_option( 380 '--seed', type='int', 381 help=('Seed value for pseudo-random generator. Same seed value generates ' 382 'the same sequence of events. Seed is randomized by default.')) 383 option_parser.add_option( 384 '--extra-args', default='', 385 help=('String of other args to pass to the command verbatim ' 386 '[default: "%default"].')) 387 388 AddCommonOptions(option_parser) 389 390 391 def ProcessMonkeyTestOptions(options, error_func): 392 """Processes all monkey test options. 393 394 Args: 395 options: optparse.Options object. 396 error_func: Function to call with the error message in case of an error. 397 398 Returns: 399 A MonkeyOptions named tuple which contains all options relevant to 400 monkey tests. 401 """ 402 if not options.package_name: 403 error_func('Package name is required.') 404 405 category = options.category 406 if category: 407 category = options.category.split(',') 408 409 return monkey_test_options.MonkeyOptions( 410 options.build_type, 411 options.verbose_count, 412 options.package_name, 413 options.activity_name, 414 options.event_count, 415 category, 416 options.throttle, 417 options.seed, 418 options.extra_args) 419 420 421 def _RunGTests(options, error_func): 422 """Subcommand of RunTestsCommands which runs gtests.""" 423 ProcessGTestOptions(options) 424 425 exit_code = 0 426 for suite_name in options.suite_name: 427 # TODO(gkanwar): Move this into ProcessGTestOptions once we require -s for 428 # the gtest command. 429 gtest_options = gtest_test_options.GTestOptions( 430 options.build_type, 431 options.tool, 432 options.cleanup_test_files, 433 options.push_deps, 434 options.test_filter, 435 options.test_arguments, 436 options.timeout, 437 suite_name) 438 runner_factory, tests = gtest_setup.Setup(gtest_options) 439 440 results, test_exit_code = test_dispatcher.RunTests( 441 tests, runner_factory, False, options.test_device, 442 shard=True, 443 build_type=options.build_type, 444 test_timeout=None, 445 num_retries=options.num_retries) 446 447 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: 448 exit_code = test_exit_code 449 450 report_results.LogFull( 451 results=results, 452 test_type='Unit test', 453 test_package=suite_name, 454 build_type=options.build_type, 455 flakiness_server=options.flakiness_dashboard_server) 456 457 if os.path.isdir(constants.ISOLATE_DEPS_DIR): 458 shutil.rmtree(constants.ISOLATE_DEPS_DIR) 459 460 return exit_code 461 462 463 def _RunInstrumentationTests(options, error_func): 464 """Subcommand of RunTestsCommands which runs instrumentation tests.""" 465 instrumentation_options = ProcessInstrumentationOptions(options, error_func) 466 467 results = base_test_result.TestRunResults() 468 exit_code = 0 469 470 if options.run_java_tests: 471 runner_factory, tests = instrumentation_setup.Setup(instrumentation_options) 472 473 test_results, exit_code = test_dispatcher.RunTests( 474 tests, runner_factory, options.wait_for_debugger, 475 options.test_device, 476 shard=True, 477 build_type=options.build_type, 478 test_timeout=None, 479 num_retries=options.num_retries) 480 481 results.AddTestRunResults(test_results) 482 483 if options.run_python_tests: 484 runner_factory, tests = host_driven_setup.InstrumentationSetup( 485 options.python_test_root, options.official_build, 486 instrumentation_options) 487 488 if tests: 489 test_results, test_exit_code = test_dispatcher.RunTests( 490 tests, runner_factory, False, 491 options.test_device, 492 shard=True, 493 build_type=options.build_type, 494 test_timeout=None, 495 num_retries=options.num_retries) 496 497 results.AddTestRunResults(test_results) 498 499 # Only allow exit code escalation 500 if test_exit_code and exit_code != constants.ERROR_EXIT_CODE: 501 exit_code = test_exit_code 502 503 report_results.LogFull( 504 results=results, 505 test_type='Instrumentation', 506 test_package=os.path.basename(options.test_apk), 507 annotation=options.annotations, 508 build_type=options.build_type, 509 flakiness_server=options.flakiness_dashboard_server) 510 511 return exit_code 512 513 514 def _RunUIAutomatorTests(options, error_func): 515 """Subcommand of RunTestsCommands which runs uiautomator tests.""" 516 uiautomator_options = ProcessUIAutomatorOptions(options, error_func) 517 518 runner_factory, tests = uiautomator_setup.Setup(uiautomator_options) 519 520 results, exit_code = test_dispatcher.RunTests( 521 tests, runner_factory, False, options.test_device, 522 shard=True, 523 build_type=options.build_type, 524 test_timeout=None, 525 num_retries=options.num_retries) 526 527 report_results.LogFull( 528 results=results, 529 test_type='UIAutomator', 530 test_package=os.path.basename(options.test_jar), 531 annotation=options.annotations, 532 build_type=options.build_type, 533 flakiness_server=options.flakiness_dashboard_server) 534 535 return exit_code 536 537 538 def _RunMonkeyTests(options, error_func): 539 """Subcommand of RunTestsCommands which runs monkey tests.""" 540 monkey_options = ProcessMonkeyTestOptions(options, error_func) 541 542 runner_factory, tests = monkey_setup.Setup(monkey_options) 543 544 results, exit_code = test_dispatcher.RunTests( 545 tests, runner_factory, False, None, shard=False, test_timeout=None) 546 547 report_results.LogFull( 548 results=results, 549 test_type='Monkey', 550 test_package='Monkey', 551 build_type=options.build_type) 552 553 return exit_code 554 555 556 557 def RunTestsCommand(command, options, args, option_parser): 558 """Checks test type and dispatches to the appropriate function. 559 560 Args: 561 command: String indicating the command that was received to trigger 562 this function. 563 options: optparse options dictionary. 564 args: List of extra args from optparse. 565 option_parser: optparse.OptionParser object. 566 567 Returns: 568 Integer indicated exit code. 569 570 Raises: 571 Exception: Unknown command name passed in, or an exception from an 572 individual test runner. 573 """ 574 575 # Check for extra arguments 576 if len(args) > 2: 577 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[2:]))) 578 return constants.ERROR_EXIT_CODE 579 580 ProcessCommonOptions(options) 581 582 if command == 'gtest': 583 return _RunGTests(options, option_parser.error) 584 elif command == 'instrumentation': 585 return _RunInstrumentationTests(options, option_parser.error) 586 elif command == 'uiautomator': 587 return _RunUIAutomatorTests(options, option_parser.error) 588 elif command == 'monkey': 589 return _RunMonkeyTests(options, option_parser.error) 590 else: 591 raise Exception('Unknown test type.') 592 593 594 def HelpCommand(command, options, args, option_parser): 595 """Display help for a certain command, or overall help. 596 597 Args: 598 command: String indicating the command that was received to trigger 599 this function. 600 options: optparse options dictionary. 601 args: List of extra args from optparse. 602 option_parser: optparse.OptionParser object. 603 604 Returns: 605 Integer indicated exit code. 606 """ 607 # If we don't have any args, display overall help 608 if len(args) < 3: 609 option_parser.print_help() 610 return 0 611 # If we have too many args, print an error 612 if len(args) > 3: 613 option_parser.error('Unrecognized arguments: %s' % (' '.join(args[3:]))) 614 return constants.ERROR_EXIT_CODE 615 616 command = args[2] 617 618 if command not in VALID_COMMANDS: 619 option_parser.error('Unrecognized command.') 620 621 # Treat the help command as a special case. We don't care about showing a 622 # specific help page for itself. 623 if command == 'help': 624 option_parser.print_help() 625 return 0 626 627 VALID_COMMANDS[command].add_options_func(option_parser) 628 option_parser.usage = '%prog ' + command + ' [options]' 629 option_parser.command_list = None 630 option_parser.print_help() 631 632 return 0 633 634 635 # Define a named tuple for the values in the VALID_COMMANDS dictionary so the 636 # syntax is a bit prettier. The tuple is two functions: (add options, run 637 # command). 638 CommandFunctionTuple = collections.namedtuple( 639 'CommandFunctionTuple', ['add_options_func', 'run_command_func']) 640 VALID_COMMANDS = { 641 'gtest': CommandFunctionTuple(AddGTestOptions, RunTestsCommand), 642 'instrumentation': CommandFunctionTuple( 643 AddInstrumentationTestOptions, RunTestsCommand), 644 'uiautomator': CommandFunctionTuple( 645 AddUIAutomatorTestOptions, RunTestsCommand), 646 'monkey': CommandFunctionTuple( 647 AddMonkeyTestOptions, RunTestsCommand), 648 'help': CommandFunctionTuple(lambda option_parser: None, HelpCommand) 649 } 650 651 652 class CommandOptionParser(optparse.OptionParser): 653 """Wrapper class for OptionParser to help with listing commands.""" 654 655 def __init__(self, *args, **kwargs): 656 self.command_list = kwargs.pop('command_list', []) 657 self.example = kwargs.pop('example', '') 658 optparse.OptionParser.__init__(self, *args, **kwargs) 659 660 #override 661 def get_usage(self): 662 normal_usage = optparse.OptionParser.get_usage(self) 663 command_list = self.get_command_list() 664 example = self.get_example() 665 return self.expand_prog_name(normal_usage + example + command_list) 666 667 #override 668 def get_command_list(self): 669 if self.command_list: 670 return '\nCommands:\n %s\n' % '\n '.join(sorted(self.command_list)) 671 return '' 672 673 def get_example(self): 674 if self.example: 675 return '\nExample:\n %s\n' % self.example 676 return '' 677 678 679 def main(argv): 680 option_parser = CommandOptionParser( 681 usage='Usage: %prog <command> [options]', 682 command_list=VALID_COMMANDS.keys()) 683 684 if len(argv) < 2 or argv[1] not in VALID_COMMANDS: 685 option_parser.error('Invalid command.') 686 command = argv[1] 687 VALID_COMMANDS[command].add_options_func(option_parser) 688 options, args = option_parser.parse_args(argv) 689 return VALID_COMMANDS[command].run_command_func( 690 command, options, args, option_parser) 691 692 693 if __name__ == '__main__': 694 sys.exit(main(sys.argv)) 695