1 #!/usr/bin/env python 2 # 3 # Copyright 2008, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Command line utility for running Android tests 18 19 runtest helps automate the instructions for building and running tests 20 - It builds the corresponding test package for the code you want to test 21 - It pushes the test package to your device or emulator 22 - It launches InstrumentationTestRunner (or similar) to run the tests you 23 specify. 24 25 runtest supports running tests whose attributes have been pre-defined in 26 _TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27 system path to the test to run (runtest --path <path>). 28 29 Do runtest --help to see full list of options. 30 """ 31 32 # Python imports 33 import glob 34 import optparse 35 import os 36 import re 37 from sets import Set 38 import sys 39 import time 40 41 # local imports 42 import adb_interface 43 import android_build 44 from coverage import coverage 45 import errors 46 import logger 47 import make_tree 48 import run_command 49 from test_defs import test_defs 50 from test_defs import test_walker 51 52 53 class TestRunner(object): 54 """Command line utility class for running pre-defined Android test(s).""" 55 56 _TEST_FILE_NAME = "test_defs.xml" 57 58 # file path to android core platform tests, relative to android build root 59 # TODO move these test data files to another directory 60 _CORE_TEST_PATH = os.path.join("development", "testrunner", 61 _TEST_FILE_NAME) 62 63 # vendor glob file path patterns to tests, relative to android 64 # build root 65 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 66 _TEST_FILE_NAME) 67 68 _RUNTEST_USAGE = ( 69 "usage: runtest.py [options] short-test-name[s]\n\n" 70 "The runtest script works in two ways. You can query it " 71 "for a list of tests, or you can launch one or more tests.") 72 73 # default value for make -jX 74 _DEFAULT_JOBS = 16 75 76 _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags" 77 _DALVIK_VERIFIER_OFF_VALUE = "v=n" 78 _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE) 79 80 # regular expression to match path to artifacts to install in make output 81 _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$') 82 83 def __init__(self): 84 # disable logging of timestamp 85 self._root_path = android_build.GetTop() 86 out_base_name = os.path.basename(android_build.GetOutDir()) 87 # regular expression to find remote device path from a file path relative 88 # to build root 89 pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$' 90 self._re_make_install_path = re.compile(pattern) 91 logger.SetTimestampLogging(False) 92 self._adb = None 93 self._known_tests = None 94 self._options = None 95 self._test_args = None 96 self._tests_to_run = None 97 98 def _ProcessOptions(self): 99 """Processes command-line options.""" 100 # TODO error messages on once-only or mutually-exclusive options. 101 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 102 self._TEST_FILE_NAME) 103 104 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 105 106 parser.add_option("-l", "--list-tests", dest="only_list_tests", 107 default=False, action="store_true", 108 help="To view the list of tests") 109 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 110 action="store_true", help="Skip build - just launch") 111 parser.add_option("-j", "--jobs", dest="make_jobs", 112 metavar="X", default=self._DEFAULT_JOBS, 113 help="Number of make jobs to use when building") 114 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 115 action="store_true", 116 help="Do not execute, just preview commands") 117 parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False, 118 action="store_true", 119 help="Do not execute, build tests and install to device only") 120 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 121 action="store_true", 122 help="Raw mode (for output to other tools)") 123 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 124 default=False, action="store_true", 125 help="Suite assignment (for details & usage see " 126 "InstrumentationTestRunner)") 127 parser.add_option("-v", "--verbose", dest="verbose", default=False, 128 action="store_true", 129 help="Increase verbosity of %s" % sys.argv[0]) 130 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 131 default=False, action="store_true", 132 help="Wait for debugger before launching tests") 133 parser.add_option("-c", "--test-class", dest="test_class", 134 help="Restrict test to a specific class") 135 parser.add_option("-m", "--test-method", dest="test_method", 136 help="Restrict test to a specific method") 137 parser.add_option("-p", "--test-package", dest="test_package", 138 help="Restrict test to a specific java package") 139 parser.add_option("-z", "--size", dest="test_size", 140 help="Restrict test to a specific test size") 141 parser.add_option("--annotation", dest="test_annotation", 142 help="Include only those tests tagged with a specific" 143 " annotation") 144 parser.add_option("--not-annotation", dest="test_not_annotation", 145 help="Exclude any tests tagged with a specific" 146 " annotation") 147 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 148 metavar="FILE", default=user_test_default, 149 help="Alternate source of user test definitions") 150 parser.add_option("-o", "--coverage", dest="coverage", 151 default=False, action="store_true", 152 help="Generate code coverage metrics for test(s)") 153 parser.add_option("--coverage-target", dest="coverage_target_path", 154 default=None, 155 help="Path to app to collect code coverage target data for.") 156 parser.add_option("-k", "--skip-permissions", dest="skip_permissions", 157 default=False, action="store_true", 158 help="Do not grant runtime permissions during test package" 159 " installation.") 160 parser.add_option("-x", "--path", dest="test_path", 161 help="Run test(s) at given file system path") 162 parser.add_option("-t", "--all-tests", dest="all_tests", 163 default=False, action="store_true", 164 help="Run all defined tests") 165 parser.add_option("--continuous", dest="continuous_tests", 166 default=False, action="store_true", 167 help="Run all tests defined as part of the continuous " 168 "test set") 169 parser.add_option("--timeout", dest="timeout", 170 default=300, help="Set a timeout limit (in sec) for " 171 "running native tests on a device (default: 300 secs)") 172 parser.add_option("--suite", dest="suite", 173 help="Run all tests defined as part of the " 174 "the given test suite") 175 parser.add_option("--user", dest="user", 176 help="The user that test apks are installing to." 177 " This is the integer user id, e.g. 0 or 10." 178 " If no user is specified, apk will be installed with" 179 " adb's default behavior, which is currently all users.") 180 parser.add_option("--install-filter", dest="filter_re", 181 help="Regular expression which generated apks have to" 182 " match to be installed to target device. Default is None" 183 " and will install all packages built. This is" 184 " useful when the test path has a lot of apks but you" 185 " only care about one.") 186 group = optparse.OptionGroup( 187 parser, "Targets", "Use these options to direct tests to a specific " 188 "Android target") 189 group.add_option("-e", "--emulator", dest="emulator", default=False, 190 action="store_true", help="use emulator") 191 group.add_option("-d", "--device", dest="device", default=False, 192 action="store_true", help="use device") 193 group.add_option("-s", "--serial", dest="serial", 194 help="use specific serial") 195 parser.add_option_group(group) 196 self._options, self._test_args = parser.parse_args() 197 198 if (not self._options.only_list_tests 199 and not self._options.all_tests 200 and not self._options.continuous_tests 201 and not self._options.suite 202 and not self._options.test_path 203 and len(self._test_args) < 1): 204 parser.print_help() 205 logger.SilentLog("at least one test name must be specified") 206 raise errors.AbortError 207 208 self._adb = adb_interface.AdbInterface() 209 if self._options.emulator: 210 self._adb.SetEmulatorTarget() 211 elif self._options.device: 212 self._adb.SetDeviceTarget() 213 elif self._options.serial is not None: 214 self._adb.SetTargetSerial(self._options.serial) 215 if self._options.verbose: 216 logger.SetVerbose(True) 217 218 if self._options.coverage_target_path: 219 self._options.coverage = True 220 221 self._known_tests = self._ReadTests() 222 223 self._options.host_lib_path = android_build.GetHostLibraryPath() 224 self._options.test_data_path = android_build.GetTestAppPath() 225 226 def _ReadTests(self): 227 """Parses the set of test definition data. 228 229 Returns: 230 A TestDefinitions object that contains the set of parsed tests. 231 Raises: 232 AbortError: If a fatal error occurred when parsing the tests. 233 """ 234 try: 235 known_tests = test_defs.TestDefinitions() 236 # only read tests when not in path mode 237 if not self._options.test_path: 238 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 239 if os.path.isfile(core_test_path): 240 known_tests.Parse(core_test_path) 241 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 242 vendor_tests_pattern = os.path.join(self._root_path, 243 self._VENDOR_TEST_PATH) 244 test_file_paths = glob.glob(vendor_tests_pattern) 245 for test_file_path in test_file_paths: 246 known_tests.Parse(test_file_path) 247 if os.path.isfile(self._options.user_tests_file): 248 known_tests.Parse(self._options.user_tests_file) 249 return known_tests 250 except errors.ParseError: 251 raise errors.AbortError 252 253 def _DumpTests(self): 254 """Prints out set of defined tests.""" 255 print "The following tests are currently defined:\n" 256 print "%-25s %-40s %s" % ("name", "build path", "description") 257 print "-" * 80 258 for test in self._known_tests: 259 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 260 test.GetDescription()) 261 print "\nSee %s for more information" % self._TEST_FILE_NAME 262 263 def _DoBuild(self): 264 logger.SilentLog("Building tests...") 265 tests = self._GetTestsToRun() 266 267 # Build and install tests that do not get granted permissions 268 self._DoPermissionAwareBuild(tests, False) 269 270 # Build and install tests that require granted permissions 271 self._DoPermissionAwareBuild(tests, True) 272 273 def _DoPermissionAwareBuild(self, tests, test_requires_permissions): 274 # turn off dalvik verifier if necessary 275 # TODO: skip turning off verifier for now, since it puts device in bad 276 # state b/14088982 277 #self._TurnOffVerifier(tests) 278 self._DoFullBuild(tests, test_requires_permissions) 279 280 target_tree = make_tree.MakeTree() 281 282 extra_args_set = [] 283 for test_suite in tests: 284 if test_suite.IsGrantedPermissions() == test_requires_permissions: 285 self._AddBuildTarget(test_suite, target_tree, extra_args_set) 286 287 if not self._options.preview: 288 self._adb.EnableAdbRoot() 289 else: 290 logger.Log("adb root") 291 292 if not target_tree.IsEmpty(): 293 if self._options.coverage: 294 coverage.EnableCoverageBuild() 295 target_tree.AddPath("external/emma") 296 297 target_list = target_tree.GetPrunedMakeList() 298 target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list] 299 target_build_string = " ".join(target_list) 300 target_dir_build_string = " ".join(target_dir_list) 301 extra_args_string = " ".join(extra_args_set) 302 303 # mmm cannot be used from python, so perform a similar operation using 304 # ONE_SHOT_MAKEFILE 305 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" GET-INSTALL-PATH all_modules %s' % ( 306 target_build_string, self._options.make_jobs, self._root_path, 307 extra_args_string) 308 # mmma equivalent, used when regular mmm fails 309 mmma_goals = [] 310 for d in target_dir_list: 311 if d.startswith("./"): 312 d = d[2:] 313 if d.endswith("/"): 314 d = d[:-1] 315 mmma_goals.append("MODULES-IN-" + d.replace("/","-")) 316 alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s %s' % ( 317 self._options.make_jobs, self._root_path, extra_args_string, " ".join(mmma_goals)) 318 319 logger.Log(cmd) 320 if not self._options.preview: 321 run_command.SetAbortOnError() 322 try: 323 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 324 ## Chances are this failed because it didn't build the dependencies 325 except errors.AbortError: 326 logger.Log("make failed. Trying to rebuild all dependencies.") 327 logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string)) 328 # Try again with mma equivalent, which will build the dependencies 329 run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600) 330 # Run mmm again to get the install paths only 331 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 332 run_command.SetAbortOnError(False) 333 logger.SilentLog(output) 334 filter_re = re.compile(self._options.filter_re) if self._options.filter_re else None 335 336 self._DoInstall(output, test_requires_permissions, filter_re=filter_re) 337 338 def _DoInstall(self, make_output, test_requires_permissions, filter_re=None): 339 """Install artifacts from build onto device. 340 341 Looks for 'install:' text from make output to find artifacts to install. 342 343 Files with the .apk extension get 'adb install'ed, all other files 344 get 'adb push'ed onto the device. 345 346 Args: 347 make_output: stdout from make command 348 """ 349 for line in make_output.split("\n"): 350 m = self._RE_MAKE_INSTALL.match(line) 351 if m: 352 # strip the 'INSTALL: <name>' from the left hand side 353 # the remaining string is a space-separated list of build-generated files 354 install_paths = m.group(2) 355 for install_path in re.split(r'\s+', install_paths): 356 if filter_re and not filter_re.match(install_path): 357 continue 358 if install_path.endswith(".apk"): 359 abs_install_path = os.path.join(self._root_path, install_path) 360 extra_flags = "" 361 if test_requires_permissions and not self._options.skip_permissions: 362 extra_flags = "-g" 363 if self._options.user: 364 extra_flags += " --user " + self._options.user 365 logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path)) 366 logger.Log(self._adb.Install(abs_install_path, extra_flags)) 367 else: 368 self._PushInstallFileToDevice(install_path) 369 370 def _PushInstallFileToDevice(self, install_path): 371 m = self._re_make_install_path.match(install_path) 372 if m: 373 remote_path = m.group(1) 374 remote_dir = os.path.dirname(remote_path) 375 logger.Log("adb shell mkdir -p %s" % remote_dir) 376 self._adb.SendShellCommand("mkdir -p %s" % remote_dir) 377 abs_install_path = os.path.join(self._root_path, install_path) 378 logger.Log("adb push %s %s" % (abs_install_path, remote_path)) 379 self._adb.Push(abs_install_path, remote_path) 380 else: 381 logger.Log("Error: Failed to recognize path of file to install %s" % install_path) 382 383 def _DoFullBuild(self, tests, test_requires_permissions): 384 """If necessary, run a full 'make' command for the tests that need it.""" 385 extra_args_set = Set() 386 387 for test in tests: 388 if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions: 389 if test.GetExtraBuildArgs(): 390 # extra args contains the args to pass to 'make' 391 extra_args_set.add(test.GetExtraBuildArgs()) 392 else: 393 logger.Log("Warning: test %s needs a full build but does not specify" 394 " extra_build_args" % test.GetName()) 395 396 # check if there is actually any tests that required a full build 397 if extra_args_set: 398 cmd = ('make -j%s %s' % (self._options.make_jobs, 399 ' '.join(list(extra_args_set)))) 400 logger.Log(cmd) 401 if not self._options.preview: 402 old_dir = os.getcwd() 403 os.chdir(self._root_path) 404 output = run_command.RunCommand(cmd, return_output=True) 405 logger.SilentLog(output) 406 os.chdir(old_dir) 407 self._DoInstall(output, test_requires_permissions) 408 409 def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): 410 if not test_suite.IsFullMake(): 411 build_dir = test_suite.GetBuildPath() 412 if self._AddBuildTargetPath(build_dir, target_tree): 413 extra_args_set.append(test_suite.GetExtraBuildArgs()) 414 for path in test_suite.GetBuildDependencies(self._options): 415 self._AddBuildTargetPath(path, target_tree) 416 417 def _AddBuildTargetPath(self, build_dir, target_tree): 418 if build_dir is not None: 419 target_tree.AddPath(build_dir) 420 return True 421 return False 422 423 def _GetTestsToRun(self): 424 """Get a list of TestSuite objects to run, based on command line args.""" 425 if self._tests_to_run: 426 return self._tests_to_run 427 428 self._tests_to_run = [] 429 if self._options.all_tests: 430 self._tests_to_run = self._known_tests.GetTests() 431 elif self._options.continuous_tests: 432 self._tests_to_run = self._known_tests.GetContinuousTests() 433 elif self._options.suite: 434 self._tests_to_run = \ 435 self._known_tests.GetTestsInSuite(self._options.suite) 436 elif self._options.test_path: 437 walker = test_walker.TestWalker() 438 self._tests_to_run = walker.FindTests(self._options.test_path) 439 440 for name in self._test_args: 441 test = self._known_tests.GetTest(name) 442 if test is None: 443 logger.Log("Error: Could not find test %s" % name) 444 self._DumpTests() 445 raise errors.AbortError 446 self._tests_to_run.append(test) 447 return self._tests_to_run 448 449 def _TurnOffVerifier(self, test_list): 450 """Turn off the dalvik verifier if needed by given tests. 451 452 If one or more tests needs dalvik verifier off, and it is not already off, 453 turns off verifier and reboots device to allow change to take effect. 454 """ 455 # hack to check if these are frameworks/base tests. If so, turn off verifier 456 # to allow framework tests to access private/protected/package-private framework api 457 framework_test = False 458 for test in test_list: 459 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 460 framework_test = True 461 if framework_test: 462 # check if verifier is off already - to avoid the reboot if not 463 # necessary 464 output = self._adb.SendShellCommand("cat /data/local.prop") 465 if not self._DALVIK_VERIFIER_OFF_PROP in output: 466 467 # Read the existing dalvik verifier flags. 468 old_prop_value = self._adb.SendShellCommand("getprop %s" \ 469 %(self._DALVIK_VERIFIER_PROP)) 470 old_prop_value = old_prop_value.strip() if old_prop_value else "" 471 472 # Append our verifier flags to existing flags 473 new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value) 474 475 # Update property now, as /data/local.prop is not read until reboot 476 logger.Log("adb shell setprop %s '%s'" \ 477 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 478 if not self._options.preview: 479 self._adb.SendShellCommand("setprop %s '%s'" \ 480 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 481 482 # Write prop to /data/local.prop 483 # Every time device is booted, it will pick up this value 484 new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value) 485 if self._options.preview: 486 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 487 % new_prop_assignment) 488 logger.Log("adb shell chmod 644 /data/local.prop") 489 else: 490 logger.Log("Turning off dalvik verifier and rebooting") 491 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 492 % new_prop_assignment) 493 494 # Reset runtime so that dalvik picks up new verifier flags from prop 495 self._ChmodRuntimeReset() 496 elif not self._options.preview: 497 # check the permissions on the file 498 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 499 if not "-rw-r--r--" in permout: 500 logger.Log("Fixing permissions on /data/local.prop and rebooting") 501 self._ChmodRuntimeReset() 502 503 def _ChmodRuntimeReset(self): 504 """Perform a chmod of /data/local.prop and reset the runtime. 505 """ 506 logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r") 507 if not self._options.preview: 508 self._adb.SendShellCommand("chmod 644 /data/local.prop") 509 510 self._adb.RuntimeReset(preview_only=self._options.preview) 511 512 if not self._options.preview: 513 self._adb.EnableAdbRoot() 514 515 516 def RunTests(self): 517 """Main entry method - executes the tests according to command line args.""" 518 try: 519 run_command.SetAbortOnError() 520 self._ProcessOptions() 521 if self._options.only_list_tests: 522 self._DumpTests() 523 return 524 525 if not self._options.skip_build: 526 self._DoBuild() 527 528 if self._options.build_install_only: 529 logger.Log("Skipping test execution (due to --build-install-only flag)") 530 return 531 532 for test_suite in self._GetTestsToRun(): 533 try: 534 test_suite.Run(self._options, self._adb) 535 except errors.WaitForResponseTimedOutError: 536 logger.Log("Timed out waiting for response") 537 538 except KeyboardInterrupt: 539 logger.Log("Exiting...") 540 except errors.AbortError, error: 541 logger.Log(error.msg) 542 logger.SilentLog("Exiting due to AbortError...") 543 except errors.WaitForResponseTimedOutError: 544 logger.Log("Timed out waiting for response") 545 546 547 def RunTests(): 548 runner = TestRunner() 549 runner.RunTests() 550 551 if __name__ == "__main__": 552 RunTests() 553
