1 #!/usr/bin/env python 2 # 3 # Copyright 2008, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Command line utility for running Android tests 18 19 runtest helps automate the instructions for building and running tests 20 - It builds the corresponding test package for the code you want to test 21 - It pushes the test package to your device or emulator 22 - It launches InstrumentationTestRunner (or similar) to run the tests you 23 specify. 24 25 runtest supports running tests whose attributes have been pre-defined in 26 _TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27 system path to the test to run (runtest --path <path>). 28 29 Do runtest --help to see full list of options. 30 """ 31 32 # Python imports 33 import glob 34 import optparse 35 import os 36 import re 37 from sets import Set 38 import sys 39 import time 40 41 # local imports 42 import adb_interface 43 import android_build 44 from coverage import coverage 45 import errors 46 import logger 47 import make_tree 48 import run_command 49 from test_defs import test_defs 50 from test_defs import test_walker 51 52 53 class TestRunner(object): 54 """Command line utility class for running pre-defined Android test(s).""" 55 56 _TEST_FILE_NAME = "test_defs.xml" 57 58 # file path to android core platform tests, relative to android build root 59 # TODO move these test data files to another directory 60 _CORE_TEST_PATH = os.path.join("development", "testrunner", 61 _TEST_FILE_NAME) 62 63 # vendor glob file path patterns to tests, relative to android 64 # build root 65 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 66 _TEST_FILE_NAME) 67 68 _RUNTEST_USAGE = ( 69 "usage: runtest.py [options] short-test-name[s]\n\n" 70 "The runtest script works in two ways. You can query it " 71 "for a list of tests, or you can launch one or more tests.") 72 73 # default value for make -jX 74 _DEFAULT_JOBS = 16 75 76 _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags" 77 _DALVIK_VERIFIER_OFF_VALUE = "v=n" 78 _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE) 79 80 # regular expression to match path to artifacts to install in make output 81 _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$') 82 83 def __init__(self): 84 # disable logging of timestamp 85 self._root_path = android_build.GetTop() 86 out_base_name = os.path.basename(android_build.GetOutDir()) 87 # regular expression to find remote device path from a file path relative 88 # to build root 89 pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$' 90 self._re_make_install_path = re.compile(pattern) 91 logger.SetTimestampLogging(False) 92 self._adb = None 93 self._known_tests = None 94 self._options = None 95 self._test_args = None 96 self._tests_to_run = None 97 98 def _ProcessOptions(self): 99 """Processes command-line options.""" 100 # TODO error messages on once-only or mutually-exclusive options. 101 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 102 self._TEST_FILE_NAME) 103 104 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 105 106 parser.add_option("-l", "--list-tests", dest="only_list_tests", 107 default=False, action="store_true", 108 help="To view the list of tests") 109 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 110 action="store_true", help="Skip build - just launch") 111 parser.add_option("-j", "--jobs", dest="make_jobs", 112 metavar="X", default=self._DEFAULT_JOBS, 113 help="Number of make jobs to use when building") 114 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 115 action="store_true", 116 help="Do not execute, just preview commands") 117 parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False, 118 action="store_true", 119 help="Do not execute, build tests and install to device only") 120 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 121 action="store_true", 122 help="Raw mode (for output to other tools)") 123 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 124 default=False, action="store_true", 125 help="Suite assignment (for details & usage see " 126 "InstrumentationTestRunner)") 127 parser.add_option("-v", "--verbose", dest="verbose", default=False, 128 action="store_true", 129 help="Increase verbosity of %s" % sys.argv[0]) 130 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 131 default=False, action="store_true", 132 help="Wait for debugger before launching tests") 133 parser.add_option("-c", "--test-class", dest="test_class", 134 help="Restrict test to a specific class") 135 parser.add_option("-m", "--test-method", dest="test_method", 136 help="Restrict test to a specific method") 137 parser.add_option("-p", "--test-package", dest="test_package", 138 help="Restrict test to a specific java package") 139 parser.add_option("-z", "--size", dest="test_size", 140 help="Restrict test to a specific test size") 141 parser.add_option("--annotation", dest="test_annotation", 142 help="Include only those tests tagged with a specific" 143 " annotation") 144 parser.add_option("--not-annotation", dest="test_not_annotation", 145 help="Exclude any tests tagged with a specific" 146 " annotation") 147 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 148 metavar="FILE", default=user_test_default, 149 help="Alternate source of user test definitions") 150 parser.add_option("-o", "--coverage", dest="coverage", 151 default=False, action="store_true", 152 help="Generate code coverage metrics for test(s)") 153 parser.add_option("--coverage-target", dest="coverage_target_path", 154 default=None, 155 help="Path to app to collect code coverage target data for.") 156 parser.add_option("-k", "--skip-permissions", dest="skip_permissions", 157 default=False, action="store_true", 158 help="Do not grant runtime permissions during test package" 159 " installation.") 160 parser.add_option("-x", "--path", dest="test_path", 161 help="Run test(s) at given file system path") 162 parser.add_option("-t", "--all-tests", dest="all_tests", 163 default=False, action="store_true", 164 help="Run all defined tests") 165 parser.add_option("--continuous", dest="continuous_tests", 166 default=False, action="store_true", 167 help="Run all tests defined as part of the continuous " 168 "test set") 169 parser.add_option("--timeout", dest="timeout", 170 default=300, help="Set a timeout limit (in sec) for " 171 "running native tests on a device (default: 300 secs)") 172 parser.add_option("--suite", dest="suite", 173 help="Run all tests defined as part of the " 174 "the given test suite") 175 parser.add_option("--user", dest="user", 176 help="The user that test apks are installing to." 177 " This is the integer user id, e.g. 0 or 10." 178 " If no user is specified, apk will be installed with" 179 " adb's default behavior, which is currently all users.") 180 parser.add_option("--install-filter", dest="filter_re", 181 help="Regular expression which generated apks have to" 182 " match to be installed to target device. Default is None" 183 " and will install all packages built. This is" 184 " useful when the test path has a lot of apks but you" 185 " only care about one.") 186 group = optparse.OptionGroup( 187 parser, "Targets", "Use these options to direct tests to a specific " 188 "Android target") 189 group.add_option("-e", "--emulator", dest="emulator", default=False, 190 action="store_true", help="use emulator") 191 group.add_option("-d", "--device", dest="device", default=False, 192 action="store_true", help="use device") 193 group.add_option("-s", "--serial", dest="serial", 194 help="use specific serial") 195 parser.add_option_group(group) 196 self._options, self._test_args = parser.parse_args() 197 198 if (not self._options.only_list_tests 199 and not self._options.all_tests 200 and not self._options.continuous_tests 201 and not self._options.suite 202 and not self._options.test_path 203 and len(self._test_args) < 1): 204 parser.print_help() 205 logger.SilentLog("at least one test name must be specified") 206 raise errors.AbortError 207 208 self._adb = adb_interface.AdbInterface() 209 if self._options.emulator: 210 self._adb.SetEmulatorTarget() 211 elif self._options.device: 212 self._adb.SetDeviceTarget() 213 elif self._options.serial is not None: 214 self._adb.SetTargetSerial(self._options.serial) 215 if self._options.verbose: 216 logger.SetVerbose(True) 217 218 if self._options.coverage_target_path: 219 self._options.coverage = True 220 221 self._known_tests = self._ReadTests() 222 223 self._options.host_lib_path = android_build.GetHostLibraryPath() 224 self._options.test_data_path = android_build.GetTestAppPath() 225 226 def _ReadTests(self): 227 """Parses the set of test definition data. 228 229 Returns: 230 A TestDefinitions object that contains the set of parsed tests. 231 Raises: 232 AbortError: If a fatal error occurred when parsing the tests. 233 """ 234 try: 235 known_tests = test_defs.TestDefinitions() 236 # only read tests when not in path mode 237 if not self._options.test_path: 238 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 239 if os.path.isfile(core_test_path): 240 known_tests.Parse(core_test_path) 241 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 242 vendor_tests_pattern = os.path.join(self._root_path, 243 self._VENDOR_TEST_PATH) 244 test_file_paths = glob.glob(vendor_tests_pattern) 245 for test_file_path in test_file_paths: 246 known_tests.Parse(test_file_path) 247 if os.path.isfile(self._options.user_tests_file): 248 known_tests.Parse(self._options.user_tests_file) 249 return known_tests 250 except errors.ParseError: 251 raise errors.AbortError 252 253 def _DumpTests(self): 254 """Prints out set of defined tests.""" 255 print "The following tests are currently defined:\n" 256 print "%-25s %-40s %s" % ("name", "build path", "description") 257 print "-" * 80 258 for test in self._known_tests: 259 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 260 test.GetDescription()) 261 print "\nSee %s for more information" % self._TEST_FILE_NAME 262 263 def _DoBuild(self): 264 logger.SilentLog("Building tests...") 265 tests = self._GetTestsToRun() 266 267 # Build and install tests that do not get granted permissions 268 self._DoPermissionAwareBuild(tests, False) 269 270 # Build and install tests that require granted permissions 271 self._DoPermissionAwareBuild(tests, True) 272 273 def _DoPermissionAwareBuild(self, tests, test_requires_permissions): 274 # turn off dalvik verifier if necessary 275 # TODO: skip turning off verifier for now, since it puts device in bad 276 # state b/14088982 277 #self._TurnOffVerifier(tests) 278 self._DoFullBuild(tests, test_requires_permissions) 279 280 target_tree = make_tree.MakeTree() 281 282 extra_args_set = [] 283 for test_suite in tests: 284 if test_suite.IsGrantedPermissions() == test_requires_permissions: 285 self._AddBuildTarget(test_suite, target_tree, extra_args_set) 286 287 if not self._options.preview: 288 self._adb.EnableAdbRoot() 289 else: 290 logger.Log("adb root") 291 292 if not target_tree.IsEmpty(): 293 if self._options.coverage: 294 coverage.EnableCoverageBuild() 295 target_tree.AddPath("external/emma") 296 297 target_list = target_tree.GetPrunedMakeList() 298 target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list] 299 target_build_string = " ".join(target_list) 300 target_dir_build_string = " ".join(target_dir_list) 301 extra_args_string = " ".join(extra_args_set) 302 303 install_path_goals = [] 304 mmma_goals = [] 305 for d in target_dir_list: 306 if d.startswith("./"): 307 d = d[2:] 308 if d.endswith("/"): 309 d = d[:-1] 310 install_path_goals.append("GET-INSTALL-PATH-IN-" + d.replace("/","-")) 311 mmma_goals.append("MODULES-IN-" + d.replace("/","-")) 312 # mmm cannot be used from python, so perform a similar operation using 313 # ONE_SHOT_MAKEFILE 314 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" %s %s %s' % ( 315 target_build_string, self._options.make_jobs, self._root_path, 316 " ".join(install_path_goals), " ".join(mmma_goals), extra_args_string) 317 # mmma cannot be used from python, so perform a similar operation 318 alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s %s' % ( 319 self._options.make_jobs, self._root_path, extra_args_string, " ".join(mmma_goals)) 320 321 logger.Log(cmd) 322 if not self._options.preview: 323 run_command.SetAbortOnError() 324 try: 325 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 326 ## Chances are this failed because it didn't build the dependencies 327 except errors.AbortError: 328 logger.Log("make failed. Trying to rebuild all dependencies.") 329 logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string)) 330 # Try again with mma equivalent, which will build the dependencies 331 run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600) 332 # Run mmm again to get the install paths only 333 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 334 run_command.SetAbortOnError(False) 335 logger.SilentLog(output) 336 filter_re = re.compile(self._options.filter_re) if self._options.filter_re else None 337 338 self._DoInstall(output, test_requires_permissions, filter_re=filter_re) 339 340 def _DoInstall(self, make_output, test_requires_permissions, filter_re=None): 341 """Install artifacts from build onto device. 342 343 Looks for 'install:' text from make output to find artifacts to install. 344 345 Files with the .apk extension get 'adb install'ed, all other files 346 get 'adb push'ed onto the device. 347 348 Args: 349 make_output: stdout from make command 350 """ 351 for line in make_output.split("\n"): 352 m = self._RE_MAKE_INSTALL.match(line) 353 if m: 354 # strip the 'INSTALL: <name>' from the left hand side 355 # the remaining string is a space-separated list of build-generated files 356 install_paths = m.group(2) 357 for install_path in re.split(r'\s+', install_paths): 358 if filter_re and not filter_re.match(install_path): 359 continue 360 if install_path.endswith(".apk"): 361 abs_install_path = os.path.join(self._root_path, install_path) 362 extra_flags = "" 363 if test_requires_permissions and not self._options.skip_permissions: 364 extra_flags = "-g" 365 if self._options.user: 366 extra_flags += " --user " + self._options.user 367 logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path)) 368 logger.Log(self._adb.Install(abs_install_path, extra_flags)) 369 else: 370 self._PushInstallFileToDevice(install_path) 371 372 def _PushInstallFileToDevice(self, install_path): 373 m = self._re_make_install_path.match(install_path) 374 if m: 375 remote_path = m.group(1) 376 remote_dir = os.path.dirname(remote_path) 377 logger.Log("adb shell mkdir -p %s" % remote_dir) 378 self._adb.SendShellCommand("mkdir -p %s" % remote_dir) 379 abs_install_path = os.path.join(self._root_path, install_path) 380 logger.Log("adb push %s %s" % (abs_install_path, remote_path)) 381 self._adb.Push(abs_install_path, remote_path) 382 else: 383 logger.Log("Error: Failed to recognize path of file to install %s" % install_path) 384 385 def _DoFullBuild(self, tests, test_requires_permissions): 386 """If necessary, run a full 'make' command for the tests that need it.""" 387 extra_args_set = Set() 388 389 for test in tests: 390 if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions: 391 if test.GetExtraBuildArgs(): 392 # extra args contains the args to pass to 'make' 393 extra_args_set.add(test.GetExtraBuildArgs()) 394 else: 395 logger.Log("Warning: test %s needs a full build but does not specify" 396 " extra_build_args" % test.GetName()) 397 398 # check if there is actually any tests that required a full build 399 if extra_args_set: 400 cmd = ('make -j%s %s' % (self._options.make_jobs, 401 ' '.join(list(extra_args_set)))) 402 logger.Log(cmd) 403 if not self._options.preview: 404 old_dir = os.getcwd() 405 os.chdir(self._root_path) 406 output = run_command.RunCommand(cmd, return_output=True) 407 logger.SilentLog(output) 408 os.chdir(old_dir) 409 self._DoInstall(output, test_requires_permissions) 410 411 def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): 412 if not test_suite.IsFullMake(): 413 build_dir = test_suite.GetBuildPath() 414 if self._AddBuildTargetPath(build_dir, target_tree): 415 extra_args_set.append(test_suite.GetExtraBuildArgs()) 416 for path in test_suite.GetBuildDependencies(self._options): 417 self._AddBuildTargetPath(path, target_tree) 418 419 def _AddBuildTargetPath(self, build_dir, target_tree): 420 if build_dir is not None: 421 target_tree.AddPath(build_dir) 422 return True 423 return False 424 425 def _GetTestsToRun(self): 426 """Get a list of TestSuite objects to run, based on command line args.""" 427 if self._tests_to_run: 428 return self._tests_to_run 429 430 self._tests_to_run = [] 431 if self._options.all_tests: 432 self._tests_to_run = self._known_tests.GetTests() 433 elif self._options.continuous_tests: 434 self._tests_to_run = self._known_tests.GetContinuousTests() 435 elif self._options.suite: 436 self._tests_to_run = \ 437 self._known_tests.GetTestsInSuite(self._options.suite) 438 elif self._options.test_path: 439 walker = test_walker.TestWalker() 440 self._tests_to_run = walker.FindTests(self._options.test_path) 441 442 for name in self._test_args: 443 test = self._known_tests.GetTest(name) 444 if test is None: 445 logger.Log("Error: Could not find test %s" % name) 446 self._DumpTests() 447 raise errors.AbortError 448 self._tests_to_run.append(test) 449 return self._tests_to_run 450 451 def _TurnOffVerifier(self, test_list): 452 """Turn off the dalvik verifier if needed by given tests. 453 454 If one or more tests needs dalvik verifier off, and it is not already off, 455 turns off verifier and reboots device to allow change to take effect. 456 """ 457 # hack to check if these are frameworks/base tests. If so, turn off verifier 458 # to allow framework tests to access private/protected/package-private framework api 459 framework_test = False 460 for test in test_list: 461 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 462 framework_test = True 463 if framework_test: 464 # check if verifier is off already - to avoid the reboot if not 465 # necessary 466 output = self._adb.SendShellCommand("cat /data/local.prop") 467 if not self._DALVIK_VERIFIER_OFF_PROP in output: 468 469 # Read the existing dalvik verifier flags. 470 old_prop_value = self._adb.SendShellCommand("getprop %s" \ 471 %(self._DALVIK_VERIFIER_PROP)) 472 old_prop_value = old_prop_value.strip() if old_prop_value else "" 473 474 # Append our verifier flags to existing flags 475 new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value) 476 477 # Update property now, as /data/local.prop is not read until reboot 478 logger.Log("adb shell setprop %s '%s'" \ 479 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 480 if not self._options.preview: 481 self._adb.SendShellCommand("setprop %s '%s'" \ 482 %(self._DALVIK_VERIFIER_PROP, new_prop_value)) 483 484 # Write prop to /data/local.prop 485 # Every time device is booted, it will pick up this value 486 new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value) 487 if self._options.preview: 488 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 489 % new_prop_assignment) 490 logger.Log("adb shell chmod 644 /data/local.prop") 491 else: 492 logger.Log("Turning off dalvik verifier and rebooting") 493 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 494 % new_prop_assignment) 495 496 # Reset runtime so that dalvik picks up new verifier flags from prop 497 self._ChmodRuntimeReset() 498 elif not self._options.preview: 499 # check the permissions on the file 500 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 501 if not "-rw-r--r--" in permout: 502 logger.Log("Fixing permissions on /data/local.prop and rebooting") 503 self._ChmodRuntimeReset() 504 505 def _ChmodRuntimeReset(self): 506 """Perform a chmod of /data/local.prop and reset the runtime. 507 """ 508 logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r") 509 if not self._options.preview: 510 self._adb.SendShellCommand("chmod 644 /data/local.prop") 511 512 self._adb.RuntimeReset(preview_only=self._options.preview) 513 514 if not self._options.preview: 515 self._adb.EnableAdbRoot() 516 517 518 def RunTests(self): 519 """Main entry method - executes the tests according to command line args.""" 520 try: 521 run_command.SetAbortOnError() 522 self._ProcessOptions() 523 if self._options.only_list_tests: 524 self._DumpTests() 525 return 526 527 if not self._options.skip_build: 528 self._DoBuild() 529 530 if self._options.build_install_only: 531 logger.Log("Skipping test execution (due to --build-install-only flag)") 532 return 533 534 for test_suite in self._GetTestsToRun(): 535 try: 536 test_suite.Run(self._options, self._adb) 537 except errors.WaitForResponseTimedOutError: 538 logger.Log("Timed out waiting for response") 539 540 except KeyboardInterrupt: 541 logger.Log("Exiting...") 542 except errors.AbortError, error: 543 logger.Log(error.msg) 544 logger.SilentLog("Exiting due to AbortError...") 545 except errors.WaitForResponseTimedOutError: 546 logger.Log("Timed out waiting for response") 547 548 549 def RunTests(): 550 runner = TestRunner() 551 runner.RunTests() 552 553 if __name__ == "__main__": 554 RunTests() 555
