1 #!/usr/bin/env python 2 # 3 # Copyright 2008, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Command line utility for running Android tests 18 19 runtest helps automate the instructions for building and running tests 20 - It builds the corresponding test package for the code you want to test 21 - It pushes the test package to your device or emulator 22 - It launches InstrumentationTestRunner (or similar) to run the tests you 23 specify. 24 25 runtest supports running tests whose attributes have been pre-defined in 26 _TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27 system path to the test to run (runtest --path <path>). 28 29 Do runtest --help to see full list of options. 30 """ 31 32 # Python imports 33 import glob 34 import optparse 35 import os 36 import re 37 from sets import Set 38 import sys 39 import time 40 41 # local imports 42 import adb_interface 43 import android_build 44 from coverage import coverage 45 import errors 46 import logger 47 import make_tree 48 import run_command 49 from test_defs import test_defs 50 from test_defs import test_walker 51 52 53 class TestRunner(object): 54 """Command line utility class for running pre-defined Android test(s).""" 55 56 _TEST_FILE_NAME = "test_defs.xml" 57 58 # file path to android core platform tests, relative to android build root 59 # TODO move these test data files to another directory 60 _CORE_TEST_PATH = os.path.join("development", "testrunner", 61 _TEST_FILE_NAME) 62 63 # vendor glob file path patterns to tests, relative to android 64 # build root 65 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 66 _TEST_FILE_NAME) 67 68 _RUNTEST_USAGE = ( 69 "usage: runtest.py [options] short-test-name[s]\n\n" 70 "The runtest script works in two ways. You can query it " 71 "for a list of tests, or you can launch one or more tests.") 72 73 # default value for make -jX 74 _DEFAULT_JOBS = 16 75 76 _DALVIK_VERIFIER_OFF_PROP = "dalvik.vm.dexopt-flags = v=n" 77 78 # regular expression to match install: statements in make output 79 _RE_MAKE_INSTALL = re.compile(r'Install:\s(.+)') 80 81 # regular expression to find remote device path from a file path relative 82 # to build root 83 _RE_MAKE_INSTALL_PATH = re.compile(r'out\/target\/product\/\w+\/(.+)$') 84 85 def __init__(self): 86 # disable logging of timestamp 87 self._root_path = android_build.GetTop() 88 logger.SetTimestampLogging(False) 89 self._adb = None 90 self._known_tests = None 91 self._options = None 92 self._test_args = None 93 self._tests_to_run = None 94 95 def _ProcessOptions(self): 96 """Processes command-line options.""" 97 # TODO error messages on once-only or mutually-exclusive options. 98 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 99 self._TEST_FILE_NAME) 100 101 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 102 103 parser.add_option("-l", "--list-tests", dest="only_list_tests", 104 default=False, action="store_true", 105 help="To view the list of tests") 106 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 107 action="store_true", help="Skip build - just launch") 108 parser.add_option("-j", "--jobs", dest="make_jobs", 109 metavar="X", default=self._DEFAULT_JOBS, 110 help="Number of make jobs to use when building") 111 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 112 action="store_true", 113 help="Do not execute, just preview commands") 114 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 115 action="store_true", 116 help="Raw mode (for output to other tools)") 117 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 118 default=False, action="store_true", 119 help="Suite assignment (for details & usage see " 120 "InstrumentationTestRunner)") 121 parser.add_option("-v", "--verbose", dest="verbose", default=False, 122 action="store_true", 123 help="Increase verbosity of %s" % sys.argv[0]) 124 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 125 default=False, action="store_true", 126 help="Wait for debugger before launching tests") 127 parser.add_option("-c", "--test-class", dest="test_class", 128 help="Restrict test to a specific class") 129 parser.add_option("-m", "--test-method", dest="test_method", 130 help="Restrict test to a specific method") 131 parser.add_option("-p", "--test-package", dest="test_package", 132 help="Restrict test to a specific java package") 133 parser.add_option("-z", "--size", dest="test_size", 134 help="Restrict test to a specific test size") 135 parser.add_option("--annotation", dest="test_annotation", 136 help="Include only those tests tagged with a specific" 137 " annotation") 138 parser.add_option("--not-annotation", dest="test_not_annotation", 139 help="Exclude any tests tagged with a specific" 140 " annotation") 141 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 142 metavar="FILE", default=user_test_default, 143 help="Alternate source of user test definitions") 144 parser.add_option("-o", "--coverage", dest="coverage", 145 default=False, action="store_true", 146 help="Generate code coverage metrics for test(s)") 147 parser.add_option("--coverage-target", dest="coverage_target_path", 148 default=None, 149 help="Path to app to collect code coverage target data for.") 150 parser.add_option("-x", "--path", dest="test_path", 151 help="Run test(s) at given file system path") 152 parser.add_option("-t", "--all-tests", dest="all_tests", 153 default=False, action="store_true", 154 help="Run all defined tests") 155 parser.add_option("--continuous", dest="continuous_tests", 156 default=False, action="store_true", 157 help="Run all tests defined as part of the continuous " 158 "test set") 159 parser.add_option("--timeout", dest="timeout", 160 default=300, help="Set a timeout limit (in sec) for " 161 "running native tests on a device (default: 300 secs)") 162 parser.add_option("--suite", dest="suite", 163 help="Run all tests defined as part of the " 164 "the given test suite") 165 group = optparse.OptionGroup( 166 parser, "Targets", "Use these options to direct tests to a specific " 167 "Android target") 168 group.add_option("-e", "--emulator", dest="emulator", default=False, 169 action="store_true", help="use emulator") 170 group.add_option("-d", "--device", dest="device", default=False, 171 action="store_true", help="use device") 172 group.add_option("-s", "--serial", dest="serial", 173 help="use specific serial") 174 parser.add_option_group(group) 175 self._options, self._test_args = parser.parse_args() 176 177 if (not self._options.only_list_tests 178 and not self._options.all_tests 179 and not self._options.continuous_tests 180 and not self._options.suite 181 and not self._options.test_path 182 and len(self._test_args) < 1): 183 parser.print_help() 184 logger.SilentLog("at least one test name must be specified") 185 raise errors.AbortError 186 187 self._adb = adb_interface.AdbInterface() 188 if self._options.emulator: 189 self._adb.SetEmulatorTarget() 190 elif self._options.device: 191 self._adb.SetDeviceTarget() 192 elif self._options.serial is not None: 193 self._adb.SetTargetSerial(self._options.serial) 194 195 if self._options.verbose: 196 logger.SetVerbose(True) 197 198 if self._options.coverage_target_path: 199 self._options.coverage = True 200 201 self._known_tests = self._ReadTests() 202 203 self._options.host_lib_path = android_build.GetHostLibraryPath() 204 self._options.test_data_path = android_build.GetTestAppPath() 205 206 def _ReadTests(self): 207 """Parses the set of test definition data. 208 209 Returns: 210 A TestDefinitions object that contains the set of parsed tests. 211 Raises: 212 AbortError: If a fatal error occurred when parsing the tests. 213 """ 214 try: 215 known_tests = test_defs.TestDefinitions() 216 # only read tests when not in path mode 217 if not self._options.test_path: 218 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 219 if os.path.isfile(core_test_path): 220 known_tests.Parse(core_test_path) 221 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 222 vendor_tests_pattern = os.path.join(self._root_path, 223 self._VENDOR_TEST_PATH) 224 test_file_paths = glob.glob(vendor_tests_pattern) 225 for test_file_path in test_file_paths: 226 known_tests.Parse(test_file_path) 227 if os.path.isfile(self._options.user_tests_file): 228 known_tests.Parse(self._options.user_tests_file) 229 return known_tests 230 except errors.ParseError: 231 raise errors.AbortError 232 233 def _DumpTests(self): 234 """Prints out set of defined tests.""" 235 print "The following tests are currently defined:\n" 236 print "%-25s %-40s %s" % ("name", "build path", "description") 237 print "-" * 80 238 for test in self._known_tests: 239 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 240 test.GetDescription()) 241 print "\nSee %s for more information" % self._TEST_FILE_NAME 242 243 def _DoBuild(self): 244 logger.SilentLog("Building tests...") 245 246 tests = self._GetTestsToRun() 247 # turn off dalvik verifier if necessary 248 self._TurnOffVerifier(tests) 249 self._DoFullBuild(tests) 250 251 target_tree = make_tree.MakeTree() 252 253 extra_args_set = [] 254 for test_suite in tests: 255 self._AddBuildTarget(test_suite, target_tree, extra_args_set) 256 257 if not self._options.preview: 258 self._adb.EnableAdbRoot() 259 else: 260 logger.Log("adb root") 261 262 if not target_tree.IsEmpty(): 263 if self._options.coverage: 264 coverage.EnableCoverageBuild() 265 target_tree.AddPath("external/emma") 266 267 target_list = target_tree.GetPrunedMakeList() 268 target_build_string = " ".join(target_list) 269 extra_args_string = " ".join(extra_args_set) 270 271 # mmm cannot be used from python, so perform a similar operation using 272 # ONE_SHOT_MAKEFILE 273 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" all_modules %s' % ( 274 target_build_string, self._options.make_jobs, self._root_path, 275 extra_args_string) 276 logger.Log(cmd) 277 if not self._options.preview: 278 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 279 self._DoInstall(output) 280 281 def _DoInstall(self, make_output): 282 """Install artifacts from build onto device. 283 284 Looks for 'install:' text from make output to find artifacts to install. 285 286 Args: 287 make_output: stdout from make command 288 """ 289 for line in make_output.split("\n"): 290 m = self._RE_MAKE_INSTALL.match(line) 291 if m: 292 install_path = m.group(1) 293 if install_path.endswith(".apk"): 294 abs_install_path = os.path.join(self._root_path, install_path) 295 logger.Log("adb install -r %s" % abs_install_path) 296 logger.Log(self._adb.Install(abs_install_path)) 297 else: 298 self._PushInstallFileToDevice(install_path) 299 300 def _PushInstallFileToDevice(self, install_path): 301 m = self._RE_MAKE_INSTALL_PATH.match(install_path) 302 if m: 303 remote_path = m.group(1) 304 abs_install_path = os.path.join(self._root_path, install_path) 305 logger.Log("adb push %s %s" % (abs_install_path, remote_path)) 306 self._adb.Push(abs_install_path, remote_path) 307 else: 308 logger.Log("Error: Failed to recognize path of file to install %s" % install_path) 309 310 def _DoFullBuild(self, tests): 311 """If necessary, run a full 'make' command for the tests that need it.""" 312 extra_args_set = Set() 313 314 # hack to build cts dependencies 315 # TODO: remove this when cts dependencies are removed 316 if self._IsCtsTests(tests): 317 # need to use make since these fail building with ONE_SHOT_MAKEFILE 318 extra_args_set.add('CtsTestStubs') 319 extra_args_set.add('android.core.tests.runner') 320 for test in tests: 321 if test.IsFullMake(): 322 if test.GetExtraBuildArgs(): 323 # extra args contains the args to pass to 'make' 324 extra_args_set.add(test.GetExtraBuildArgs()) 325 else: 326 logger.Log("Warning: test %s needs a full build but does not specify" 327 " extra_build_args" % test.GetName()) 328 329 # check if there is actually any tests that required a full build 330 if extra_args_set: 331 cmd = ('make -j%s %s' % (self._options.make_jobs, 332 ' '.join(list(extra_args_set)))) 333 logger.Log(cmd) 334 if not self._options.preview: 335 old_dir = os.getcwd() 336 os.chdir(self._root_path) 337 output = run_command.RunCommand(cmd, return_output=True) 338 os.chdir(old_dir) 339 self._DoInstall(output) 340 341 def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): 342 if not test_suite.IsFullMake(): 343 build_dir = test_suite.GetBuildPath() 344 if self._AddBuildTargetPath(build_dir, target_tree): 345 extra_args_set.append(test_suite.GetExtraBuildArgs()) 346 for path in test_suite.GetBuildDependencies(self._options): 347 self._AddBuildTargetPath(path, target_tree) 348 349 def _AddBuildTargetPath(self, build_dir, target_tree): 350 if build_dir is not None: 351 target_tree.AddPath(build_dir) 352 return True 353 return False 354 355 def _GetTestsToRun(self): 356 """Get a list of TestSuite objects to run, based on command line args.""" 357 if self._tests_to_run: 358 return self._tests_to_run 359 360 self._tests_to_run = [] 361 if self._options.all_tests: 362 self._tests_to_run = self._known_tests.GetTests() 363 elif self._options.continuous_tests: 364 self._tests_to_run = self._known_tests.GetContinuousTests() 365 elif self._options.suite: 366 self._tests_to_run = \ 367 self._known_tests.GetTestsInSuite(self._options.suite) 368 elif self._options.test_path: 369 walker = test_walker.TestWalker() 370 self._tests_to_run = walker.FindTests(self._options.test_path) 371 372 for name in self._test_args: 373 test = self._known_tests.GetTest(name) 374 if test is None: 375 logger.Log("Error: Could not find test %s" % name) 376 self._DumpTests() 377 raise errors.AbortError 378 self._tests_to_run.append(test) 379 return self._tests_to_run 380 381 def _IsCtsTests(self, test_list): 382 """Check if any cts tests are included in given list of tests to run.""" 383 for test in test_list: 384 if test.GetSuite() == 'cts': 385 return True 386 return False 387 388 def _TurnOffVerifier(self, test_list): 389 """Turn off the dalvik verifier if needed by given tests. 390 391 If one or more tests needs dalvik verifier off, and it is not already off, 392 turns off verifier and reboots device to allow change to take effect. 393 """ 394 # hack to check if these are frameworks/base tests. If so, turn off verifier 395 # to allow framework tests to access package-private framework api 396 framework_test = False 397 for test in test_list: 398 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 399 framework_test = True 400 if framework_test: 401 # check if verifier is off already - to avoid the reboot if not 402 # necessary 403 output = self._adb.SendShellCommand("cat /data/local.prop") 404 if not self._DALVIK_VERIFIER_OFF_PROP in output: 405 if self._options.preview: 406 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 407 % self._DALVIK_VERIFIER_OFF_PROP) 408 logger.Log("adb shell chmod 644 /data/local.prop") 409 logger.Log("adb reboot") 410 logger.Log("adb wait-for-device") 411 else: 412 logger.Log("Turning off dalvik verifier and rebooting") 413 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 414 % self._DALVIK_VERIFIER_OFF_PROP) 415 416 self._ChmodReboot() 417 elif not self._options.preview: 418 # check the permissions on the file 419 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 420 if not "-rw-r--r--" in permout: 421 logger.Log("Fixing permissions on /data/local.prop and rebooting") 422 self._ChmodReboot() 423 424 def _ChmodReboot(self): 425 """Perform a chmod of /data/local.prop and reboot. 426 """ 427 self._adb.SendShellCommand("chmod 644 /data/local.prop") 428 self._adb.SendCommand("reboot") 429 # wait for device to go offline 430 time.sleep(10) 431 self._adb.SendCommand("wait-for-device", timeout_time=60, 432 retry_count=3) 433 self._adb.EnableAdbRoot() 434 435 436 def RunTests(self): 437 """Main entry method - executes the tests according to command line args.""" 438 try: 439 run_command.SetAbortOnError() 440 self._ProcessOptions() 441 if self._options.only_list_tests: 442 self._DumpTests() 443 return 444 445 if not self._options.skip_build: 446 self._DoBuild() 447 448 for test_suite in self._GetTestsToRun(): 449 try: 450 test_suite.Run(self._options, self._adb) 451 except errors.WaitForResponseTimedOutError: 452 logger.Log("Timed out waiting for response") 453 454 except KeyboardInterrupt: 455 logger.Log("Exiting...") 456 except errors.AbortError, error: 457 logger.Log(error.msg) 458 logger.SilentLog("Exiting due to AbortError...") 459 except errors.WaitForResponseTimedOutError: 460 logger.Log("Timed out waiting for response") 461 462 463 def RunTests(): 464 runner = TestRunner() 465 runner.RunTests() 466 467 if __name__ == "__main__": 468 RunTests() 469
