1 #!/usr/bin/env python 2 # 3 # Copyright 2008, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Command line utility for running Android tests 18 19 runtest helps automate the instructions for building and running tests 20 - It builds the corresponding test package for the code you want to test 21 - It pushes the test package to your device or emulator 22 - It launches InstrumentationTestRunner (or similar) to run the tests you 23 specify. 24 25 runtest supports running tests whose attributes have been pre-defined in 26 _TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27 system path to the test to run (runtest --path <path>). 28 29 Do runtest --help to see full list of options. 30 """ 31 32 # Python imports 33 import glob 34 import optparse 35 import os 36 import re 37 from sets import Set 38 import sys 39 import time 40 41 # local imports 42 import adb_interface 43 import android_build 44 from coverage import coverage 45 import errors 46 import logger 47 import make_tree 48 import run_command 49 from test_defs import test_defs 50 from test_defs import test_walker 51 52 53 class TestRunner(object): 54 """Command line utility class for running pre-defined Android test(s).""" 55 56 _TEST_FILE_NAME = "test_defs.xml" 57 58 # file path to android core platform tests, relative to android build root 59 # TODO move these test data files to another directory 60 _CORE_TEST_PATH = os.path.join("development", "testrunner", 61 _TEST_FILE_NAME) 62 63 # vendor glob file path patterns to tests, relative to android 64 # build root 65 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 66 _TEST_FILE_NAME) 67 68 _RUNTEST_USAGE = ( 69 "usage: runtest.py [options] short-test-name[s]\n\n" 70 "The runtest script works in two ways. You can query it " 71 "for a list of tests, or you can launch one or more tests.") 72 73 # default value for make -jX 74 _DEFAULT_JOBS = 16 75 76 _DALVIK_VERIFIER_OFF_PROP = "dalvik.vm.dexopt-flags = v=n" 77 78 # regular expression to match path to artifacts to install in make output 79 _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s(.+)\s(.+)') 80 81 82 def __init__(self): 83 # disable logging of timestamp 84 self._root_path = android_build.GetTop() 85 out_base_name = os.path.basename(android_build.GetOutDir()) 86 # regular expression to find remote device path from a file path relative 87 # to build root 88 pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$' 89 self._re_make_install_path = re.compile(pattern) 90 logger.SetTimestampLogging(False) 91 self._adb = None 92 self._known_tests = None 93 self._options = None 94 self._test_args = None 95 self._tests_to_run = None 96 97 def _ProcessOptions(self): 98 """Processes command-line options.""" 99 # TODO error messages on once-only or mutually-exclusive options. 100 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 101 self._TEST_FILE_NAME) 102 103 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 104 105 parser.add_option("-l", "--list-tests", dest="only_list_tests", 106 default=False, action="store_true", 107 help="To view the list of tests") 108 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 109 action="store_true", help="Skip build - just launch") 110 parser.add_option("-j", "--jobs", dest="make_jobs", 111 metavar="X", default=self._DEFAULT_JOBS, 112 help="Number of make jobs to use when building") 113 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 114 action="store_true", 115 help="Do not execute, just preview commands") 116 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 117 action="store_true", 118 help="Raw mode (for output to other tools)") 119 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 120 default=False, action="store_true", 121 help="Suite assignment (for details & usage see " 122 "InstrumentationTestRunner)") 123 parser.add_option("-v", "--verbose", dest="verbose", default=False, 124 action="store_true", 125 help="Increase verbosity of %s" % sys.argv[0]) 126 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 127 default=False, action="store_true", 128 help="Wait for debugger before launching tests") 129 parser.add_option("-c", "--test-class", dest="test_class", 130 help="Restrict test to a specific class") 131 parser.add_option("-m", "--test-method", dest="test_method", 132 help="Restrict test to a specific method") 133 parser.add_option("-p", "--test-package", dest="test_package", 134 help="Restrict test to a specific java package") 135 parser.add_option("-z", "--size", dest="test_size", 136 help="Restrict test to a specific test size") 137 parser.add_option("--annotation", dest="test_annotation", 138 help="Include only those tests tagged with a specific" 139 " annotation") 140 parser.add_option("--not-annotation", dest="test_not_annotation", 141 help="Exclude any tests tagged with a specific" 142 " annotation") 143 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 144 metavar="FILE", default=user_test_default, 145 help="Alternate source of user test definitions") 146 parser.add_option("-o", "--coverage", dest="coverage", 147 default=False, action="store_true", 148 help="Generate code coverage metrics for test(s)") 149 parser.add_option("--coverage-target", dest="coverage_target_path", 150 default=None, 151 help="Path to app to collect code coverage target data for.") 152 parser.add_option("-x", "--path", dest="test_path", 153 help="Run test(s) at given file system path") 154 parser.add_option("-t", "--all-tests", dest="all_tests", 155 default=False, action="store_true", 156 help="Run all defined tests") 157 parser.add_option("--continuous", dest="continuous_tests", 158 default=False, action="store_true", 159 help="Run all tests defined as part of the continuous " 160 "test set") 161 parser.add_option("--timeout", dest="timeout", 162 default=300, help="Set a timeout limit (in sec) for " 163 "running native tests on a device (default: 300 secs)") 164 parser.add_option("--suite", dest="suite", 165 help="Run all tests defined as part of the " 166 "the given test suite") 167 group = optparse.OptionGroup( 168 parser, "Targets", "Use these options to direct tests to a specific " 169 "Android target") 170 group.add_option("-e", "--emulator", dest="emulator", default=False, 171 action="store_true", help="use emulator") 172 group.add_option("-d", "--device", dest="device", default=False, 173 action="store_true", help="use device") 174 group.add_option("-s", "--serial", dest="serial", 175 help="use specific serial") 176 parser.add_option_group(group) 177 self._options, self._test_args = parser.parse_args() 178 179 if (not self._options.only_list_tests 180 and not self._options.all_tests 181 and not self._options.continuous_tests 182 and not self._options.suite 183 and not self._options.test_path 184 and len(self._test_args) < 1): 185 parser.print_help() 186 logger.SilentLog("at least one test name must be specified") 187 raise errors.AbortError 188 189 self._adb = adb_interface.AdbInterface() 190 if self._options.emulator: 191 self._adb.SetEmulatorTarget() 192 elif self._options.device: 193 self._adb.SetDeviceTarget() 194 elif self._options.serial is not None: 195 self._adb.SetTargetSerial(self._options.serial) 196 197 if self._options.verbose: 198 logger.SetVerbose(True) 199 200 if self._options.coverage_target_path: 201 self._options.coverage = True 202 203 self._known_tests = self._ReadTests() 204 205 self._options.host_lib_path = android_build.GetHostLibraryPath() 206 self._options.test_data_path = android_build.GetTestAppPath() 207 208 def _ReadTests(self): 209 """Parses the set of test definition data. 210 211 Returns: 212 A TestDefinitions object that contains the set of parsed tests. 213 Raises: 214 AbortError: If a fatal error occurred when parsing the tests. 215 """ 216 try: 217 known_tests = test_defs.TestDefinitions() 218 # only read tests when not in path mode 219 if not self._options.test_path: 220 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 221 if os.path.isfile(core_test_path): 222 known_tests.Parse(core_test_path) 223 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 224 vendor_tests_pattern = os.path.join(self._root_path, 225 self._VENDOR_TEST_PATH) 226 test_file_paths = glob.glob(vendor_tests_pattern) 227 for test_file_path in test_file_paths: 228 known_tests.Parse(test_file_path) 229 if os.path.isfile(self._options.user_tests_file): 230 known_tests.Parse(self._options.user_tests_file) 231 return known_tests 232 except errors.ParseError: 233 raise errors.AbortError 234 235 def _DumpTests(self): 236 """Prints out set of defined tests.""" 237 print "The following tests are currently defined:\n" 238 print "%-25s %-40s %s" % ("name", "build path", "description") 239 print "-" * 80 240 for test in self._known_tests: 241 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 242 test.GetDescription()) 243 print "\nSee %s for more information" % self._TEST_FILE_NAME 244 245 def _DoBuild(self): 246 logger.SilentLog("Building tests...") 247 248 tests = self._GetTestsToRun() 249 # turn off dalvik verifier if necessary 250 self._TurnOffVerifier(tests) 251 self._DoFullBuild(tests) 252 253 target_tree = make_tree.MakeTree() 254 255 extra_args_set = [] 256 for test_suite in tests: 257 self._AddBuildTarget(test_suite, target_tree, extra_args_set) 258 259 if not self._options.preview: 260 self._adb.EnableAdbRoot() 261 else: 262 logger.Log("adb root") 263 264 if not target_tree.IsEmpty(): 265 if self._options.coverage: 266 coverage.EnableCoverageBuild() 267 target_tree.AddPath("external/emma") 268 269 target_list = target_tree.GetPrunedMakeList() 270 target_build_string = " ".join(target_list) 271 extra_args_string = " ".join(extra_args_set) 272 273 # mmm cannot be used from python, so perform a similar operation using 274 # ONE_SHOT_MAKEFILE 275 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" GET-INSTALL-PATH all_modules %s' % ( 276 target_build_string, self._options.make_jobs, self._root_path, 277 extra_args_string) 278 logger.Log(cmd) 279 if not self._options.preview: 280 output = run_command.RunCommand(cmd, return_output=True, timeout_time=600) 281 logger.SilentLog(output) 282 self._DoInstall(output) 283 284 def _DoInstall(self, make_output): 285 """Install artifacts from build onto device. 286 287 Looks for 'install:' text from make output to find artifacts to install. 288 289 Args: 290 make_output: stdout from make command 291 """ 292 for line in make_output.split("\n"): 293 m = self._RE_MAKE_INSTALL.match(line) 294 if m: 295 install_path = m.group(2) 296 if install_path.endswith(".apk"): 297 abs_install_path = os.path.join(self._root_path, install_path) 298 logger.Log("adb install -r %s" % abs_install_path) 299 logger.Log(self._adb.Install(abs_install_path)) 300 else: 301 self._PushInstallFileToDevice(install_path) 302 303 def _PushInstallFileToDevice(self, install_path): 304 m = self._re_make_install_path.match(install_path) 305 if m: 306 remote_path = m.group(1) 307 remote_dir = os.path.dirname(remote_path) 308 logger.Log("adb shell mkdir -p %s" % remote_dir) 309 self._adb.SendShellCommand("mkdir -p %s" % remote_dir) 310 abs_install_path = os.path.join(self._root_path, install_path) 311 logger.Log("adb push %s %s" % (abs_install_path, remote_path)) 312 self._adb.Push(abs_install_path, remote_path) 313 else: 314 logger.Log("Error: Failed to recognize path of file to install %s" % install_path) 315 316 def _DoFullBuild(self, tests): 317 """If necessary, run a full 'make' command for the tests that need it.""" 318 extra_args_set = Set() 319 320 # hack to build cts dependencies 321 # TODO: remove this when cts dependencies are removed 322 is_cts = self._IsCtsTests(tests) 323 if is_cts: 324 # need to use make since these fail building with ONE_SHOT_MAKEFILE 325 extra_args_set.add('CtsTestStubs') 326 extra_args_set.add('android.core.tests.runner') 327 for test in tests: 328 if test.IsFullMake(): 329 if test.GetExtraBuildArgs(): 330 # extra args contains the args to pass to 'make' 331 extra_args_set.add(test.GetExtraBuildArgs()) 332 else: 333 logger.Log("Warning: test %s needs a full build but does not specify" 334 " extra_build_args" % test.GetName()) 335 336 # check if there is actually any tests that required a full build 337 if extra_args_set: 338 cmd = ('make -j%s %s' % (self._options.make_jobs, 339 ' '.join(list(extra_args_set)))) 340 logger.Log(cmd) 341 if not self._options.preview: 342 old_dir = os.getcwd() 343 os.chdir(self._root_path) 344 output = run_command.RunCommand(cmd, return_output=True) 345 logger.SilentLog(output) 346 os.chdir(old_dir) 347 self._DoInstall(output) 348 if is_cts: 349 # hack! hardcode install of CtsTestStubs 350 out = android_build.GetTestAppPath() 351 abs_install_path = os.path.join(out, "CtsTestStubs.apk") 352 logger.Log("adb install -r %s" % abs_install_path) 353 logger.Log(self._adb.Install(abs_install_path)) 354 355 def _AddBuildTarget(self, test_suite, target_tree, extra_args_set): 356 if not test_suite.IsFullMake(): 357 build_dir = test_suite.GetBuildPath() 358 if self._AddBuildTargetPath(build_dir, target_tree): 359 extra_args_set.append(test_suite.GetExtraBuildArgs()) 360 for path in test_suite.GetBuildDependencies(self._options): 361 self._AddBuildTargetPath(path, target_tree) 362 363 def _AddBuildTargetPath(self, build_dir, target_tree): 364 if build_dir is not None: 365 target_tree.AddPath(build_dir) 366 return True 367 return False 368 369 def _GetTestsToRun(self): 370 """Get a list of TestSuite objects to run, based on command line args.""" 371 if self._tests_to_run: 372 return self._tests_to_run 373 374 self._tests_to_run = [] 375 if self._options.all_tests: 376 self._tests_to_run = self._known_tests.GetTests() 377 elif self._options.continuous_tests: 378 self._tests_to_run = self._known_tests.GetContinuousTests() 379 elif self._options.suite: 380 self._tests_to_run = \ 381 self._known_tests.GetTestsInSuite(self._options.suite) 382 elif self._options.test_path: 383 walker = test_walker.TestWalker() 384 self._tests_to_run = walker.FindTests(self._options.test_path) 385 386 for name in self._test_args: 387 test = self._known_tests.GetTest(name) 388 if test is None: 389 logger.Log("Error: Could not find test %s" % name) 390 self._DumpTests() 391 raise errors.AbortError 392 self._tests_to_run.append(test) 393 return self._tests_to_run 394 395 def _IsCtsTests(self, test_list): 396 """Check if any cts tests are included in given list of tests to run.""" 397 for test in test_list: 398 if test.GetSuite() == 'cts': 399 return True 400 return False 401 402 def _TurnOffVerifier(self, test_list): 403 """Turn off the dalvik verifier if needed by given tests. 404 405 If one or more tests needs dalvik verifier off, and it is not already off, 406 turns off verifier and reboots device to allow change to take effect. 407 """ 408 # hack to check if these are frameworks/base tests. If so, turn off verifier 409 # to allow framework tests to access package-private framework api 410 framework_test = False 411 for test in test_list: 412 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 413 framework_test = True 414 if framework_test: 415 # check if verifier is off already - to avoid the reboot if not 416 # necessary 417 output = self._adb.SendShellCommand("cat /data/local.prop") 418 if not self._DALVIK_VERIFIER_OFF_PROP in output: 419 if self._options.preview: 420 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 421 % self._DALVIK_VERIFIER_OFF_PROP) 422 logger.Log("adb shell chmod 644 /data/local.prop") 423 logger.Log("adb reboot") 424 logger.Log("adb wait-for-device") 425 else: 426 logger.Log("Turning off dalvik verifier and rebooting") 427 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 428 % self._DALVIK_VERIFIER_OFF_PROP) 429 430 self._ChmodReboot() 431 elif not self._options.preview: 432 # check the permissions on the file 433 permout = self._adb.SendShellCommand("ls -l /data/local.prop") 434 if not "-rw-r--r--" in permout: 435 logger.Log("Fixing permissions on /data/local.prop and rebooting") 436 self._ChmodReboot() 437 438 def _ChmodReboot(self): 439 """Perform a chmod of /data/local.prop and reboot. 440 """ 441 self._adb.SendShellCommand("chmod 644 /data/local.prop") 442 self._adb.SendCommand("reboot") 443 # wait for device to go offline 444 time.sleep(10) 445 self._adb.SendCommand("wait-for-device", timeout_time=60, 446 retry_count=3) 447 self._adb.EnableAdbRoot() 448 449 450 def RunTests(self): 451 """Main entry method - executes the tests according to command line args.""" 452 try: 453 run_command.SetAbortOnError() 454 self._ProcessOptions() 455 if self._options.only_list_tests: 456 self._DumpTests() 457 return 458 459 if not self._options.skip_build: 460 self._DoBuild() 461 462 for test_suite in self._GetTestsToRun(): 463 try: 464 test_suite.Run(self._options, self._adb) 465 except errors.WaitForResponseTimedOutError: 466 logger.Log("Timed out waiting for response") 467 468 except KeyboardInterrupt: 469 logger.Log("Exiting...") 470 except errors.AbortError, error: 471 logger.Log(error.msg) 472 logger.SilentLog("Exiting due to AbortError...") 473 except errors.WaitForResponseTimedOutError: 474 logger.Log("Timed out waiting for response") 475 476 477 def RunTests(): 478 runner = TestRunner() 479 runner.RunTests() 480 481 if __name__ == "__main__": 482 RunTests() 483
