1 #!/usr/bin/python2.4 2 # 3 # Copyright 2008, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Command line utility for running Android tests 18 19 runtest helps automate the instructions for building and running tests 20 - It builds the corresponding test package for the code you want to test 21 - It pushes the test package to your device or emulator 22 - It launches InstrumentationTestRunner (or similar) to run the tests you 23 specify. 24 25 runtest supports running tests whose attributes have been pre-defined in 26 _TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27 system path to the test to run (runtest --path <path>). 28 29 Do runtest --help to see full list of options. 30 """ 31 32 # Python imports 33 import glob 34 import optparse 35 import os 36 from sets import Set 37 import sys 38 39 # local imports 40 import adb_interface 41 import android_build 42 import coverage 43 import errors 44 import logger 45 import run_command 46 from test_defs import test_defs 47 from test_defs import test_walker 48 49 50 class TestRunner(object): 51 """Command line utility class for running pre-defined Android test(s).""" 52 53 _TEST_FILE_NAME = "test_defs.xml" 54 55 # file path to android core platform tests, relative to android build root 56 # TODO move these test data files to another directory 57 _CORE_TEST_PATH = os.path.join("development", "testrunner", 58 _TEST_FILE_NAME) 59 60 # vendor glob file path patterns to tests, relative to android 61 # build root 62 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 63 _TEST_FILE_NAME) 64 65 _RUNTEST_USAGE = ( 66 "usage: runtest.py [options] short-test-name[s]\n\n" 67 "The runtest script works in two ways. You can query it " 68 "for a list of tests, or you can launch one or more tests.") 69 70 # default value for make -jX 71 _DEFAULT_JOBS = 4 72 73 _DALVIK_VERIFIER_OFF_PROP = "dalvik.vm.dexopt-flags = v=n" 74 75 def __init__(self): 76 # disable logging of timestamp 77 self._root_path = android_build.GetTop() 78 logger.SetTimestampLogging(False) 79 self._adb = None 80 self._known_tests = None 81 self._options = None 82 self._test_args = None 83 self._tests_to_run = None 84 85 def _ProcessOptions(self): 86 """Processes command-line options.""" 87 # TODO error messages on once-only or mutually-exclusive options. 88 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 89 self._TEST_FILE_NAME) 90 91 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 92 93 parser.add_option("-l", "--list-tests", dest="only_list_tests", 94 default=False, action="store_true", 95 help="To view the list of tests") 96 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 97 action="store_true", help="Skip build - just launch") 98 parser.add_option("-j", "--jobs", dest="make_jobs", 99 metavar="X", default=self._DEFAULT_JOBS, 100 help="Number of make jobs to use when building") 101 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 102 action="store_true", 103 help="Do not execute, just preview commands") 104 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 105 action="store_true", 106 help="Raw mode (for output to other tools)") 107 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 108 default=False, action="store_true", 109 help="Suite assignment (for details & usage see " 110 "InstrumentationTestRunner)") 111 parser.add_option("-v", "--verbose", dest="verbose", default=False, 112 action="store_true", 113 help="Increase verbosity of %s" % sys.argv[0]) 114 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 115 default=False, action="store_true", 116 help="Wait for debugger before launching tests") 117 parser.add_option("-c", "--test-class", dest="test_class", 118 help="Restrict test to a specific class") 119 parser.add_option("-m", "--test-method", dest="test_method", 120 help="Restrict test to a specific method") 121 parser.add_option("-p", "--test-package", dest="test_package", 122 help="Restrict test to a specific java package") 123 parser.add_option("-z", "--size", dest="test_size", 124 help="Restrict test to a specific test size") 125 parser.add_option("--annotation", dest="test_annotation", 126 help="Include only those tests tagged with a specific" 127 " annotation") 128 parser.add_option("--not-annotation", dest="test_not_annotation", 129 help="Exclude any tests tagged with a specific" 130 " annotation") 131 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 132 metavar="FILE", default=user_test_default, 133 help="Alternate source of user test definitions") 134 parser.add_option("-o", "--coverage", dest="coverage", 135 default=False, action="store_true", 136 help="Generate code coverage metrics for test(s)") 137 parser.add_option("-x", "--path", dest="test_path", 138 help="Run test(s) at given file system path") 139 parser.add_option("-t", "--all-tests", dest="all_tests", 140 default=False, action="store_true", 141 help="Run all defined tests") 142 parser.add_option("--continuous", dest="continuous_tests", 143 default=False, action="store_true", 144 help="Run all tests defined as part of the continuous " 145 "test set") 146 parser.add_option("--timeout", dest="timeout", 147 default=300, help="Set a timeout limit (in sec) for " 148 "running native tests on a device (default: 300 secs)") 149 parser.add_option("--suite", dest="suite", 150 help="Run all tests defined as part of the " 151 "the given test suite") 152 group = optparse.OptionGroup( 153 parser, "Targets", "Use these options to direct tests to a specific " 154 "Android target") 155 group.add_option("-e", "--emulator", dest="emulator", default=False, 156 action="store_true", help="use emulator") 157 group.add_option("-d", "--device", dest="device", default=False, 158 action="store_true", help="use device") 159 group.add_option("-s", "--serial", dest="serial", 160 help="use specific serial") 161 parser.add_option_group(group) 162 self._options, self._test_args = parser.parse_args() 163 164 if (not self._options.only_list_tests 165 and not self._options.all_tests 166 and not self._options.continuous_tests 167 and not self._options.suite 168 and not self._options.test_path 169 and len(self._test_args) < 1): 170 parser.print_help() 171 logger.SilentLog("at least one test name must be specified") 172 raise errors.AbortError 173 174 self._adb = adb_interface.AdbInterface() 175 if self._options.emulator: 176 self._adb.SetEmulatorTarget() 177 elif self._options.device: 178 self._adb.SetDeviceTarget() 179 elif self._options.serial is not None: 180 self._adb.SetTargetSerial(self._options.serial) 181 182 if self._options.verbose: 183 logger.SetVerbose(True) 184 185 self._known_tests = self._ReadTests() 186 187 self._options.host_lib_path = android_build.GetHostLibraryPath() 188 self._options.test_data_path = android_build.GetTestAppPath() 189 190 def _ReadTests(self): 191 """Parses the set of test definition data. 192 193 Returns: 194 A TestDefinitions object that contains the set of parsed tests. 195 Raises: 196 AbortError: If a fatal error occurred when parsing the tests. 197 """ 198 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 199 try: 200 known_tests = test_defs.TestDefinitions() 201 known_tests.Parse(core_test_path) 202 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 203 vendor_tests_pattern = os.path.join(self._root_path, 204 self._VENDOR_TEST_PATH) 205 test_file_paths = glob.glob(vendor_tests_pattern) 206 for test_file_path in test_file_paths: 207 known_tests.Parse(test_file_path) 208 if os.path.isfile(self._options.user_tests_file): 209 known_tests.Parse(self._options.user_tests_file) 210 return known_tests 211 except errors.ParseError: 212 raise errors.AbortError 213 214 def _DumpTests(self): 215 """Prints out set of defined tests.""" 216 print "The following tests are currently defined:\n" 217 print "%-25s %-40s %s" % ("name", "build path", "description") 218 print "-" * 80 219 for test in self._known_tests: 220 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 221 test.GetDescription()) 222 print "\nSee %s for more information" % self._TEST_FILE_NAME 223 224 def _DoBuild(self): 225 logger.SilentLog("Building tests...") 226 target_set = Set() 227 extra_args_set = Set() 228 tests = self._GetTestsToRun() 229 for test_suite in tests: 230 self._AddBuildTarget(test_suite, target_set, extra_args_set) 231 232 if not self._options.preview: 233 self._adb.EnableAdbRoot() 234 else: 235 logger.Log("adb root") 236 rebuild_libcore = False 237 if target_set: 238 if self._options.coverage: 239 coverage.EnableCoverageBuild() 240 # hack to remove core library intermediates 241 # hack is needed because: 242 # 1. EMMA_INSTRUMENT changes what source files to include in libcore 243 # but it does not trigger a rebuild 244 # 2. there's no target (like "clear-intermediates") to remove the files 245 # decently 246 rebuild_libcore = not coverage.TestDeviceCoverageSupport(self._adb) 247 if rebuild_libcore: 248 cmd = "rm -rf %s" % os.path.join( 249 self._root_path, 250 "out/target/common/obj/JAVA_LIBRARIES/core_intermediates/") 251 logger.Log(cmd) 252 run_command.RunCommand(cmd, return_output=False) 253 254 # hack to build cts dependencies 255 # TODO: remove this when build dependency support added to runtest or 256 # cts dependencies are removed 257 if self._IsCtsTests(tests): 258 # need to use make since these fail building with ONE_SHOT_MAKEFILE 259 cmd = ('make -j%s CtsTestStubs android.core.tests.runner' % 260 self._options.make_jobs) 261 logger.Log(cmd) 262 if not self._options.preview: 263 old_dir = os.getcwd() 264 os.chdir(self._root_path) 265 run_command.RunCommand(cmd, return_output=False) 266 os.chdir(old_dir) 267 # turn off dalvik verifier if necessary 268 self._TurnOffVerifier(tests) 269 target_build_string = " ".join(list(target_set)) 270 extra_args_string = " ".join(list(extra_args_set)) 271 # mmm cannot be used from python, so perform a similar operation using 272 # ONE_SHOT_MAKEFILE 273 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" files %s' % ( 274 target_build_string, self._options.make_jobs, self._root_path, 275 extra_args_string) 276 logger.Log(cmd) 277 278 if self._options.preview: 279 # in preview mode, just display to the user what command would have been 280 # run 281 logger.Log("adb sync") 282 else: 283 # set timeout for build to 10 minutes, since libcore may need to 284 # be rebuilt 285 run_command.RunCommand(cmd, return_output=False, timeout_time=600) 286 logger.Log("Syncing to device...") 287 self._adb.Sync(runtime_restart=rebuild_libcore) 288 289 def _AddBuildTarget(self, test_suite, target_set, extra_args_set): 290 build_dir = test_suite.GetBuildPath() 291 if self._AddBuildTargetPath(build_dir, target_set): 292 extra_args_set.add(test_suite.GetExtraBuildArgs()) 293 for path in test_suite.GetBuildDependencies(self._options): 294 self._AddBuildTargetPath(path, target_set) 295 296 def _AddBuildTargetPath(self, build_dir, target_set): 297 if build_dir is not None: 298 build_file_path = os.path.join(build_dir, "Android.mk") 299 if os.path.isfile(os.path.join(self._root_path, build_file_path)): 300 target_set.add(build_file_path) 301 return True 302 else: 303 logger.Log("%s has no Android.mk, skipping" % build_dir) 304 return False 305 306 def _GetTestsToRun(self): 307 """Get a list of TestSuite objects to run, based on command line args.""" 308 if self._tests_to_run: 309 return self._tests_to_run 310 311 self._tests_to_run = [] 312 if self._options.all_tests: 313 self._tests_to_run = self._known_tests.GetTests() 314 elif self._options.continuous_tests: 315 self._tests_to_run = self._known_tests.GetContinuousTests() 316 elif self._options.suite: 317 self._tests_to_run = \ 318 self._known_tests.GetTestsInSuite(self._options.suite) 319 elif self._options.test_path: 320 walker = test_walker.TestWalker() 321 self._tests_to_run = walker.FindTests(self._options.test_path) 322 323 for name in self._test_args: 324 test = self._known_tests.GetTest(name) 325 if test is None: 326 logger.Log("Error: Could not find test %s" % name) 327 self._DumpTests() 328 raise errors.AbortError 329 self._tests_to_run.append(test) 330 return self._tests_to_run 331 332 def _IsCtsTests(self, test_list): 333 """Check if any cts tests are included in given list of tests to run.""" 334 for test in test_list: 335 if test.GetSuite() == 'cts': 336 return True 337 return False 338 339 def _TurnOffVerifier(self, test_list): 340 """Turn off the dalvik verifier if needed by given tests. 341 342 If one or more tests needs dalvik verifier off, and it is not already off, 343 turns off verifier and reboots device to allow change to take effect. 344 """ 345 # hack to check if these are framework/base tests. If so, turn off verifier 346 # to allow framework tests to access package-private framework api 347 framework_test = False 348 for test in test_list: 349 if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]): 350 framework_test = True 351 if framework_test: 352 # check if verifier is off already - to avoid the reboot if not 353 # necessary 354 output = self._adb.SendShellCommand("cat /data/local.prop") 355 if not self._DALVIK_VERIFIER_OFF_PROP in output: 356 if self._options.preview: 357 logger.Log("adb shell \"echo %s >> /data/local.prop\"" 358 % self._DALVIK_VERIFIER_OFF_PROP) 359 logger.Log("adb reboot") 360 logger.Log("adb wait-for-device") 361 else: 362 logger.Log("Turning off dalvik verifier and rebooting") 363 self._adb.SendShellCommand("\"echo %s >> /data/local.prop\"" 364 % self._DALVIK_VERIFIER_OFF_PROP) 365 self._adb.SendCommand("reboot") 366 self._adb.SendCommand("wait-for-device", timeout_time=60, 367 retry_count=3) 368 369 def RunTests(self): 370 """Main entry method - executes the tests according to command line args.""" 371 try: 372 run_command.SetAbortOnError() 373 self._ProcessOptions() 374 if self._options.only_list_tests: 375 self._DumpTests() 376 return 377 378 if not self._options.skip_build: 379 self._DoBuild() 380 381 for test_suite in self._GetTestsToRun(): 382 try: 383 test_suite.Run(self._options, self._adb) 384 except errors.WaitForResponseTimedOutError: 385 logger.Log("Timed out waiting for response") 386 387 except KeyboardInterrupt: 388 logger.Log("Exiting...") 389 except errors.AbortError, error: 390 logger.Log(error.msg) 391 logger.SilentLog("Exiting due to AbortError...") 392 except errors.WaitForResponseTimedOutError: 393 logger.Log("Timed out waiting for response") 394 395 396 def RunTests(): 397 runner = TestRunner() 398 runner.RunTests() 399 400 if __name__ == "__main__": 401 RunTests() 402
