1 #!/usr/bin/python2.4 2 # 3 # Copyright 2008, The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Command line utility for running Android tests 18 19 runtest helps automate the instructions for building and running tests 20 - It builds the corresponding test package for the code you want to test 21 - It pushes the test package to your device or emulator 22 - It launches InstrumentationTestRunner (or similar) to run the tests you 23 specify. 24 25 runtest supports running tests whose attributes have been pre-defined in 26 _TEST_FILE_NAME files, (runtest <testname>), or by specifying the file 27 system path to the test to run (runtest --path <path>). 28 29 Do runtest --help to see full list of options. 30 """ 31 32 # Python imports 33 import glob 34 import optparse 35 import os 36 from sets import Set 37 import sys 38 39 # local imports 40 import adb_interface 41 import android_build 42 import coverage 43 import errors 44 import logger 45 import run_command 46 from test_defs import test_defs 47 from test_defs import test_walker 48 49 50 class TestRunner(object): 51 """Command line utility class for running pre-defined Android test(s).""" 52 53 _TEST_FILE_NAME = "test_defs.xml" 54 55 # file path to android core platform tests, relative to android build root 56 # TODO move these test data files to another directory 57 _CORE_TEST_PATH = os.path.join("development", "testrunner", 58 _TEST_FILE_NAME) 59 60 # vendor glob file path patterns to tests, relative to android 61 # build root 62 _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo", 63 _TEST_FILE_NAME) 64 65 _RUNTEST_USAGE = ( 66 "usage: runtest.py [options] short-test-name[s]\n\n" 67 "The runtest script works in two ways. You can query it " 68 "for a list of tests, or you can launch one or more tests.") 69 70 # default value for make -jX 71 _DEFAULT_JOBS = 4 72 73 def __init__(self): 74 # disable logging of timestamp 75 self._root_path = android_build.GetTop() 76 logger.SetTimestampLogging(False) 77 self._adb = None 78 self._known_tests = None 79 self._options = None 80 self._test_args = None 81 self._tests_to_run = None 82 83 def _ProcessOptions(self): 84 """Processes command-line options.""" 85 # TODO error messages on once-only or mutually-exclusive options. 86 user_test_default = os.path.join(os.environ.get("HOME"), ".android", 87 self._TEST_FILE_NAME) 88 89 parser = optparse.OptionParser(usage=self._RUNTEST_USAGE) 90 91 parser.add_option("-l", "--list-tests", dest="only_list_tests", 92 default=False, action="store_true", 93 help="To view the list of tests") 94 parser.add_option("-b", "--skip-build", dest="skip_build", default=False, 95 action="store_true", help="Skip build - just launch") 96 parser.add_option("-j", "--jobs", dest="make_jobs", 97 metavar="X", default=self._DEFAULT_JOBS, 98 help="Number of make jobs to use when building") 99 parser.add_option("-n", "--skip_execute", dest="preview", default=False, 100 action="store_true", 101 help="Do not execute, just preview commands") 102 parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False, 103 action="store_true", 104 help="Raw mode (for output to other tools)") 105 parser.add_option("-a", "--suite-assign", dest="suite_assign_mode", 106 default=False, action="store_true", 107 help="Suite assignment (for details & usage see " 108 "InstrumentationTestRunner)") 109 parser.add_option("-v", "--verbose", dest="verbose", default=False, 110 action="store_true", 111 help="Increase verbosity of %s" % sys.argv[0]) 112 parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger", 113 default=False, action="store_true", 114 help="Wait for debugger before launching tests") 115 parser.add_option("-c", "--test-class", dest="test_class", 116 help="Restrict test to a specific class") 117 parser.add_option("-m", "--test-method", dest="test_method", 118 help="Restrict test to a specific method") 119 parser.add_option("-p", "--test-package", dest="test_package", 120 help="Restrict test to a specific java package") 121 parser.add_option("-z", "--size", dest="test_size", 122 help="Restrict test to a specific test size") 123 parser.add_option("--annotation", dest="test_annotation", 124 help="Include only those tests tagged with a specific" 125 " annotation") 126 parser.add_option("--not-annotation", dest="test_not_annotation", 127 help="Exclude any tests tagged with a specific" 128 " annotation") 129 parser.add_option("-u", "--user-tests-file", dest="user_tests_file", 130 metavar="FILE", default=user_test_default, 131 help="Alternate source of user test definitions") 132 parser.add_option("-o", "--coverage", dest="coverage", 133 default=False, action="store_true", 134 help="Generate code coverage metrics for test(s)") 135 parser.add_option("-x", "--path", dest="test_path", 136 help="Run test(s) at given file system path") 137 parser.add_option("-t", "--all-tests", dest="all_tests", 138 default=False, action="store_true", 139 help="Run all defined tests") 140 parser.add_option("--continuous", dest="continuous_tests", 141 default=False, action="store_true", 142 help="Run all tests defined as part of the continuous " 143 "test set") 144 parser.add_option("--timeout", dest="timeout", 145 default=300, help="Set a timeout limit (in sec) for " 146 "running native tests on a device (default: 300 secs)") 147 parser.add_option("--suite", dest="suite", 148 help="Run all tests defined as part of the " 149 "the given test suite") 150 group = optparse.OptionGroup( 151 parser, "Targets", "Use these options to direct tests to a specific " 152 "Android target") 153 group.add_option("-e", "--emulator", dest="emulator", default=False, 154 action="store_true", help="use emulator") 155 group.add_option("-d", "--device", dest="device", default=False, 156 action="store_true", help="use device") 157 group.add_option("-s", "--serial", dest="serial", 158 help="use specific serial") 159 parser.add_option_group(group) 160 161 self._options, self._test_args = parser.parse_args() 162 163 if (not self._options.only_list_tests 164 and not self._options.all_tests 165 and not self._options.continuous_tests 166 and not self._options.suite 167 and not self._options.test_path 168 and len(self._test_args) < 1): 169 parser.print_help() 170 logger.SilentLog("at least one test name must be specified") 171 raise errors.AbortError 172 173 self._adb = adb_interface.AdbInterface() 174 if self._options.emulator: 175 self._adb.SetEmulatorTarget() 176 elif self._options.device: 177 self._adb.SetDeviceTarget() 178 elif self._options.serial is not None: 179 self._adb.SetTargetSerial(self._options.serial) 180 181 if self._options.verbose: 182 logger.SetVerbose(True) 183 184 self._known_tests = self._ReadTests() 185 186 self._options.host_lib_path = android_build.GetHostLibraryPath() 187 self._options.test_data_path = android_build.GetTestAppPath() 188 189 def _ReadTests(self): 190 """Parses the set of test definition data. 191 192 Returns: 193 A TestDefinitions object that contains the set of parsed tests. 194 Raises: 195 AbortError: If a fatal error occurred when parsing the tests. 196 """ 197 core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH) 198 try: 199 known_tests = test_defs.TestDefinitions() 200 known_tests.Parse(core_test_path) 201 # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths 202 vendor_tests_pattern = os.path.join(self._root_path, 203 self._VENDOR_TEST_PATH) 204 test_file_paths = glob.glob(vendor_tests_pattern) 205 for test_file_path in test_file_paths: 206 known_tests.Parse(test_file_path) 207 if os.path.isfile(self._options.user_tests_file): 208 known_tests.Parse(self._options.user_tests_file) 209 return known_tests 210 except errors.ParseError: 211 raise errors.AbortError 212 213 def _DumpTests(self): 214 """Prints out set of defined tests.""" 215 print "The following tests are currently defined:\n" 216 print "%-25s %-40s %s" % ("name", "build path", "description") 217 print "-" * 80 218 for test in self._known_tests: 219 print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(), 220 test.GetDescription()) 221 print "\nSee %s for more information" % self._TEST_FILE_NAME 222 223 def _DoBuild(self): 224 logger.SilentLog("Building tests...") 225 target_set = Set() 226 extra_args_set = Set() 227 tests = self._GetTestsToRun() 228 for test_suite in tests: 229 self._AddBuildTarget(test_suite, target_set, extra_args_set) 230 231 if target_set: 232 if self._options.coverage: 233 coverage.EnableCoverageBuild() 234 235 # hack to build cts dependencies 236 # TODO: remove this when build dependency support added to runtest or 237 # cts dependencies are removed 238 if self._IsCtsTests(tests): 239 # need to use make since these fail building with ONE_SHOT_MAKEFILE 240 cmd = ('make -j%s CtsTestStubs android.core.tests.runner' % 241 self._options.make_jobs) 242 logger.Log(cmd) 243 if not self._options.preview: 244 old_dir = os.getcwd() 245 os.chdir(self._root_path) 246 run_command.RunCommand(cmd, return_output=False) 247 os.chdir(old_dir) 248 target_build_string = " ".join(list(target_set)) 249 extra_args_string = " ".join(list(extra_args_set)) 250 # mmm cannot be used from python, so perform a similar operation using 251 # ONE_SHOT_MAKEFILE 252 cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" files %s' % ( 253 target_build_string, self._options.make_jobs, self._root_path, 254 extra_args_string) 255 logger.Log(cmd) 256 257 if self._options.preview: 258 # in preview mode, just display to the user what command would have been 259 # run 260 logger.Log("adb sync") 261 else: 262 run_command.RunCommand(cmd, return_output=False) 263 logger.Log("Syncing to device...") 264 self._adb.Sync() 265 266 def _AddBuildTarget(self, test_suite, target_set, extra_args_set): 267 build_dir = test_suite.GetBuildPath() 268 if self._AddBuildTargetPath(build_dir, target_set): 269 extra_args_set.add(test_suite.GetExtraBuildArgs()) 270 for path in test_suite.GetBuildDependencies(self._options): 271 self._AddBuildTargetPath(path, target_set) 272 273 def _AddBuildTargetPath(self, build_dir, target_set): 274 if build_dir is not None: 275 build_file_path = os.path.join(build_dir, "Android.mk") 276 if os.path.isfile(os.path.join(self._root_path, build_file_path)): 277 target_set.add(build_file_path) 278 return True 279 else: 280 logger.Log("%s has no Android.mk, skipping" % build_dir) 281 return False 282 283 def _GetTestsToRun(self): 284 """Get a list of TestSuite objects to run, based on command line args.""" 285 if self._tests_to_run: 286 return self._tests_to_run 287 288 self._tests_to_run = [] 289 if self._options.all_tests: 290 self._tests_to_run = self._known_tests.GetTests() 291 elif self._options.continuous_tests: 292 self._tests_to_run = self._known_tests.GetContinuousTests() 293 elif self._options.suite: 294 self._tests_to_run = \ 295 self._known_tests.GetTestsInSuite(self._options.suite) 296 elif self._options.test_path: 297 walker = test_walker.TestWalker() 298 self._tests_to_run = walker.FindTests(self._options.test_path) 299 300 for name in self._test_args: 301 test = self._known_tests.GetTest(name) 302 if test is None: 303 logger.Log("Error: Could not find test %s" % name) 304 self._DumpTests() 305 raise errors.AbortError 306 self._tests_to_run.append(test) 307 return self._tests_to_run 308 309 def _IsCtsTests(self, test_list): 310 """Check if any cts tests are included in given list of tests to run.""" 311 for test in test_list: 312 if test.GetSuite() == 'cts': 313 return True 314 return False 315 316 def RunTests(self): 317 """Main entry method - executes the tests according to command line args.""" 318 try: 319 run_command.SetAbortOnError() 320 self._ProcessOptions() 321 if self._options.only_list_tests: 322 self._DumpTests() 323 return 324 325 if not self._options.skip_build: 326 self._DoBuild() 327 328 for test_suite in self._GetTestsToRun(): 329 try: 330 test_suite.Run(self._options, self._adb) 331 except errors.WaitForResponseTimedOutError: 332 logger.Log("Timed out waiting for response") 333 334 except KeyboardInterrupt: 335 logger.Log("Exiting...") 336 except errors.AbortError, error: 337 logger.Log(error.msg) 338 logger.SilentLog("Exiting due to AbortError...") 339 except errors.WaitForResponseTimedOutError: 340 logger.Log("Timed out waiting for response") 341 342 343 def RunTests(): 344 runner = TestRunner() 345 runner.RunTests() 346 347 if __name__ == "__main__": 348 RunTests() 349
