1 # 2 # Copyright (C) 2016 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 17 from future import standard_library 18 standard_library.install_aliases() 19 20 import copy 21 import importlib 22 import inspect 23 import logging 24 import os 25 import pkgutil 26 import signal 27 import sys 28 29 from vts.runners.host import base_test 30 from vts.runners.host import config_parser 31 from vts.runners.host import keys 32 from vts.runners.host import logger 33 from vts.runners.host import records 34 from vts.runners.host import signals 35 from vts.runners.host import utils 36 37 38 def main(): 39 """Execute the test class in a test module. 40 41 This is to be used in a test script's main so the script can be executed 42 directly. It will discover all the classes that inherit from BaseTestClass 43 and excute them. all the test results will be aggregated into one. 44 45 A VTS host-driven test case has three args: 46 1st arg: the path of a test case config file. 47 2nd arg: the serial ID of a target device (device config). 48 3rd arg: the path of a test case data dir. 49 50 Returns: 51 The TestResult object that holds the results of the test run. 52 """ 53 test_classes = [] 54 main_module_members = sys.modules["__main__"] 55 for _, module_member in main_module_members.__dict__.items(): 56 if inspect.isclass(module_member): 57 if issubclass(module_member, base_test.BaseTestClass): 58 test_classes.append(module_member) 59 # TODO(angli): Need to handle the case where more than one test class is in 60 # a test script. The challenge is to handle multiple configs and how to do 61 # default config in this case. 62 if len(test_classes) != 1: 63 logging.error("Expected 1 test class per file, found %s.", 64 len(test_classes)) 65 sys.exit(1) 66 test_result = runTestClass(test_classes[0]) 67 return test_result 68 69 70 def runTestClass(test_class): 71 """Execute one test class. 72 73 This will create a TestRunner, execute one test run with one test class. 74 75 Args: 76 test_class: The test class to instantiate and execute. 77 78 Returns: 79 The TestResult object that holds the results of the test run. 80 """ 81 test_cls_name = test_class.__name__ 82 if len(sys.argv) < 2: 83 logging.warning("Missing a configuration file. Using the default.") 84 test_configs = [config_parser.GetDefaultConfig(test_cls_name)] 85 else: 86 try: 87 config_path = sys.argv[1] 88 baseline_config = config_parser.GetDefaultConfig(test_cls_name) 89 baseline_config[keys.ConfigKeys.KEY_TESTBED] = [ 90 baseline_config[keys.ConfigKeys.KEY_TESTBED] 91 ] 92 test_configs = config_parser.load_test_config_file( 93 config_path, baseline_config=baseline_config) 94 except IndexError: 95 logging.error("No valid config file found.") 96 sys.exit(1) 97 98 test_identifiers = [(test_cls_name, None)] 99 100 for config in test_configs: 101 tr = TestRunner(config, test_identifiers) 102 tr.parseTestConfig(config) 103 try: 104 # Create console signal handler to make sure TestRunner is stopped 105 # in the event of termination. 106 handler = config_parser.gen_term_signal_handler([tr]) 107 signal.signal(signal.SIGTERM, handler) 108 signal.signal(signal.SIGINT, handler) 109 tr.runTestClass(test_class, None) 110 finally: 111 tr.stop() 112 return tr.results 113 114 115 class TestRunner(object): 116 """The class that instantiates test classes, executes test cases, and 117 report results. 118 119 Attributes: 120 self.test_run_info: A dictionary containing the information needed by 121 test classes for this test run, including params, 122 controllers, and other objects. All of these will 123 be passed to test classes. 124 self.test_configs: A dictionary that is the original test configuration 125 passed in by user. 126 self.id: A string that is the unique identifier of this test run. 127 self.log_path: A string representing the path of the dir under which 128 all logs from this test run should be written. 129 self.controller_registry: A dictionary that holds the controller 130 objects used in a test run. 131 self.controller_destructors: A dictionary that holds the controller 132 distructors. Keys are controllers' names. 133 self.run_list: A list of tuples specifying what tests to run. 134 self.results: The test result object used to record the results of 135 this test run. 136 self.running: A boolean signifies whether this test run is ongoing or 137 not. 138 """ 139 140 def __init__(self, test_configs, run_list): 141 self.test_run_info = {} 142 self.test_run_info[keys.ConfigKeys.IKEY_DATA_FILE_PATH] = getattr( 143 test_configs, keys.ConfigKeys.IKEY_DATA_FILE_PATH, "./") 144 self.test_configs = test_configs 145 self.testbed_configs = self.test_configs[keys.ConfigKeys.KEY_TESTBED] 146 self.testbed_name = self.testbed_configs[ 147 keys.ConfigKeys.KEY_TESTBED_NAME] 148 start_time = logger.getLogFileTimestamp() 149 self.id = "{}@{}".format(self.testbed_name, start_time) 150 # log_path should be set before parsing configs. 151 l_path = os.path.join(self.test_configs[keys.ConfigKeys.KEY_LOG_PATH], 152 self.testbed_name, start_time) 153 self.log_path = os.path.abspath(l_path) 154 logger.setupTestLogger(self.log_path, self.testbed_name) 155 self.controller_registry = {} 156 self.controller_destructors = {} 157 self.run_list = run_list 158 self.results = records.TestResult() 159 self.running = False 160 161 def __enter__(self): 162 return self 163 164 def __exit__(self, *args): 165 self.stop() 166 167 def importTestModules(self, test_paths): 168 """Imports test classes from test scripts. 169 170 1. Locate all .py files under test paths. 171 2. Import the .py files as modules. 172 3. Find the module members that are test classes. 173 4. Categorize the test classes by name. 174 175 Args: 176 test_paths: A list of directory paths where the test files reside. 177 178 Returns: 179 A dictionary where keys are test class name strings, values are 180 actual test classes that can be instantiated. 181 """ 182 183 def is_testfile_name(name, ext): 184 if ext == ".py": 185 if name.endswith("Test") or name.endswith("_test"): 186 return True 187 return False 188 189 file_list = utils.find_files(test_paths, is_testfile_name) 190 test_classes = {} 191 for path, name, _ in file_list: 192 sys.path.append(path) 193 try: 194 module = importlib.import_module(name) 195 except: 196 for test_cls_name, _ in self.run_list: 197 alt_name = name.replace('_', '').lower() 198 alt_cls_name = test_cls_name.lower() 199 # Only block if a test class on the run list causes an 200 # import error. We need to check against both naming 201 # conventions: AaaBbb and aaa_bbb. 202 if name == test_cls_name or alt_name == alt_cls_name: 203 msg = ("Encountered error importing test class %s, " 204 "abort.") % test_cls_name 205 # This exception is logged here to help with debugging 206 # under py2, because "raise X from Y" syntax is only 207 # supported under py3. 208 logging.exception(msg) 209 raise USERError(msg) 210 continue 211 for member_name in dir(module): 212 if not member_name.startswith("__"): 213 if member_name.endswith("Test"): 214 test_class = getattr(module, member_name) 215 if inspect.isclass(test_class): 216 test_classes[member_name] = test_class 217 return test_classes 218 219 def verifyControllerModule(self, module): 220 """Verifies a module object follows the required interface for 221 controllers. 222 223 Args: 224 module: An object that is a controller module. This is usually 225 imported with import statements or loaded by importlib. 226 227 Raises: 228 ControllerError is raised if the module does not match the vts.runners.host 229 controller interface, or one of the required members is null. 230 """ 231 required_attributes = ("create", "destroy", 232 "VTS_CONTROLLER_CONFIG_NAME") 233 for attr in required_attributes: 234 if not hasattr(module, attr): 235 raise signals.ControllerError( 236 ("Module %s missing required " 237 "controller module attribute %s.") % (module.__name__, 238 attr)) 239 if not getattr(module, attr): 240 raise signals.ControllerError( 241 ("Controller interface %s in %s " 242 "cannot be null.") % (attr, module.__name__)) 243 244 def registerController(self, module, start_services=True): 245 """Registers a controller module for a test run. 246 247 This declares a controller dependency of this test class. If the target 248 module exists and matches the controller interface, the controller 249 module will be instantiated with corresponding configs in the test 250 config file. The module should be imported first. 251 252 Params: 253 module: A module that follows the controller module interface. 254 start_services: boolean, controls whether services (e.g VTS agent) 255 are started on the target. 256 257 Returns: 258 A list of controller objects instantiated from controller_module. 259 260 Raises: 261 ControllerError is raised if no corresponding config can be found, 262 or if the controller module has already been registered. 263 """ 264 logging.info("cwd: %s", os.getcwd()) 265 logging.info("adb devices: %s", module.list_adb_devices()) 266 self.verifyControllerModule(module) 267 module_ref_name = module.__name__.split('.')[-1] 268 if module_ref_name in self.controller_registry: 269 raise signals.ControllerError( 270 ("Controller module %s has already " 271 "been registered. It can not be " 272 "registered again.") % module_ref_name) 273 # Create controller objects. 274 create = module.create 275 module_config_name = module.VTS_CONTROLLER_CONFIG_NAME 276 if module_config_name not in self.testbed_configs: 277 raise signals.ControllerError(("No corresponding config found for" 278 " %s") % module_config_name) 279 try: 280 # Make a deep copy of the config to pass to the controller module, 281 # in case the controller module modifies the config internally. 282 original_config = self.testbed_configs[module_config_name] 283 controller_config = copy.deepcopy(original_config) 284 logging.info("controller_config: %s", controller_config) 285 if "use_vts_agent" not in self.testbed_configs: 286 objects = create(controller_config, start_services) 287 else: 288 objects = create(controller_config, 289 self.testbed_configs["use_vts_agent"]) 290 except: 291 logging.exception(("Failed to initialize objects for controller " 292 "%s, abort!"), module_config_name) 293 raise 294 if not isinstance(objects, list): 295 raise ControllerError(("Controller module %s did not return a list" 296 " of objects, abort.") % module_ref_name) 297 self.controller_registry[module_ref_name] = objects 298 logging.debug("Found %d objects for controller %s", len(objects), 299 module_config_name) 300 destroy_func = module.destroy 301 self.controller_destructors[module_ref_name] = destroy_func 302 return objects 303 304 def unregisterControllers(self): 305 """Destroy controller objects and clear internal registry. 306 307 This will be called at the end of each TestRunner.run call. 308 """ 309 for name, destroy in self.controller_destructors.items(): 310 try: 311 logging.debug("Destroying %s.", name) 312 dut = self.controller_destructors[name][0] 313 destroy(self.controller_registry[name]) 314 except: 315 logging.exception("Exception occurred destroying %s.", name) 316 self.controller_registry = {} 317 self.controller_destructors = {} 318 319 def parseTestConfig(self, test_configs): 320 """Parses the test configuration and unpacks objects and parameters 321 into a dictionary to be passed to test classes. 322 323 Args: 324 test_configs: A json object representing the test configurations. 325 """ 326 self.test_run_info[ 327 keys.ConfigKeys.IKEY_TESTBED_NAME] = self.testbed_name 328 # Unpack other params. 329 self.test_run_info["registerController"] = self.registerController 330 self.test_run_info[keys.ConfigKeys.IKEY_LOG_PATH] = self.log_path 331 user_param_pairs = [] 332 for item in test_configs.items(): 333 if item[0] not in keys.ConfigKeys.RESERVED_KEYS: 334 user_param_pairs.append(item) 335 self.test_run_info[keys.ConfigKeys.IKEY_USER_PARAM] = copy.deepcopy( 336 dict(user_param_pairs)) 337 338 def runTestClass(self, test_cls, test_cases=None): 339 """Instantiates and executes a test class. 340 341 If test_cases is None, the test cases listed by self.tests will be 342 executed instead. If self.tests is empty as well, no test case in this 343 test class will be executed. 344 345 Args: 346 test_cls: The test class to be instantiated and executed. 347 test_cases: List of test case names to execute within the class. 348 349 Returns: 350 A tuple, with the number of cases passed at index 0, and the total 351 number of test cases at index 1. 352 """ 353 self.running = True 354 with test_cls(self.test_run_info) as test_cls_instance: 355 try: 356 cls_result = test_cls_instance.run(test_cases) 357 self.results += cls_result 358 except signals.TestAbortAll as e: 359 self.results += e.results 360 raise e 361 362 def run(self): 363 """Executes test cases. 364 365 This will instantiate controller and test classes, and execute test 366 classes. This can be called multiple times to repeatly execute the 367 requested test cases. 368 369 A call to TestRunner.stop should eventually happen to conclude the life 370 cycle of a TestRunner. 371 372 Args: 373 test_classes: A dictionary where the key is test class name, and 374 the values are actual test classes. 375 """ 376 if not self.running: 377 self.running = True 378 # Initialize controller objects and pack appropriate objects/params 379 # to be passed to test class. 380 self.parseTestConfig(self.test_configs) 381 t_configs = self.test_configs[keys.ConfigKeys.KEY_TEST_PATHS] 382 test_classes = self.importTestModules(t_configs) 383 logging.debug("Executing run list %s.", self.run_list) 384 try: 385 for test_cls_name, test_case_names in self.run_list: 386 if not self.running: 387 break 388 if test_case_names: 389 logging.debug("Executing test cases %s in test class %s.", 390 test_case_names, test_cls_name) 391 else: 392 logging.debug("Executing test class %s", test_cls_name) 393 try: 394 test_cls = test_classes[test_cls_name] 395 except KeyError: 396 raise USERError( 397 ("Unable to locate class %s in any of the test " 398 "paths specified.") % test_cls_name) 399 try: 400 self.runTestClass(test_cls, test_case_names) 401 except signals.TestAbortAll as e: 402 logging.warning( 403 ("Abort all subsequent test classes. Reason: " 404 "%s"), e) 405 raise 406 finally: 407 self.unregisterControllers() 408 409 def stop(self): 410 """Releases resources from test run. Should always be called after 411 TestRunner.run finishes. 412 413 This function concludes a test run and writes out a test report. 414 """ 415 if self.running: 416 msg = "\nSummary for test run %s: %s\n" % (self.id, 417 self.results.summary()) 418 self._writeResultsJsonString() 419 logging.info(msg.strip()) 420 logger.killTestLogger(logging.getLogger()) 421 self.running = False 422 423 def _writeResultsJsonString(self): 424 """Writes out a json file with the test result info for easy parsing. 425 """ 426 path = os.path.join(self.log_path, "test_run_summary.json") 427 with open(path, 'w') as f: 428 f.write(self.results.jsonString()) 429
