1 #!/usr/bin/env python3.4 2 # 3 # Copyright 2016 - The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 from future import standard_library 18 standard_library.install_aliases() 19 20 import argparse 21 import copy 22 import importlib 23 import inspect 24 import logging 25 import os 26 import pkgutil 27 import sys 28 29 from acts import base_test 30 from acts import config_parser 31 from acts import keys 32 from acts import logger 33 from acts import records 34 from acts import signals 35 from acts import utils 36 37 38 def main(): 39 """Execute the test class in a test module. 40 41 This is the default entry point for running a test script file directly. 42 In this case, only one test class in a test script is allowed. 43 44 To make your test script executable, add the following to your file: 45 46 from acts import test_runner 47 ... 48 if __name__ == "__main__": 49 test_runner.main() 50 51 If you want to implement your own cli entry point, you could use function 52 execute_one_test_class(test_class, test_config, test_identifier) 53 """ 54 # Parse cli args. 55 parser = argparse.ArgumentParser(description="ACTS Test Executable.") 56 parser.add_argument( 57 '-c', 58 '--config', 59 nargs=1, 60 type=str, 61 required=True, 62 metavar="<PATH>", 63 help="Path to the test configuration file.") 64 parser.add_argument( 65 '--test_case', 66 nargs='+', 67 type=str, 68 metavar="[test_a test_b...]", 69 help="A list of test case names in the test script.") 70 parser.add_argument( 71 '-tb', 72 '--test_bed', 73 nargs='+', 74 type=str, 75 metavar="[<TEST BED NAME1> <TEST BED NAME2> ...]", 76 help="Specify which test beds to run tests on.") 77 args = parser.parse_args(sys.argv[1:]) 78 # Load test config file. 79 test_configs = config_parser.load_test_config_file(args.config[0], 80 args.test_bed) 81 # Find the test class in the test script. 82 test_class = _find_test_class() 83 test_class_name = test_class.__name__ 84 # Parse test case specifiers if exist. 85 test_case_names = None 86 if args.test_case: 87 test_case_names = args.test_case 88 test_identifier = [(test_class_name, test_case_names)] 89 # Execute the test class with configs. 90 ok = True 91 for config in test_configs: 92 try: 93 result = execute_one_test_class(test_class, config, 94 test_identifier) 95 ok = result and ok 96 except signals.TestAbortAll: 97 pass 98 except: 99 logging.exception("Error occurred when executing test bed %s", 100 config[keys.Config.key_testbed.value]) 101 ok = False 102 if not ok: 103 sys.exit(1) 104 105 106 def _find_test_class(): 107 """Finds the test class in a test script. 108 109 Walk through module memebers and find the subclass of BaseTestClass. Only 110 one subclass is allowed in a test script. 111 112 Returns: 113 The test class in the test module. 114 """ 115 test_classes = [] 116 main_module_members = sys.modules["__main__"] 117 for _, module_member in main_module_members.__dict__.items(): 118 if inspect.isclass(module_member): 119 if issubclass(module_member, base_test.BaseTestClass): 120 test_classes.append(module_member) 121 if len(test_classes) != 1: 122 logging.error("Expected 1 test class per file, found %s.", 123 [t.__name__ for t in test_classes]) 124 sys.exit(1) 125 return test_classes[0] 126 127 128 def execute_one_test_class(test_class, test_config, test_identifier): 129 """Executes one specific test class. 130 131 You could call this function in your own cli test entry point if you choose 132 not to use act.py or test_runner.main. 133 134 Args: 135 test_class: A subclass of acts.base_test.BaseTestClass that has the test 136 logic to be executed. 137 test_config: A dict representing one set of configs for a test run. 138 test_identifier: A list of tuples specifying which test cases to run in 139 the test class. 140 141 Returns: 142 True if all tests passed without any error, False otherwise. 143 144 Raises: 145 If signals.TestAbortAll is raised by a test run, pipe it through. 146 """ 147 tr = TestRunner(test_config, test_identifier) 148 try: 149 tr.run(test_class) 150 return tr.results.is_all_pass 151 except signals.TestAbortAll: 152 raise 153 except: 154 logging.exception("Exception when executing %s.", tr.testbed_name) 155 finally: 156 tr.stop() 157 158 159 class TestRunner(object): 160 """The class that instantiates test classes, executes test cases, and 161 report results. 162 163 Attributes: 164 self.test_run_info: A dictionary containing the information needed by 165 test classes for this test run, including params, 166 controllers, and other objects. All of these will 167 be passed to test classes. 168 self.test_configs: A dictionary that is the original test configuration 169 passed in by user. 170 self.id: A string that is the unique identifier of this test run. 171 self.log_path: A string representing the path of the dir under which 172 all logs from this test run should be written. 173 self.log: The logger object used throughout this test run. 174 self.controller_registry: A dictionary that holds the controller 175 objects used in a test run. 176 self.controller_destructors: A dictionary that holds the controller 177 distructors. Keys are controllers' names. 178 self.test_classes: A dictionary where we can look up the test classes 179 by name to instantiate. 180 self.run_list: A list of tuples specifying what tests to run. 181 self.results: The test result object used to record the results of 182 this test run. 183 self.running: A boolean signifies whether this test run is ongoing or 184 not. 185 """ 186 187 def __init__(self, test_configs, run_list): 188 self.test_run_info = {} 189 self.test_configs = test_configs 190 self.testbed_configs = self.test_configs[keys.Config.key_testbed.value] 191 self.testbed_name = self.testbed_configs[ 192 keys.Config.key_testbed_name.value] 193 start_time = logger.get_log_file_timestamp() 194 self.id = "{}@{}".format(self.testbed_name, start_time) 195 # log_path should be set before parsing configs. 196 l_path = os.path.join( 197 self.test_configs[keys.Config.key_log_path.value], 198 self.testbed_name, start_time) 199 self.log_path = os.path.abspath(l_path) 200 logger.setup_test_logger(self.log_path, self.testbed_name) 201 self.log = logging.getLogger() 202 self.controller_registry = {} 203 self.controller_destructors = {} 204 if self.test_configs.get(keys.Config.key_random.value): 205 test_case_iterations = self.test_configs.get( 206 keys.Config.key_test_case_iterations.value, 10) 207 self.log.info( 208 "Campaign randomizer is enabled with test_case_iterations %s", 209 test_case_iterations) 210 self.run_list = config_parser.test_randomizer( 211 run_list, test_case_iterations=test_case_iterations) 212 self.write_test_campaign() 213 else: 214 self.run_list = run_list 215 self.results = records.TestResult() 216 self.running = False 217 218 def import_test_modules(self, test_paths): 219 """Imports test classes from test scripts. 220 221 1. Locate all .py files under test paths. 222 2. Import the .py files as modules. 223 3. Find the module members that are test classes. 224 4. Categorize the test classes by name. 225 226 Args: 227 test_paths: A list of directory paths where the test files reside. 228 229 Returns: 230 A dictionary where keys are test class name strings, values are 231 actual test classes that can be instantiated. 232 """ 233 234 def is_testfile_name(name, ext): 235 if ext == ".py": 236 if name.endswith("Test") or name.endswith("_test"): 237 return True 238 return False 239 240 file_list = utils.find_files(test_paths, is_testfile_name) 241 test_classes = {} 242 for path, name, _ in file_list: 243 sys.path.append(path) 244 try: 245 module = importlib.import_module(name) 246 except: 247 for test_cls_name, _ in self.run_list: 248 alt_name = name.replace('_', '').lower() 249 alt_cls_name = test_cls_name.lower() 250 # Only block if a test class on the run list causes an 251 # import error. We need to check against both naming 252 # conventions: AaaBbb and aaa_bbb. 253 if name == test_cls_name or alt_name == alt_cls_name: 254 msg = ("Encountered error importing test class %s, " 255 "abort.") % test_cls_name 256 # This exception is logged here to help with debugging 257 # under py2, because "raise X from Y" syntax is only 258 # supported under py3. 259 self.log.exception(msg) 260 raise ValueError(msg) 261 continue 262 for member_name in dir(module): 263 if not member_name.startswith("__"): 264 if member_name.endswith("Test"): 265 test_class = getattr(module, member_name) 266 if inspect.isclass(test_class): 267 test_classes[member_name] = test_class 268 return test_classes 269 270 def _import_builtin_controllers(self): 271 """Import built-in controller modules. 272 273 Go through the testbed configs, find any built-in controller configs 274 and import the corresponding controller module from acts.controllers 275 package. 276 277 TODO(angli): Remove this when all scripts change to explicitly declare 278 controller dependency. 279 280 Returns: 281 A list of controller modules. 282 """ 283 builtin_controllers = [] 284 for ctrl_name in keys.Config.builtin_controller_names.value: 285 if ctrl_name in self.testbed_configs: 286 module_name = keys.get_module_name(ctrl_name) 287 module = importlib.import_module("acts.controllers.%s" % 288 module_name) 289 builtin_controllers.append(module) 290 return builtin_controllers 291 292 @staticmethod 293 def verify_controller_module(module): 294 """Verifies a module object follows the required interface for 295 controllers. 296 297 Args: 298 module: An object that is a controller module. This is usually 299 imported with import statements or loaded by importlib. 300 301 Raises: 302 ControllerError is raised if the module does not match the ACTS 303 controller interface, or one of the required members is null. 304 """ 305 required_attributes = ("create", "destroy", 306 "ACTS_CONTROLLER_CONFIG_NAME") 307 for attr in required_attributes: 308 if not hasattr(module, attr): 309 raise signals.ControllerError( 310 ("Module %s missing required " 311 "controller module attribute %s.") % (module.__name__, 312 attr)) 313 if not getattr(module, attr): 314 raise signals.ControllerError( 315 "Controller interface %s in %s cannot be null." % ( 316 attr, module.__name__)) 317 318 def register_controller(self, module, required=True): 319 """Registers an ACTS controller module for a test run. 320 321 An ACTS controller module is a Python lib that can be used to control 322 a device, service, or equipment. To be ACTS compatible, a controller 323 module needs to have the following members: 324 325 def create(configs): 326 [Required] Creates controller objects from configurations. 327 Args: 328 configs: A list of serialized data like string/dict. Each 329 element of the list is a configuration for a 330 controller object. 331 Returns: 332 A list of objects. 333 334 def destroy(objects): 335 [Required] Destroys controller objects created by the create 336 function. Each controller object shall be properly cleaned up 337 and all the resources held should be released, e.g. memory 338 allocation, sockets, file handlers etc. 339 Args: 340 A list of controller objects created by the create function. 341 342 def get_info(objects): 343 [Optional] Gets info from the controller objects used in a test 344 run. The info will be included in test_result_summary.json under 345 the key "ControllerInfo". Such information could include unique 346 ID, version, or anything that could be useful for describing the 347 test bed and debugging. 348 Args: 349 objects: A list of controller objects created by the create 350 function. 351 Returns: 352 A list of json serializable objects, each represents the 353 info of a controller object. The order of the info object 354 should follow that of the input objects. 355 356 Registering a controller module declares a test class's dependency the 357 controller. If the module config exists and the module matches the 358 controller interface, controller objects will be instantiated with 359 corresponding configs. The module should be imported first. 360 361 Args: 362 module: A module that follows the controller module interface. 363 required: A bool. If True, failing to register the specified 364 controller module raises exceptions. If False, returns 365 None upon failures. 366 367 Returns: 368 A list of controller objects instantiated from controller_module, or 369 None. 370 371 Raises: 372 When required is True, ControllerError is raised if no corresponding 373 config can be found. 374 Regardless of the value of "required", ControllerError is raised if 375 the controller module has already been registered or any other error 376 occurred in the registration process. 377 """ 378 TestRunner.verify_controller_module(module) 379 try: 380 # If this is a builtin controller module, use the default ref name. 381 module_ref_name = module.ACTS_CONTROLLER_REFERENCE_NAME 382 builtin = True 383 except AttributeError: 384 # Or use the module's name 385 builtin = False 386 module_ref_name = module.__name__.split('.')[-1] 387 if module_ref_name in self.controller_registry: 388 raise signals.ControllerError( 389 ("Controller module %s has already been registered. It can not" 390 " be registered again.") % module_ref_name) 391 # Create controller objects. 392 create = module.create 393 module_config_name = module.ACTS_CONTROLLER_CONFIG_NAME 394 if module_config_name not in self.testbed_configs: 395 if required: 396 raise signals.ControllerError( 397 "No corresponding config found for %s" % 398 module_config_name) 399 self.log.warning( 400 "No corresponding config found for optional controller %s", 401 module_config_name) 402 return None 403 try: 404 # Make a deep copy of the config to pass to the controller module, 405 # in case the controller module modifies the config internally. 406 original_config = self.testbed_configs[module_config_name] 407 controller_config = copy.deepcopy(original_config) 408 objects = create(controller_config) 409 except: 410 self.log.exception( 411 "Failed to initialize objects for controller %s, abort!", 412 module_config_name) 413 raise 414 if not isinstance(objects, list): 415 raise signals.ControllerError( 416 "Controller module %s did not return a list of objects, abort." 417 % module_ref_name) 418 self.controller_registry[module_ref_name] = objects 419 # Collect controller information and write to test result. 420 # Implementation of "get_info" is optional for a controller module. 421 if hasattr(module, "get_info"): 422 controller_info = module.get_info(objects) 423 self.log.info("Controller %s: %s", module_config_name, 424 controller_info) 425 self.results.add_controller_info(module_config_name, 426 controller_info) 427 else: 428 self.log.warning("No controller info obtained for %s", 429 module_config_name) 430 # TODO(angli): After all tests move to register_controller, stop 431 # tracking controller objs in test_run_info. 432 if builtin: 433 self.test_run_info[module_ref_name] = objects 434 self.log.debug("Found %d objects for controller %s", len(objects), 435 module_config_name) 436 destroy_func = module.destroy 437 self.controller_destructors[module_ref_name] = destroy_func 438 return objects 439 440 def unregister_controllers(self): 441 """Destroy controller objects and clear internal registry. 442 443 This will be called at the end of each TestRunner.run call. 444 """ 445 for name, destroy in self.controller_destructors.items(): 446 try: 447 self.log.debug("Destroying %s.", name) 448 destroy(self.controller_registry[name]) 449 except: 450 self.log.exception("Exception occurred destroying %s.", name) 451 self.controller_registry = {} 452 self.controller_destructors = {} 453 454 def parse_config(self, test_configs): 455 """Parses the test configuration and unpacks objects and parameters 456 into a dictionary to be passed to test classes. 457 458 Args: 459 test_configs: A json object representing the test configurations. 460 """ 461 self.test_run_info[ 462 keys.Config.ikey_testbed_name.value] = self.testbed_name 463 # Unpack other params. 464 self.test_run_info["register_controller"] = self.register_controller 465 self.test_run_info[keys.Config.ikey_logpath.value] = self.log_path 466 self.test_run_info[keys.Config.ikey_logger.value] = self.log 467 cli_args = test_configs.get(keys.Config.ikey_cli_args.value) 468 self.test_run_info[keys.Config.ikey_cli_args.value] = cli_args 469 user_param_pairs = [] 470 for item in test_configs.items(): 471 if item[0] not in keys.Config.reserved_keys.value: 472 user_param_pairs.append(item) 473 self.test_run_info[keys.Config.ikey_user_param.value] = copy.deepcopy( 474 dict(user_param_pairs)) 475 476 def set_test_util_logs(self, module=None): 477 """Sets the log object to each test util module. 478 479 This recursively include all modules under acts.test_utils and sets the 480 main test logger to each module. 481 482 Args: 483 module: A module under acts.test_utils. 484 """ 485 # Initial condition of recursion. 486 if not module: 487 module = importlib.import_module("acts.test_utils") 488 # Somehow pkgutil.walk_packages is not working for me. 489 # Using iter_modules for now. 490 pkg_iter = pkgutil.iter_modules(module.__path__, module.__name__ + '.') 491 for _, module_name, ispkg in pkg_iter: 492 m = importlib.import_module(module_name) 493 if ispkg: 494 self.set_test_util_logs(module=m) 495 else: 496 self.log.debug("Setting logger to test util module %s", 497 module_name) 498 setattr(m, "log", self.log) 499 500 def run_test_class(self, test_cls_name, test_cases=None): 501 """Instantiates and executes a test class. 502 503 If test_cases is None, the test cases listed by self.tests will be 504 executed instead. If self.tests is empty as well, no test case in this 505 test class will be executed. 506 507 Args: 508 test_cls_name: Name of the test class to execute. 509 test_cases: List of test case names to execute within the class. 510 511 Raises: 512 ValueError is raised if the requested test class could not be found 513 in the test_paths directories. 514 """ 515 try: 516 test_cls = self.test_classes[test_cls_name] 517 except KeyError: 518 self.log.info( 519 "Cannot find test class %s skipping for now." % test_cls_name) 520 record = records.TestResultRecord("*all*", test_cls_name) 521 record.test_skip(signals.TestSkip("Test class does not exist.")) 522 self.results.add_record(record) 523 return 524 if self.test_configs.get(keys.Config.key_random.value) or ( 525 "Preflight" in test_cls_name) or "Postflight" in test_cls_name: 526 test_case_iterations = 1 527 else: 528 test_case_iterations = self.test_configs.get( 529 keys.Config.key_test_case_iterations.value, 1) 530 531 with test_cls(self.test_run_info) as test_cls_instance: 532 try: 533 cls_result = test_cls_instance.run(test_cases, 534 test_case_iterations) 535 self.results += cls_result 536 except signals.TestAbortAll as e: 537 self.results += e.results 538 raise e 539 540 def run(self, test_class=None): 541 """Executes test cases. 542 543 This will instantiate controller and test classes, and execute test 544 classes. This can be called multiple times to repeatly execute the 545 requested test cases. 546 547 A call to TestRunner.stop should eventually happen to conclude the life 548 cycle of a TestRunner. 549 550 Args: 551 test_class: The python module of a test class. If provided, run this 552 class; otherwise, import modules in under test_paths 553 based on run_list. 554 """ 555 if not self.running: 556 self.running = True 557 # Initialize controller objects and pack appropriate objects/params 558 # to be passed to test class. 559 self.parse_config(self.test_configs) 560 if test_class: 561 self.test_classes = {test_class.__name__: test_class} 562 else: 563 t_paths = self.test_configs[keys.Config.key_test_paths.value] 564 self.test_classes = self.import_test_modules(t_paths) 565 self.log.debug("Executing run list %s.", self.run_list) 566 for test_cls_name, test_case_names in self.run_list: 567 if not self.running: 568 break 569 if test_case_names: 570 self.log.debug("Executing test cases %s in test class %s.", 571 test_case_names, test_cls_name) 572 else: 573 self.log.debug("Executing test class %s", test_cls_name) 574 try: 575 # Import and register the built-in controller modules specified 576 # in testbed config. 577 for module in self._import_builtin_controllers(): 578 self.register_controller(module) 579 self.run_test_class(test_cls_name, test_case_names) 580 except signals.TestAbortAll as e: 581 self.log.warning( 582 "Abort all subsequent test classes. Reason: %s", e) 583 raise 584 finally: 585 self.unregister_controllers() 586 587 def stop(self): 588 """Releases resources from test run. Should always be called after 589 TestRunner.run finishes. 590 591 This function concludes a test run and writes out a test report. 592 """ 593 if self.running: 594 msg = "\nSummary for test run %s: %s\n" % ( 595 self.id, self.results.summary_str()) 596 self._write_results_json_str() 597 self.log.info(msg.strip()) 598 logger.kill_test_logger(self.log) 599 self.running = False 600 601 def _write_results_json_str(self): 602 """Writes out a json file with the test result info for easy parsing. 603 604 TODO(angli): This should be replaced by standard log record mechanism. 605 """ 606 path = os.path.join(self.log_path, "test_run_summary.json") 607 with open(path, 'w') as f: 608 f.write(self.results.json_str()) 609 610 def write_test_campaign(self): 611 """Log test campaign file.""" 612 path = os.path.join(self.log_path, "test_campaign.log") 613 with open(path, 'w') as f: 614 for test_class, test_cases in self.run_list: 615 f.write("%s:\n%s" % (test_class, ",\n".join(test_cases))) 616 f.write("\n\n") 617
