1 #!/usr/bin/env python3.4 2 # 3 # Copyright 2016 - The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 import os 18 import time 19 import traceback 20 21 from acts import asserts 22 from acts import keys 23 from acts import logger 24 from acts import records 25 from acts import signals 26 from acts import tracelogger 27 from acts import utils 28 29 # Macro strings for test result reporting 30 TEST_CASE_TOKEN = "[Test Case]" 31 RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 32 33 34 class Error(Exception): 35 """Raised for exceptions that occured in BaseTestClass.""" 36 37 38 class BaseTestClass(object): 39 """Base class for all test classes to inherit from. 40 41 This class gets all the controller objects from test_runner and executes 42 the test cases requested within itself. 43 44 Most attributes of this class are set at runtime based on the configuration 45 provided. 46 47 Attributes: 48 tests: A list of strings, each representing a test case name. 49 TAG: A string used to refer to a test class. Default is the test class 50 name. 51 log: A logger object used for logging. 52 results: A records.TestResult object for aggregating test results from 53 the execution of test cases. 54 current_test_name: A string that's the name of the test case currently 55 being executed. If no test is executing, this should 56 be None. 57 """ 58 59 TAG = None 60 61 def __init__(self, configs): 62 self.tests = [] 63 if not self.TAG: 64 self.TAG = self.__class__.__name__ 65 # Set all the controller objects and params. 66 for name, value in configs.items(): 67 setattr(self, name, value) 68 self.results = records.TestResult() 69 self.current_test_name = None 70 self.log = tracelogger.TraceLogger(self.log) 71 if 'android_devices' in self.__dict__: 72 for ad in self.android_devices: 73 if ad.droid: 74 utils.set_location_service(ad, False) 75 utils.sync_device_time(ad) 76 77 def __enter__(self): 78 return self 79 80 def __exit__(self, *args): 81 self._exec_func(self.clean_up) 82 83 def unpack_userparams(self, 84 req_param_names=[], 85 opt_param_names=[], 86 **kwargs): 87 """An optional function that unpacks user defined parameters into 88 individual variables. 89 90 After unpacking, the params can be directly accessed with self.xxx. 91 92 If a required param is not provided, an exception is raised. If an 93 optional param is not provided, a warning line will be logged. 94 95 To provide a param, add it in the config file or pass it in as a kwarg. 96 If a param appears in both the config file and kwarg, the value in the 97 config file is used. 98 99 User params from the config file can also be directly accessed in 100 self.user_params. 101 102 Args: 103 req_param_names: A list of names of the required user params. 104 opt_param_names: A list of names of the optional user params. 105 **kwargs: Arguments that provide default values. 106 e.g. unpack_userparams(required_list, opt_list, arg_a="hello") 107 self.arg_a will be "hello" unless it is specified again in 108 required_list or opt_list. 109 110 Raises: 111 Error is raised if a required user params is not provided. 112 """ 113 for k, v in kwargs.items(): 114 if k in self.user_params: 115 v = self.user_params[k] 116 setattr(self, k, v) 117 for name in req_param_names: 118 if hasattr(self, name): 119 continue 120 if name not in self.user_params: 121 raise Error(("Missing required user param '%s' in test " 122 "configuration.") % name) 123 setattr(self, name, self.user_params[name]) 124 for name in opt_param_names: 125 if hasattr(self, name): 126 continue 127 if name in self.user_params: 128 setattr(self, name, self.user_params[name]) 129 else: 130 self.log.warning(("Missing optional user param '%s' in " 131 "configuration, continue."), name) 132 133 capablity_of_devices = utils.CapablityPerDevice 134 if "additional_energy_info_models" in self.user_params: 135 self.energy_info_models = (capablity_of_devices.energy_info_models 136 + self.additional_energy_info_models) 137 else: 138 self.energy_info_models = capablity_of_devices.energy_info_models 139 self.user_params["energy_info_models"] = self.energy_info_models 140 141 if "additional_tdls_models" in self.user_params: 142 self.tdls_models = (capablity_of_devices.energy_info_models + 143 self.additional_tdls_models) 144 else: 145 self.tdls_models = capablity_of_devices.energy_info_models 146 self.user_params["tdls_models"] = self.tdls_models 147 148 def _setup_class(self): 149 """Proxy function to guarantee the base implementation of setup_class 150 is called. 151 """ 152 return self.setup_class() 153 154 def setup_class(self): 155 """Setup function that will be called before executing any test case in 156 the test class. 157 158 To signal setup failure, return False or raise an exception. If 159 exceptions were raised, the stack trace would appear in log, but the 160 exceptions would not propagate to upper levels. 161 162 Implementation is optional. 163 """ 164 165 def teardown_class(self): 166 """Teardown function that will be called after all the selected test 167 cases in the test class have been executed. 168 169 Implementation is optional. 170 """ 171 172 def _setup_test(self, test_name): 173 """Proxy function to guarantee the base implementation of setup_test is 174 called. 175 """ 176 self.current_test_name = test_name 177 try: 178 # Write test start token to adb log if android device is attached. 179 for ad in self.android_devices: 180 ad.droid.logV("%s BEGIN %s" % (TEST_CASE_TOKEN, test_name)) 181 except: 182 pass 183 return self.setup_test() 184 185 def setup_test(self): 186 """Setup function that will be called every time before executing each 187 test case in the test class. 188 189 To signal setup failure, return False or raise an exception. If 190 exceptions were raised, the stack trace would appear in log, but the 191 exceptions would not propagate to upper levels. 192 193 Implementation is optional. 194 """ 195 196 def _teardown_test(self, test_name): 197 """Proxy function to guarantee the base implementation of teardown_test 198 is called. 199 """ 200 try: 201 # Write test end token to adb log if android device is attached. 202 for ad in self.android_devices: 203 ad.droid.logV("%s END %s" % (TEST_CASE_TOKEN, test_name)) 204 except: 205 pass 206 try: 207 self.teardown_test() 208 finally: 209 self.current_test_name = None 210 211 def teardown_test(self): 212 """Teardown function that will be called every time a test case has 213 been executed. 214 215 Implementation is optional. 216 """ 217 218 def _on_fail(self, record): 219 """Proxy function to guarantee the base implementation of on_fail is 220 called. 221 222 Args: 223 record: The records.TestResultRecord object for the failed test 224 case. 225 """ 226 test_name = record.test_name 227 if record.details: 228 self.log.error(record.details) 229 begin_time = logger.epoch_to_log_line_timestamp(record.begin_time) 230 self.log.info(RESULT_LINE_TEMPLATE, test_name, record.result) 231 self.on_fail(test_name, begin_time) 232 233 def on_fail(self, test_name, begin_time): 234 """A function that is executed upon a test case failure. 235 236 User implementation is optional. 237 238 Args: 239 test_name: Name of the test that triggered this function. 240 begin_time: Logline format timestamp taken when the test started. 241 """ 242 243 def _on_pass(self, record): 244 """Proxy function to guarantee the base implementation of on_pass is 245 called. 246 247 Args: 248 record: The records.TestResultRecord object for the passed test 249 case. 250 """ 251 test_name = record.test_name 252 begin_time = logger.epoch_to_log_line_timestamp(record.begin_time) 253 msg = record.details 254 if msg: 255 self.log.info(msg) 256 self.log.info(RESULT_LINE_TEMPLATE, test_name, record.result) 257 self.on_pass(test_name, begin_time) 258 259 def on_pass(self, test_name, begin_time): 260 """A function that is executed upon a test case passing. 261 262 Implementation is optional. 263 264 Args: 265 test_name: Name of the test that triggered this function. 266 begin_time: Logline format timestamp taken when the test started. 267 """ 268 269 def _on_skip(self, record): 270 """Proxy function to guarantee the base implementation of on_skip is 271 called. 272 273 Args: 274 record: The records.TestResultRecord object for the skipped test 275 case. 276 """ 277 test_name = record.test_name 278 begin_time = logger.epoch_to_log_line_timestamp(record.begin_time) 279 self.log.info(RESULT_LINE_TEMPLATE, test_name, record.result) 280 self.log.info("Reason to skip: %s", record.details) 281 self.on_skip(test_name, begin_time) 282 283 def on_skip(self, test_name, begin_time): 284 """A function that is executed upon a test case being skipped. 285 286 Implementation is optional. 287 288 Args: 289 test_name: Name of the test that triggered this function. 290 begin_time: Logline format timestamp taken when the test started. 291 """ 292 293 def _on_blocked(self, record): 294 """Proxy function to guarantee the base implementation of on_blocked 295 is called. 296 297 Args: 298 record: The records.TestResultRecord object for the blocked test 299 case. 300 """ 301 test_name = record.test_name 302 begin_time = logger.epoch_to_log_line_timestamp(record.begin_time) 303 self.log.info(RESULT_LINE_TEMPLATE, test_name, record.result) 304 self.log.info("Reason to block: %s", record.details) 305 self.on_blocked(test_name, begin_time) 306 307 def on_blocked(self, test_name, begin_time): 308 """A function that is executed upon a test begin skipped. 309 310 Args: 311 test_name: Name of the test that triggered this function. 312 begin_time: Logline format timestamp taken when the test started. 313 """ 314 315 def _on_exception(self, record): 316 """Proxy function to guarantee the base implementation of on_exception 317 is called. 318 319 Args: 320 record: The records.TestResultRecord object for the failed test 321 case. 322 """ 323 test_name = record.test_name 324 self.log.exception(record.details) 325 begin_time = logger.epoch_to_log_line_timestamp(record.begin_time) 326 self.on_exception(test_name, begin_time) 327 328 def on_exception(self, test_name, begin_time): 329 """A function that is executed upon an unhandled exception from a test 330 case. 331 332 Implementation is optional. 333 334 Args: 335 test_name: Name of the test that triggered this function. 336 begin_time: Logline format timestamp taken when the test started. 337 """ 338 339 def _exec_procedure_func(self, func, tr_record): 340 """Executes a procedure function like on_pass, on_fail etc. 341 342 This function will alternate the 'Result' of the test's record if 343 exceptions happened when executing the procedure function. 344 345 This will let signals.TestAbortAll through so abort_all works in all 346 procedure functions. 347 348 Args: 349 func: The procedure function to be executed. 350 tr_record: The TestResultRecord object associated with the test 351 case executed. 352 """ 353 try: 354 func(tr_record) 355 except signals.TestAbortAll: 356 raise 357 except Exception as e: 358 self.log.exception("Exception happened when executing %s for %s.", 359 func.__name__, self.current_test_name) 360 tr_record.add_error(func.__name__, e) 361 362 def exec_one_testcase(self, test_name, test_func, args, **kwargs): 363 """Executes one test case and update test results. 364 365 Executes one test case, create a records.TestResultRecord object with 366 the execution information, and add the record to the test class's test 367 results. 368 369 Args: 370 test_name: Name of the test. 371 test_func: The test function. 372 args: A tuple of params. 373 kwargs: Extra kwargs. 374 """ 375 is_generate_trigger = False 376 tr_record = records.TestResultRecord(test_name, self.TAG) 377 tr_record.test_begin() 378 self.log.info("%s %s", TEST_CASE_TOKEN, test_name) 379 verdict = None 380 try: 381 try: 382 if hasattr(self, 'android_devices'): 383 for ad in self.android_devices: 384 if not ad.is_adb_logcat_on: 385 ad.start_adb_logcat(cont_logcat_file=True) 386 ret = self._setup_test(test_name) 387 asserts.assert_true(ret is not False, 388 "Setup for %s failed." % test_name) 389 if args or kwargs: 390 verdict = test_func(*args, **kwargs) 391 else: 392 verdict = test_func() 393 finally: 394 try: 395 self._teardown_test(test_name) 396 except signals.TestAbortAll: 397 raise 398 except Exception as e: 399 self.log.error(traceback.format_exc()) 400 tr_record.add_error("teardown_test", e) 401 self._exec_procedure_func(self._on_exception, tr_record) 402 except (signals.TestFailure, AssertionError) as e: 403 self.log.error(e) 404 tr_record.test_fail(e) 405 self._exec_procedure_func(self._on_fail, tr_record) 406 except signals.TestSkip as e: 407 # Test skipped. 408 tr_record.test_skip(e) 409 self._exec_procedure_func(self._on_skip, tr_record) 410 except (signals.TestAbortClass, signals.TestAbortAll) as e: 411 # Abort signals, pass along. 412 tr_record.test_fail(e) 413 raise e 414 except signals.TestPass as e: 415 # Explicit test pass. 416 tr_record.test_pass(e) 417 self._exec_procedure_func(self._on_pass, tr_record) 418 except signals.TestSilent as e: 419 # This is a trigger test for generated tests, suppress reporting. 420 is_generate_trigger = True 421 self.results.requested.remove(test_name) 422 except signals.TestBlocked as e: 423 tr_record.test_blocked(e) 424 self._exec_procedure_func(self._on_blocked, tr_record) 425 except Exception as e: 426 self.log.error(traceback.format_exc()) 427 # Exception happened during test. 428 tr_record.test_unknown(e) 429 self._exec_procedure_func(self._on_exception, tr_record) 430 self._exec_procedure_func(self._on_fail, tr_record) 431 else: 432 # Keep supporting return False for now. 433 # TODO(angli): Deprecate return False support. 434 if verdict or (verdict is None): 435 # Test passed. 436 tr_record.test_pass() 437 self._exec_procedure_func(self._on_pass, tr_record) 438 return 439 # Test failed because it didn't return True. 440 # This should be removed eventually. 441 tr_record.test_fail() 442 self._exec_procedure_func(self._on_fail, tr_record) 443 finally: 444 if not is_generate_trigger: 445 self.results.add_record(tr_record) 446 447 def run_generated_testcases(self, 448 test_func, 449 settings, 450 args=None, 451 kwargs=None, 452 tag="", 453 name_func=None, 454 format_args=False): 455 """Runs generated test cases. 456 457 Generated test cases are not written down as functions, but as a list 458 of parameter sets. This way we reduce code repetition and improve 459 test case scalability. 460 461 Args: 462 test_func: The common logic shared by all these generated test 463 cases. This function should take at least one argument, 464 which is a parameter set. 465 settings: A list of strings representing parameter sets. These are 466 usually json strings that get loaded in the test_func. 467 args: Iterable of additional position args to be passed to 468 test_func. 469 kwargs: Dict of additional keyword args to be passed to test_func 470 tag: Name of this group of generated test cases. Ignored if 471 name_func is provided and operates properly. 472 name_func: A function that takes a test setting and generates a 473 proper test name. The test name should be shorter than 474 utils.MAX_FILENAME_LEN. Names over the limit will be 475 truncated. 476 format_args: If True, args will be appended as the first argument 477 in the args list passed to test_func. 478 479 Returns: 480 A list of settings that did not pass. 481 """ 482 args = args or () 483 kwargs = kwargs or {} 484 failed_settings = [] 485 486 for setting in settings: 487 test_name = "{} {}".format(tag, setting) 488 489 if name_func: 490 try: 491 test_name = name_func(setting, *args, **kwargs) 492 except: 493 self.log.exception(("Failed to get test name from " 494 "test_func. Fall back to default %s"), 495 test_name) 496 497 self.results.requested.append(test_name) 498 499 if len(test_name) > utils.MAX_FILENAME_LEN: 500 test_name = test_name[:utils.MAX_FILENAME_LEN] 501 502 previous_success_cnt = len(self.results.passed) 503 504 if format_args: 505 self.exec_one_testcase(test_name, test_func, 506 args + (setting, ), **kwargs) 507 else: 508 self.exec_one_testcase(test_name, test_func, 509 (setting, ) + args, **kwargs) 510 511 if len(self.results.passed) - previous_success_cnt != 1: 512 failed_settings.append(setting) 513 514 return failed_settings 515 516 def _exec_func(self, func, *args): 517 """Executes a function with exception safeguard. 518 519 This will let signals.TestAbortAll through so abort_all works in all 520 procedure functions. 521 522 Args: 523 func: Function to be executed. 524 args: Arguments to be passed to the function. 525 526 Returns: 527 Whatever the function returns, or False if unhandled exception 528 occured. 529 """ 530 try: 531 return func(*args) 532 except signals.TestAbortAll: 533 raise 534 except: 535 self.log.exception("Exception happened when executing %s in %s.", 536 func.__name__, self.TAG) 537 return False 538 539 def _get_all_test_names(self): 540 """Finds all the function names that match the test case naming 541 convention in this class. 542 543 Returns: 544 A list of strings, each is a test case name. 545 """ 546 test_names = [] 547 for name in dir(self): 548 if name.startswith("test_"): 549 test_names.append(name) 550 return test_names 551 552 def _get_test_funcs(self, test_names): 553 """Obtain the actual functions of test cases based on test names. 554 555 Args: 556 test_names: A list of strings, each string is a test case name. 557 558 Returns: 559 A list of tuples of (string, function). String is the test case 560 name, function is the actual test case function. 561 562 Raises: 563 Error is raised if the test name does not follow 564 naming convention "test_*". This can only be caused by user input 565 here. 566 """ 567 test_funcs = [] 568 for test_name in test_names: 569 test_funcs.append(self._get_test_func(test_name)) 570 571 return test_funcs 572 573 def _get_test_func(self, test_name): 574 """Obtain the actual function of test cases based on the test name. 575 576 Args: 577 test_name: String, The name of the test. 578 579 Returns: 580 A tuples of (string, function). String is the test case 581 name, function is the actual test case function. 582 583 Raises: 584 Error is raised if the test name does not follow 585 naming convention "test_*". This can only be caused by user input 586 here. 587 """ 588 if not test_name.startswith("test_"): 589 raise Error(("Test case name %s does not follow naming " 590 "convention test_*, abort.") % test_name) 591 try: 592 return test_name, getattr(self, test_name) 593 except: 594 595 def test_skip_func(*args, **kwargs): 596 raise signals.TestSkip("Test %s does not exist" % test_name) 597 598 self.log.info("Test case %s not found in %s.", test_name, self.TAG) 599 return test_name, test_skip_func 600 601 def _block_all_test_cases(self, tests): 602 """ 603 Block all passed in test cases. 604 Args: 605 tests: The tests to block. 606 """ 607 for test_name, test_func in tests: 608 signal = signals.TestBlocked("Failed class setup") 609 record = records.TestResultRecord(test_name, self.TAG) 610 record.test_begin() 611 if hasattr(test_func, 'gather'): 612 signal.extras = test_func.gather() 613 record.test_blocked(signal) 614 self.results.add_record(record) 615 self._on_blocked(record) 616 617 def run(self, test_names=None, test_case_iterations=1): 618 """Runs test cases within a test class by the order they appear in the 619 execution list. 620 621 One of these test cases lists will be executed, shown here in priority 622 order: 623 1. The test_names list, which is passed from cmd line. Invalid names 624 are guarded by cmd line arg parsing. 625 2. The self.tests list defined in test class. Invalid names are 626 ignored. 627 3. All function that matches test case naming convention in the test 628 class. 629 630 Args: 631 test_names: A list of string that are test case names requested in 632 cmd line. 633 634 Returns: 635 The test results object of this class. 636 """ 637 self.log.info("==========> %s <==========", self.TAG) 638 # Devise the actual test cases to run in the test class. 639 if not test_names: 640 if self.tests: 641 # Specified by run list in class. 642 test_names = list(self.tests) 643 else: 644 # No test case specified by user, execute all in the test class 645 test_names = self._get_all_test_names() 646 self.results.requested = test_names 647 tests = self._get_test_funcs(test_names) 648 # A TestResultRecord used for when setup_class fails. 649 # Setup for the class. 650 try: 651 if self._setup_class() is False: 652 self.log.error("Failed to setup %s.", self.TAG) 653 self._block_all_test_cases(tests) 654 return self.results 655 except Exception as e: 656 self.log.exception("Failed to setup %s.", self.TAG) 657 self._exec_func(self.teardown_class) 658 self._block_all_test_cases(tests) 659 return self.results 660 # Run tests in order. 661 try: 662 for test_name, test_func in tests: 663 for _ in range(test_case_iterations): 664 self.exec_one_testcase(test_name, test_func, self.cli_args) 665 return self.results 666 except signals.TestAbortClass: 667 return self.results 668 except signals.TestAbortAll as e: 669 # Piggy-back test results on this exception object so we don't lose 670 # results from this test class. 671 setattr(e, "results", self.results) 672 raise e 673 finally: 674 self._exec_func(self.teardown_class) 675 self.log.info("Summary for test class %s: %s", self.TAG, 676 self.results.summary_str()) 677 678 def clean_up(self): 679 """A function that is executed upon completion of all tests cases 680 selected in the test class. 681 682 This function should clean up objects initialized in the constructor by 683 user. 684 """ 685 686 def _take_bug_report(self, test_name, begin_time): 687 if "no_bug_report_on_fail" in self.user_params: 688 return 689 690 # magical sleep to ensure the runtime restart or reboot begins 691 time.sleep(1) 692 for ad in self.android_devices: 693 try: 694 ad.adb.wait_for_device() 695 ad.take_bug_report(test_name, begin_time) 696 bugreport_path = os.path.join(ad.log_path, test_name) 697 utils.create_dir(bugreport_path) 698 ad.check_crash_report(True, test_name) 699 if getattr(ad, "qxdm_always_on", False): 700 ad.log.info("Pull QXDM Logs") 701 ad.pull_files(["/data/vendor/radio/diag_logs/logs/"], 702 bugreport_path) 703 except Exception as e: 704 ad.log.error( 705 "Failed to take a bug report for %s with error %s", 706 test_name, e) 707 708 def _reboot_device(self, ad): 709 ad.log.info("Rebooting device.") 710 ad = ad.reboot() 711 712 def _cleanup_logger_sessions(self): 713 for (logger, session) in self.logger_sessions: 714 self.log.info("Resetting a diagnostic session %s, %s", logger, 715 session) 716 logger.reset() 717 self.logger_sessions = [] 718 719 def _pull_diag_logs(self, test_name, begin_time): 720 for (logger, session) in self.logger_sessions: 721 self.log.info("Pulling diagnostic session %s", logger) 722 logger.stop(session) 723 diag_path = os.path.join(self.log_path, begin_time) 724 utils.create_dir(diag_path) 725 logger.pull(session, diag_path) 726