1 # 2 # Copyright (C) 2016 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 # 16 17 import logging 18 import os 19 import re 20 import sys 21 22 from vts.proto import VtsReportMessage_pb2 as ReportMsg 23 from vts.runners.host import asserts 24 from vts.runners.host import const 25 from vts.runners.host import errors 26 from vts.runners.host import keys 27 from vts.runners.host import logger 28 from vts.runners.host import records 29 from vts.runners.host import signals 30 from vts.runners.host import utils 31 from vts.utils.python.controllers import android_device 32 from vts.utils.python.common import filter_utils 33 from vts.utils.python.common import list_utils 34 from vts.utils.python.coverage import coverage_utils 35 from vts.utils.python.coverage import sancov_utils 36 from vts.utils.python.precondition import precondition_utils 37 from vts.utils.python.profiling import profiling_utils 38 from vts.utils.python.reporting import log_uploading_utils 39 from vts.utils.python.systrace import systrace_utils 40 from vts.utils.python.web import feature_utils 41 from vts.utils.python.web import web_utils 42 43 from acts import signals as acts_signals 44 45 # Macro strings for test result reporting 46 TEST_CASE_TOKEN = "[Test Case]" 47 RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 48 STR_TEST = "test" 49 STR_GENERATE = "generate" 50 _REPORT_MESSAGE_FILE_NAME = "report_proto.msg" 51 _BUG_REPORT_FILE_PREFIX = "bugreport" 52 _BUG_REPORT_FILE_EXTENSION = ".zip" 53 _LOGCAT_FILE_PREFIX = "logcat" 54 _LOGCAT_FILE_EXTENSION = ".txt" 55 _ANDROID_DEVICES = '_android_devices' 56 _REASON_TO_SKIP_ALL_TESTS = '_reason_to_skip_all_tests' 57 _SETUP_RETRY_NUMBER = 5 58 59 LOGCAT_BUFFERS = [ 60 'radio', 61 'events', 62 'main', 63 'system', 64 'crash' 65 ] 66 67 68 class BaseTestClass(object): 69 """Base class for all test classes to inherit from. 70 71 This class gets all the controller objects from test_runner and executes 72 the test cases requested within itself. 73 74 Most attributes of this class are set at runtime based on the configuration 75 provided. 76 77 Attributes: 78 android_devices: A list of AndroidDevice object, representing android 79 devices. 80 test_module_name: A string representing the test module name. 81 tests: A list of strings, each representing a test case name. 82 log: A logger object used for logging. 83 results: A records.TestResult object for aggregating test results from 84 the execution of test cases. 85 _current_record: A records.TestResultRecord object for the test case 86 currently being executed. If no test is running, this 87 should be None. 88 include_filer: A list of string, each representing a test case name to 89 include. 90 exclude_filer: A list of string, each representing a test case name to 91 exclude. Has no effect if include_filer is not empty. 92 abi_name: String, name of abi in use 93 abi_bitness: String, bitness of abi in use 94 web: WebFeature, object storing web feature util for test run 95 coverage: CoverageFeature, object storing coverage feature util for test run 96 sancov: SancovFeature, object storing sancov feature util for test run 97 profiling: ProfilingFeature, object storing profiling feature util for test run 98 _bug_report_on_failure: bool, whether to catch bug report at the end 99 of failed test cases. Default is False 100 _logcat_on_failure: bool, whether to dump logcat at the end 101 of failed test cases. Default is True 102 test_filter: Filter object to filter test names. 103 """ 104 105 def __init__(self, configs): 106 self.tests = [] 107 # Set all the controller objects and params. 108 for name, value in configs.items(): 109 setattr(self, name, value) 110 self.results = records.TestResult() 111 self.log = logger.LoggerProxy() 112 self._current_record = None 113 114 # Setup test filters 115 self.include_filter = self.getUserParam( 116 [ 117 keys.ConfigKeys.KEY_TEST_SUITE, 118 keys.ConfigKeys.KEY_INCLUDE_FILTER 119 ], 120 default_value=[]) 121 self.exclude_filter = self.getUserParam( 122 [ 123 keys.ConfigKeys.KEY_TEST_SUITE, 124 keys.ConfigKeys.KEY_EXCLUDE_FILTER 125 ], 126 default_value=[]) 127 128 # TODO(yuexima): remove include_filter and exclude_filter from class attributes 129 # after confirming all modules no longer have reference to them 130 self.include_filter = list_utils.ExpandItemDelimiters( 131 list_utils.ItemsToStr(self.include_filter), ',') 132 self.exclude_filter = list_utils.ExpandItemDelimiters( 133 list_utils.ItemsToStr(self.exclude_filter), ',') 134 exclude_over_include = self.getUserParam( 135 keys.ConfigKeys.KEY_EXCLUDE_OVER_INCLUDE, default_value=None) 136 self.test_module_name = self.getUserParam( 137 keys.ConfigKeys.KEY_TESTBED_NAME, 138 log_warning_and_continue_if_not_found=True, 139 default_value=self.__class__.__name__) 140 self.test_filter = filter_utils.Filter( 141 self.include_filter, 142 self.exclude_filter, 143 enable_regex=True, 144 exclude_over_include=exclude_over_include, 145 enable_negative_pattern=True, 146 enable_module_name_prefix_matching=True, 147 module_name=self.test_module_name, 148 expand_bitness=True) 149 logging.info('Test filter: %s' % self.test_filter) 150 151 # TODO: get abi information differently for multi-device support. 152 # Set other optional parameters 153 self.abi_name = self.getUserParam( 154 keys.ConfigKeys.IKEY_ABI_NAME, default_value=None) 155 self.abi_bitness = self.getUserParam( 156 keys.ConfigKeys.IKEY_ABI_BITNESS, default_value=None) 157 self.skip_on_32bit_abi = self.getUserParam( 158 keys.ConfigKeys.IKEY_SKIP_ON_32BIT_ABI, default_value=False) 159 self.skip_on_64bit_abi = self.getUserParam( 160 keys.ConfigKeys.IKEY_SKIP_ON_64BIT_ABI, default_value=False) 161 self.run_32bit_on_64bit_abi = self.getUserParam( 162 keys.ConfigKeys.IKEY_RUN_32BIT_ON_64BIT_ABI, default_value=False) 163 self.web = web_utils.WebFeature(self.user_params) 164 self.coverage = coverage_utils.CoverageFeature( 165 self.user_params, web=self.web) 166 self.sancov = sancov_utils.SancovFeature( 167 self.user_params, web=self.web) 168 self.profiling = profiling_utils.ProfilingFeature( 169 self.user_params, web=self.web) 170 self.systrace = systrace_utils.SystraceFeature( 171 self.user_params, web=self.web) 172 self.log_uploading = log_uploading_utils.LogUploadingFeature( 173 self.user_params, web=self.web) 174 self.collect_tests_only = self.getUserParam( 175 keys.ConfigKeys.IKEY_COLLECT_TESTS_ONLY, default_value=False) 176 self.run_as_vts_self_test = self.getUserParam( 177 keys.ConfigKeys.RUN_AS_VTS_SELFTEST, default_value=False) 178 self.run_as_compliance_test = self.getUserParam( 179 keys.ConfigKeys.RUN_AS_COMPLIANCE_TEST, default_value=False) 180 self._bug_report_on_failure = self.getUserParam( 181 keys.ConfigKeys.IKEY_BUG_REPORT_ON_FAILURE, default_value=False) 182 self._logcat_on_failure = self.getUserParam( 183 keys.ConfigKeys.IKEY_LOGCAT_ON_FAILURE, default_value=True) 184 185 @property 186 def android_devices(self): 187 """Returns a list of AndroidDevice objects""" 188 if not hasattr(self, _ANDROID_DEVICES): 189 setattr(self, _ANDROID_DEVICES, 190 self.registerController(android_device)) 191 return getattr(self, _ANDROID_DEVICES) 192 193 @android_devices.setter 194 def android_devices(self, devices): 195 """Set the list of AndroidDevice objects""" 196 setattr(self, _ANDROID_DEVICES, devices) 197 198 def __enter__(self): 199 return self 200 201 def __exit__(self, *args): 202 self._exec_func(self.cleanUp) 203 204 def unpack_userparams(self, req_param_names=[], opt_param_names=[], **kwargs): 205 """Wrapper for test cases using ACTS runner API.""" 206 return self.getUserParams(req_param_names, opt_param_names, **kwargs) 207 208 def getUserParams(self, req_param_names=[], opt_param_names=[], **kwargs): 209 """Unpacks user defined parameters in test config into individual 210 variables. 211 212 Instead of accessing the user param with self.user_params["xxx"], the 213 variable can be directly accessed with self.xxx. 214 215 A missing required param will raise an exception. If an optional param 216 is missing, an INFO line will be logged. 217 218 Args: 219 req_param_names: A list of names of the required user params. 220 opt_param_names: A list of names of the optional user params. 221 **kwargs: Arguments that provide default values. 222 e.g. getUserParams(required_list, opt_list, arg_a="hello") 223 self.arg_a will be "hello" unless it is specified again in 224 required_list or opt_list. 225 226 Raises: 227 BaseTestError is raised if a required user params is missing from 228 test config. 229 """ 230 for k, v in kwargs.items(): 231 setattr(self, k, v) 232 for name in req_param_names: 233 if name not in self.user_params: 234 raise errors.BaseTestError(("Missing required user param '%s' " 235 "in test configuration.") % name) 236 setattr(self, name, self.user_params[name]) 237 for name in opt_param_names: 238 if name not in self.user_params: 239 logging.info(("Missing optional user param '%s' in " 240 "configuration, continue."), name) 241 else: 242 setattr(self, name, self.user_params[name]) 243 244 def getUserParam(self, 245 param_name, 246 error_if_not_found=False, 247 log_warning_and_continue_if_not_found=False, 248 default_value=None, 249 to_str=False): 250 """Get the value of a single user parameter. 251 252 This method returns the value of specified user parameter. 253 Note: this method will not automatically set attribute using the parameter name and value. 254 255 Args: 256 param_name: string or list of string, denoting user parameter names. If provided 257 a single string, self.user_params["<param_name>"] will be accessed. 258 If provided multiple strings, 259 self.user_params["<param_name1>"]["<param_name2>"]["<param_name3>"]... 260 will be accessed. 261 error_if_not_found: bool, whether to raise error if parameter not exists. Default: 262 False 263 log_warning_and_continue_if_not_found: bool, log a warning message if parameter value 264 not found. 265 default_value: object, default value to return if not found. If error_if_not_found is 266 True, this parameter has no effect. Default: None 267 to_str: boolean, whether to convert the result object to string if not None. 268 Note, strings passing in from java json config are usually unicode. 269 270 Returns: 271 object, value of the specified parameter name chain if exists; 272 <default_value> if not exists. 273 """ 274 275 def ToStr(return_value): 276 """Check to_str option and convert to string if not None""" 277 if to_str and return_value is not None: 278 return str(return_value) 279 return return_value 280 281 if not param_name: 282 if error_if_not_found: 283 raise errors.BaseTestError("empty param_name provided") 284 logging.error("empty param_name") 285 return ToStr(default_value) 286 287 if not isinstance(param_name, list): 288 param_name = [param_name] 289 290 curr_obj = self.user_params 291 for param in param_name: 292 if param not in curr_obj: 293 msg = "Missing user param '%s' in test configuration." % param_name 294 if error_if_not_found: 295 raise errors.BaseTestError(msg) 296 elif log_warning_and_continue_if_not_found: 297 logging.warn(msg) 298 return ToStr(default_value) 299 curr_obj = curr_obj[param] 300 301 return ToStr(curr_obj) 302 303 def _setUpClass(self): 304 """Proxy function to guarantee the base implementation of setUpClass 305 is called. 306 """ 307 if not precondition_utils.MeetFirstApiLevelPrecondition(self): 308 self.skipAllTests("The device's first API level doesn't meet the " 309 "precondition.") 310 return self.setUpClass() 311 312 def setUpClass(self): 313 """Setup function that will be called before executing any test case in 314 the test class. 315 316 To signal setup failure, return False or raise an exception. If 317 exceptions were raised, the stack trace would appear in log, but the 318 exceptions would not propagate to upper levels. 319 320 Implementation is optional. 321 """ 322 pass 323 324 def _tearDownClass(self): 325 """Proxy function to guarantee the base implementation of tearDownClass 326 is called. 327 """ 328 ret = self.tearDownClass() 329 if self.log_uploading.enabled: 330 self.log_uploading.UploadLogs() 331 if self.web.enabled: 332 message_b = self.web.GenerateReportMessage(self.results.requested, 333 self.results.executed) 334 else: 335 message_b = '' 336 337 report_proto_path = os.path.join(logging.log_path, 338 _REPORT_MESSAGE_FILE_NAME) 339 340 if message_b: 341 logging.info('Result proto message path: %s', report_proto_path) 342 343 with open(report_proto_path, "wb") as f: 344 f.write(message_b) 345 346 return ret 347 348 def tearDownClass(self): 349 """Teardown function that will be called after all the selected test 350 cases in the test class have been executed. 351 352 Implementation is optional. 353 """ 354 pass 355 356 def _testEntry(self, test_record): 357 """Internal function to be called upon entry of a test case. 358 359 Args: 360 test_record: The TestResultRecord object for the test case going to 361 be executed. 362 """ 363 self._current_record = test_record 364 if self.web.enabled: 365 self.web.AddTestReport(test_record.test_name) 366 367 def _setUp(self, test_name): 368 """Proxy function to guarantee the base implementation of setUp is 369 called. 370 """ 371 if self.systrace.enabled: 372 self.systrace.StartSystrace() 373 return self.setUp() 374 375 def setUp(self): 376 """Setup function that will be called every time before executing each 377 test case in the test class. 378 379 To signal setup failure, return False or raise an exception. If 380 exceptions were raised, the stack trace would appear in log, but the 381 exceptions would not propagate to upper levels. 382 383 Implementation is optional. 384 """ 385 386 def _testExit(self): 387 """Internal function to be called upon exit of a test.""" 388 self._current_record = None 389 390 def _tearDown(self, test_name): 391 """Proxy function to guarantee the base implementation of tearDown 392 is called. 393 """ 394 if self.systrace.enabled: 395 self.systrace.ProcessAndUploadSystrace(test_name) 396 self.tearDown() 397 398 def tearDown(self): 399 """Teardown function that will be called every time a test case has 400 been executed. 401 402 Implementation is optional. 403 """ 404 405 def _onFail(self): 406 """Proxy function to guarantee the base implementation of onFail is 407 called. 408 """ 409 record = self._current_record 410 logging.error(record.details) 411 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 412 logging.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 413 if self.web.enabled: 414 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_FAIL) 415 self.onFail(record.test_name, begin_time) 416 if self._bug_report_on_failure: 417 self.DumpBugReport( 418 '%s-%s' % (self.test_module_name, record.test_name)) 419 if self._logcat_on_failure: 420 self.DumpLogcat('%s-%s' % (self.test_module_name, record.test_name)) 421 422 def onFail(self, test_name, begin_time): 423 """A function that is executed upon a test case failure. 424 425 User implementation is optional. 426 427 Args: 428 test_name: Name of the test that triggered this function. 429 begin_time: Logline format timestamp taken when the test started. 430 """ 431 432 def _onPass(self): 433 """Proxy function to guarantee the base implementation of onPass is 434 called. 435 """ 436 record = self._current_record 437 test_name = record.test_name 438 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 439 msg = record.details 440 if msg: 441 logging.info(msg) 442 logging.info(RESULT_LINE_TEMPLATE, test_name, record.result) 443 if self.web.enabled: 444 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_PASS) 445 self.onPass(test_name, begin_time) 446 447 def onPass(self, test_name, begin_time): 448 """A function that is executed upon a test case passing. 449 450 Implementation is optional. 451 452 Args: 453 test_name: Name of the test that triggered this function. 454 begin_time: Logline format timestamp taken when the test started. 455 """ 456 457 def _onSkip(self): 458 """Proxy function to guarantee the base implementation of onSkip is 459 called. 460 """ 461 record = self._current_record 462 test_name = record.test_name 463 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 464 logging.info(RESULT_LINE_TEMPLATE, test_name, record.result) 465 logging.info("Reason to skip: %s", record.details) 466 if self.web.enabled: 467 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_SKIP) 468 self.onSkip(test_name, begin_time) 469 470 def onSkip(self, test_name, begin_time): 471 """A function that is executed upon a test case being skipped. 472 473 Implementation is optional. 474 475 Args: 476 test_name: Name of the test that triggered this function. 477 begin_time: Logline format timestamp taken when the test started. 478 """ 479 480 def _onSilent(self): 481 """Proxy function to guarantee the base implementation of onSilent is 482 called. 483 """ 484 record = self._current_record 485 test_name = record.test_name 486 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 487 if self.web.enabled: 488 self.web.SetTestResult(None) 489 self.onSilent(test_name, begin_time) 490 491 def onSilent(self, test_name, begin_time): 492 """A function that is executed upon a test case being marked as silent. 493 494 Implementation is optional. 495 496 Args: 497 test_name: Name of the test that triggered this function. 498 begin_time: Logline format timestamp taken when the test started. 499 """ 500 501 def _onException(self): 502 """Proxy function to guarantee the base implementation of onException 503 is called. 504 """ 505 record = self._current_record 506 test_name = record.test_name 507 logging.exception(record.details) 508 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 509 if self.web.enabled: 510 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_EXCEPTION) 511 self.onException(test_name, begin_time) 512 if self._bug_report_on_failure: 513 self.DumpBugReport( 514 '%s-%s' % (self.test_module_name, record.test_name)) 515 if self._logcat_on_failure: 516 self.DumpLogcat('%s-%s' % (self.test_module_name, record.test_name)) 517 518 def onException(self, test_name, begin_time): 519 """A function that is executed upon an unhandled exception from a test 520 case. 521 522 Implementation is optional. 523 524 Args: 525 test_name: Name of the test that triggered this function. 526 begin_time: Logline format timestamp taken when the test started. 527 """ 528 529 def _exec_procedure_func(self, func): 530 """Executes a procedure function like onPass, onFail etc. 531 532 This function will alternate the 'Result' of the test's record if 533 exceptions happened when executing the procedure function. 534 535 This will let signals.TestAbortAll through so abortAll works in all 536 procedure functions. 537 538 Args: 539 func: The procedure function to be executed. 540 """ 541 record = self._current_record 542 if record is None: 543 logging.error("Cannot execute %s. No record for current test.", 544 func.__name__) 545 return 546 try: 547 func() 548 except (signals.TestAbortAll, acts_signals.TestAbortAll) as e: 549 raise signals.TestAbortAll, e, sys.exc_info()[2] 550 except Exception as e: 551 logging.exception("Exception happened when executing %s for %s.", 552 func.__name__, record.test_name) 553 record.addError(func.__name__, e) 554 555 def addTableToResult(self, name, rows): 556 """Adds a table to current test record. 557 558 A subclass can call this method to add a table to _current_record when 559 running test cases. 560 561 Args: 562 name: String, the table name. 563 rows: A 2-dimensional list which contains the data. 564 """ 565 self._current_record.addTable(name, rows) 566 567 def filterOneTest(self, test_name, test_filter=None): 568 """Check test filters for a test name. 569 570 The first layer of filter is user defined test filters: 571 if a include filter is not empty, only tests in include filter will 572 be executed regardless whether they are also in exclude filter. Else 573 if include filter is empty, only tests not in exclude filter will be 574 executed. 575 576 The second layer of filter is checking whether skipAllTests method is 577 called. If the flag is set, this method raises signals.TestSkip. 578 579 The third layer of filter is checking abi bitness: 580 if a test has a suffix indicating the intended architecture bitness, 581 and the current abi bitness information is available, non matching tests 582 will be skipped. By our convention, this function will look for bitness in suffix 583 formated as "32bit", "32Bit", "32BIT", or 64 bit equivalents. 584 585 This method assumes const.SUFFIX_32BIT and const.SUFFIX_64BIT are in lower cases. 586 587 Args: 588 test_name: string, name of a test 589 test_filter: Filter object, test filter 590 591 Raises: 592 signals.TestSilent if a test should not be executed 593 signals.TestSkip if a test should be logged but not be executed 594 """ 595 self._filterOneTestThroughTestFilter(test_name, test_filter) 596 self._filterOneTestThroughAbiBitness(test_name) 597 598 def _filterOneTestThroughTestFilter(self, test_name, test_filter=None): 599 """Check test filter for the given test name. 600 601 Args: 602 test_name: string, name of a test 603 604 Raises: 605 signals.TestSilent if a test should not be executed 606 signals.TestSkip if a test should be logged but not be executed 607 """ 608 if not test_filter: 609 test_filter = self.test_filter 610 611 if not test_filter.Filter(test_name): 612 raise signals.TestSilent("Test case '%s' did not pass filters.") 613 614 if self.isSkipAllTests(): 615 raise signals.TestSkip(self.getSkipAllTestsReason()) 616 617 def _filterOneTestThroughAbiBitness(self, test_name): 618 """Check test filter for the given test name. 619 620 Args: 621 test_name: string, name of a test 622 623 Raises: 624 signals.TestSilent if a test should not be executed 625 """ 626 asserts.skipIf( 627 self.abi_bitness and 628 ((self.skip_on_32bit_abi is True) and self.abi_bitness == "32") or 629 ((self.skip_on_64bit_abi is True) and self.abi_bitness == "64") or 630 (test_name.lower().endswith(const.SUFFIX_32BIT) and 631 self.abi_bitness != "32") or 632 (test_name.lower().endswith(const.SUFFIX_64BIT) and 633 self.abi_bitness != "64" and not self.run_32bit_on_64bit_abi), 634 "Test case '{}' excluded as ABI bitness is {}.".format( 635 test_name, self.abi_bitness)) 636 637 def execOneTest(self, test_name, test_func, args, **kwargs): 638 """Executes one test case and update test results. 639 640 Executes one test case, create a records.TestResultRecord object with 641 the execution information, and add the record to the test class's test 642 results. 643 644 Args: 645 test_name: Name of the test. 646 test_func: The test function. 647 args: A tuple of params. 648 kwargs: Extra kwargs. 649 """ 650 is_silenced = False 651 tr_record = records.TestResultRecord(test_name, self.test_module_name) 652 tr_record.testBegin() 653 logging.info("%s %s", TEST_CASE_TOKEN, test_name) 654 verdict = None 655 finished = False 656 try: 657 ret = self._testEntry(tr_record) 658 asserts.assertTrue(ret is not False, 659 "Setup test entry for %s failed." % test_name) 660 self.filterOneTest(test_name) 661 if self.collect_tests_only: 662 asserts.explicitPass("Collect tests only.") 663 664 try: 665 ret = self._setUp(test_name) 666 asserts.assertTrue(ret is not False, 667 "Setup for %s failed." % test_name) 668 669 if args or kwargs: 670 verdict = test_func(*args, **kwargs) 671 else: 672 verdict = test_func() 673 finished = True 674 finally: 675 self._tearDown(test_name) 676 except (signals.TestFailure, acts_signals.TestFailure, AssertionError) as e: 677 tr_record.testFail(e) 678 self._exec_procedure_func(self._onFail) 679 finished = True 680 except (signals.TestSkip, acts_signals.TestSkip) as e: 681 # Test skipped. 682 tr_record.testSkip(e) 683 self._exec_procedure_func(self._onSkip) 684 finished = True 685 except (signals.TestAbortClass, acts_signals.TestAbortClass) as e: 686 # Abort signals, pass along. 687 tr_record.testFail(e) 688 finished = True 689 raise signals.TestAbortClass, e, sys.exc_info()[2] 690 except (signals.TestAbortAll, acts_signals.TestAbortAll) as e: 691 # Abort signals, pass along. 692 tr_record.testFail(e) 693 finished = True 694 raise signals.TestAbortAll, e, sys.exc_info()[2] 695 except (signals.TestPass, acts_signals.TestPass) as e: 696 # Explicit test pass. 697 tr_record.testPass(e) 698 self._exec_procedure_func(self._onPass) 699 finished = True 700 except (signals.TestSilent, acts_signals.TestSilent) as e: 701 # Suppress test reporting. 702 is_silenced = True 703 self._exec_procedure_func(self._onSilent) 704 self.results.removeRecord(tr_record) 705 finished = True 706 except Exception as e: 707 # Exception happened during test. 708 logging.exception(e) 709 tr_record.testError(e) 710 self._exec_procedure_func(self._onException) 711 self._exec_procedure_func(self._onFail) 712 finished = True 713 else: 714 # Keep supporting return False for now. 715 # TODO(angli): Deprecate return False support. 716 if verdict or (verdict is None): 717 # Test passed. 718 tr_record.testPass() 719 self._exec_procedure_func(self._onPass) 720 return 721 # Test failed because it didn't return True. 722 # This should be removed eventually. 723 tr_record.testFail() 724 self._exec_procedure_func(self._onFail) 725 finished = True 726 finally: 727 if not finished: 728 for device in self.android_devices: 729 # if shell has not been set up yet 730 if device.shell is not None: 731 device.shell.DisableShell() 732 733 logging.error('Test timed out.') 734 tr_record.testError() 735 self._exec_procedure_func(self._onException) 736 self._exec_procedure_func(self._onFail) 737 738 if not is_silenced: 739 self.results.addRecord(tr_record) 740 self._testExit() 741 742 def runGeneratedTests(self, 743 test_func, 744 settings, 745 args=None, 746 kwargs=None, 747 tag="", 748 name_func=None): 749 """Runs generated test cases. 750 751 Generated test cases are not written down as functions, but as a list 752 of parameter sets. This way we reduce code repetition and improve 753 test case scalability. 754 755 Args: 756 test_func: The common logic shared by all these generated test 757 cases. This function should take at least one argument, 758 which is a parameter set. 759 settings: A list of strings representing parameter sets. These are 760 usually json strings that get loaded in the test_func. 761 args: Iterable of additional position args to be passed to 762 test_func. 763 kwargs: Dict of additional keyword args to be passed to test_func 764 tag: Name of this group of generated test cases. Ignored if 765 name_func is provided and operates properly. 766 name_func: A function that takes a test setting and generates a 767 proper test name. The test name should be shorter than 768 utils.MAX_FILENAME_LEN. Names over the limit will be 769 truncated. 770 771 Returns: 772 A list of settings that did not pass. 773 """ 774 args = args or () 775 kwargs = kwargs or {} 776 failed_settings = [] 777 778 def GenerateTestName(setting): 779 test_name = "{} {}".format(tag, setting) 780 if name_func: 781 try: 782 test_name = name_func(setting, *args, **kwargs) 783 except: 784 logging.exception(("Failed to get test name from " 785 "test_func. Fall back to default %s"), 786 test_name) 787 788 if len(test_name) > utils.MAX_FILENAME_LEN: 789 test_name = test_name[:utils.MAX_FILENAME_LEN] 790 791 return test_name 792 793 for setting in settings: 794 test_name = GenerateTestName(setting) 795 796 tr_record = records.TestResultRecord(test_name, self.test_module_name) 797 self.results.requested.append(tr_record) 798 799 for setting in settings: 800 test_name = GenerateTestName(setting) 801 previous_success_cnt = len(self.results.passed) 802 803 self.execOneTest(test_name, test_func, (setting, ) + args, **kwargs) 804 if len(self.results.passed) - previous_success_cnt != 1: 805 failed_settings.append(setting) 806 807 return failed_settings 808 809 def _exec_func(self, func, *args): 810 """Executes a function with exception safeguard. 811 812 This will let signals.TestAbortAll through so abortAll works in all 813 procedure functions. 814 815 Args: 816 func: Function to be executed. 817 args: Arguments to be passed to the function. 818 819 Returns: 820 Whatever the function returns, or False if non-caught exception 821 occurred. 822 """ 823 try: 824 return func(*args) 825 except (signals.TestAbortAll, acts_signals.TestAbortAll) as e: 826 raise signals.TestAbortAll, e, sys.exc_info()[2] 827 except: 828 logging.exception("Exception happened when executing %s in %s.", 829 func.__name__, self.test_module_name) 830 return False 831 832 def _get_all_test_names(self): 833 """Finds all the function names that match the test case naming 834 convention in this class. 835 836 Returns: 837 A list of strings, each is a test case name. 838 """ 839 test_names = [] 840 for name in dir(self): 841 if name.startswith(STR_TEST) or name.startswith(STR_GENERATE): 842 attr_func = getattr(self, name) 843 if hasattr(attr_func, "__call__"): 844 test_names.append(name) 845 return test_names 846 847 def _get_test_funcs(self, test_names): 848 """Obtain the actual functions of test cases based on test names. 849 850 Args: 851 test_names: A list of strings, each string is a test case name. 852 853 Returns: 854 A list of tuples of (string, function). String is the test case 855 name, function is the actual test case function. 856 857 Raises: 858 errors.USERError is raised if the test name does not follow 859 naming convention "test_*". This can only be caused by user input 860 here. 861 """ 862 test_funcs = [] 863 for test_name in test_names: 864 if not hasattr(self, test_name): 865 logging.warning("%s does not have test case %s.", 866 self.test_module_name, test_name) 867 elif (test_name.startswith(STR_TEST) or 868 test_name.startswith(STR_GENERATE)): 869 test_funcs.append((test_name, getattr(self, test_name))) 870 else: 871 msg = ("Test case name %s does not follow naming convention " 872 "test*, abort.") % test_name 873 raise errors.USERError(msg) 874 875 return test_funcs 876 877 def getTests(self, test_names=None): 878 """Get the test cases within a test class. 879 880 Args: 881 test_names: A list of string that are test case names requested in 882 cmd line. 883 884 Returns: 885 A list of tuples of (string, function). String is the test case 886 name, function is the actual test case function. 887 """ 888 if not test_names: 889 if self.tests: 890 # Specified by run list in class. 891 test_names = list(self.tests) 892 else: 893 # No test case specified by user, execute all in the test class 894 test_names = self._get_all_test_names() 895 896 tests = self._get_test_funcs(test_names) 897 return tests 898 899 def runTests(self, tests): 900 """Run tests and collect test results. 901 902 Args: 903 tests: A list of tests to be run. 904 905 Returns: 906 The test results object of this class. 907 """ 908 # Setup for the class with retry. 909 for i in xrange(_SETUP_RETRY_NUMBER): 910 try: 911 if self._setUpClass() is False: 912 raise signals.TestFailure( 913 "Failed to setup %s." % self.test_module_name) 914 else: 915 break 916 except Exception as e: 917 logging.exception("Failed to setup %s.", self.test_module_name) 918 if i + 1 == _SETUP_RETRY_NUMBER: 919 self.results.failClass(self.test_module_name, e) 920 self._exec_func(self._tearDownClass) 921 return self.results 922 else: 923 # restart services before retry setup. 924 for device in self.android_devices: 925 logging.info("restarting service on device %s", device.serial) 926 device.stopServices() 927 device.startServices() 928 929 # Run tests in order. 930 try: 931 # Check if module is running in self test mode. 932 if self.run_as_vts_self_test: 933 logging.info('setUpClass function was executed successfully.') 934 self.results.passClass(self.test_module_name) 935 return self.results 936 937 for test_name, test_func in tests: 938 if test_name.startswith(STR_GENERATE): 939 logging.info( 940 "Executing generated test trigger function '%s'", 941 test_name) 942 test_func() 943 logging.info("Finished '%s'", test_name) 944 else: 945 self.execOneTest(test_name, test_func, None) 946 if self.isSkipAllTests() and not self.results.executed: 947 self.results.skipClass( 948 self.test_module_name, 949 "All test cases skipped; unable to find any test case.") 950 return self.results 951 except (signals.TestAbortClass, acts_signals.TestAbortClass): 952 logging.info("Received TestAbortClass signal") 953 return self.results 954 except (signals.TestAbortAll, acts_signals.TestAbortAll) as e: 955 logging.info("Received TestAbortAll signal") 956 # Piggy-back test results on this exception object so we don't lose 957 # results from this test class. 958 setattr(e, "results", self.results) 959 raise signals.TestAbortAll, e, sys.exc_info()[2] 960 except Exception as e: 961 # Exception happened during test. 962 logging.exception(e) 963 raise e 964 finally: 965 self._exec_func(self._tearDownClass) 966 if self.web.enabled: 967 name, timestamp = self.web.GetTestModuleKeys() 968 self.results.setTestModuleKeys(name, timestamp) 969 logging.info("Summary for test class %s: %s", 970 self.test_module_name, self.results.summary()) 971 972 def run(self, test_names=None): 973 """Runs test cases within a test class by the order they appear in the 974 execution list. 975 976 One of these test cases lists will be executed, shown here in priority 977 order: 978 1. The test_names list, which is passed from cmd line. Invalid names 979 are guarded by cmd line arg parsing. 980 2. The self.tests list defined in test class. Invalid names are 981 ignored. 982 3. All function that matches test case naming convention in the test 983 class. 984 985 Args: 986 test_names: A list of string that are test case names requested in 987 cmd line. 988 989 Returns: 990 The test results object of this class. 991 """ 992 logging.info("==========> %s <==========", self.test_module_name) 993 # Devise the actual test cases to run in the test class. 994 tests = self.getTests(test_names) 995 996 if not self.run_as_vts_self_test: 997 self.results.requested = [ 998 records.TestResultRecord(test_name, self.test_module_name) 999 for test_name,_ in tests if test_name.startswith(STR_TEST) 1000 ] 1001 return self.runTests(tests) 1002 1003 def cleanUp(self): 1004 """A function that is executed upon completion of all tests cases 1005 selected in the test class. 1006 1007 This function should clean up objects initialized in the constructor by 1008 user. 1009 """ 1010 1011 def DumpBugReport(self, prefix=''): 1012 """Get device bugreport through adb command. 1013 1014 Args: 1015 prefix: string, file name prefix. Usually in format of 1016 <test_module>-<test_case> 1017 """ 1018 if prefix: 1019 prefix = re.sub('[^\w\-_\. ]', '_', prefix) + '_' 1020 1021 for device in self.android_devices: 1022 file_name = (prefix 1023 + _BUG_REPORT_FILE_PREFIX 1024 + '_%s' % device.serial 1025 + _BUG_REPORT_FILE_EXTENSION) 1026 1027 file_path = os.path.join(logging.log_path, 1028 file_name) 1029 1030 logging.info('Catching bugreport %s...' % file_path) 1031 device.adb.bugreport(file_path) 1032 1033 def skipAllTests(self, msg): 1034 """Skip all test cases. 1035 1036 This method is usually called in setup functions when a precondition 1037 to the test module is not met. 1038 1039 Args: 1040 msg: string, reason why tests are skipped. If set to None or empty 1041 string, a default message will be used (not recommended) 1042 """ 1043 if not msg: 1044 msg = "No reason provided." 1045 1046 setattr(self, _REASON_TO_SKIP_ALL_TESTS, msg) 1047 1048 def isSkipAllTests(self): 1049 """Returns whether all tests are set to be skipped. 1050 1051 Note: If all tests are being skipped not due to skipAllTests 1052 being called, or there is no tests defined, this method will 1053 still return False (since skipAllTests is not called.) 1054 1055 Returns: 1056 bool, True if skipAllTests has been called; False otherwise. 1057 """ 1058 return self.getSkipAllTestsReason() is not None 1059 1060 def getSkipAllTestsReason(self): 1061 """Returns the reason why all tests are skipped. 1062 1063 Note: If all tests are being skipped not due to skipAllTests 1064 being called, or there is no tests defined, this method will 1065 still return None (since skipAllTests is not called.) 1066 1067 Returns: 1068 String, reason why tests are skipped. None if skipAllTests 1069 is not called. 1070 """ 1071 return getattr(self, _REASON_TO_SKIP_ALL_TESTS, None) 1072 1073 def DumpLogcat(self, prefix=''): 1074 """Dumps device logcat outputs to log directory. 1075 1076 Args: 1077 prefix: string, file name prefix. Usually in format of 1078 <test_module>-<test_case> 1079 """ 1080 if prefix: 1081 prefix = re.sub('[^\w\-_\. ]', '_', prefix) + '_' 1082 1083 for device in self.android_devices: 1084 for buffer in LOGCAT_BUFFERS: 1085 file_name = (prefix 1086 + _LOGCAT_FILE_PREFIX 1087 + '_%s_' % buffer 1088 + device.serial 1089 + _LOGCAT_FILE_EXTENSION) 1090 1091 file_path = os.path.join(logging.log_path, 1092 file_name) 1093 1094 logging.info('Dumping logcat %s...' % file_path) 1095 device.adb.logcat('-b', buffer, '-d', '>', file_path) 1096
