1 #!/usr/bin/env python3.4 2 # 3 # Copyright 2016 - The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 """This module is where all the record definitions and record containers live. 17 """ 18 19 import json 20 import logging 21 import pprint 22 23 from acts import logger 24 from acts import signals 25 from acts import utils 26 27 28 class TestResultEnums(object): 29 """Enums used for TestResultRecord class. 30 31 Includes the tokens to mark test result with, and the string names for each 32 field in TestResultRecord. 33 """ 34 35 RECORD_NAME = "Test Name" 36 RECORD_CLASS = "Test Class" 37 RECORD_BEGIN_TIME = "Begin Time" 38 RECORD_END_TIME = "End Time" 39 RECORD_LOG_BEGIN_TIME = "Log Begin Time" 40 RECORD_LOG_END_TIME = "Log End Time" 41 RECORD_RESULT = "Result" 42 RECORD_UID = "UID" 43 RECORD_EXTRAS = "Extras" 44 RECORD_ADDITIONAL_ERRORS = "Extra Errors" 45 RECORD_DETAILS = "Details" 46 TEST_RESULT_PASS = "PASS" 47 TEST_RESULT_FAIL = "FAIL" 48 TEST_RESULT_SKIP = "SKIP" 49 TEST_RESULT_BLOCKED = "BLOCKED" 50 TEST_RESULT_UNKNOWN = "UNKNOWN" 51 52 53 class TestResultRecord(object): 54 """A record that holds the information of a test case execution. 55 56 Attributes: 57 test_name: A string representing the name of the test case. 58 begin_time: Epoch timestamp of when the test case started. 59 end_time: Epoch timestamp of when the test case ended. 60 self.uid: Unique identifier of a test case. 61 self.result: Test result, PASS/FAIL/SKIP. 62 self.extras: User defined extra information of the test result. 63 self.details: A string explaining the details of the test case. 64 """ 65 66 def __init__(self, t_name, t_class=None): 67 self.test_name = t_name 68 self.test_class = t_class 69 self.begin_time = None 70 self.end_time = None 71 self.log_begin_time = None 72 self.log_end_time = None 73 self.uid = None 74 self.result = None 75 self.extras = None 76 self.details = None 77 self.additional_errors = {} 78 79 def test_begin(self): 80 """Call this when the test case it records begins execution. 81 82 Sets the begin_time of this record. 83 """ 84 self.begin_time = utils.get_current_epoch_time() 85 self.log_begin_time = logger.epoch_to_log_line_timestamp( 86 self.begin_time) 87 88 def _test_end(self, result, e): 89 """Class internal function to signal the end of a test case execution. 90 91 Args: 92 result: One of the TEST_RESULT enums in TestResultEnums. 93 e: A test termination signal (usually an exception object). It can 94 be any exception instance or of any subclass of 95 acts.signals.TestSignal. 96 """ 97 self.end_time = utils.get_current_epoch_time() 98 self.log_end_time = logger.epoch_to_log_line_timestamp(self.end_time) 99 self.result = result 100 if self.additional_errors: 101 self.result = TestResultEnums.TEST_RESULT_UNKNOWN 102 if isinstance(e, signals.TestSignal): 103 self.details = e.details 104 self.extras = e.extras 105 elif e: 106 self.details = str(e) 107 108 def test_pass(self, e=None): 109 """To mark the test as passed in this record. 110 111 Args: 112 e: An instance of acts.signals.TestPass. 113 """ 114 self._test_end(TestResultEnums.TEST_RESULT_PASS, e) 115 116 def test_fail(self, e=None): 117 """To mark the test as failed in this record. 118 119 Only test_fail does instance check because we want "assert xxx" to also 120 fail the test same way assert_true does. 121 122 Args: 123 e: An exception object. It can be an instance of AssertionError or 124 acts.base_test.TestFailure. 125 """ 126 self._test_end(TestResultEnums.TEST_RESULT_FAIL, e) 127 128 def test_skip(self, e=None): 129 """To mark the test as skipped in this record. 130 131 Args: 132 e: An instance of acts.signals.TestSkip. 133 """ 134 self._test_end(TestResultEnums.TEST_RESULT_SKIP, e) 135 136 def test_blocked(self, e=None): 137 """To mark the test as blocked in this record. 138 139 Args: 140 e: An instance of acts.signals.Test 141 """ 142 self._test_end(TestResultEnums.TEST_RESULT_BLOCKED, e) 143 144 def test_unknown(self, e=None): 145 """To mark the test as unknown in this record. 146 147 Args: 148 e: An exception object. 149 """ 150 self._test_end(TestResultEnums.TEST_RESULT_UNKNOWN, e) 151 152 def add_error(self, tag, e): 153 """Add extra error happened during a test mark the test result as 154 UNKNOWN. 155 156 If an error is added the test record, the record's result is equivalent 157 to the case where an uncaught exception happened. 158 159 Args: 160 tag: A string describing where this error came from, e.g. 'on_pass'. 161 e: An exception object. 162 """ 163 self.result = TestResultEnums.TEST_RESULT_UNKNOWN 164 self.additional_errors[tag] = str(e) 165 166 def __str__(self): 167 d = self.to_dict() 168 l = ["%s = %s" % (k, v) for k, v in d.items()] 169 s = ', '.join(l) 170 return s 171 172 def __repr__(self): 173 """This returns a short string representation of the test record.""" 174 t = utils.epoch_to_human_time(self.begin_time) 175 return "%s %s %s" % (t, self.test_name, self.result) 176 177 def to_dict(self): 178 """Gets a dictionary representing the content of this class. 179 180 Returns: 181 A dictionary representing the content of this class. 182 """ 183 d = {} 184 d[TestResultEnums.RECORD_NAME] = self.test_name 185 d[TestResultEnums.RECORD_CLASS] = self.test_class 186 d[TestResultEnums.RECORD_BEGIN_TIME] = self.begin_time 187 d[TestResultEnums.RECORD_END_TIME] = self.end_time 188 d[TestResultEnums.RECORD_LOG_BEGIN_TIME] = self.log_begin_time 189 d[TestResultEnums.RECORD_LOG_END_TIME] = self.log_end_time 190 d[TestResultEnums.RECORD_RESULT] = self.result 191 d[TestResultEnums.RECORD_UID] = self.uid 192 d[TestResultEnums.RECORD_EXTRAS] = self.extras 193 d[TestResultEnums.RECORD_DETAILS] = self.details 194 d[TestResultEnums.RECORD_ADDITIONAL_ERRORS] = self.additional_errors 195 return d 196 197 def json_str(self): 198 """Converts this test record to a string in json format. 199 200 Format of the json string is: 201 { 202 'Test Name': <test name>, 203 'Begin Time': <epoch timestamp>, 204 'Details': <details>, 205 ... 206 } 207 208 Returns: 209 A json-format string representing the test record. 210 """ 211 return json.dumps(self.to_dict()) 212 213 214 class TestResult(object): 215 """A class that contains metrics of a test run. 216 217 This class is essentially a container of TestResultRecord objects. 218 219 Attributes: 220 self.requested: A list of strings, each is the name of a test requested 221 by user. 222 self.failed: A list of records for tests failed. 223 self.executed: A list of records for tests that were actually executed. 224 self.passed: A list of records for tests passed. 225 self.skipped: A list of records for tests skipped. 226 self.unknown: A list of records for tests with unknown result token. 227 """ 228 229 def __init__(self): 230 self.requested = [] 231 self.failed = [] 232 self.executed = [] 233 self.passed = [] 234 self.skipped = [] 235 self.blocked = [] 236 self.unknown = [] 237 self.controller_info = {} 238 self.post_run_data = {} 239 self.extras = {} 240 241 def __add__(self, r): 242 """Overrides '+' operator for TestResult class. 243 244 The add operator merges two TestResult objects by concatenating all of 245 their lists together. 246 247 Args: 248 r: another instance of TestResult to be added 249 250 Returns: 251 A TestResult instance that's the sum of two TestResult instances. 252 """ 253 if not isinstance(r, TestResult): 254 raise TypeError("Operand %s of type %s is not a TestResult." % 255 (r, type(r))) 256 sum_result = TestResult() 257 for name in sum_result.__dict__: 258 r_value = getattr(r, name) 259 l_value = getattr(self, name) 260 if isinstance(r_value, list): 261 setattr(sum_result, name, l_value + r_value) 262 elif isinstance(r_value, dict): 263 # '+' operator for TestResult is only valid when multiple 264 # TestResult objs were created in the same test run, which means 265 # the controller info would be the same across all of them. 266 # TODO(angli): have a better way to validate this situation. 267 setattr(sum_result, name, l_value) 268 return sum_result 269 270 def add_controller_info(self, name, info): 271 try: 272 json.dumps(info) 273 except TypeError: 274 logging.warning(("Controller info for %s is not JSON serializable!" 275 " Coercing it to string.") % name) 276 self.controller_info[name] = str(info) 277 return 278 self.controller_info[name] = info 279 280 def set_extra_data(self, name, info): 281 try: 282 json.dumps(info) 283 except TypeError: 284 logging.warning("Controller info for %s is not JSON serializable! " 285 "Coercing it to string." % name) 286 info = str(info) 287 self.extras[name] = info 288 289 def add_record(self, record): 290 """Adds a test record to test result. 291 292 A record is considered executed once it's added to the test result. 293 294 Args: 295 record: A test record object to add. 296 """ 297 if record.result == TestResultEnums.TEST_RESULT_FAIL: 298 self.executed.append(record) 299 self.failed.append(record) 300 elif record.result == TestResultEnums.TEST_RESULT_SKIP: 301 self.skipped.append(record) 302 elif record.result == TestResultEnums.TEST_RESULT_PASS: 303 self.executed.append(record) 304 self.passed.append(record) 305 elif record.result == TestResultEnums.TEST_RESULT_BLOCKED: 306 self.blocked.append(record) 307 else: 308 self.executed.append(record) 309 self.unknown.append(record) 310 311 @property 312 def is_all_pass(self): 313 """True if no tests failed or threw errors, False otherwise.""" 314 num_of_failures = (len(self.failed) + 315 len(self.unknown) + 316 len(self.blocked)) 317 return num_of_failures == 0 318 319 def json_str(self): 320 """Converts this test result to a string in json format. 321 322 Format of the json string is: 323 { 324 "Results": [ 325 {<executed test record 1>}, 326 {<executed test record 2>}, 327 ... 328 ], 329 "Summary": <summary dict> 330 } 331 332 Returns: 333 A json-format string representing the test results. 334 """ 335 d = {} 336 d["ControllerInfo"] = self.controller_info 337 d["Results"] = [record.to_dict() for record in self.executed] 338 d["Summary"] = self.summary_dict() 339 d["Extras"] = self.extras 340 json_str = json.dumps(d, indent=4, sort_keys=True) 341 return json_str 342 343 def summary_str(self): 344 """Gets a string that summarizes the stats of this test result. 345 346 The summary rovides the counts of how many test cases fall into each 347 category, like "Passed", "Failed" etc. 348 349 Format of the string is: 350 Requested <int>, Executed <int>, ... 351 352 Returns: 353 A summary string of this test result. 354 """ 355 l = ["%s %s" % (k, v) for k, v in self.summary_dict().items()] 356 # Sort the list so the order is the same every time. 357 msg = ", ".join(sorted(l)) 358 return msg 359 360 def summary_dict(self): 361 """Gets a dictionary that summarizes the stats of this test result. 362 363 The summary rovides the counts of how many test cases fall into each 364 category, like "Passed", "Failed" etc. 365 366 Returns: 367 A dictionary with the stats of this test result. 368 """ 369 d = {} 370 d["ControllerInfo"] = self.controller_info 371 d["Requested"] = len(self.requested) 372 d["Executed"] = len(self.executed) 373 d["Passed"] = len(self.passed) 374 d["Failed"] = len(self.failed) 375 d["Skipped"] = len(self.skipped) 376 d["Blocked"] = len(self.blocked) 377 d["Unknown"] = len(self.unknown) 378 return d 379
