1 #!/usr/bin/python 2 # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 7 """Parses and displays the contents of one or more autoserv result directories. 8 9 This script parses the contents of one or more autoserv results folders and 10 generates test reports. 11 """ 12 13 import datetime 14 import glob 15 import logging 16 import operator 17 import optparse 18 import os 19 import re 20 import sys 21 22 import common 23 try: 24 # Ensure the chromite site-package is installed. 25 from chromite.lib import terminal 26 except ImportError: 27 import subprocess 28 build_externals_path = os.path.join( 29 os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 30 'utils', 'build_externals.py') 31 subprocess.check_call([build_externals_path, 'chromiterepo']) 32 # Restart the script so python now finds the autotest site-packages. 33 sys.exit(os.execv(__file__, sys.argv)) 34 35 36 _STDOUT_IS_TTY = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() 37 38 39 def Die(message_format, *args, **kwargs): 40 """Log a message and kill the current process. 41 42 @param message_format: string for logging.error. 43 44 """ 45 logging.error(message_format, *args, **kwargs) 46 sys.exit(1) 47 48 49 class CrashWaiver: 50 """Represents a crash that we want to ignore for now.""" 51 def __init__(self, signals, deadline, url, person): 52 self.signals = signals 53 self.deadline = datetime.datetime.strptime(deadline, '%Y-%b-%d') 54 self.issue_url = url 55 self.suppressor = person 56 57 # List of crashes which are okay to ignore. This list should almost always be 58 # empty. If you add an entry, include the bug URL and your name, something like 59 # 'crashy':CrashWaiver( 60 # ['sig 11'], '2011-Aug-18', 'http://crosbug/123456', 'developer'), 61 62 _CRASH_WHITELIST = { 63 } 64 65 66 class ResultCollector(object): 67 """Collects status and performance data from an autoserv results dir.""" 68 69 def __init__(self, collect_perf=True, collect_attr=False, 70 collect_info=False, escape_error=False, 71 whitelist_chrome_crashes=False): 72 """Initialize ResultsCollector class. 73 74 @param collect_perf: Should perf keyvals be collected? 75 @param collect_attr: Should attr keyvals be collected? 76 @param collect_info: Should info keyvals be collected? 77 @param escape_error: Escape error message text for tools. 78 @param whitelist_chrome_crashes: Treat Chrome crashes as non-fatal. 79 80 """ 81 self._collect_perf = collect_perf 82 self._collect_attr = collect_attr 83 self._collect_info = collect_info 84 self._escape_error = escape_error 85 self._whitelist_chrome_crashes = whitelist_chrome_crashes 86 87 def _CollectPerf(self, testdir): 88 """Parses keyval file under testdir and return the perf keyval pairs. 89 90 @param testdir: autoserv test result directory path. 91 92 @return dict of perf keyval pairs. 93 94 """ 95 if not self._collect_perf: 96 return {} 97 return self._CollectKeyval(testdir, 'perf') 98 99 def _CollectAttr(self, testdir): 100 """Parses keyval file under testdir and return the attr keyval pairs. 101 102 @param testdir: autoserv test result directory path. 103 104 @return dict of attr keyval pairs. 105 106 """ 107 if not self._collect_attr: 108 return {} 109 return self._CollectKeyval(testdir, 'attr') 110 111 def _CollectKeyval(self, testdir, keyword): 112 """Parses keyval file under testdir. 113 114 If testdir contains a result folder, process the keyval file and return 115 a dictionary of perf keyval pairs. 116 117 @param testdir: The autoserv test result directory. 118 @param keyword: The keyword of keyval, either 'perf' or 'attr'. 119 120 @return If the perf option is disabled or the there's no keyval file 121 under testdir, returns an empty dictionary. Otherwise, returns 122 a dictionary of parsed keyvals. Duplicate keys are uniquified 123 by their instance number. 124 125 """ 126 keyval = {} 127 keyval_file = os.path.join(testdir, 'results', 'keyval') 128 if not os.path.isfile(keyval_file): 129 return keyval 130 131 instances = {} 132 133 for line in open(keyval_file): 134 match = re.search(r'^(.+){%s}=(.+)$' % keyword, line) 135 if match: 136 key = match.group(1) 137 val = match.group(2) 138 139 # If the same key name was generated multiple times, uniquify 140 # all instances other than the first one by adding the instance 141 # count to the key name. 142 key_inst = key 143 instance = instances.get(key, 0) 144 if instance: 145 key_inst = '%s{%d}' % (key, instance) 146 instances[key] = instance + 1 147 148 keyval[key_inst] = val 149 150 return keyval 151 152 def _CollectCrashes(self, status_raw): 153 """Parses status_raw file for crashes. 154 155 Saves crash details if crashes are discovered. If a whitelist is 156 present, only records whitelisted crashes. 157 158 @param status_raw: The contents of the status.log or status file from 159 the test. 160 161 @return a list of crash entries to be reported. 162 163 """ 164 crashes = [] 165 regex = re.compile( 166 'Received crash notification for ([-\w]+).+ (sig \d+)') 167 chrome_regex = re.compile(r'^supplied_[cC]hrome|^chrome$') 168 for match in regex.finditer(status_raw): 169 w = _CRASH_WHITELIST.get(match.group(1)) 170 if (self._whitelist_chrome_crashes and 171 chrome_regex.match(match.group(1))): 172 print '@@@STEP_WARNINGS@@@' 173 print '%s crashed with %s' % (match.group(1), match.group(2)) 174 elif (w is not None and match.group(2) in w.signals and 175 w.deadline > datetime.datetime.now()): 176 print 'Ignoring crash in %s for waiver that expires %s' % ( 177 match.group(1), w.deadline.strftime('%Y-%b-%d')) 178 else: 179 crashes.append('%s %s' % match.groups()) 180 return crashes 181 182 def _CollectInfo(self, testdir, custom_info): 183 """Parses *_info files under testdir/sysinfo/var/log. 184 185 If the sysinfo/var/log/*info files exist, save information that shows 186 hw, ec and bios version info. 187 188 This collection of extra info is disabled by default (this funtion is 189 a no-op). It is enabled only if the --info command-line option is 190 explicitly supplied. Normal job parsing does not supply this option. 191 192 @param testdir: The autoserv test result directory. 193 @param custom_info: Dictionary to collect detailed ec/bios info. 194 195 @return a dictionary of info that was discovered. 196 197 """ 198 if not self._collect_info: 199 return {} 200 info = custom_info 201 202 sysinfo_dir = os.path.join(testdir, 'sysinfo', 'var', 'log') 203 for info_file, info_keys in {'ec_info.txt': ['fw_version'], 204 'bios_info.txt': ['fwid', 205 'hwid']}.iteritems(): 206 info_file_path = os.path.join(sysinfo_dir, info_file) 207 if not os.path.isfile(info_file_path): 208 continue 209 # Some example raw text that might be matched include: 210 # 211 # fw_version | snow_v1.1.332-cf20b3e 212 # fwid = Google_Snow.2711.0.2012_08_06_1139 # Active firmware ID 213 # hwid = DAISY TEST A-A 9382 # Hardware ID 214 info_regex = re.compile(r'^(%s)\s*[|=]\s*(.*)' % 215 '|'.join(info_keys)) 216 with open(info_file_path, 'r') as f: 217 for line in f: 218 line = line.strip() 219 line = line.split('#')[0] 220 match = info_regex.match(line) 221 if match: 222 info[match.group(1)] = str(match.group(2)).strip() 223 return info 224 225 def _CollectEndTimes(self, status_raw, status_re='', is_end=True): 226 """Helper to match and collect timestamp and localtime. 227 228 Preferred to locate timestamp and localtime with an 229 'END GOOD test_name...' line. However, aborted tests occasionally fail 230 to produce this line and then need to scrape timestamps from the 'START 231 test_name...' line. 232 233 @param status_raw: multi-line text to search. 234 @param status_re: status regex to seek (e.g. GOOD|FAIL) 235 @param is_end: if True, search for 'END' otherwise 'START'. 236 237 @return Tuple of timestamp, localtime retrieved from the test status 238 log. 239 240 """ 241 timestamp = '' 242 localtime = '' 243 244 localtime_re = r'\w+\s+\w+\s+[:\w]+' 245 match_filter = ( 246 r'^\s*%s\s+(?:%s).*timestamp=(\d*).*localtime=(%s).*$' % ( 247 'END' if is_end else 'START', status_re, localtime_re)) 248 matches = re.findall(match_filter, status_raw, re.MULTILINE) 249 if matches: 250 # There may be multiple lines with timestamp/localtime info. 251 # The last one found is selected because it will reflect the end 252 # time. 253 for i in xrange(len(matches)): 254 timestamp_, localtime_ = matches[-(i+1)] 255 if not timestamp or timestamp_ > timestamp: 256 timestamp = timestamp_ 257 localtime = localtime_ 258 return timestamp, localtime 259 260 def _CheckExperimental(self, testdir): 261 """Parses keyval file and return the value of `experimental`. 262 263 @param testdir: The result directory that has the keyval file. 264 265 @return The value of 'experimental', which is a boolean value indicating 266 whether it is an experimental test or not. 267 268 """ 269 keyval_file = os.path.join(testdir, 'keyval') 270 if not os.path.isfile(keyval_file): 271 return False 272 273 with open(keyval_file) as f: 274 for line in f: 275 match = re.match(r'experimental=(.+)', line) 276 if match: 277 return match.group(1) == 'True' 278 else: 279 return False 280 281 282 def _CollectResult(self, testdir, results, is_experimental=False): 283 """Collects results stored under testdir into a dictionary. 284 285 The presence/location of status files (status.log, status and 286 job_report.html) varies depending on whether the job is a simple 287 client test, simple server test, old-style suite or new-style 288 suite. For example: 289 -In some cases a single job_report.html may exist but many times 290 multiple instances are produced in a result tree. 291 -Most tests will produce a status.log but client tests invoked 292 by a server test will only emit a status file. 293 294 The two common criteria that seem to define the presence of a 295 valid test result are: 296 1. Existence of a 'status.log' or 'status' file. Note that if both a 297 'status.log' and 'status' file exist for a test, the 'status' file 298 is always a subset of the 'status.log' fle contents. 299 2. Presence of a 'debug' directory. 300 301 In some cases multiple 'status.log' files will exist where the parent 302 'status.log' contains the contents of multiple subdirectory 'status.log' 303 files. Parent and subdirectory 'status.log' files are always expected 304 to agree on the outcome of a given test. 305 306 The test results discovered from the 'status*' files are included 307 in the result dictionary. The test directory name and a test directory 308 timestamp/localtime are saved to be used as sort keys for the results. 309 310 The value of 'is_experimental' is included in the result dictionary. 311 312 @param testdir: The autoserv test result directory. 313 @param results: A list to which a populated test-result-dictionary will 314 be appended if a status file is found. 315 @param is_experimental: A boolean value indicating whether the result 316 directory is for an experimental test. 317 318 """ 319 status_file = os.path.join(testdir, 'status.log') 320 if not os.path.isfile(status_file): 321 status_file = os.path.join(testdir, 'status') 322 if not os.path.isfile(status_file): 323 return 324 325 # Status is True if GOOD, else False for all others. 326 status = False 327 error_msg = None 328 status_raw = open(status_file, 'r').read() 329 failure_tags = 'ABORT|ERROR|FAIL' 330 warning_tag = 'WARN|TEST_NA' 331 failure = re.search(r'%s' % failure_tags, status_raw) 332 warning = re.search(r'%s' % warning_tag, status_raw) and not failure 333 good = (re.search(r'GOOD.+completed successfully', status_raw) and 334 not (failure or warning)) 335 336 # We'd like warnings to allow the tests to pass, but still gather info. 337 if good or warning: 338 status = True 339 340 if not good: 341 match = re.search(r'^\t+(%s|%s)\t(.+)' % (failure_tags, 342 warning_tag), 343 status_raw, re.MULTILINE) 344 if match: 345 failure_type = match.group(1) 346 reason = match.group(2).split('\t')[4] 347 if self._escape_error: 348 reason = re.escape(reason) 349 error_msg = ': '.join([failure_type, reason]) 350 351 # Grab the timestamp - can be used for sorting the test runs. 352 # Grab the localtime - may be printed to enable line filtering by date. 353 # Designed to match a line like this: 354 # END GOOD testname ... timestamp=1347324321 localtime=Sep 10 17:45:21 355 status_re = r'GOOD|%s|%s' % (failure_tags, warning_tag) 356 timestamp, localtime = self._CollectEndTimes(status_raw, status_re) 357 # Hung tests will occasionally skip printing the END line so grab 358 # a default timestamp from the START line in those cases. 359 if not timestamp: 360 timestamp, localtime = self._CollectEndTimes(status_raw, 361 is_end=False) 362 363 results.append({ 364 'testdir': testdir, 365 'crashes': self._CollectCrashes(status_raw), 366 'status': status, 367 'error_msg': error_msg, 368 'localtime': localtime, 369 'timestamp': timestamp, 370 'perf': self._CollectPerf(testdir), 371 'attr': self._CollectAttr(testdir), 372 'info': self._CollectInfo(testdir, {'localtime': localtime, 373 'timestamp': timestamp}), 374 'experimental': is_experimental}) 375 376 def RecursivelyCollectResults(self, resdir, parent_experimental_tag=False): 377 """Recursively collect results into a list of dictionaries. 378 379 Only recurses into directories that possess a 'debug' subdirectory 380 because anything else is not considered a 'test' directory. 381 382 The value of 'experimental' in keyval file is used to determine whether 383 the result is for an experimental test. If it is, all its sub 384 directories are considered to be experimental tests too. 385 386 @param resdir: results/test directory to parse results from and recurse 387 into. 388 @param parent_experimental_tag: A boolean value, used to keep track of 389 whether its parent directory is for an experimental test. 390 391 @return List of dictionaries of results. 392 393 """ 394 results = [] 395 is_experimental = (parent_experimental_tag or 396 self._CheckExperimental(resdir)) 397 self._CollectResult(resdir, results, is_experimental) 398 for testdir in glob.glob(os.path.join(resdir, '*')): 399 # Remove false positives that are missing a debug dir. 400 if not os.path.exists(os.path.join(testdir, 'debug')): 401 continue 402 403 results.extend(self.RecursivelyCollectResults( 404 testdir, is_experimental)) 405 return results 406 407 408 class ReportGenerator(object): 409 """Collects and displays data from autoserv results directories. 410 411 This class collects status and performance data from one or more autoserv 412 result directories and generates test reports. 413 """ 414 415 _KEYVAL_INDENT = 2 416 _STATUS_STRINGS = {'hr': {'pass': '[ PASSED ]', 'fail': '[ FAILED ]'}, 417 'csv': {'pass': 'PASS', 'fail': 'FAIL'}} 418 419 def __init__(self, options, args): 420 self._options = options 421 self._args = args 422 self._color = terminal.Color(options.color) 423 self._results = [] 424 425 def _CollectAllResults(self): 426 """Parses results into the self._results list. 427 428 Builds a list (self._results) where each entry is a dictionary of 429 result data from one test (which may contain other tests). Each 430 dictionary will contain values such as: test folder, status, localtime, 431 crashes, error_msg, perf keyvals [optional], info [optional]. 432 433 """ 434 collector = ResultCollector( 435 collect_perf=self._options.perf, 436 collect_attr=self._options.attr, 437 collect_info=self._options.info, 438 escape_error=self._options.escape_error, 439 whitelist_chrome_crashes=self._options.whitelist_chrome_crashes) 440 441 for resdir in self._args: 442 if not os.path.isdir(resdir): 443 Die('%r does not exist', resdir) 444 self._results.extend(collector.RecursivelyCollectResults(resdir)) 445 446 if not self._results: 447 Die('no test directories found') 448 449 def _GenStatusString(self, status): 450 """Given a bool indicating success or failure, return the right string. 451 452 Also takes --csv into account, returns old-style strings if it is set. 453 454 @param status: True or False, indicating success or failure. 455 456 @return The appropriate string for printing.. 457 458 """ 459 success = 'pass' if status else 'fail' 460 if self._options.csv: 461 return self._STATUS_STRINGS['csv'][success] 462 return self._STATUS_STRINGS['hr'][success] 463 464 def _Indent(self, msg): 465 """Given a message, indents it appropriately. 466 467 @param msg: string to indent. 468 @return indented version of msg. 469 470 """ 471 return ' ' * self._KEYVAL_INDENT + msg 472 473 def _GetTestColumnWidth(self): 474 """Returns the test column width based on the test data. 475 476 The test results are aligned by discovering the longest width test 477 directory name or perf key stored in the list of result dictionaries. 478 479 @return The width for the test column. 480 481 """ 482 width = 0 483 for result in self._results: 484 width = max(width, len(result['testdir'])) 485 perf = result.get('perf') 486 if perf: 487 perf_key_width = len(max(perf, key=len)) 488 width = max(width, perf_key_width + self._KEYVAL_INDENT) 489 return width 490 491 def _PrintDashLine(self, width): 492 """Prints a line of dashes as a separator in output. 493 494 @param width: an integer. 495 """ 496 if not self._options.csv: 497 print ''.ljust(width + len(self._STATUS_STRINGS['hr']['pass']), '-') 498 499 def _PrintEntries(self, entries): 500 """Prints a list of strings, delimited based on --csv flag. 501 502 @param entries: a list of strings, entities to output. 503 504 """ 505 delimiter = ',' if self._options.csv else ' ' 506 print delimiter.join(entries) 507 508 def _PrintErrors(self, test, error_msg): 509 """Prints an indented error message, unless the --csv flag is set. 510 511 @param test: the name of a test with which to prefix the line. 512 @param error_msg: a message to print. None is allowed, but ignored. 513 514 """ 515 if not self._options.csv and error_msg: 516 self._PrintEntries([test, self._Indent(error_msg)]) 517 518 def _PrintErrorLogs(self, test, test_string): 519 """Prints the error log for |test| if --debug is set. 520 521 @param test: the name of a test suitable for embedding in a path 522 @param test_string: the name of a test with which to prefix the line. 523 524 """ 525 if self._options.print_debug: 526 debug_file_regex = os.path.join( 527 'results.', test, 'debug', 528 '%s*.ERROR' % os.path.basename(test)) 529 for path in glob.glob(debug_file_regex): 530 try: 531 with open(path) as fh: 532 for line in fh: 533 # Ensure line is not just WS. 534 if len(line.lstrip()) <= 0: 535 continue 536 self._PrintEntries( 537 [test_string, self._Indent(line.rstrip())]) 538 except IOError: 539 print 'Could not open %s' % path 540 541 def _PrintResultDictKeyVals(self, test_entry, result_dict): 542 """Formatted print a dict of keyvals like 'perf' or 'info'. 543 544 This function emits each keyval on a single line for uncompressed 545 review. The 'perf' dictionary contains performance keyvals while the 546 'info' dictionary contains ec info, bios info and some test timestamps. 547 548 @param test_entry: The unique name of the test (dir) - matches other 549 test output. 550 @param result_dict: A dict of keyvals to be presented. 551 552 """ 553 if not result_dict: 554 return 555 dict_keys = result_dict.keys() 556 dict_keys.sort() 557 width = self._GetTestColumnWidth() 558 for dict_key in dict_keys: 559 if self._options.csv: 560 key_entry = dict_key 561 else: 562 key_entry = dict_key.ljust(width - self._KEYVAL_INDENT) 563 key_entry = key_entry.rjust(width) 564 value_entry = self._color.Color( 565 self._color.BOLD, result_dict[dict_key]) 566 self._PrintEntries([test_entry, key_entry, value_entry]) 567 568 def _GetSortedTests(self): 569 """Sort the test result dicts in preparation for results printing. 570 571 By default sorts the results directionaries by their test names. 572 However, when running long suites, it is useful to see if an early test 573 has wedged the system and caused the remaining tests to abort/fail. The 574 datetime-based chronological sorting allows this view. 575 576 Uses the --sort-chron command line option to control. 577 578 """ 579 if self._options.sort_chron: 580 # Need to reverse sort the test dirs to ensure the suite folder 581 # shows at the bottom. Because the suite folder shares its datetime 582 # with the last test it shows second-to-last without the reverse 583 # sort first. 584 tests = sorted(self._results, key=operator.itemgetter('testdir'), 585 reverse=True) 586 tests = sorted(tests, key=operator.itemgetter('timestamp')) 587 else: 588 tests = sorted(self._results, key=operator.itemgetter('testdir')) 589 return tests 590 591 # TODO(zamorzaev): reuse this method in _GetResultsForHTMLReport to avoid 592 # code copying. 593 def _GetDedupedResults(self): 594 """Aggregate results from multiple retries of the same test.""" 595 deduped_results = {} 596 for test in self._GetSortedTests(): 597 test_details_matched = re.search(r'(.*)results-(\d[0-9]*)-(.*)', 598 test['testdir']) 599 if not test_details_matched: 600 continue 601 602 log_dir, test_number, test_name = test_details_matched.groups() 603 if (test_name in deduped_results and 604 deduped_results[test_name].get('status')): 605 # Already have a successfull (re)try. 606 continue 607 608 deduped_results[test_name] = test 609 return deduped_results.values() 610 611 def _GetResultsForHTMLReport(self): 612 """Return cleaned results for HTML report.!""" 613 import copy 614 tests = copy.deepcopy(self._GetSortedTests()) 615 pass_tag = "Pass" 616 fail_tag = "Fail" 617 na_tag = "NA" 618 count = 0 619 html_results = {} 620 for test_status in tests: 621 individual_tc_results = {} 622 test_details_matched = re.search(r'(.*)results-(\d[0-9]*)-(.*)', 623 test_status['testdir']) 624 if not test_details_matched: 625 continue 626 log_dir = test_details_matched.group(1) 627 test_number = test_details_matched.group(2) 628 test_name = test_details_matched.group(3) 629 if '/' in test_name: 630 test_name = test_name.split('/')[0] 631 if test_status['error_msg'] is None: 632 test_status['error_msg'] = '' 633 if not html_results.has_key(test_name): 634 count = count + 1 635 # Arranging the results in an order 636 individual_tc_results['status'] = test_status['status'] 637 individual_tc_results['error_msg'] = test_status['error_msg'] 638 individual_tc_results['s_no'] = count 639 individual_tc_results['crashes'] = test_status['crashes'] 640 641 # Add <b> and </b> tag for the good format in the report. 642 individual_tc_results['attempts'] = \ 643 '<b>test_result_number: %s - %s</b> : %s' % ( 644 test_number, log_dir, test_status['error_msg']) 645 html_results[test_name] = individual_tc_results 646 else: 647 648 # If test found already then we are using the previous data 649 # instead of creating two different html rows. If existing 650 # status is False then needs to be updated 651 if html_results[test_name]['status'] is False: 652 html_results[test_name]['status'] = test_status['status'] 653 html_results[test_name]['error_msg'] = test_status[ 654 'error_msg'] 655 html_results[test_name]['crashes'] = \ 656 html_results[test_name]['crashes'] + test_status[ 657 'crashes'] 658 html_results[test_name]['attempts'] = \ 659 html_results[test_name]['attempts'] + \ 660 '</br><b>test_result_number : %s - %s</b> : %s' % ( 661 test_number, log_dir, test_status['error_msg']) 662 663 # Re-formating the dictionary as s_no as key. So that we can have 664 # ordered data at the end 665 sorted_html_results = {} 666 for key in html_results.keys(): 667 sorted_html_results[str(html_results[key]['s_no'])] = \ 668 html_results[key] 669 sorted_html_results[str(html_results[key]['s_no'])]['test'] = key 670 671 # Mapping the Test case status if True->Pass, False->Fail and if 672 # True and the error message then NA 673 for key in sorted_html_results.keys(): 674 if sorted_html_results[key]['status']: 675 if sorted_html_results[key]['error_msg'] != '': 676 sorted_html_results[key]['status'] = na_tag 677 else: 678 sorted_html_results[key]['status'] = pass_tag 679 else: 680 sorted_html_results[key]['status'] = fail_tag 681 682 return sorted_html_results 683 684 def GenerateReportHTML(self): 685 """Generate clean HTMl report for the results.""" 686 687 results = self._GetResultsForHTMLReport() 688 html_table_header = """ <th>S.No</th> 689 <th>Test</th> 690 <th>Status</th> 691 <th>Error Message</th> 692 <th>Crashes</th> 693 <th>Attempts</th> 694 """ 695 passed_tests = len([key for key in results.keys() if results[key][ 696 'status'].lower() == 'pass']) 697 failed_tests = len([key for key in results.keys() if results[key][ 698 'status'].lower() == 'fail']) 699 na_tests = len([key for key in results.keys() if results[key][ 700 'status'].lower() == 'na']) 701 total_tests = passed_tests + failed_tests + na_tests 702 703 # Sort the keys 704 ordered_keys = sorted([int(key) for key in results.keys()]) 705 html_table_body = '' 706 for key in ordered_keys: 707 key = str(key) 708 if results[key]['status'].lower() == 'pass': 709 color = 'LimeGreen' 710 elif results[key]['status'].lower() == 'na': 711 color = 'yellow' 712 else: 713 color = 'red' 714 html_table_body = html_table_body + """<tr> 715 <td>%s</td> 716 <td>%s</td> 717 <td 718 style="background-color:%s;"> 719 %s</td> 720 <td>%s</td> 721 <td>%s</td> 722 <td>%s</td></tr>""" % \ 723 (key, results[key]['test'], 724 color, 725 results[key]['status'], 726 results[key]['error_msg'], 727 results[key]['crashes'], 728 results[key]['attempts']) 729 html_page = """ 730 <!DOCTYPE html> 731 <html lang="en"> 732 <head> 733 <title>Automation Results</title> 734 <meta charset="utf-8"> 735 <meta name="viewport" content="width=device-width,initial-scale=1"> 736 <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css"> 737 <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.3.1/jquery.min.js"></script> 738 <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js"></script> 739 </head> 740 <body> 741 <div class="container"> 742 <h2>Automation Report</h2> 743 <table class="table table-bordered" border="1"> 744 <thead> 745 <tr style="background-color:LightSkyBlue;"> 746 \n%s 747 </tr> 748 </thead> 749 <tbody> 750 \n%s 751 </tbody> 752 </table> 753 <div class="row"> 754 <div class="col-sm-4">Passed: <b>%d</b></div> 755 <div class="col-sm-4">Failed: <b>%d</b></div> 756 <div class="col-sm-4">NA: <b>%d</b></div> 757 </div> 758 <div class="row"> 759 <div class="col-sm-4">Total: <b>%d</b></div> 760 </div> 761 </div> 762 </body> 763 </html> 764 765 """ % (html_table_header, html_table_body, passed_tests, 766 failed_tests, na_tests, total_tests) 767 with open(os.path.join(self._options.html_report_dir, 768 "test_report.html"), 'w') as html_file: 769 html_file.write(html_page) 770 771 def _GenerateReportText(self): 772 """Prints a result report to stdout. 773 774 Prints a result table to stdout. Each row of the table contains the 775 test result directory and the test result (PASS, FAIL). If the perf 776 option is enabled, each test entry is followed by perf keyval entries 777 from the test results. 778 779 """ 780 tests = self._GetSortedTests() 781 width = self._GetTestColumnWidth() 782 783 crashes = {} 784 tests_pass = 0 785 self._PrintDashLine(width) 786 787 for result in tests: 788 testdir = result['testdir'] 789 test_entry = testdir if self._options.csv else testdir.ljust(width) 790 791 status_entry = self._GenStatusString(result['status']) 792 if result['status']: 793 color = self._color.GREEN 794 tests_pass += 1 795 else: 796 color = self._color.RED 797 798 test_entries = [test_entry, self._color.Color(color, status_entry)] 799 800 info = result.get('info', {}) 801 info.update(result.get('attr', {})) 802 if self._options.csv and (self._options.info or self._options.attr): 803 if info: 804 test_entries.extend(['%s=%s' % (k, info[k]) 805 for k in sorted(info.keys())]) 806 if not result['status'] and result['error_msg']: 807 test_entries.append('reason="%s"' % result['error_msg']) 808 809 self._PrintEntries(test_entries) 810 self._PrintErrors(test_entry, result['error_msg']) 811 812 # Print out error log for failed tests. 813 if not result['status']: 814 self._PrintErrorLogs(testdir, test_entry) 815 816 # Emit the perf keyvals entries. There will be no entries if the 817 # --no-perf option is specified. 818 self._PrintResultDictKeyVals(test_entry, result['perf']) 819 820 # Determine that there was a crash during this test. 821 if result['crashes']: 822 for crash in result['crashes']: 823 if not crash in crashes: 824 crashes[crash] = set([]) 825 crashes[crash].add(testdir) 826 827 # Emit extra test metadata info on separate lines if not --csv. 828 if not self._options.csv: 829 self._PrintResultDictKeyVals(test_entry, info) 830 831 self._PrintDashLine(width) 832 833 if not self._options.csv: 834 total_tests = len(tests) 835 percent_pass = 100 * tests_pass / total_tests 836 pass_str = '%d/%d (%d%%)' % (tests_pass, total_tests, percent_pass) 837 print 'Total PASS: ' + self._color.Color(self._color.BOLD, pass_str) 838 839 if self._options.crash_detection: 840 print '' 841 if crashes: 842 print self._color.Color(self._color.RED, 843 'Crashes detected during testing:') 844 self._PrintDashLine(width) 845 846 for crash_name, crashed_tests in sorted(crashes.iteritems()): 847 print self._color.Color(self._color.RED, crash_name) 848 for crashed_test in crashed_tests: 849 print self._Indent(crashed_test) 850 851 self._PrintDashLine(width) 852 print ('Total unique crashes: ' + 853 self._color.Color(self._color.BOLD, str(len(crashes)))) 854 855 # Sometimes the builders exit before these buffers are flushed. 856 sys.stderr.flush() 857 sys.stdout.flush() 858 859 def Run(self): 860 """Runs report generation.""" 861 self._CollectAllResults() 862 if not self._options.just_status_code: 863 self._GenerateReportText() 864 if self._options.html: 865 print "\nLogging the data into test_report.html file." 866 try: 867 self.GenerateReportHTML() 868 except Exception as e: 869 print "Failed to generate HTML report %s" % str(e) 870 for d in self._GetDedupedResults(): 871 if d['experimental'] and self._options.ignore_experimental_tests: 872 continue 873 if not d['status'] or ( 874 self._options.crash_detection and d['crashes']): 875 sys.exit(1) 876 877 878 def main(): 879 usage = 'Usage: %prog [options] result-directories...' 880 parser = optparse.OptionParser(usage=usage) 881 parser.add_option('--color', dest='color', action='store_true', 882 default=_STDOUT_IS_TTY, 883 help='Use color for text reports [default if TTY stdout]') 884 parser.add_option('--no-color', dest='color', action='store_false', 885 help='Don\'t use color for text reports') 886 parser.add_option('--no-crash-detection', dest='crash_detection', 887 action='store_false', default=True, 888 help='Don\'t report crashes or error out when detected') 889 parser.add_option('--csv', dest='csv', action='store_true', 890 help='Output test result in CSV format. ' 891 'Implies --no-debug --no-crash-detection.') 892 parser.add_option('--html', dest='html', action='store_true', 893 help='To generate HTML File. ' 894 'Implies --no-debug --no-crash-detection.') 895 parser.add_option('--html-report-dir', dest='html_report_dir', 896 action='store', default=None, help='Path to generate ' 897 'html report') 898 parser.add_option('--info', dest='info', action='store_true', 899 default=False, 900 help='Include info keyvals in the report') 901 parser.add_option('--escape-error', dest='escape_error', 902 action='store_true', default=False, 903 help='Escape error message text for tools.') 904 parser.add_option('--perf', dest='perf', action='store_true', 905 default=True, 906 help='Include perf keyvals in the report [default]') 907 parser.add_option('--attr', dest='attr', action='store_true', 908 default=False, 909 help='Include attr keyvals in the report') 910 parser.add_option('--no-perf', dest='perf', action='store_false', 911 help='Don\'t include perf keyvals in the report') 912 parser.add_option('--sort-chron', dest='sort_chron', action='store_true', 913 default=False, 914 help='Sort results by datetime instead of by test name.') 915 parser.add_option('--no-debug', dest='print_debug', action='store_false', 916 default=True, 917 help='Don\'t print out logs when tests fail.') 918 parser.add_option('--whitelist_chrome_crashes', 919 dest='whitelist_chrome_crashes', 920 action='store_true', default=False, 921 help='Treat Chrome crashes as non-fatal.') 922 parser.add_option('--ignore_experimental_tests', 923 dest='ignore_experimental_tests', 924 action='store_true', default=False, 925 help='If set, experimental test results will not ' 926 'influence the exit code.') 927 parser.add_option('--just_status_code', 928 dest='just_status_code', 929 action='store_true', default=False, 930 help='Skip generating a report, just return status code.') 931 932 (options, args) = parser.parse_args() 933 934 if not args: 935 parser.print_help() 936 Die('no result directories provided') 937 938 if options.csv and (options.print_debug or options.crash_detection): 939 Warning('Forcing --no-debug --no-crash-detection') 940 options.print_debug = False 941 options.crash_detection = False 942 943 report_options = ['color', 'csv', 'info', 'escape_error', 'perf', 'attr', 944 'sort_chron', 'print_debug', 'html', 'html_report_dir'] 945 if options.just_status_code and any( 946 getattr(options, opt) for opt in report_options): 947 Warning('Passed --just_status_code and incompatible options %s' % 948 ' '.join(opt for opt in report_options if getattr(options,opt))) 949 950 generator = ReportGenerator(options, args) 951 generator.Run() 952 953 954 if __name__ == '__main__': 955 main() 956
