1 #!/usr/bin/python 2 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Report summarizer of internal test pass% from running many tests in LTP. 7 8 LTP is the Linux Test Project from http://ltp.sourceforge.net/. 9 10 This script serves to summarize the results of a test run by LTP test 11 infrastructure. LTP frequently runs >1000 tests so summarizing the results 12 by result-type and count is useful. This script is invoked by the ltp.py 13 wrapper in Autotest as a post-processing step to summarize the LTP run results 14 in the Autotest log file. 15 16 This script may be invoked by the command-line as follows: 17 18 $ ./parse_ltp_out.py -l /mypath/ltp.out 19 """ 20 21 import optparse 22 import os 23 import re 24 import sys 25 26 27 SUMMARY_BORDER = 80 * '-' 28 # Prefix char used in summaries: 29 # +: sums into 'passing' 30 # -: sums into 'notpassing' 31 TEST_FILTERS = {'TPASS': '+Pass', 'TFAIL': '-Fail', 'TBROK': '-Broken', 32 'TCONF': '-Config error', 'TRETR': 'Retired', 33 'TWARN': '+Warning'} 34 35 36 def parse_args(argv): 37 """Setup command line parsing options. 38 39 @param argv: command-line arguments. 40 41 @return parsed option result from optparse. 42 """ 43 parser = optparse.OptionParser('Usage: %prog --ltp-out-file=/path/ltp.out') 44 parser.add_option( 45 '-l', '--ltp-out-file', 46 help='[required] Path and file name for ltp.out [default: %default]', 47 dest='ltp_out_file', 48 default=None) 49 parser.add_option( 50 '-t', '--timings', 51 help='Show test timings in buckets [default: %default]', 52 dest='test_timings', action='store_true', 53 default=False) 54 options, args = parser.parse_args() 55 if not options.ltp_out_file: 56 parser.error('You must supply a value for --ltp-out-file.') 57 58 return options 59 60 61 def _filter_and_count(ltp_out_file, test_filters): 62 """Utility function to count lines that match certain filters. 63 64 @param ltp_out_file: human-readable output file from LTP -p (ltp.out). 65 @param test_filters: dict of the tags to match and corresponding print tags. 66 67 @return a dictionary with counts of the lines that matched each tag. 68 """ 69 marker_line = '^<<<%s>>>$' 70 status_line_re = re.compile('^\w+ +\d+ +(\w+) +: +\w+') 71 filter_accumulator = dict.fromkeys(test_filters.keys(), 0) 72 parse_states = ( 73 {'filters': {}, 74 'terminator': re.compile(marker_line % 'test_output')}, 75 {'filters': filter_accumulator, 76 'terminator': re.compile(marker_line % 'execution_status')}) 77 78 # Simple 2-state state machine. 79 state_test_active = False 80 with open(ltp_out_file) as f: 81 for line in f: 82 state_index = int(state_test_active) 83 if re.match(parse_states[state_index]['terminator'], line): 84 # This state is terminated - proceed to next. 85 state_test_active = not state_test_active 86 else: 87 # Determine if this line matches any of the sought tags. 88 m = re.match(status_line_re, line) 89 if m and m.group(1) in parse_states[state_index]['filters']: 90 parse_states[state_index]['filters'][m.group(1)] += 1 91 return filter_accumulator 92 93 94 def _print_summary(filters, accumulator): 95 """Utility function to print the summary of the parsing of ltp.out. 96 97 Prints a count of each type of test result, then a %pass-rate score. 98 99 @param filters: map of tags sought and corresponding print headers. 100 @param accumulator: counts of test results with same keys as filters. 101 """ 102 print SUMMARY_BORDER 103 print 'Linux Test Project (LTP) Run Summary:' 104 print SUMMARY_BORDER 105 # Size the header to the largest printable tag. 106 fmt = '%%%ss: %%s' % max(map(lambda x: len(x), filters.values())) 107 for k in sorted(filters): 108 print fmt % (filters[k], accumulator[k]) 109 110 print SUMMARY_BORDER 111 # These calculations from ltprun-summary.sh script. 112 pass_count = sum([accumulator[k] for k in filters if filters[k][0] == '+']) 113 notpass_count = sum([accumulator[k] for k in filters 114 if filters[k][0] == '-']) 115 total_count = pass_count + notpass_count 116 if total_count: 117 score = float(pass_count) / float(total_count) * 100.0 118 else: 119 score = 0.0 120 print 'SCORE.ltp: %.2f' % score 121 print SUMMARY_BORDER 122 123 124 def _filter_times(ltp_out_file): 125 """Utility function to count lines that match certain filters. 126 127 @param ltp_out_file: human-readable output file from LTP -p (ltp.out). 128 129 @return a dictionary with test tags and corresponding times. 130 The dictionary is a set of buckets of tests based on the test 131 duration: 132 0: [tests that recoreded 0sec runtimes], 133 1: [tests that recorded runtimes from 0-60sec], ... 134 2: [tests that recorded runtimes from 61-120sec], ... 135 """ 136 test_tag_line_re = re.compile('^tag=(\w+)\s+stime=(\d+)$') 137 test_duration_line_re = re.compile('^duration=(\d+)\s+.*') 138 filter_accumulator = {} 139 with open(ltp_out_file) as f: 140 previous_tag = None 141 previous_time_s = 0 142 recorded_tags = set() 143 for line in f: 144 tag_matches = re.match(test_tag_line_re, line) 145 if tag_matches: 146 current_tag = tag_matches.group(1) 147 if previous_tag: 148 if previous_tag in recorded_tags: 149 print 'WARNING: duplicate tag found: %s.' % previous_tag 150 previous_tag = current_tag 151 continue 152 duration_matches = re.match(test_duration_line_re, line) 153 if duration_matches: 154 duration = int(duration_matches.group(1)) 155 if not previous_tag: 156 print 'WARNING: duration without a tag: %s.' % duration 157 continue 158 if duration != 0: 159 duration = int(duration / 60) + 1 160 test_list = filter_accumulator.setdefault(duration, []) 161 test_list.append(previous_tag) 162 return filter_accumulator 163 164 165 def _print_timings(accumulator): 166 """Utility function to print the summary of the parsing of ltp.out. 167 168 Prints a count of each type of test result, then a %pass-rate score. 169 170 Args: 171 @param accumulator: counts of test results 172 """ 173 print SUMMARY_BORDER 174 print 'Linux Test Project (LTP) Timing Summary:' 175 print SUMMARY_BORDER 176 for test_limit in sorted(accumulator.keys()): 177 print '<=%smin: %s tags: %s' % ( 178 test_limit, len(accumulator[test_limit]), 179 ', '.join(sorted(accumulator[test_limit]))) 180 print '' 181 print SUMMARY_BORDER 182 return 183 184 185 def summarize(ltp_out_file, test_timings=None): 186 """Scan detailed output from LTP run for summary test status reporting. 187 188 Looks for all possible test result types know to LTP: pass, fail, broken, 189 config error, retired and warning. Prints a summary. 190 191 @param ltp_out_file: human-readable output file from LTP -p (ltp.out). 192 @param test_timings: if True, emit an ordered summary of run timings of 193 tests. 194 """ 195 if not os.path.isfile(ltp_out_file): 196 print 'Unable to locate %s.' % ltp_out_file 197 return 198 199 _print_summary(TEST_FILTERS, _filter_and_count(ltp_out_file, TEST_FILTERS)) 200 if test_timings: 201 _print_timings(_filter_times(ltp_out_file)) 202 203 204 def main(argv): 205 """ Parse the human-readable logs from an LTP run and print a summary. 206 207 @param argv: command-line arguments. 208 """ 209 options = parse_args(argv) 210 summarize(options.ltp_out_file, options.test_timings) 211 212 213 if __name__ == '__main__': 214 main(sys.argv) 215
