Home | History | Annotate | Download | only in ltp
      1 #!/usr/bin/python
      2 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Report summarizer of internal test pass% from running many tests in LTP.
      7 
      8 LTP is the Linux Test Project from http://ltp.sourceforge.net/.
      9 
     10 This script serves to summarize the results of a test run by LTP test
     11 infrastructure.  LTP frequently runs >1000 tests so summarizing the results
     12 by result-type and count is useful. This script is invoked by the ltp.py
     13 wrapper in Autotest as a post-processing step to summarize the LTP run results
     14 in the Autotest log file.
     15 
     16 This script may be invoked by the command-line as follows:
     17 
     18 $ ./parse_ltp_out.py -l /mypath/ltp.out
     19 """
     20 
     21 import optparse
     22 import os
     23 import re
     24 import sys
     25 
     26 
     27 SUMMARY_BORDER = 80 * '-'
     28 # Prefix char used in summaries:
     29 # +: sums into 'passing'
     30 # -: sums into 'notpassing'
     31 TEST_FILTERS = {'TPASS': '+Pass', 'TFAIL': '-Fail', 'TBROK': '-Broken',
     32                 'TCONF': '-Config error', 'TRETR': 'Retired',
     33                 'TWARN': '+Warning'}
     34 
     35 
     36 def parse_args(argv):
     37     """Setup command line parsing options.
     38 
     39     Args:
     40         argv: command-line arguments.
     41 
     42     Returns:
     43         parsed option result from optparse.
     44     """
     45     parser = optparse.OptionParser('Usage: %prog --ltp-out-file=/path/ltp.out')
     46     parser.add_option(
     47         '-l', '--ltp-out-file',
     48         help='[required] Path and file name for ltp.out [default: %default]',
     49         dest='ltp_out_file',
     50         default=None)
     51     parser.add_option(
     52         '-t', '--timings',
     53         help='Show test timings in buckets [default: %default]',
     54         dest='test_timings', action='store_true',
     55         default=False)
     56     options, args = parser.parse_args()
     57     if not options.ltp_out_file:
     58         parser.error('You must supply a value for --ltp-out-file.')
     59 
     60     return options
     61 
     62 
     63 def _filter_and_count(ltp_out_file, test_filters):
     64     """Utility function to count lines that match certain filters.
     65 
     66     Args:
     67         ltp_out_file: human-readable output file from LTP -p (ltp.out).
     68         test_filters: dict of the tags to match and corresponding print tags.
     69 
     70     Returns:
     71         A dictionary with counts of the lines that matched each tag.
     72     """
     73     marker_line = '^<<<%s>>>$'
     74     status_line_re = re.compile('^\w+ +\d+ +(\w+) +: +\w+')
     75     filter_accumulator = dict.fromkeys(test_filters.keys(), 0)
     76     parse_states = (
     77         {'filters': {},
     78          'terminator': re.compile(marker_line % 'test_output')},
     79         {'filters': filter_accumulator,
     80          'terminator': re.compile(marker_line % 'execution_status')})
     81 
     82     # Simple 2-state state machine.
     83     state_test_active = False
     84     with open(ltp_out_file) as f:
     85         for line in f:
     86             state_index = int(state_test_active)
     87             if re.match(parse_states[state_index]['terminator'], line):
     88                 # This state is terminated - proceed to next.
     89                 state_test_active = not state_test_active
     90             else:
     91                 # Determine if this line matches any of the sought tags.
     92                 m = re.match(status_line_re, line)
     93                 if m and m.group(1) in parse_states[state_index]['filters']:
     94                     parse_states[state_index]['filters'][m.group(1)] += 1
     95     return filter_accumulator
     96 
     97 
     98 def _print_summary(filters, accumulator):
     99     """Utility function to print the summary of the parsing of ltp.out.
    100 
    101     Prints a count of each type of test result, then a %pass-rate score.
    102 
    103     Args:
    104         filters: map of tags sought and corresponding print headers.
    105         accumulator: counts of test results with same keys as filters.
    106     """
    107     print SUMMARY_BORDER
    108     print 'Linux Test Project (LTP) Run Summary:'
    109     print SUMMARY_BORDER
    110     # Size the header to the largest printable tag.
    111     fmt = '%%%ss: %%s' % max(map(lambda x: len(x), filters.values()))
    112     for k in sorted(filters):
    113          print fmt % (filters[k], accumulator[k])
    114 
    115     print SUMMARY_BORDER
    116     # These calculations from ltprun-summary.sh script.
    117     pass_count = sum([accumulator[k] for k in filters if filters[k][0] == '+'])
    118     notpass_count = sum([accumulator[k] for k in filters
    119                                         if filters[k][0] == '-'])
    120     total_count = pass_count + notpass_count
    121     if total_count:
    122       score = float(pass_count) / float(total_count) * 100.0
    123     else:
    124       score = 0.0
    125     print 'SCORE.ltp: %.2f' % score
    126     print SUMMARY_BORDER
    127 
    128 
    129 def _filter_times(ltp_out_file):
    130     """Utility function to count lines that match certain filters.
    131 
    132     Args:
    133         ltp_out_file: human-readable output file from LTP -p (ltp.out).
    134 
    135     Returns:
    136         A dictionary with test tags and corresponding times.  The dictionary is
    137         a set of buckets of tests based on the test duration:
    138           0: [tests that recoreded 0sec runtimes],
    139           1: [tests that recorded runtimes from 0-60sec], ...
    140           2: [tests that recorded runtimes from 61-120sec], ...
    141     """
    142     test_tag_line_re = re.compile('^tag=(\w+)\s+stime=(\d+)$')
    143     test_duration_line_re = re.compile('^duration=(\d+)\s+.*')
    144     filter_accumulator = {}
    145     with open(ltp_out_file) as f:
    146         previous_tag = None
    147         previous_time_s = 0
    148         recorded_tags = set()
    149         for line in f:
    150             tag_matches = re.match(test_tag_line_re, line)
    151             if tag_matches:
    152                 current_tag = tag_matches.group(1)
    153                 if previous_tag:
    154                     if previous_tag in recorded_tags:
    155                         print 'WARNING: duplicate tag found: %s.' % previous_tag
    156                 previous_tag = current_tag
    157                 continue
    158             duration_matches = re.match(test_duration_line_re, line)
    159             if duration_matches:
    160                 duration = int(duration_matches.group(1))
    161                 if not previous_tag:
    162                     print 'WARNING: duration without a tag: %s.' % duration
    163                     continue
    164                 if duration != 0:
    165                     duration = int(duration / 60) + 1
    166                 test_list = filter_accumulator.setdefault(duration, [])
    167                 test_list.append(previous_tag)
    168     return filter_accumulator
    169 
    170 
    171 def _print_timings(accumulator):
    172     """Utility function to print the summary of the parsing of ltp.out.
    173 
    174     Prints a count of each type of test result, then a %pass-rate score.
    175 
    176     Args:
    177         filters: map of tags sought and corresponding print headers.
    178         accumulator: counts of test results with same keys as filters.
    179     """
    180     print SUMMARY_BORDER
    181     print 'Linux Test Project (LTP) Timing Summary:'
    182     print SUMMARY_BORDER
    183     for test_limit in sorted(accumulator.keys()):
    184         print '<=%smin: %s tags: %s' % (
    185             test_limit, len(accumulator[test_limit]),
    186             ', '.join(sorted(accumulator[test_limit])))
    187         print ''
    188     print SUMMARY_BORDER
    189     return
    190 
    191 
    192 def summarize(ltp_out_file, test_timings=None):
    193     """Scan detailed output from LTP run for summary test status reporting.
    194 
    195     Looks for all possible test result types know to LTP: pass, fail, broken,
    196     config error, retired and warning.  Prints a summary.
    197 
    198     Args:
    199         ltp_out_file: human-readable output file from LTP -p (ltp.out).
    200         test_timings: if True, emit an ordered summary of run timings of tests.
    201     """
    202     if not os.path.isfile(ltp_out_file):
    203         print 'Unable to locate %s.' % ltp_out_file
    204         return
    205 
    206     _print_summary(TEST_FILTERS, _filter_and_count(ltp_out_file, TEST_FILTERS))
    207     if test_timings:
    208       _print_timings(_filter_times(ltp_out_file))
    209 
    210 
    211 def main(argv):
    212     """ Parse the human-readable logs from an LTP run and print a summary.
    213 
    214     Args:
    215         argv: command-line arguments.
    216     """
    217     options = parse_args(argv)
    218     summarize(options.ltp_out_file, options.test_timings)
    219 
    220 
    221 if __name__ == '__main__':
    222     main(sys.argv)
    223