Home | History | Annotate | Download | only in bench
      1 #!/usr/bin/env python
      2 '''
      3 Created on May 16, 2011
      4 
      5 @author: bungeman
      6 '''
      7 import sys
      8 import getopt
      9 import bench_util
     10 
     11 def usage():
     12     """Prints simple usage information."""
     13 
     14     print '-o <file> the old bench output file.'
     15     print '-n <file> the new bench output file.'
     16     print '-h causes headers to be output.'
     17     print '-s <stat> the type of statistical analysis used'
     18     print '   Not specifying is the same as -s "avg".'
     19     print '  avg: average of all data points'
     20     print '  min: minimum of all data points'
     21     print '  med: median of all data points'
     22     print '  25th: twenty-fifth percentile for all data points'
     23     print '-f <fieldSpec> which fields to output and in what order.'
     24     print '   Not specifying is the same as -f "bctondp".'
     25     print '  b: bench'
     26     print '  c: config'
     27     print '  t: time type'
     28     print '  o: old time'
     29     print '  n: new time'
     30     print '  d: diff'
     31     print '  p: percent diff'
     32     print '-t use tab delimited format for output.'
     33     print '--match <bench> only matches benches which begin with <bench>.'
     34 
     35 class BenchDiff:
     36     """A compare between data points produced by bench.
     37 
     38     (BenchDataPoint, BenchDataPoint)"""
     39     def __init__(self, old, new):
     40         self.old = old
     41         self.new = new
     42         self.diff = old.time - new.time
     43         diffp = 0
     44         if old.time != 0:
     45             diffp = self.diff / old.time
     46         self.diffp = diffp
     47 
     48     def __repr__(self):
     49         return "BenchDiff(%s, %s)" % (
     50                    str(self.new),
     51                    str(self.old),
     52                )
     53 
     54 def main():
     55     """Parses command line and writes output."""
     56 
     57     try:
     58         opts, _ = getopt.getopt(sys.argv[1:], "f:o:n:s:ht", ['match='])
     59     except getopt.GetoptError, err:
     60         print str(err) 
     61         usage()
     62         sys.exit(2)
     63 
     64     old = None
     65     new = None
     66     column_format = ""
     67     header_format = ""
     68     columns = 'bctondp'
     69     header = False
     70     stat_type = "avg"
     71     use_tabs = False
     72     match_bench = None;
     73 
     74     for option, value in opts:
     75         if option == "-o":
     76             old = value
     77         elif option == "-n":
     78             new = value
     79         elif option == "-h":
     80             header = True
     81         elif option == "-f":
     82             columns = value
     83         elif option == "-s":
     84             stat_type = value
     85         elif option == "-t":
     86             use_tabs = True
     87         elif option == "--match":
     88             match_bench = value
     89         else:
     90             usage()
     91             assert False, "unhandled option"
     92 
     93     if old is None or new is None:
     94         usage()
     95         sys.exit(2)
     96 
     97     old_benches = bench_util.parse({}, open(old, 'r'), stat_type)
     98     new_benches = bench_util.parse({}, open(new, 'r'), stat_type)
     99 
    100     bench_diffs = []
    101     for old_bench in old_benches:
    102         #filter benches by the match criteria
    103         if match_bench and not old_bench.bench.startswith(match_bench):
    104             continue
    105 
    106         #filter new_benches for benches that match old_bench
    107         new_bench_match = [bench for bench in new_benches
    108             if old_bench.bench == bench.bench and
    109                old_bench.config == bench.config and
    110                old_bench.time_type == bench.time_type
    111         ]
    112         if (len(new_bench_match) < 1):
    113             continue
    114         bench_diffs.append(BenchDiff(old_bench, new_bench_match[0]))
    115 
    116     if use_tabs:
    117         column_formats = {
    118             'b' : '{bench}\t',
    119             'c' : '{config}\t',
    120             't' : '{time_type}\t',
    121             'o' : '{old_time: 0.2f}\t',
    122             'n' : '{new_time: 0.2f}\t',
    123             'd' : '{diff: 0.2f}\t',
    124             'p' : '{diffp: 0.1%}\t',
    125         }
    126         header_formats = {
    127             'b' : '{bench}\t',
    128             'c' : '{config}\t',
    129             't' : '{time_type}\t',
    130             'o' : '{old_time}\t',
    131             'n' : '{new_time}\t',
    132             'd' : '{diff}\t',
    133             'p' : '{diffp}\t',
    134         }
    135     else:
    136         bench_max_len = max(map(lambda b: len(b.old.bench), bench_diffs))
    137         config_max_len = max(map(lambda b: len(b.old.config), bench_diffs))
    138         column_formats = {
    139             'b' : '{bench: >%d} ' % (bench_max_len),
    140             'c' : '{config: <%d} ' % (config_max_len),
    141             't' : '{time_type: <4} ',
    142             'o' : '{old_time: >10.2f} ',
    143             'n' : '{new_time: >10.2f} ',
    144             'd' : '{diff: >+10.2f} ',
    145             'p' : '{diffp: >+8.1%} ',
    146         }
    147         header_formats = {
    148             'b' : '{bench: >%d} ' % (bench_max_len),
    149             'c' : '{config: <%d} ' % (config_max_len),
    150             't' : '{time_type: <4} ',
    151             'o' : '{old_time: >10} ',
    152             'n' : '{new_time: >10} ',
    153             'd' : '{diff: >10} ',
    154             'p' : '{diffp: >8} ',
    155         }
    156 
    157     for column_char in columns:
    158         if column_formats[column_char]:
    159             column_format += column_formats[column_char]
    160             header_format += header_formats[column_char]
    161         else:
    162             usage()
    163             sys.exit(2)
    164 
    165     if header:
    166         print header_format.format(
    167             bench='bench'
    168             , config='conf'
    169             , time_type='time'
    170             , old_time='old'
    171             , new_time='new'
    172             , diff='diff'
    173             , diffp='diffP'
    174         )
    175 
    176     bench_diffs.sort(key=lambda d : [d.diffp,
    177                                      d.old.bench,
    178                                      d.old.config,
    179                                      d.old.time_type,
    180                                     ])
    181     for bench_diff in bench_diffs:
    182         print column_format.format(
    183             bench=bench_diff.old.bench.strip()
    184             , config=bench_diff.old.config.strip()
    185             , time_type=bench_diff.old.time_type
    186             , old_time=bench_diff.old.time
    187             , new_time=bench_diff.new.time
    188             , diff=bench_diff.diff
    189             , diffp=bench_diff.diffp
    190         )
    191 
    192 if __name__ == "__main__":
    193     main()
    194