Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 
      3 """
      4 compare.py - versatile benchmark output compare tool
      5 """
      6 
      7 import argparse
      8 from argparse import ArgumentParser
      9 import sys
     10 import gbench
     11 from gbench import util, report
     12 from gbench.util import *
     13 
     14 
     15 def check_inputs(in1, in2, flags):
     16     """
     17     Perform checking on the user provided inputs and diagnose any abnormalities
     18     """
     19     in1_kind, in1_err = classify_input_file(in1)
     20     in2_kind, in2_err = classify_input_file(in2)
     21     output_file = find_benchmark_flag('--benchmark_out=', flags)
     22     output_type = find_benchmark_flag('--benchmark_out_format=', flags)
     23     if in1_kind == IT_Executable and in2_kind == IT_Executable and output_file:
     24         print(("WARNING: '--benchmark_out=%s' will be passed to both "
     25                "benchmarks causing it to be overwritten") % output_file)
     26     if in1_kind == IT_JSON and in2_kind == IT_JSON and len(flags) > 0:
     27         print("WARNING: passing optional flags has no effect since both "
     28               "inputs are JSON")
     29     if output_type is not None and output_type != 'json':
     30         print(("ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
     31                " is not supported.") % output_type)
     32         sys.exit(1)
     33 
     34 
     35 def create_parser():
     36     parser = ArgumentParser(
     37         description='versatile benchmark output compare tool')
     38     subparsers = parser.add_subparsers(
     39         help='This tool has multiple modes of operation:',
     40         dest='mode')
     41 
     42     parser_a = subparsers.add_parser(
     43         'benchmarks',
     44         help='The most simple use-case, compare all the output of these two benchmarks')
     45     baseline = parser_a.add_argument_group(
     46         'baseline', 'The benchmark baseline')
     47     baseline.add_argument(
     48         'test_baseline',
     49         metavar='test_baseline',
     50         type=argparse.FileType('r'),
     51         nargs=1,
     52         help='A benchmark executable or JSON output file')
     53     contender = parser_a.add_argument_group(
     54         'contender', 'The benchmark that will be compared against the baseline')
     55     contender.add_argument(
     56         'test_contender',
     57         metavar='test_contender',
     58         type=argparse.FileType('r'),
     59         nargs=1,
     60         help='A benchmark executable or JSON output file')
     61     parser_a.add_argument(
     62         'benchmark_options',
     63         metavar='benchmark_options',
     64         nargs=argparse.REMAINDER,
     65         help='Arguments to pass when running benchmark executables')
     66 
     67     parser_b = subparsers.add_parser(
     68         'filters', help='Compare filter one with the filter two of benchmark')
     69     baseline = parser_b.add_argument_group(
     70         'baseline', 'The benchmark baseline')
     71     baseline.add_argument(
     72         'test',
     73         metavar='test',
     74         type=argparse.FileType('r'),
     75         nargs=1,
     76         help='A benchmark executable or JSON output file')
     77     baseline.add_argument(
     78         'filter_baseline',
     79         metavar='filter_baseline',
     80         type=str,
     81         nargs=1,
     82         help='The first filter, that will be used as baseline')
     83     contender = parser_b.add_argument_group(
     84         'contender', 'The benchmark that will be compared against the baseline')
     85     contender.add_argument(
     86         'filter_contender',
     87         metavar='filter_contender',
     88         type=str,
     89         nargs=1,
     90         help='The second filter, that will be compared against the baseline')
     91     parser_b.add_argument(
     92         'benchmark_options',
     93         metavar='benchmark_options',
     94         nargs=argparse.REMAINDER,
     95         help='Arguments to pass when running benchmark executables')
     96 
     97     parser_c = subparsers.add_parser(
     98         'benchmarksfiltered',
     99         help='Compare filter one of first benchmark with filter two of the second benchmark')
    100     baseline = parser_c.add_argument_group(
    101         'baseline', 'The benchmark baseline')
    102     baseline.add_argument(
    103         'test_baseline',
    104         metavar='test_baseline',
    105         type=argparse.FileType('r'),
    106         nargs=1,
    107         help='A benchmark executable or JSON output file')
    108     baseline.add_argument(
    109         'filter_baseline',
    110         metavar='filter_baseline',
    111         type=str,
    112         nargs=1,
    113         help='The first filter, that will be used as baseline')
    114     contender = parser_c.add_argument_group(
    115         'contender', 'The benchmark that will be compared against the baseline')
    116     contender.add_argument(
    117         'test_contender',
    118         metavar='test_contender',
    119         type=argparse.FileType('r'),
    120         nargs=1,
    121         help='The second benchmark executable or JSON output file, that will be compared against the baseline')
    122     contender.add_argument(
    123         'filter_contender',
    124         metavar='filter_contender',
    125         type=str,
    126         nargs=1,
    127         help='The second filter, that will be compared against the baseline')
    128     parser_c.add_argument(
    129         'benchmark_options',
    130         metavar='benchmark_options',
    131         nargs=argparse.REMAINDER,
    132         help='Arguments to pass when running benchmark executables')
    133 
    134     return parser
    135 
    136 
    137 def main():
    138     # Parse the command line flags
    139     parser = create_parser()
    140     args, unknown_args = parser.parse_known_args()
    141     assert not unknown_args
    142     benchmark_options = args.benchmark_options
    143 
    144     if args.mode == 'benchmarks':
    145         test_baseline = args.test_baseline[0].name
    146         test_contender = args.test_contender[0].name
    147         filter_baseline = ''
    148         filter_contender = ''
    149 
    150         # NOTE: if test_baseline == test_contender, you are analyzing the stdev
    151 
    152         description = 'Comparing %s to %s' % (test_baseline, test_contender)
    153     elif args.mode == 'filters':
    154         test_baseline = args.test[0].name
    155         test_contender = args.test[0].name
    156         filter_baseline = args.filter_baseline[0]
    157         filter_contender = args.filter_contender[0]
    158 
    159         # NOTE: if filter_baseline == filter_contender, you are analyzing the
    160         # stdev
    161 
    162         description = 'Comparing %s to %s (from %s)' % (
    163             filter_baseline, filter_contender, args.test[0].name)
    164     elif args.mode == 'benchmarksfiltered':
    165         test_baseline = args.test_baseline[0].name
    166         test_contender = args.test_contender[0].name
    167         filter_baseline = args.filter_baseline[0]
    168         filter_contender = args.filter_contender[0]
    169 
    170         # NOTE: if test_baseline == test_contender and
    171         # filter_baseline == filter_contender, you are analyzing the stdev
    172 
    173         description = 'Comparing %s (from %s) to %s (from %s)' % (
    174             filter_baseline, test_baseline, filter_contender, test_contender)
    175     else:
    176         # should never happen
    177         print("Unrecognized mode of operation: '%s'" % args.mode)
    178         exit(1)
    179 
    180     check_inputs(test_baseline, test_contender, benchmark_options)
    181 
    182     options_baseline = []
    183     options_contender = []
    184 
    185     if filter_baseline and filter_contender:
    186         options_baseline = ['--benchmark_filter=%s' % filter_baseline]
    187         options_contender = ['--benchmark_filter=%s' % filter_contender]
    188 
    189     # Run the benchmarks and report the results
    190     json1 = json1_orig = gbench.util.run_or_load_benchmark(
    191         test_baseline, benchmark_options + options_baseline)
    192     json2 = json2_orig = gbench.util.run_or_load_benchmark(
    193         test_contender, benchmark_options + options_contender)
    194 
    195     # Now, filter the benchmarks so that the difference report can work
    196     if filter_baseline and filter_contender:
    197         replacement = '[%s vs. %s]' % (filter_baseline, filter_contender)
    198         json1 = gbench.report.filter_benchmark(
    199             json1_orig, filter_baseline, replacement)
    200         json2 = gbench.report.filter_benchmark(
    201             json2_orig, filter_contender, replacement)
    202 
    203     # Diff and output
    204     output_lines = gbench.report.generate_difference_report(json1, json2)
    205     print(description)
    206     for ln in output_lines:
    207         print(ln)
    208 
    209 
    210 import unittest
    211 
    212 
    213 class TestParser(unittest.TestCase):
    214     def setUp(self):
    215         self.parser = create_parser()
    216         testInputs = os.path.join(
    217             os.path.dirname(
    218                 os.path.realpath(__file__)),
    219             'gbench',
    220             'Inputs')
    221         self.testInput0 = os.path.join(testInputs, 'test_baseline_run1.json')
    222         self.testInput1 = os.path.join(testInputs, 'test_baseline_run2.json')
    223 
    224     def test_benchmarks_basic(self):
    225         parsed = self.parser.parse_args(
    226             ['benchmarks', self.testInput0, self.testInput1])
    227         self.assertEqual(parsed.mode, 'benchmarks')
    228         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
    229         self.assertEqual(parsed.test_contender[0].name, self.testInput1)
    230         self.assertFalse(parsed.benchmark_options)
    231 
    232     def test_benchmarks_with_remainder(self):
    233         parsed = self.parser.parse_args(
    234             ['benchmarks', self.testInput0, self.testInput1, 'd'])
    235         self.assertEqual(parsed.mode, 'benchmarks')
    236         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
    237         self.assertEqual(parsed.test_contender[0].name, self.testInput1)
    238         self.assertEqual(parsed.benchmark_options, ['d'])
    239 
    240     def test_benchmarks_with_remainder_after_doubleminus(self):
    241         parsed = self.parser.parse_args(
    242             ['benchmarks', self.testInput0, self.testInput1, '--', 'e'])
    243         self.assertEqual(parsed.mode, 'benchmarks')
    244         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
    245         self.assertEqual(parsed.test_contender[0].name, self.testInput1)
    246         self.assertEqual(parsed.benchmark_options, ['e'])
    247 
    248     def test_filters_basic(self):
    249         parsed = self.parser.parse_args(
    250             ['filters', self.testInput0, 'c', 'd'])
    251         self.assertEqual(parsed.mode, 'filters')
    252         self.assertEqual(parsed.test[0].name, self.testInput0)
    253         self.assertEqual(parsed.filter_baseline[0], 'c')
    254         self.assertEqual(parsed.filter_contender[0], 'd')
    255         self.assertFalse(parsed.benchmark_options)
    256 
    257     def test_filters_with_remainder(self):
    258         parsed = self.parser.parse_args(
    259             ['filters', self.testInput0, 'c', 'd', 'e'])
    260         self.assertEqual(parsed.mode, 'filters')
    261         self.assertEqual(parsed.test[0].name, self.testInput0)
    262         self.assertEqual(parsed.filter_baseline[0], 'c')
    263         self.assertEqual(parsed.filter_contender[0], 'd')
    264         self.assertEqual(parsed.benchmark_options, ['e'])
    265 
    266     def test_filters_with_remainder_after_doubleminus(self):
    267         parsed = self.parser.parse_args(
    268             ['filters', self.testInput0, 'c', 'd', '--', 'f'])
    269         self.assertEqual(parsed.mode, 'filters')
    270         self.assertEqual(parsed.test[0].name, self.testInput0)
    271         self.assertEqual(parsed.filter_baseline[0], 'c')
    272         self.assertEqual(parsed.filter_contender[0], 'd')
    273         self.assertEqual(parsed.benchmark_options, ['f'])
    274 
    275     def test_benchmarksfiltered_basic(self):
    276         parsed = self.parser.parse_args(
    277             ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e'])
    278         self.assertEqual(parsed.mode, 'benchmarksfiltered')
    279         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
    280         self.assertEqual(parsed.filter_baseline[0], 'c')
    281         self.assertEqual(parsed.test_contender[0].name, self.testInput1)
    282         self.assertEqual(parsed.filter_contender[0], 'e')
    283         self.assertFalse(parsed.benchmark_options)
    284 
    285     def test_benchmarksfiltered_with_remainder(self):
    286         parsed = self.parser.parse_args(
    287             ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', 'f'])
    288         self.assertEqual(parsed.mode, 'benchmarksfiltered')
    289         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
    290         self.assertEqual(parsed.filter_baseline[0], 'c')
    291         self.assertEqual(parsed.test_contender[0].name, self.testInput1)
    292         self.assertEqual(parsed.filter_contender[0], 'e')
    293         self.assertEqual(parsed.benchmark_options[0], 'f')
    294 
    295     def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
    296         parsed = self.parser.parse_args(
    297             ['benchmarksfiltered', self.testInput0, 'c', self.testInput1, 'e', '--', 'g'])
    298         self.assertEqual(parsed.mode, 'benchmarksfiltered')
    299         self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
    300         self.assertEqual(parsed.filter_baseline[0], 'c')
    301         self.assertEqual(parsed.test_contender[0].name, self.testInput1)
    302         self.assertEqual(parsed.filter_contender[0], 'e')
    303         self.assertEqual(parsed.benchmark_options[0], 'g')
    304 
    305 
    306 if __name__ == '__main__':
    307     # unittest.main()
    308     main()
    309 
    310 # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
    311 # kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
    312 # kate: indent-mode python; remove-trailing-spaces modified;
    313