Home | History | Annotate | Download | only in bench 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be found
      4 # in the LICENSE file.
      5 
      6 """ Analyze recent bench data from graphs, and output suggested ranges.
      7 
      8 This script reads and parses Skia benchmark values from the xhtml files
      9 generated by bench_graph_svg.py, and outputs an html file containing suggested
     10 bench ranges to use in bench_expectations.txt, with analytical plots.
     11 """
     12 
     13 __author__ = 'bensong (at] google.com (Ben Chen)'
     14 
     15 import getopt
     16 import math
     17 import re
     18 import sys
     19 import urllib
     20 from datetime import datetime
     21 
     22 
     23 # Constants for calculating suggested bench ranges.
     24 WINDOW = 5  # Moving average sliding window size.
     25 # We use moving average as expected bench value, and calculate average Variance
     26 # of bench from the moving average. Set range to be [X_UB * Variance above
     27 # moving average, X_LB * Variance below moving average] of latest revision.
     28 X_UB = 4.0
     29 X_LB = 5.0
     30 
     31 # List of platforms.
     32 PLATFORMS = ['GalaxyNexus_4-1_Float_Release',
     33              'Mac_Float_NoDebug_32',
     34              'Mac_Float_NoDebug_64',
     35              'MacMiniLion_Float_NoDebug_32',
     36              'MacMiniLion_Float_NoDebug_64',
     37              'Nexus7_4-1_Float_Release',
     38              'Shuttle_Ubuntu12_ATI5770_Float_Release_64',
     39              'Shuttle_Win7_Intel_Float_Release_32',
     40              'Shuttle_Win7_Intel_Float_Release_64',
     41              'Xoom_4-1_Float_Release'
     42             ]
     43 
     44 # List of bench representation algorithms. Flag "-a" is chosen from the list.
     45 ALGS = ['25th', 'avg', 'med', 'min']
     46 
     47 # Regular expressions for parsing bench/revision values.
     48 HEIGHT_RE = 'height (\d+\.\d+) corresponds to bench value (\d+\.\d+).-->'
     49 REV_RE = '<rect id="(\d+)" x="(\d+\.\d+)" y="'  # Revision corresponding x.
     50 LINE_RE = '<polyline id="(.*)".*points="(.*)"/>'  # Bench value lines.
     51 
     52 # Bench graph url pattern.
     53 INPUT_URL_TEMPLATE = ('http://chromium-skia-gm.commondatastorage.googleapis.com'
     54                       '/graph-Skia_%s-2.xhtml')
     55 
     56 # Output HTML elements and templates.
     57 HTML_HEAD = ('<html><head><title>Skia Bench Expected Ranges</title>'
     58              '<script type="text/javascript" src="https://skia.googlecode.com/'
     59              'svn/buildbot/dygraph-combined.js"></script></head><body>Please '
     60              'adjust values as appropriate and update benches to monitor in '
     61              'bench/bench_expectations.txt.<br><br>')
     62 HTML_SUFFIX = '</body></html>'
     63 GRAPH_PREFIX = ('<br>%s<br><div id="%s" style="width:400px;height:200px"></div>'
     64                 '<script type="text/javascript">g%s=new Dygraph('
     65                 'document.getElementById("%s"),"rev,bench,alert\\n')
     66 GRAPH_SUFFIX = ('",{customBars: true,"alert":{strokeWidth:0.0,drawPoints:true,'
     67                 'pointSize:4,highlightCircleSize:6}});</script>')
     68 
     69 
     70 def Usage():
     71   """Prints flag usage information."""
     72   print '-a <representation-algorithm>: defaults to "25th".'
     73   print '  If set, must be one of the list element in ALGS defined above.'
     74   print '-b <bench-prefix>: prefix of matching bench names to analyze.'
     75   print '  Only include benchmarks whose names start with this string.'
     76   print '  Cannot be empty, because there are too many benches overall.'
     77   print '-o <file>: html output filename. Output to STDOUT if not set.'
     78   print '-p <platform-prefix>: prefix of platform names to analyze.'
     79   print '  PLATFORMS has list of matching candidates. Matches all if not set.'
     80 
     81 def GetBenchValues(page, bench_prefix):
     82   """Returns a dict of matching bench values from the given xhtml page.
     83   Args:
     84     page: substring used to construct the specific bench graph URL to fetch.
     85     bench_prefix: only benches starting with this string will be included.
     86 
     87   Returns:
     88     a dict mapping benchmark name and revision combinations to bench values.
     89   """
     90   height = None
     91   max_bench = None
     92   height_scale = None
     93   revisions = []
     94   x_axes = []  # For calculating corresponding revisions.
     95   val_dic = {}  # dict[bench_name][revision] -> bench_value
     96 
     97   lines = urllib.urlopen(INPUT_URL_TEMPLATE % page).readlines()
     98   for line in lines:
     99     height_match = re.search(HEIGHT_RE, line)
    100     if height_match:
    101       height = float(height_match.group(1))
    102       max_bench = float(height_match.group(2))
    103       height_scale = max_bench / height
    104 
    105     rev_match = re.search(REV_RE, line)
    106     if rev_match:
    107       revisions.append(int(rev_match.group(1)))
    108       x_axes.append(float(rev_match.group(2)))
    109 
    110     line_match = re.search(LINE_RE, line)
    111     if not line_match:
    112       continue
    113     bench = line_match.group(1)
    114     bench = bench[:bench.find('_{')]
    115     if not bench.startswith(bench_prefix):
    116       continue
    117     if bench not in val_dic:
    118       val_dic[bench] = {}
    119 
    120     vals = line_match.group(2).strip().split(' ')
    121     if len(vals) < WINDOW:  # Too few bench data points; skip.
    122       continue
    123     for val in vals:
    124       x, y = [float(i) for i in val.split(',')]
    125       for i in range(len(x_axes)):
    126         if x <= x_axes[i]:  # Found corresponding bench revision.
    127           break
    128       val_dic[bench][revisions[i]] = float(
    129           '%.3f' % ((height - y) * height_scale))
    130 
    131   return val_dic
    132 
    133 def CreateBenchOutput(page, bench, val_dic):
    134   """Returns output for the given page and bench data in dict.
    135   Args:
    136     page: substring of bench graph webpage, to indicate the bench platform.
    137     bench: name of the benchmark to process.
    138     val_dic: dict[bench_name][revision] -> bench_value.
    139 
    140   Returns:
    141     string of html/javascript as part of the whole script output for the bench.
    142   """
    143   revs = val_dic[bench].keys()
    144   revs.sort()
    145   # Uses moving average to calculate expected bench variance, then sets
    146   # expectations and ranges accordingly.
    147   variances = []
    148   moving_avgs = []
    149   points = []
    150   for rev in revs:
    151     points.append(val_dic[bench][rev])
    152     if len(points) >= WINDOW:
    153       moving_avgs.append(sum(points[-WINDOW:]) / WINDOW)
    154       variances.append(abs(points[-1] - moving_avgs[-1]))
    155     else:  # For the first WINDOW-1 points, cannot calculate moving average.
    156       moving_avgs.append(points[-1])  # Uses actual value as estimates.
    157       variances.append(0)
    158   if len(variances) >= WINDOW:
    159     for i in range(WINDOW - 1):
    160       # Backfills estimated variances for the first WINDOW-1 points.
    161       variances[i] = variances[WINDOW - 1]
    162 
    163   avg_var = sum(variances) / len(variances)
    164   for val in variances:  # Removes outlier variances. Only does one iter.
    165     if val > min(X_LB, X_UB) * avg_var:
    166       variances.remove(val)
    167   avg_var = sum(variances) / len(variances)
    168 
    169   graph_id = '%s_%s' % (bench, page.replace('-', '_'))
    170   expectations = '%s,%s,%.2f,%.2f,%.2f' % (bench, page, moving_avgs[-1],
    171                                            moving_avgs[-1] - X_LB * avg_var,
    172                                            moving_avgs[-1] + X_UB * avg_var)
    173   out = GRAPH_PREFIX % (expectations, graph_id, graph_id, graph_id)
    174   for i in range(len(revs)):
    175     out += '%s,%.2f;%.2f;%.2f,' % (revs[i], moving_avgs[i] - X_LB * avg_var,
    176                                    points[i], moving_avgs[i] + X_UB * avg_var)
    177     if (points[i] > moving_avgs[i] + X_UB * avg_var or
    178         points[i] < moving_avgs[i] - X_LB * avg_var):  # Mark as alert point.
    179       out += '%.2f;%.2f;%.2f\\n' % (points[i], points[i], points[i])
    180     else:
    181       out += 'NaN;NaN;NaN\\n'
    182 
    183   return out
    184 
    185 def main():
    186   """Parses flags and outputs analysis results."""
    187   try:
    188     opts, _ = getopt.getopt(sys.argv[1:], 'a:b:o:p:')
    189   except getopt.GetoptError, err:
    190     Usage()
    191     sys.exit(2)
    192 
    193   alg = '25th'
    194   bench_prefix = None
    195   out_file = None
    196   platform_prefix = ''
    197   for option, value in opts:
    198     if option == '-a':
    199       if value not in ALGS:
    200         raise Exception('Invalid flag -a (%s): must be set to one of %s.' %
    201                         (value, str(ALGS)))
    202       alg = value
    203     elif option == '-b':
    204       bench_prefix = value
    205     elif option == '-o':
    206       out_file = value
    207     elif option == '-p':
    208       platform_prefix = value
    209     else:
    210       Usage()
    211       raise Exception('Error handling flags.')
    212 
    213   if not bench_prefix:
    214     raise Exception('Must provide nonempty Flag -b (bench name prefix).')
    215 
    216   pages = []
    217   for platform in PLATFORMS:
    218     if not platform.startswith(platform_prefix):
    219       continue
    220     pages.append('%s-%s' % (platform, alg))
    221 
    222   if not pages:  # No matching platform found.
    223     raise Exception('Flag -p (platform prefix: %s) does not match any of %s.' %
    224                     (platform_prefix, str(PLATFORMS)))
    225 
    226   body = ''
    227   # Iterates through bench graph xhtml pages for oututting matching benches.
    228   for page in pages:
    229     bench_value_dict = GetBenchValues(page, bench_prefix)
    230     for bench in bench_value_dict:
    231       body += CreateBenchOutput(page, bench, bench_value_dict) + GRAPH_SUFFIX
    232 
    233   if not body:
    234     raise Exception('No bench outputs. Most likely there are no matching bench'
    235                     ' prefix (%s) in Flags -b for platforms %s.\nPlease also '
    236                     'check if the bench graph URLs are valid at %s.' % (
    237                         bench_prefix, str(PLATFORMS), INPUT_URL_TEMPLATE))
    238   if out_file:
    239     f = open(out_file, 'w+')
    240     f.write(HTML_HEAD + body + HTML_SUFFIX)
    241     f.close()
    242   else:
    243     print HTML_HEAD + body + HTML_SUFFIX
    244 
    245 
    246 if '__main__' == __name__:
    247   main()
    248