Home | History | Annotate | Download | only in bench 
      1 #!/usr/bin/env python
      2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be found
      4 # in the LICENSE file.
      5 
      6 """ Analyze recent SkPicture bench data, and output suggested ranges.
      7 
      8 The outputs can be edited and pasted to bench_expectations.txt to trigger
      9 buildbot alerts if the actual benches are out of range. Details are documented
     10 in the .txt file.
     11 
     12 Currently the easiest way to update bench_expectations.txt is to delete all skp
     13 bench lines, run this script, and redirect outputs (">>") to be added to the
     14 .txt file.
     15 TODO(bensong): find a better way for updating the bench lines in place.
     16 
     17 Note: since input data are stored in Google Storage, you will need to set up
     18 the corresponding library.
     19 See http://developers.google.com/storage/docs/gspythonlibrary for details.
     20 """
     21 
     22 __author__ = 'bensong (at] google.com (Ben Chen)'
     23 
     24 import bench_util
     25 import boto
     26 import cStringIO
     27 import optparse
     28 import re
     29 import shutil
     30 
     31 from oauth2_plugin import oauth2_plugin
     32 
     33 
     34 # Ratios for calculating suggested picture bench upper and lower bounds.
     35 BENCH_UB = 1.1  # Allow for 10% room for normal variance on the up side.
     36 BENCH_LB = 0.85
     37 
     38 # Further allow for a fixed amount of noise. This is especially useful for
     39 # benches of smaller absolute value. Keeping this value small will not affect
     40 # performance tunings.
     41 BENCH_ALLOWED_NOISE = 10
     42 
     43 # List of platforms to track.
     44 PLATFORMS = ['Mac_Float_Bench_32',
     45              'Nexus10_4-1_Float_Bench_32',
     46              'Shuttle_Ubuntu12_ATI5770_Float_Bench_32',
     47             ]
     48 
     49 # Filter for configs of no interest. They are old config names replaced by more
     50 # specific ones.
     51 CONFIGS_TO_FILTER = ['gpu', 'raster']
     52 
     53 # Template for gsutil uri.
     54 GOOGLE_STORAGE_URI_SCHEME = 'gs'
     55 URI_BUCKET = 'chromium-skia-gm'
     56 
     57 # Constants for optparse.
     58 USAGE_STRING = 'USAGE: %s [options]'
     59 HOWTO_STRING = """
     60 Feel free to revise PLATFORMS for your own needs. The default is the most common
     61 combination that we care most about. Platforms that did not run bench_pictures
     62 in the given revision range will not have corresponding outputs.
     63 Please check http://go/skpbench to choose a range that fits your needs.
     64 BENCH_UB, BENCH_LB and BENCH_ALLOWED_NOISE can be changed to expand or narrow
     65 the permitted bench ranges without triggering buidbot alerts.
     66 """
     67 HELP_STRING = """
     68 Outputs expectation picture bench ranges for the latest revisions for the given
     69 revision range. For instance, --rev_range=6000:6000 will return only bench
     70 ranges for the bots that ran bench_pictures at rev 6000; --rev-range=6000:7000
     71 may have multiple bench data points for each bench configuration, and the code
     72 returns bench data for the latest revision of all available (closer to 7000).
     73 """ + HOWTO_STRING
     74 
     75 OPTION_REVISION_RANGE = '--rev-range'
     76 OPTION_REVISION_RANGE_SHORT = '-r'
     77 # Bench bench representation algorithm flag.
     78 OPTION_REPRESENTATION_ALG = '--algorithm'
     79 OPTION_REPRESENTATION_ALG_SHORT = '-a'
     80 
     81 # List of valid representation algorithms.
     82 REPRESENTATION_ALGS = ['avg', 'min', 'med', '25th']
     83 
     84 def OutputSkpBenchExpectations(rev_min, rev_max, representation_alg):
     85   """Reads skp bench data from google storage, and outputs expectations.
     86 
     87   Ignores data with revisions outside [rev_min, rev_max] integer range. For
     88   bench data with multiple revisions, we use higher revisions to calculate
     89   expected bench values.
     90   Uses the provided representation_alg for calculating bench representations.
     91   """
     92   expectation_dic = {}
     93   uri = boto.storage_uri(URI_BUCKET, GOOGLE_STORAGE_URI_SCHEME)
     94   for obj in uri.get_bucket():
     95     # Filters out non-skp-bench files.
     96     if (not obj.name.startswith('perfdata/Skia_') or
     97         obj.name.find('_data_skp_') < 0):
     98       continue
     99     # Ignores uninterested platforms.
    100     platform = obj.name.split('/')[1][5:]  # Removes "Skia_" prefix.
    101     if platform not in PLATFORMS:
    102       continue
    103     # Filters by revision.
    104     for rev in range(rev_min, rev_max + 1):
    105       if '_r%s_' % rev not in obj.name:
    106         continue
    107 
    108     contents = cStringIO.StringIO()
    109     obj.get_file(contents)
    110     for point in bench_util.parse('', contents.getvalue().split('\n'),
    111                                   representation_alg):
    112       if point.config in CONFIGS_TO_FILTER:
    113         continue
    114       # TODO(bensong): the filtering below is only needed during skp generation
    115       # system transitioning. Change it once the new system (bench name starts
    116       # with http) is stable for the switch-over, and delete it once we
    117       # deprecate the old ones.
    118       if point.bench.startswith('http'):
    119         continue
    120 
    121       key = '%s_%s_%s,%s-%s' % (point.bench, point.config, point.time_type,
    122                                 platform, representation_alg)
    123       # It is fine to have later revisions overwrite earlier benches, since we
    124       # only use the latest bench within revision range to set expectations.
    125       expectation_dic[key] = point.time
    126   keys = expectation_dic.keys()
    127   keys.sort()
    128   for key in keys:
    129     bench_val = expectation_dic[key]
    130     # Prints out expectation lines.
    131     print '%s,%.3f,%.3f,%.3f' % (key, bench_val,
    132                                  bench_val * BENCH_LB - BENCH_ALLOWED_NOISE,
    133                                  bench_val * BENCH_UB + BENCH_ALLOWED_NOISE)
    134 
    135 def main():
    136   """Parses flags and outputs expected Skia picture bench results."""
    137   parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING)
    138   parser.add_option(OPTION_REVISION_RANGE_SHORT, OPTION_REVISION_RANGE,
    139       dest='rev_range',
    140       help='(Mandatory) revision range separated by ":", e.g., 6000:6005')
    141   parser.add_option(OPTION_REPRESENTATION_ALG_SHORT, OPTION_REPRESENTATION_ALG,
    142       dest='alg', default='25th',
    143       help=('Bench representation algorithm. One of '
    144             '%s. Default to "25th".' % str(REPRESENTATION_ALGS)))
    145   (options, args) = parser.parse_args()
    146   if options.rev_range:
    147     range_match = re.search('(\d+)\:(\d+)', options.rev_range)
    148     if not range_match:
    149       parser.error('Wrong format for rev-range [%s]' % options.rev_range)
    150     else:
    151       rev_min = int(range_match.group(1))
    152       rev_max = int(range_match.group(2))
    153       OutputSkpBenchExpectations(rev_min, rev_max, options.alg)
    154   else:
    155     parser.error('Please provide mandatory flag %s' % OPTION_REVISION_RANGE)
    156 
    157 
    158 if '__main__' == __name__:
    159   main()
    160