1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be found 4 # in the LICENSE file. 5 6 """ Analyze recent SkPicture or Microbench data, and output suggested ranges. 7 8 The outputs can be edited and pasted to bench_expectations_<builder>.txt to 9 trigger buildbot alerts if the actual benches are out of range. Details are 10 documented in the corresponding .txt file for each builder. 11 12 Currently the easiest way to batch update bench_expectations_<builder>.txt is to 13 delete all bench lines, run this script, and redirect outputs (">>") to be added 14 to the corresponding .txt file for each perf builder. 15 You can also just manually change a few lines of interest, of course. 16 17 Note: since input data are stored in Google Storage, you will need to set up 18 the corresponding library. 19 See http://developers.google.com/storage/docs/gspythonlibrary for details. 20 """ 21 22 __author__ = 'bensong (at] google.com (Ben Chen)' 23 24 import bench_util 25 import boto 26 import cStringIO 27 import optparse 28 import re 29 import shutil 30 31 from oauth2_plugin import oauth2_plugin 32 33 34 # Ratios for calculating suggested picture bench upper and lower bounds. 35 BENCH_UB = 1.1 # Allow for 10% room for normal variance on the up side. 36 BENCH_LB = 0.9 37 38 # Further allow for a fixed amount of noise. This is especially useful for 39 # benches of smaller absolute value. Keeping this value small will not affect 40 # performance tunings. 41 BENCH_ALLOWED_NOISE = 10 42 43 # Name prefix for benchmark builders. 44 BENCH_BUILDER_PREFIX = 'Perf-' 45 46 # List of platforms to track. Feel free to change it to meet your needs. 47 PLATFORMS = ['Perf-Mac10.8-MacMini4.1-GeForce320M-x86-Release', 48 'Perf-Android-Nexus7-Tegra3-Arm7-Release', 49 'Perf-Ubuntu12-ShuttleA-ATI5770-x86-Release', 50 'Perf-Win7-ShuttleA-HD2000-x86-Release', 51 ] 52 53 # Filter for configs of no interest. They are old config names replaced by more 54 # specific ones. 55 CONFIGS_TO_FILTER = ['gpu', 'raster'] 56 57 # Template for gsutil uri. 58 GOOGLE_STORAGE_URI_SCHEME = 'gs' 59 URI_BUCKET = 'chromium-skia-gm' 60 61 # Constants for optparse. 62 USAGE_STRING = 'USAGE: %s [options]' 63 HOWTO_STRING = """ 64 Feel free to revise PLATFORMS for your own needs. The default is the most common 65 combination that we care most about. Platforms that did not run bench_pictures 66 or benchmain in the given revision range will not have corresponding outputs. 67 Please check http://go/skpbench to choose a range that fits your needs. 68 BENCH_UB, BENCH_LB and BENCH_ALLOWED_NOISE can be changed to expand or narrow 69 the permitted bench ranges without triggering buidbot alerts. 70 """ 71 HELP_STRING = """ 72 Outputs expectation picture bench ranges for the latest revisions for the given 73 revision range. For instance, --rev_range=6000:6000 will return only bench 74 ranges for the bots that ran benches at rev 6000; --rev-range=6000:7000 75 may have multiple bench data points for each bench configuration, and the code 76 returns bench data for the latest revision of all available (closer to 7000). 77 """ + HOWTO_STRING 78 79 OPTION_REVISION_RANGE = '--rev-range' 80 OPTION_REVISION_RANGE_SHORT = '-r' 81 # Bench representation algorithm flag. 82 OPTION_REPRESENTATION_ALG = '--algorithm' 83 OPTION_REPRESENTATION_ALG_SHORT = '-a' 84 # Bench type to examine. Either 'micro' or 'skp'. 85 OPTION_BENCH_TYPE = '--bench-type' 86 OPTION_BENCH_TYPE_SHORT = '-b' 87 88 # List of valid bench types. 89 BENCH_TYPES = ['micro', 'skp'] 90 # List of valid representation algorithms. 91 REPRESENTATION_ALGS = ['avg', 'min', 'med', '25th'] 92 93 def OutputBenchExpectations(bench_type, rev_min, rev_max, representation_alg): 94 """Reads bench data from google storage, and outputs expectations. 95 96 Ignores data with revisions outside [rev_min, rev_max] integer range. For 97 bench data with multiple revisions, we use higher revisions to calculate 98 expected bench values. 99 bench_type is either 'micro' or 'skp', according to the flag '-b'. 100 Uses the provided representation_alg for calculating bench representations. 101 """ 102 if bench_type not in BENCH_TYPES: 103 raise Exception('Not valid bench_type! (%s)' % BENCH_TYPES) 104 expectation_dic = {} 105 uri = boto.storage_uri(URI_BUCKET, GOOGLE_STORAGE_URI_SCHEME) 106 for obj in uri.get_bucket(): 107 # Filters out non-bench files. 108 if ((not obj.name.startswith('perfdata/%s' % BENCH_BUILDER_PREFIX) and 109 not obj.name.startswith( 110 'playback/perfdata/%s' % BENCH_BUILDER_PREFIX)) or 111 obj.name.find('_data') < 0): 112 continue 113 if ((bench_type == 'micro' and obj.name.find('_data_skp_') > 0) or 114 (bench_type == 'skp' and obj.name.find('_skp_') < 0)): 115 # Skips wrong bench type. 116 continue 117 # Ignores uninterested platforms. 118 platform = obj.name.split('/')[1] 119 if not platform.startswith(BENCH_BUILDER_PREFIX): 120 platform = obj.name.split('/')[2] 121 if not platform.startswith(BENCH_BUILDER_PREFIX): 122 continue # Ignores non-platform object 123 if platform not in PLATFORMS: 124 continue 125 # Filters by revision. 126 to_filter = True 127 for rev in range(rev_min, rev_max + 1): 128 if '_r%s_' % rev in obj.name: 129 to_filter = False 130 break 131 if to_filter: 132 continue 133 contents = cStringIO.StringIO() 134 obj.get_file(contents) 135 for point in bench_util.parse('', contents.getvalue().split('\n'), 136 representation_alg): 137 if point.config in CONFIGS_TO_FILTER: 138 continue 139 140 key = '%s_%s_%s,%s-%s' % (point.bench, point.config, point.time_type, 141 platform, representation_alg) 142 # It is fine to have later revisions overwrite earlier benches, since we 143 # only use the latest bench within revision range to set expectations. 144 expectation_dic[key] = point.time 145 keys = expectation_dic.keys() 146 keys.sort() 147 for key in keys: 148 bench_val = expectation_dic[key] 149 # Prints out expectation lines. 150 print '%s,%.3f,%.3f,%.3f' % (key, bench_val, 151 bench_val * BENCH_LB - BENCH_ALLOWED_NOISE, 152 bench_val * BENCH_UB + BENCH_ALLOWED_NOISE) 153 154 def main(): 155 """Parses flags and outputs expected Skia bench results.""" 156 parser = optparse.OptionParser(USAGE_STRING % '%prog' + HELP_STRING) 157 parser.add_option(OPTION_REVISION_RANGE_SHORT, OPTION_REVISION_RANGE, 158 dest='rev_range', 159 help='(Mandatory) revision range separated by ":", e.g., 6000:6005') 160 parser.add_option(OPTION_BENCH_TYPE_SHORT, OPTION_BENCH_TYPE, 161 dest='bench_type', default='skp', 162 help=('Bench type, either "skp" or "micro". Default to "skp".')) 163 parser.add_option(OPTION_REPRESENTATION_ALG_SHORT, OPTION_REPRESENTATION_ALG, 164 dest='alg', default='25th', 165 help=('Bench representation algorithm. One of ' 166 '%s. Default to "25th".' % str(REPRESENTATION_ALGS))) 167 (options, args) = parser.parse_args() 168 if options.rev_range: 169 range_match = re.search('(\d+)\:(\d+)', options.rev_range) 170 if not range_match: 171 parser.error('Wrong format for rev-range [%s]' % options.rev_range) 172 else: 173 rev_min = int(range_match.group(1)) 174 rev_max = int(range_match.group(2)) 175 OutputBenchExpectations(options.bench_type, rev_min, rev_max, options.alg) 176 else: 177 parser.error('Please provide mandatory flag %s' % OPTION_REVISION_RANGE) 178 179 180 if '__main__' == __name__: 181 main() 182
