1 #!/usr/bin/env python2.7 2 # Copyright 2015 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Script to pull chromium.perf step timings from chrome-infra-stats API. 7 8 Currently this pulls the list of steps per builder. For each step, if it is not 9 a setup step, we get the step stats for the last 20 runs for that builder. 10 11 The API documentation for chrome-infra-stats is at: 12 https://apis-explorer.appspot.com/apis-explorer/? 13 base=https://chrome-infra-stats.appspot.com/_ah/api#p/ 14 """ 15 16 import csv 17 import datetime 18 import json 19 import sys 20 import urllib 21 import urllib2 22 23 24 BUILDER_STEPS_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/' 25 'masters/chromium.perf/%s') 26 27 28 STEP_ACTIVE_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/' 29 'steps/last/chromium.perf/%s/%s/1') 30 31 32 STEP_STATS_URL = ('https://chrome-infra-stats.appspot.com/_ah/api/stats/v1/' 33 'stats/last/chromium.perf/%s/%s/20') 34 35 36 IGNORED_STEPS = [ 37 'List Perf Tests', 38 'Sharded Perf Tests', 39 'authorize_adb_devices', 40 'bot_update', 41 'build__schedule__time__', 42 'clean local files', 43 'cleanup_temp', 44 'device_status_check', 45 'extract build', 46 'gclient runhooks', 47 'get compile targets for scripts', 48 'get perf test list', 49 'gsutil download_build_product', 50 'host_info', 51 'install ChromeShell.apk', 52 'json.output cache', 53 'json.output cache', 54 'overall__build__result__', 55 'overall__queued__time__', 56 'provision_devices', 57 'read test spec', 58 'rmtree build directory', 59 'setup_build', 60 'spawn_logcat_monitor', 61 'stack_tool_for_tombstones', 62 'stack_tool_with_logcat_dump', 63 'steps', 64 'test_report', 65 'unzip_build_product', 66 'update_scripts' 67 ] 68 69 KNOWN_TESTERS_LIST = [ 70 'Android Nexus4 Perf', 71 'Android Nexus5 Perf', 72 'Android Nexus6 Perf', 73 'Android Nexus10 Perf', 74 'Android Nexus7v2 Perf', 75 'Android One Perf', 76 'Linux Perf (1)', 77 'Linux Perf (2)', 78 'Linux Perf (3)', 79 'Linux Perf (4)', 80 'Linux Perf (5)', 81 'Mac 10.8 Perf (1)', 82 'Mac 10.8 Perf (2)', 83 'Mac 10.8 Perf (3)', 84 'Mac 10.8 Perf (4)', 85 'Mac 10.8 Perf (5)', 86 'Mac 10.9 Perf (1)', 87 'Mac 10.9 Perf (2)', 88 'Mac 10.9 Perf (3)', 89 'Mac 10.9 Perf (4)', 90 'Mac 10.9 Perf (5)', 91 'Win 7 ATI GPU Perf', 92 'Win 7 Intel GPU Perf', 93 'Win 7 Low-End Perf (1)', 94 'Win 7 Low-End Perf (2)', 95 'Win 7 Nvidia GPU Perf', 96 'Win 7 Perf (1)', 97 'Win 7 Perf (2)', 98 'Win 7 Perf (3)', 99 'Win 7 Perf (4)', 100 'Win 7 Perf (5)', 101 'Win 7 x64 Perf (1)', 102 'Win 7 x64 Perf (2)', 103 'Win 8 Perf (1)', 104 'Win 8 Perf (2)', 105 'Win XP Perf (1)', 106 'Win XP Perf (2)', 107 'Win XP Perf (3)', 108 'Win XP Perf (4)', 109 'Win XP Perf (5)' 110 ] 111 112 113 USAGE = 'Usage: chrome-perf-step-timings.py <outfilename>' 114 115 116 def main(): 117 if len(sys.argv) != 2: 118 print USAGE 119 sys.exit(0) 120 outfilename = sys.argv[1] 121 122 threshold_time = datetime.datetime.now() - datetime.timedelta(days=2) 123 124 col_names = [('builder', 'step', 'run_count', 'stddev', 'mean', 'maximum', 125 'median', 'seventyfive', 'ninety', 'ninetynine')] 126 with open(outfilename, 'wb') as f: 127 writer = csv.writer(f) 128 writer.writerows(col_names) 129 130 for builder in KNOWN_TESTERS_LIST: 131 step_timings = [] 132 url = BUILDER_STEPS_URL % urllib.quote(builder) 133 response = urllib2.urlopen(url) 134 results = json.load(response) 135 steps = results['steps'] 136 steps.sort() # to group tests and their references together. 137 for step in steps: 138 if step in IGNORED_STEPS: 139 continue 140 url = STEP_ACTIVE_URL % (urllib.quote(builder), urllib.quote(step)) 141 response = urllib2.urlopen(url) 142 results = json.load(response) 143 if ('step_records' not in results.keys() or 144 len(results['step_records']) == 0): 145 continue 146 first_record = results['step_records'][0] 147 last_step_time = datetime.datetime.strptime( 148 first_record['step_start'], "%Y-%m-%dT%H:%M:%S.%f") 149 # ignore steps that did not run for more than 2 days 150 if last_step_time < threshold_time: 151 continue 152 url = STEP_STATS_URL % (urllib.quote(builder), urllib.quote(step)) 153 response = urllib2.urlopen(url) 154 results = json.load(response) 155 step_timings.append( 156 [builder, step, results['count'], results['stddev'], 157 results['mean'], results['maximum'], results['median'], 158 results['seventyfive'], results['ninety'], 159 results['ninetynine']]) 160 with open(outfilename, 'ab') as f: 161 writer = csv.writer(f) 162 writer.writerows(step_timings) 163 164 165 if __name__ == '__main__': 166 main() 167
