1 #!/usr/bin/env python 2 3 4 # Copyright (c) 2014 The Chromium Authors. All rights reserved. 5 # Use of this source code is governed by a BSD-style license that can be 6 # found in the LICENSE file. 7 8 9 """greenify.py: standalone script to correct flaky bench expectations. 10 11 Requires Rietveld credentials on the running machine. 12 13 Usage: 14 Copy script to a separate dir outside Skia repo. The script will create a 15 skia dir on the first run to host the repo, and will create/delete 16 temp dirs as needed. 17 ./greenify.py --url <the stdio url from failed CheckForRegressions step> 18 """ 19 20 import argparse 21 import filecmp 22 import os 23 import re 24 import shutil 25 import subprocess 26 import time 27 import urllib2 28 29 30 # Regular expression for matching exception data. 31 EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) ' 32 'vs (\d+.\d+), ') 33 EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE) 34 35 36 def clean_dir(d): 37 if os.path.exists(d): 38 shutil.rmtree(d) 39 os.makedirs(d) 40 41 def checkout_or_update_skia(repo_dir): 42 status = True 43 old_cwd = os.getcwd() 44 os.chdir(repo_dir) 45 print 'CHECK SKIA REPO...' 46 if subprocess.call(['git', 'pull'], 47 stderr=subprocess.PIPE): 48 print 'Checking out Skia from git, please be patient...' 49 os.chdir(old_cwd) 50 clean_dir(repo_dir) 51 os.chdir(repo_dir) 52 if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch', 53 'https://skia.googlesource.com/skia.git', '.']): 54 status = False 55 subprocess.call(['git', 'checkout', 'master']) 56 subprocess.call(['git', 'pull']) 57 os.chdir(old_cwd) 58 return status 59 60 def git_commit_expectations(repo_dir, exp_dir, bot, build, commit): 61 commit_msg = """Greenify bench bot %s at build %s 62 63 TBR=bsalomon (at] google.com 64 65 Bypassing trybots: 66 NOTRY=true""" % (bot, build) 67 old_cwd = os.getcwd() 68 os.chdir(repo_dir) 69 upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks', 70 '--bypass-watchlists', '-m', commit_msg] 71 if commit: 72 upload.append('--use-commit-queue') 73 branch = exp_dir[exp_dir.rfind('/') + 1:] 74 filename = 'bench_expectations_%s.txt' % bot 75 cmds = ([['git', 'checkout', 'master'], 76 ['git', 'pull'], 77 ['git', 'checkout', '-b', branch, '-t', 'origin/master'], 78 ['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'], 79 ['git', 'add', 'expectations/bench/' + filename], 80 ['git', 'commit', '-m', commit_msg], 81 upload, 82 ['git', 'checkout', 'master'], 83 ['git', 'branch', '-D', branch], 84 ]) 85 status = True 86 for cmd in cmds: 87 print 'Running ' + ' '.join(cmd) 88 if subprocess.call(cmd): 89 print 'FAILED. Please check if skia git repo is present.' 90 subprocess.call(['git', 'checkout', 'master']) 91 status = False 92 break 93 os.chdir(old_cwd) 94 return status 95 96 def delete_dirs(li): 97 for d in li: 98 print 'Deleting directory %s' % d 99 shutil.rmtree(d) 100 101 def widen_bench_ranges(url, bot, repo_dir, exp_dir): 102 fname = 'bench_expectations_%s.txt' % bot 103 src = os.path.join(repo_dir, 'expectations', 'bench', fname) 104 if not os.path.isfile(src): 105 print 'This bot has no expectations! %s' % bot 106 return False 107 row_dic = {} 108 for l in urllib2.urlopen(url).read().split('\n'): 109 data = EXCEPTION_RE_COMPILED.search(l) 110 if data: 111 row = data.group(1) 112 lb = float(data.group(2)) 113 ub = float(data.group(3)) 114 actual = float(data.group(4)) 115 exp = float(data.group(5)) 116 avg = (actual + exp) / 2 117 shift = avg - exp 118 lb = lb + shift 119 ub = ub + shift 120 # In case outlier really fluctuates a lot 121 if actual < lb: 122 lb = actual - abs(shift) * 0.1 + 0.5 123 elif actual > ub: 124 ub = actual + abs(shift) * 0.1 + 0.5 125 row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub) 126 if not row_dic: 127 print 'NO out-of-range benches found at %s' % url 128 return False 129 130 changed = 0 131 li = [] 132 for l in open(src).readlines(): 133 parts = l.strip().split(',') 134 if parts[0].startswith('#') or len(parts) != 5: 135 li.append(l.strip()) 136 continue 137 if ','.join(parts[:2]) in row_dic: 138 li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])]) 139 changed += 1 140 else: 141 li.append(l.strip()) 142 if not changed: 143 print 'Not in source file:\n' + '\n'.join(row_dic.keys()) 144 return False 145 146 dst = os.path.join(exp_dir, fname) 147 with open(dst, 'w+') as f: 148 f.write('\n'.join(li)) 149 return True 150 151 152 def main(): 153 d = os.path.dirname(os.path.abspath(__file__)) 154 os.chdir(d) 155 if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE): 156 print 'Please copy script to a separate dir outside git repos to use.' 157 return 158 ts_str = '%s' % time.time() 159 160 parser = argparse.ArgumentParser() 161 parser.add_argument('--url', 162 help='Broken bench build CheckForRegressions page url.') 163 parser.add_argument('--commit', action='store_true', 164 help='Whether to commit changes automatically.') 165 args = parser.parse_args() 166 repo_dir = os.path.join(d, 'skia') 167 if not os.path.exists(repo_dir): 168 os.makedirs(repo_dir) 169 if not checkout_or_update_skia(repo_dir): 170 print 'ERROR setting up Skia repo at %s' % repo_dir 171 return 1 172 173 file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py') 174 if not filecmp.cmp(__file__, file_in_repo): 175 shutil.copy(file_in_repo, __file__) 176 print 'Updated this script from repo; please run again.' 177 return 178 179 if not args.url: 180 raise Exception('Please provide a url with broken CheckForRegressions.') 181 path = args.url.split('/') 182 if len(path) != 11 or not path[6].isdigit(): 183 raise Exception('Unexpected url format: %s' % args.url) 184 bot = path[4] 185 build = path[6] 186 commit = False 187 if args.commit: 188 commit = True 189 190 exp_dir = os.path.join(d, 'exp' + ts_str) 191 clean_dir(exp_dir) 192 if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir): 193 print 'NO bench exceptions found! %s' % args.url 194 elif not git_commit_expectations( 195 repo_dir, exp_dir, bot, build, commit): 196 print 'ERROR uploading expectations using git.' 197 elif not commit: 198 print 'CL created. Please take a look at the link above.' 199 else: 200 print 'New bench baselines should be in CQ now.' 201 delete_dirs([exp_dir]) 202 203 204 if __name__ == "__main__": 205 main() 206
