Home | History | Annotate | Download | only in benchtools
      1 #!/usr/bin/env python
      2 
      3 
      4 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
      5 # Use of this source code is governed by a BSD-style license that can be
      6 # found in the LICENSE file.
      7 
      8 
      9 """greenify.py: standalone script to correct flaky bench expectations.
     10 
     11     Requires Rietveld credentials on the running machine.
     12 
     13     Usage:
     14       Copy script to a separate dir outside Skia repo. The script will create a
     15           skia dir on the first run to host the repo, and will create/delete
     16           temp dirs as needed.
     17       ./greenify.py --url <the stdio url from failed CheckForRegressions step>
     18 """
     19 
     20 import argparse
     21 import filecmp
     22 import os
     23 import re
     24 import shutil
     25 import subprocess
     26 import time
     27 import urllib2
     28 
     29 
     30 # Regular expression for matching exception data.
     31 EXCEPTION_RE = ('Bench (\S+) out of range \[(\d+.\d+), (\d+.\d+)\] \((\d+.\d+) '
     32                 'vs (\d+.\d+), ')
     33 EXCEPTION_RE_COMPILED = re.compile(EXCEPTION_RE)
     34 
     35 
     36 def clean_dir(d):
     37   if os.path.exists(d):
     38     shutil.rmtree(d)
     39   os.makedirs(d)
     40 
     41 def checkout_or_update_skia(repo_dir):
     42   status = True
     43   old_cwd = os.getcwd()
     44   os.chdir(repo_dir)
     45   print 'CHECK SKIA REPO...'
     46   if subprocess.call(['git', 'pull'],
     47                      stderr=subprocess.PIPE):
     48     print 'Checking out Skia from git, please be patient...'
     49     os.chdir(old_cwd)
     50     clean_dir(repo_dir)
     51     os.chdir(repo_dir)
     52     if subprocess.call(['git', 'clone', '-q', '--depth=50', '--single-branch',
     53                         'https://skia.googlesource.com/skia.git', '.']):
     54       status = False
     55   subprocess.call(['git', 'checkout', 'master'])
     56   subprocess.call(['git', 'pull'])
     57   os.chdir(old_cwd)
     58   return status
     59 
     60 def git_commit_expectations(repo_dir, exp_dir, bot, build, commit):
     61   commit_msg = """Greenify bench bot %s at build %s
     62 
     63 TBR=bsalomon (at] google.com
     64 
     65 Bypassing trybots:
     66 NOTRY=true""" % (bot, build)
     67   old_cwd = os.getcwd()
     68   os.chdir(repo_dir)
     69   upload = ['git', 'cl', 'upload', '-f', '--bypass-hooks',
     70             '--bypass-watchlists', '-m', commit_msg]
     71   if commit:
     72     upload.append('--use-commit-queue')
     73   branch = exp_dir[exp_dir.rfind('/') + 1:]
     74   filename = 'bench_expectations_%s.txt' % bot
     75   cmds = ([['git', 'checkout', 'master'],
     76            ['git', 'pull'],
     77            ['git', 'checkout', '-b', branch, '-t', 'origin/master'],
     78            ['cp', '%s/%s' % (exp_dir, filename), 'expectations/bench'],
     79            ['git', 'add', 'expectations/bench/' + filename],
     80            ['git', 'commit', '-m', commit_msg],
     81            upload,
     82            ['git', 'checkout', 'master'],
     83            ['git', 'branch', '-D', branch],
     84           ])
     85   status = True
     86   for cmd in cmds:
     87     print 'Running ' + ' '.join(cmd)
     88     if subprocess.call(cmd):
     89       print 'FAILED. Please check if skia git repo is present.'
     90       subprocess.call(['git', 'checkout', 'master'])
     91       status = False
     92       break
     93   os.chdir(old_cwd)
     94   return status
     95 
     96 def delete_dirs(li):
     97   for d in li:
     98     print 'Deleting directory %s' % d
     99     shutil.rmtree(d)
    100 
    101 def widen_bench_ranges(url, bot, repo_dir, exp_dir):
    102   fname = 'bench_expectations_%s.txt' % bot
    103   src = os.path.join(repo_dir, 'expectations', 'bench', fname)
    104   if not os.path.isfile(src):
    105     print 'This bot has no expectations! %s' % bot
    106     return False
    107   row_dic = {}
    108   for l in urllib2.urlopen(url).read().split('\n'):
    109     data = EXCEPTION_RE_COMPILED.search(l)
    110     if data:
    111       row = data.group(1)
    112       lb = float(data.group(2))
    113       ub = float(data.group(3))
    114       actual = float(data.group(4))
    115       exp = float(data.group(5))
    116       avg = (actual + exp) / 2
    117       shift = avg - exp
    118       lb = lb + shift
    119       ub = ub + shift
    120       # In case outlier really fluctuates a lot
    121       if actual < lb:
    122         lb = actual - abs(shift) * 0.1 + 0.5
    123       elif actual > ub:
    124         ub = actual + abs(shift) * 0.1 + 0.5
    125       row_dic[row] = '%.2f,%.2f,%.2f' % (avg, lb, ub)
    126   if not row_dic:
    127     print 'NO out-of-range benches found at %s' % url
    128     return False
    129 
    130   changed = 0
    131   li = []
    132   for l in open(src).readlines():
    133     parts = l.strip().split(',')
    134     if parts[0].startswith('#') or len(parts) != 5:
    135       li.append(l.strip())
    136       continue
    137     if ','.join(parts[:2]) in row_dic:
    138       li.append(','.join(parts[:2]) + ',' + row_dic[','.join(parts[:2])])
    139       changed += 1
    140     else:
    141       li.append(l.strip())
    142   if not changed:
    143     print 'Not in source file:\n' + '\n'.join(row_dic.keys())
    144     return False
    145 
    146   dst = os.path.join(exp_dir, fname)
    147   with open(dst, 'w+') as f:
    148     f.write('\n'.join(li))
    149   return True
    150 
    151 
    152 def main():
    153   d = os.path.dirname(os.path.abspath(__file__))
    154   os.chdir(d)
    155   if not subprocess.call(['git', 'rev-parse'], stderr=subprocess.PIPE):
    156     print 'Please copy script to a separate dir outside git repos to use.'
    157     return
    158   ts_str = '%s' % time.time()
    159 
    160   parser = argparse.ArgumentParser()
    161   parser.add_argument('--url',
    162                       help='Broken bench build CheckForRegressions page url.')
    163   parser.add_argument('--commit', action='store_true',
    164                       help='Whether to commit changes automatically.')
    165   args = parser.parse_args()
    166   repo_dir = os.path.join(d, 'skia')
    167   if not os.path.exists(repo_dir):
    168     os.makedirs(repo_dir)
    169   if not checkout_or_update_skia(repo_dir):
    170     print 'ERROR setting up Skia repo at %s' % repo_dir
    171     return 1
    172 
    173   file_in_repo = os.path.join(d, 'skia/experimental/benchtools/greenify.py')
    174   if not filecmp.cmp(__file__, file_in_repo):
    175     shutil.copy(file_in_repo, __file__)
    176     print 'Updated this script from repo; please run again.'
    177     return
    178 
    179   if not args.url:
    180     raise Exception('Please provide a url with broken CheckForRegressions.')
    181   path = args.url.split('/')
    182   if len(path) != 11 or not path[6].isdigit():
    183     raise Exception('Unexpected url format: %s' % args.url)
    184   bot = path[4]
    185   build = path[6]
    186   commit = False
    187   if args.commit:
    188     commit = True
    189 
    190   exp_dir = os.path.join(d, 'exp' + ts_str)
    191   clean_dir(exp_dir)
    192   if not widen_bench_ranges(args.url, bot, repo_dir, exp_dir):
    193     print 'NO bench exceptions found! %s' % args.url
    194   elif not git_commit_expectations(
    195       repo_dir, exp_dir, bot, build, commit):
    196     print 'ERROR uploading expectations using git.'
    197   elif not commit:
    198     print 'CL created. Please take a look at the link above.'
    199   else:
    200     print 'New bench baselines should be in CQ now.'
    201   delete_dirs([exp_dir])
    202 
    203 
    204 if __name__ == "__main__":
    205   main()
    206