1 #!/usr/bin/pyton 2 3 # Copyright 2017 Google Inc. 4 # 5 # Use of this source code is governed by a BSD-style license that can be 6 # found in the LICENSE file. 7 8 import os 9 import sys 10 import subprocess 11 import multiprocessing 12 13 from argparse import ArgumentParser 14 15 16 README = """ 17 Simply run 18 \033[36m 19 python {0} TEST_GIT_BRANCH 20 \033[0m 21 to see if TEST_GIT_BRANCH has performance regressions against master in 8888. 22 23 To compare a specific config with svg and skp resources included, add --config 24 and --extraarg option. For exampe, 25 \033[36m 26 python {0} TEST_GIT_BRANCH --config gl \\ 27 --extraarg "--svgs ~/Desktop/bots/svgs --skps ~/Desktop/bots/skps" 28 \033[0m 29 For more options, please see 30 31 python {0} --help 32 """.format(__file__) 33 34 35 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) 36 AB_SCRIPT = "ab.py" 37 38 39 def parse_args(): 40 if len(sys.argv) <= 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help': 41 print README 42 43 parser = ArgumentParser( 44 description='Noiselessly (hence calm) becnhmark a git branch against ' + 45 'another baseline branch (e.g., master) using multiple ' + 46 ' nanobench runs.' 47 ) 48 49 default_threads = max(1, multiprocessing.cpu_count() / 2); 50 default_skiadir = os.path.normpath(CURRENT_DIR + "/../../") 51 52 config_help = ( 53 'nanobench config; we currently support only one config ' 54 'at a time (default: %(default)s)') 55 reps_help = ( 56 'initial repititions of the nanobench run; this may be ' 57 'overridden when we have many threads (default: %(default)s)') 58 extraarg_help = ( 59 'nanobench args (example: --svgs ~/Desktop/bots/svgs --skps ' 60 '~/Desktop/bots/skps)') 61 baseline_help = ( 62 'baseline branch to compare against (default: %(default)s)') 63 basearg_help = ( 64 'nanobench arg for the baseline branch; if not given, we use ' 65 ' the same arg for both the test branch and the baseline branch') 66 threads_help = ( 67 'number of threads to be used (default: %(default)s); ' 68 'for GPU config, this will always be 1') 69 no_compile_help = ( 70 'whether NOT to compile nanobench and copy it to WRITEDIR ' 71 '(i.e., reuse previous nanobench compiled)') 72 skip_base_help = ( 73 'whether NOT to run nanobench on baseline branch ' 74 '(i.e., reuse previous baseline measurements)') 75 noinit_help = ( 76 'whether to skip initial nanobench runs (default: %(default)s)') 77 branch_help = ( 78 "the test branch to benchmark; if it's 'modified', we'll benchmark the " 79 "current modified code against 'git stash'.") 80 81 definitions = [ 82 # argname, type, default value, help 83 ['--config', str, '8888', config_help], 84 ['--skiadir', str, default_skiadir, 'default: %(default)s'], 85 ['--ninjadir', str, 'out/Release', 'default: %(default)s'], 86 ['--writedir', str, '/var/tmp', 'default: %(default)s'], 87 ['--extraarg', str, '', extraarg_help], 88 ['--baseline', str, 'master', baseline_help], 89 ['--basearg', str, '', basearg_help], 90 ['--reps', int, 2, reps_help], 91 ['--threads', int, default_threads, threads_help], 92 ] 93 94 for d in definitions: 95 parser.add_argument(d[0], type=d[1], default=d[2], help=d[3]) 96 97 parser.add_argument('branch', type=str, help=branch_help) 98 parser.add_argument('--no-compile', dest='no_compile', action="store_true", 99 help=no_compile_help) 100 parser.add_argument('--skip-base', dest='skipbase', action="store_true", 101 help=skip_base_help) 102 parser.add_argument('--noinit', dest='noinit', action="store_true", 103 help=noinit_help) 104 parser.add_argument('--concise', dest='concise', action="store_true", 105 help="If set, no verbose thread info will be printed.") 106 parser.set_defaults(no_compile=False); 107 parser.set_defaults(skipbase=False); 108 parser.set_defaults(noinit=False); 109 parser.set_defaults(concise=False); 110 111 # Additional args for bots 112 BHELP = "bot specific options" 113 parser.add_argument('--githash', type=str, help=BHELP) 114 parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP) 115 116 args = parser.parse_args() 117 if not args.basearg: 118 args.basearg = args.extraarg 119 120 return args 121 122 123 def nano_path(args, branch): 124 return args.writedir + '/nanobench_' + branch 125 126 127 def compile_branch(args, branch): 128 print "Compiling branch %s" % args.branch 129 130 commands = [ 131 ['git', 'checkout', branch], 132 ['gclient', 'sync'], 133 ['ninja', '-C', args.ninjadir, 'nanobench'], 134 ['cp', args.ninjadir + '/nanobench', nano_path(args, branch)] 135 ] 136 for command in commands: 137 subprocess.check_call(command, cwd=args.skiadir) 138 139 140 def compile_modified(args): 141 print "Compiling modified code" 142 subprocess.check_call( 143 ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 144 subprocess.check_call( 145 ['cp', args.ninjadir + '/nanobench', nano_path(args, args.branch)], 146 cwd=args.skiadir) 147 148 print "Compiling stashed code" 149 stash_output = subprocess.check_output(['git', 'stash'], cwd=args.skiadir) 150 if 'No local changes to save' in stash_output: 151 subprocess.check_call(['git', 'reset', 'HEAD^', '--soft']) 152 subprocess.check_call(['git', 'stash']) 153 154 subprocess.check_call(['gclient', 'sync'], cwd=args.skiadir) 155 subprocess.check_call( 156 ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 157 subprocess.check_call( 158 ['cp', args.ninjadir + '/nanobench', nano_path(args, args.baseline)], 159 cwd=args.skiadir) 160 subprocess.check_call(['git', 'stash', 'pop'], cwd=args.skiadir) 161 162 def compile_nanobench(args): 163 if args.branch == 'modified': 164 compile_modified(args) 165 else: 166 compile_branch(args, args.branch) 167 compile_branch(args, args.baseline) 168 169 170 def main(): 171 args = parse_args() 172 173 # copy in case that it will be gone after git branch switching 174 orig_ab_name = CURRENT_DIR + "/" + AB_SCRIPT 175 temp_ab_name = args.writedir + "/" + AB_SCRIPT 176 subprocess.check_call(['cp', orig_ab_name, temp_ab_name]) 177 178 if not args.no_compile: 179 compile_nanobench(args) 180 181 command = [ 182 'python', 183 temp_ab_name, 184 args.writedir, 185 args.branch + ("_A" if args.branch == args.baseline else ""), 186 args.baseline + ("_B" if args.branch == args.baseline else ""), 187 nano_path(args, args.branch), 188 nano_path(args, args.baseline), 189 args.extraarg, 190 args.basearg, 191 str(args.reps), 192 "true" if args.skipbase else "false", 193 args.config, 194 str(args.threads if args.config in ["8888", "565"] else 1), 195 "true" if args.noinit else "false" 196 ] 197 198 if args.githash: 199 command += ['--githash', args.githash] 200 if args.keys: 201 command += (['--keys'] + args.keys) 202 203 if args.concise: 204 command.append("--concise") 205 206 p = subprocess.Popen(command, cwd=args.skiadir) 207 try: 208 p.wait() 209 except KeyboardInterrupt: 210 try: 211 p.terminate() 212 except OSError as e: 213 print e 214 215 216 if __name__ == "__main__": 217 main() 218
