1 #!/usr/bin/pyton 2 3 # Copyright 2017 Google Inc. 4 # 5 # Use of this source code is governed by a BSD-style license that can be 6 # found in the LICENSE file. 7 8 import os 9 import sys 10 import subprocess 11 import multiprocessing 12 13 from argparse import ArgumentParser 14 15 16 README = """ 17 Simply run 18 \033[36m 19 python {0} TEST_GIT_BRANCH 20 \033[0m 21 to see if TEST_GIT_BRANCH has performance regressions against master in 8888. 22 23 To compare a specific config with svg and skp resources included, add --config 24 and --extraarg option. For exampe, 25 \033[36m 26 python {0} TEST_GIT_BRANCH --config gl \\ 27 --extraarg "--svgs ~/Desktop/bots/svgs --skps ~/Desktop/bots/skps" 28 \033[0m 29 For more options, please see 30 31 python {0} --help 32 """.format(__file__) 33 34 35 CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) 36 AB_SCRIPT = "ab.py" 37 38 39 def parse_args(): 40 if len(sys.argv) <= 1 or sys.argv[1] == '-h' or sys.argv[1] == '--help': 41 print README 42 43 parser = ArgumentParser( 44 description='Noiselessly (hence calm) becnhmark a git branch against ' + 45 'another baseline branch (e.g., master) using multiple ' + 46 ' nanobench runs.' 47 ) 48 49 default_threads = max(1, multiprocessing.cpu_count() / 2); 50 default_skiadir = os.path.normpath(CURRENT_DIR + "/../../") 51 52 config_help = ( 53 'nanobench config; we currently support only one config ' 54 'at a time (default: %(default)s)') 55 reps_help = ( 56 'initial repititions of the nanobench run; this may be ' 57 'overridden when we have many threads (default: %(default)s)') 58 extraarg_help = ( 59 'nanobench args (example: --svgs ~/Desktop/bots/svgs --skps ' 60 '~/Desktop/bots/skps)') 61 baseline_help = ( 62 'baseline branch to compare against (default: %(default)s)') 63 basearg_help = ( 64 'nanobench arg for the baseline branch; if not given, we use ' 65 ' the same arg for both the test branch and the baseline branch') 66 threads_help = ( 67 'number of threads to be used (default: %(default)s); ' 68 'for GPU config, this will always be 1') 69 no_compile_help = ( 70 'whether NOT to compile nanobench and copy it to WRITEDIR ' 71 '(i.e., reuse previous nanobench compiled)') 72 skip_base_help = ( 73 'whether NOT to run nanobench on baseline branch ' 74 '(i.e., reuse previous baseline measurements)') 75 noinit_help = ( 76 'whether to skip initial nanobench runs (default: %(default)s)') 77 branch_help = ( 78 "the test branch to benchmark; if it's 'modified', we'll benchmark the " 79 "current modified code against 'git stash'.") 80 81 definitions = [ 82 # argname, type, default value, help 83 ['--config', str, '8888', config_help], 84 ['--skiadir', str, default_skiadir, 'default: %(default)s'], 85 ['--ninjadir', str, 'out/Release', 'default: %(default)s'], 86 ['--writedir', str, '/var/tmp', 'default: %(default)s'], 87 ['--extraarg', str, '', extraarg_help], 88 ['--baseline', str, 'master', baseline_help], 89 ['--basearg', str, '', basearg_help], 90 ['--reps', int, 2, reps_help], 91 ['--threads', int, default_threads, threads_help], 92 ] 93 94 for d in definitions: 95 parser.add_argument(d[0], type=d[1], default=d[2], help=d[3]) 96 97 parser.add_argument('branch', type=str, help=branch_help) 98 parser.add_argument('--no-compile', dest='no_compile', action="store_true", 99 help=no_compile_help) 100 parser.add_argument('--skip-base', dest='skipbase', action="store_true", 101 help=skip_base_help) 102 parser.add_argument('--noinit', dest='noinit', action="store_true", 103 help=noinit_help) 104 parser.add_argument('--concise', dest='concise', action="store_true", 105 help="If set, no verbose thread info will be printed.") 106 parser.set_defaults(no_compile=False); 107 parser.set_defaults(skipbase=False); 108 parser.set_defaults(noinit=False); 109 parser.set_defaults(concise=False); 110 111 # Additional args for bots 112 BHELP = "bot specific options" 113 parser.add_argument('--githash', type=str, help=BHELP) 114 parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP) 115 116 args = parser.parse_args() 117 if not args.basearg: 118 args.basearg = args.extraarg 119 120 return args 121 122 123 def nano_path(args, branch): 124 return args.writedir + '/nanobench_' + branch 125 126 127 def compile_branch(args, branch): 128 print "Compiling branch %s" % args.branch 129 130 commands = [ 131 ['git', 'checkout', branch], 132 ['ninja', '-C', args.ninjadir, 'nanobench'], 133 ['cp', args.ninjadir + '/nanobench', nano_path(args, branch)] 134 ] 135 for command in commands: 136 subprocess.check_call(command, cwd=args.skiadir) 137 138 139 def compile_modified(args): 140 print "Compiling modified code" 141 subprocess.check_call( 142 ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 143 subprocess.check_call( 144 ['cp', args.ninjadir + '/nanobench', nano_path(args, args.branch)], 145 cwd=args.skiadir) 146 147 print "Compiling stashed code" 148 stash_output = subprocess.check_output(['git', 'stash'], cwd=args.skiadir) 149 if 'No local changes to save' in stash_output: 150 subprocess.check_call(['git', 'reset', 'HEAD^', '--soft']) 151 subprocess.check_call(['git', 'stash']) 152 153 subprocess.check_call(['gclient', 'sync'], cwd=args.skiadir) 154 subprocess.check_call( 155 ['ninja', '-C', args.ninjadir, 'nanobench'], cwd=args.skiadir) 156 subprocess.check_call( 157 ['cp', args.ninjadir + '/nanobench', nano_path(args, args.baseline)], 158 cwd=args.skiadir) 159 subprocess.check_call(['git', 'stash', 'pop'], cwd=args.skiadir) 160 161 def compile_nanobench(args): 162 if args.branch == 'modified': 163 compile_modified(args) 164 else: 165 compile_branch(args, args.branch) 166 compile_branch(args, args.baseline) 167 168 169 def main(): 170 args = parse_args() 171 172 # copy in case that it will be gone after git branch switching 173 orig_ab_name = CURRENT_DIR + "/" + AB_SCRIPT 174 temp_ab_name = args.writedir + "/" + AB_SCRIPT 175 subprocess.check_call(['cp', orig_ab_name, temp_ab_name]) 176 177 if not args.no_compile: 178 compile_nanobench(args) 179 180 command = [ 181 'python', 182 temp_ab_name, 183 args.writedir, 184 args.branch + ("_A" if args.branch == args.baseline else ""), 185 args.baseline + ("_B" if args.branch == args.baseline else ""), 186 nano_path(args, args.branch), 187 nano_path(args, args.baseline), 188 args.extraarg, 189 args.basearg, 190 str(args.reps), 191 "true" if args.skipbase else "false", 192 args.config, 193 str(args.threads if args.config in ["8888", "565"] else 1), 194 "true" if args.noinit else "false" 195 ] 196 197 if args.githash: 198 command += ['--githash', args.githash] 199 if args.keys: 200 command += (['--keys'] + args.keys) 201 202 if args.concise: 203 command.append("--concise") 204 205 p = subprocess.Popen(command, cwd=args.skiadir) 206 try: 207 p.wait() 208 except KeyboardInterrupt: 209 try: 210 p.terminate() 211 except OSError as e: 212 print e 213 214 215 if __name__ == "__main__": 216 main() 217
