1 #!/usr/bin/env python 2 3 # Copyright 2016 Google Inc. 4 # 5 # Use of this source code is governed by a BSD-style license that can be 6 # found in the LICENSE file. 7 8 from __future__ import print_function 9 from _adb import Adb 10 from _benchresult import BenchResult 11 from _hardware import HardwareException, Hardware 12 from argparse import ArgumentParser 13 from multiprocessing import Queue 14 from threading import Thread, Timer 15 import collections 16 import glob 17 import math 18 import re 19 import subprocess 20 import sys 21 import time 22 23 __argparse = ArgumentParser(description=""" 24 25 Executes the skpbench binary with various configs and skps. 26 27 Also monitors the output in order to filter out and re-run results that have an 28 unacceptable stddev. 29 30 """) 31 32 __argparse.add_argument('skpbench', 33 help="path to the skpbench binary") 34 __argparse.add_argument('--adb', 35 action='store_true', help="execute skpbench over adb") 36 __argparse.add_argument('--adb_binary', default='adb', 37 help="The name of the adb binary to use.") 38 __argparse.add_argument('-s', '--device-serial', 39 help="if using adb, ID of the specific device to target " 40 "(only required if more than 1 device is attached)") 41 __argparse.add_argument('-m', '--max-stddev', 42 type=float, default=4, 43 help="initial max allowable relative standard deviation") 44 __argparse.add_argument('-x', '--suffix', 45 help="suffix to append on config (e.g. '_before', '_after')") 46 __argparse.add_argument('-w','--write-path', 47 help="directory to save .png proofs to disk.") 48 __argparse.add_argument('-v','--verbosity', 49 type=int, default=1, help="level of verbosity (0=none to 5=debug)") 50 __argparse.add_argument('-d', '--duration', 51 type=int, help="number of milliseconds to run each benchmark") 52 __argparse.add_argument('-l', '--sample-ms', 53 type=int, help="duration of a sample (minimum)") 54 __argparse.add_argument('--gpu', 55 action='store_true', 56 help="perform timing on the gpu clock instead of cpu (gpu work only)") 57 __argparse.add_argument('--fps', 58 action='store_true', help="use fps instead of ms") 59 __argparse.add_argument('--pr', 60 help="comma- or space-separated list of GPU path renderers, including: " 61 "[[~]all [~]default [~]dashline [~]nvpr [~]msaa [~]aaconvex " 62 "[~]aalinearizing [~]small [~]tess]") 63 __argparse.add_argument('--nocache', 64 action='store_true', help="disable caching of path mask textures") 65 __argparse.add_argument('-c', '--config', 66 default='gl', help="comma- or space-separated list of GPU configs") 67 __argparse.add_argument('-a', '--resultsfile', 68 help="optional file to append results into") 69 __argparse.add_argument('skps', 70 nargs='+', 71 help=".skp files or directories to expand for .skp files") 72 73 FLAGS = __argparse.parse_args() 74 if FLAGS.adb: 75 import _adb_path as _path 76 _path.init(FLAGS.device_serial, FLAGS.adb_binary) 77 else: 78 import _os_path as _path 79 80 def dump_commandline_if_verbose(commandline): 81 if FLAGS.verbosity >= 5: 82 quoted = ['\'%s\'' % re.sub(r'([\\\'])', r'\\\1', x) for x in commandline] 83 print(' '.join(quoted), file=sys.stderr) 84 85 86 class StddevException(Exception): 87 pass 88 89 class Message: 90 READLINE = 0, 91 POLL_HARDWARE = 1, 92 EXIT = 2 93 def __init__(self, message, value=None): 94 self.message = message 95 self.value = value 96 97 class SubprocessMonitor(Thread): 98 def __init__(self, queue, proc): 99 self._queue = queue 100 self._proc = proc 101 Thread.__init__(self) 102 103 def run(self): 104 """Runs on the background thread.""" 105 for line in iter(self._proc.stdout.readline, b''): 106 self._queue.put(Message(Message.READLINE, line.decode('utf-8').rstrip())) 107 self._queue.put(Message(Message.EXIT)) 108 109 class SKPBench: 110 ARGV = [FLAGS.skpbench, '--verbosity', str(FLAGS.verbosity)] 111 if FLAGS.duration: 112 ARGV.extend(['--duration', str(FLAGS.duration)]) 113 if FLAGS.sample_ms: 114 ARGV.extend(['--sampleMs', str(FLAGS.sample_ms)]) 115 if FLAGS.gpu: 116 ARGV.extend(['--gpuClock', 'true']) 117 if FLAGS.fps: 118 ARGV.extend(['--fps', 'true']) 119 if FLAGS.pr: 120 ARGV.extend(['--pr'] + re.split(r'[ ,]', FLAGS.pr)) 121 if FLAGS.nocache: 122 ARGV.extend(['--cachePathMasks', 'false']) 123 if FLAGS.adb: 124 if FLAGS.device_serial is None: 125 ARGV[:0] = [FLAGS.adb_binary, 'shell'] 126 else: 127 ARGV[:0] = [FLAGS.adb_binary, '-s', FLAGS.device_serial, 'shell'] 128 129 @classmethod 130 def get_header(cls, outfile=sys.stdout): 131 commandline = cls.ARGV + ['--duration', '0'] 132 dump_commandline_if_verbose(commandline) 133 out = subprocess.check_output(commandline, stderr=subprocess.STDOUT) 134 return out.rstrip() 135 136 @classmethod 137 def run_warmup(cls, warmup_time, config): 138 if not warmup_time: 139 return 140 print('running %i second warmup...' % warmup_time, file=sys.stderr) 141 commandline = cls.ARGV + ['--duration', str(warmup_time * 1000), 142 '--config', config, 143 '--skp', 'warmup'] 144 dump_commandline_if_verbose(commandline) 145 output = subprocess.check_output(commandline, stderr=subprocess.STDOUT) 146 147 # validate the warmup run output. 148 for line in output.decode('utf-8').split('\n'): 149 match = BenchResult.match(line.rstrip()) 150 if match and match.bench == 'warmup': 151 return 152 raise Exception('Invalid warmup output:\n%s' % output) 153 154 def __init__(self, skp, config, max_stddev, best_result=None): 155 self.skp = skp 156 self.config = config 157 self.max_stddev = max_stddev 158 self.best_result = best_result 159 self._queue = Queue() 160 self._proc = None 161 self._monitor = None 162 self._hw_poll_timer = None 163 164 def __enter__(self): 165 return self 166 167 def __exit__(self, exception_type, exception_value, traceback): 168 if self._proc: 169 self.terminate() 170 if self._hw_poll_timer: 171 self._hw_poll_timer.cancel() 172 173 def execute(self, hardware): 174 hardware.sanity_check() 175 self._schedule_hardware_poll() 176 177 commandline = self.ARGV + ['--config', self.config, 178 '--skp', self.skp, 179 '--suppressHeader', 'true'] 180 if FLAGS.write_path: 181 pngfile = _path.join(FLAGS.write_path, self.config, 182 _path.basename(self.skp) + '.png') 183 commandline.extend(['--png', pngfile]) 184 dump_commandline_if_verbose(commandline) 185 self._proc = subprocess.Popen(commandline, stdout=subprocess.PIPE, 186 stderr=subprocess.STDOUT) 187 self._monitor = SubprocessMonitor(self._queue, self._proc) 188 self._monitor.start() 189 190 while True: 191 message = self._queue.get() 192 if message.message == Message.READLINE: 193 result = BenchResult.match(message.value) 194 if result: 195 hardware.sanity_check() 196 self._process_result(result) 197 elif hardware.filter_line(message.value): 198 print(message.value, file=sys.stderr) 199 continue 200 if message.message == Message.POLL_HARDWARE: 201 hardware.sanity_check() 202 self._schedule_hardware_poll() 203 continue 204 if message.message == Message.EXIT: 205 self._monitor.join() 206 self._proc.wait() 207 if self._proc.returncode != 0: 208 raise Exception("skpbench exited with nonzero exit code %i" % 209 self._proc.returncode) 210 self._proc = None 211 break 212 213 def _schedule_hardware_poll(self): 214 if self._hw_poll_timer: 215 self._hw_poll_timer.cancel() 216 self._hw_poll_timer = \ 217 Timer(1, lambda: self._queue.put(Message(Message.POLL_HARDWARE))) 218 self._hw_poll_timer.start() 219 220 def _process_result(self, result): 221 if not self.best_result or result.stddev <= self.best_result.stddev: 222 self.best_result = result 223 elif FLAGS.verbosity >= 2: 224 print("reusing previous result for %s/%s with lower stddev " 225 "(%s%% instead of %s%%)." % 226 (result.config, result.bench, self.best_result.stddev, 227 result.stddev), file=sys.stderr) 228 if self.max_stddev and self.best_result.stddev > self.max_stddev: 229 raise StddevException() 230 231 def terminate(self): 232 if self._proc: 233 self._proc.terminate() 234 self._monitor.join() 235 self._proc.wait() 236 self._proc = None 237 238 def emit_result(line, resultsfile=None): 239 print(line) 240 sys.stdout.flush() 241 if resultsfile: 242 print(line, file=resultsfile) 243 resultsfile.flush() 244 245 def run_benchmarks(configs, skps, hardware, resultsfile=None): 246 hasheader = False 247 benches = collections.deque([(skp, config, FLAGS.max_stddev) 248 for skp in skps 249 for config in configs]) 250 while benches: 251 try: 252 with hardware: 253 SKPBench.run_warmup(hardware.warmup_time, configs[0]) 254 if not hasheader: 255 emit_result(SKPBench.get_header(), resultsfile) 256 hasheader = True 257 while benches: 258 benchargs = benches.popleft() 259 with SKPBench(*benchargs) as skpbench: 260 try: 261 skpbench.execute(hardware) 262 if skpbench.best_result: 263 emit_result(skpbench.best_result.format(FLAGS.suffix), 264 resultsfile) 265 else: 266 print("WARNING: no result for %s with config %s" % 267 (skpbench.skp, skpbench.config), file=sys.stderr) 268 269 except StddevException: 270 retry_max_stddev = skpbench.max_stddev * math.sqrt(2) 271 if FLAGS.verbosity >= 1: 272 print("stddev is too high for %s/%s (%s%%, max=%.2f%%), " 273 "re-queuing with max=%.2f%%." % 274 (skpbench.best_result.config, skpbench.best_result.bench, 275 skpbench.best_result.stddev, skpbench.max_stddev, 276 retry_max_stddev), 277 file=sys.stderr) 278 benches.append((skpbench.skp, skpbench.config, retry_max_stddev, 279 skpbench.best_result)) 280 281 except HardwareException as exception: 282 skpbench.terminate() 283 if FLAGS.verbosity >= 4: 284 hardware.print_debug_diagnostics() 285 if FLAGS.verbosity >= 1: 286 print("%s; rebooting and taking a %i second nap..." % 287 (exception.message, exception.sleeptime), file=sys.stderr) 288 benches.appendleft(benchargs) # retry the same bench next time. 289 raise # wake hw up from benchmarking mode before the nap. 290 291 except HardwareException as exception: 292 time.sleep(exception.sleeptime) 293 294 def main(): 295 # Delimiter is ',' or ' ', skip if nested inside parens (e.g. gpu(a=b,c=d)). 296 DELIMITER = r'[, ](?!(?:[^(]*\([^)]*\))*[^()]*\))' 297 configs = re.split(DELIMITER, FLAGS.config) 298 skps = _path.find_skps(FLAGS.skps) 299 300 if FLAGS.adb: 301 adb = Adb(FLAGS.device_serial, FLAGS.adb_binary, 302 echo=(FLAGS.verbosity >= 5)) 303 model = adb.check('getprop ro.product.model').strip() 304 if model == 'Pixel C': 305 from _hardware_pixel_c import HardwarePixelC 306 hardware = HardwarePixelC(adb) 307 elif model == 'Pixel': 308 from _hardware_pixel import HardwarePixel 309 hardware = HardwarePixel(adb) 310 elif model == 'Nexus 6P': 311 from _hardware_nexus_6p import HardwareNexus6P 312 hardware = HardwareNexus6P(adb) 313 else: 314 from _hardware_android import HardwareAndroid 315 print("WARNING: %s: don't know how to monitor this hardware; results " 316 "may be unreliable." % model, file=sys.stderr) 317 hardware = HardwareAndroid(adb) 318 else: 319 hardware = Hardware() 320 321 if FLAGS.resultsfile: 322 with open(FLAGS.resultsfile, mode='a+') as resultsfile: 323 run_benchmarks(configs, skps, hardware, resultsfile=resultsfile) 324 else: 325 run_benchmarks(configs, skps, hardware) 326 327 328 if __name__ == '__main__': 329 main() 330
