1 #!/usr/bin/env python2 2 # 3 # Copyright 2017 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 # 7 # pylint: disable=cros-logging-import 8 """Transforms skia benchmark results to ones that crosperf can understand.""" 9 10 from __future__ import print_function 11 12 import itertools 13 import logging 14 import json 15 import sys 16 17 # Turn the logging level to INFO before importing other autotest 18 # code, to avoid having failed import logging messages confuse the 19 # test_droid user. 20 logging.basicConfig(level=logging.INFO) 21 22 # All of the results we care about, by name. 23 # Each of these *must* end in _ns, _us, _ms, or _s, since all the metrics we 24 # collect (so far) are related to time, and we alter the results based on the 25 # suffix of these strings (so we don't have 0.000421ms per sample, for example) 26 _RESULT_RENAMES = { 27 'memset32_100000_640_480_nonrendering': 'memset_time_ms', 28 'path_equality_50%_640_480_nonrendering': 'path_equality_ns', 29 'sort_qsort_backward_640_480_nonrendering': 'qsort_us' 30 } 31 32 33 def _GetFamiliarName(name): 34 r = _RESULT_RENAMES[name] 35 return r if r else name 36 37 38 def _IsResultInteresting(name): 39 return name in _RESULT_RENAMES 40 41 42 def _GetTimeMultiplier(label_name): 43 """Given a time (in milliseconds), normalize it to what label_name expects. 44 45 "What label_name expects" meaning "we pattern match against the last few 46 non-space chars in label_name." 47 48 This expects the time unit to be separated from anything else by '_'. 49 """ 50 ms_mul = 1000 * 1000. 51 endings = [('_ns', 1), ('_us', 1000), ('_ms', ms_mul), ('_s', ms_mul * 1000)] 52 for end, mul in endings: 53 if label_name.endswith(end): 54 return ms_mul / mul 55 raise ValueError('Unknown ending in "%s"; expecting one of %s' % 56 (label_name, [end for end, _ in endings])) 57 58 59 def _GetTimeDenom(ms): 60 """Given a list of times (in milliseconds), find a sane time unit for them. 61 62 Returns the unit name, and `ms` normalized to that time unit. 63 64 >>> _GetTimeDenom([1, 2, 3]) 65 ('ms', [1.0, 2.0, 3.0]) 66 >>> _GetTimeDenom([.1, .2, .3]) 67 ('us', [100.0, 200.0, 300.0]) 68 """ 69 70 ms_mul = 1000 * 1000 71 units = [('us', 1000), ('ms', ms_mul), ('s', ms_mul * 1000)] 72 for name, mul in reversed(units): 73 normalized = [float(t) * ms_mul / mul for t in ms] 74 average = sum(normalized) / len(normalized) 75 if all(n > 0.1 for n in normalized) and average >= 1: 76 return name, normalized 77 78 normalized = [float(t) * ms_mul for t in ms] 79 return 'ns', normalized 80 81 82 def _TransformBenchmarks(raw_benchmarks): 83 # We get {"results": {"bench_name": Results}} 84 # where 85 # Results = {"config_name": {"samples": [float], etc.}} 86 # 87 # We want {"data": {"skia": [[BenchmarkData]]}, 88 # "platforms": ["platform1, ..."]} 89 # where 90 # BenchmarkData = {"bench_name": bench_samples[N], ..., "retval": 0} 91 # 92 # Note that retval is awkward -- crosperf's JSON reporter reports the result 93 # as a failure if it's not there. Everything else treats it like a 94 # statistic... 95 benchmarks = raw_benchmarks['results'] 96 results = [] 97 for bench_name, bench_result in benchmarks.iteritems(): 98 try: 99 for cfg_name, keyvals in bench_result.iteritems(): 100 # Some benchmarks won't have timing data (either it won't exist at all, 101 # or it'll be empty); skip them. 102 samples = keyvals.get('samples') 103 if not samples: 104 continue 105 106 bench_name = '%s_%s' % (bench_name, cfg_name) 107 if not _IsResultInteresting(bench_name): 108 continue 109 110 friendly_name = _GetFamiliarName(bench_name) 111 if len(results) < len(samples): 112 results.extend({ 113 'retval': 0 114 } for _ in xrange(len(samples) - len(results))) 115 116 time_mul = _GetTimeMultiplier(friendly_name) 117 for sample, app in itertools.izip(samples, results): 118 assert friendly_name not in app 119 app[friendly_name] = sample * time_mul 120 except (KeyError, ValueError) as e: 121 logging.error('While converting "%s" (key: %s): %s', 122 bench_result, bench_name, e.message) 123 raise 124 125 # Realistically, [results] should be multiple results, where each entry in the 126 # list is the result for a different label. Because we only deal with one 127 # label at the moment, we need to wrap it in its own list. 128 return results 129 130 131 if __name__ == '__main__': 132 133 def _GetUserFile(argv): 134 if not argv or argv[0] == '-': 135 return sys.stdin 136 return open(argv[0]) 137 138 def _Main(): 139 with _GetUserFile(sys.argv[1:]) as in_file: 140 obj = json.load(in_file) 141 output = _TransformBenchmarks(obj) 142 json.dump(output, sys.stdout) 143 144 _Main() 145
