1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Parse data from benchmark_runs for tabulator.""" 5 6 from __future__ import print_function 7 8 import errno 9 import json 10 import os 11 import re 12 import sys 13 14 from cros_utils import misc 15 16 _TELEMETRY_RESULT_DEFAULTS_FILE = 'default-telemetry-results.json' 17 _DUP_KEY_REGEX = re.compile(r'(\w+)\{(\d+)\}') 18 19 20 def _AdjustIteration(benchmarks, max_dup, bench): 21 """Adjust the interation numbers if they have keys like ABCD{i}.""" 22 for benchmark in benchmarks: 23 if benchmark.name != bench or benchmark.iteration_adjusted: 24 continue 25 benchmark.iteration_adjusted = True 26 benchmark.iterations *= (max_dup + 1) 27 28 29 def _GetMaxDup(data): 30 """Find the maximum i inside ABCD{i}. 31 32 data should be a [[[Key]]], where Key is a string that may look like 33 ABCD{i}. 34 """ 35 max_dup = 0 36 for label in data: 37 for run in label: 38 for key in run: 39 match = _DUP_KEY_REGEX.match(key) 40 if match: 41 max_dup = max(max_dup, int(match.group(2))) 42 return max_dup 43 44 45 def _Repeat(func, times): 46 """Returns the result of running func() n times.""" 47 return [func() for _ in xrange(times)] 48 49 50 def _GetNonDupLabel(max_dup, runs): 51 """Create new list for the runs of the same label. 52 53 Specifically, this will split out keys like foo{0}, foo{1} from one run into 54 their own runs. For example, given a run like: 55 {"foo": 1, "bar{0}": 2, "baz": 3, "qux{1}": 4, "pirate{0}": 5} 56 57 You'll get: 58 [{"foo": 1, "baz": 3}, {"bar": 2, "pirate": 5}, {"qux": 4}] 59 60 Hands back the lists of transformed runs, all concatenated together. 61 """ 62 new_runs = [] 63 for run in runs: 64 new_run = {} 65 added_runs = _Repeat(dict, max_dup) 66 for key, value in run.iteritems(): 67 match = _DUP_KEY_REGEX.match(key) 68 if not match: 69 new_run[key] = value 70 else: 71 new_key, index_str = match.groups() 72 added_runs[int(index_str)-1][new_key] = str(value) 73 new_runs.append(new_run) 74 new_runs += added_runs 75 return new_runs 76 77 78 def _DuplicatePass(result, benchmarks): 79 """Properly expands keys like `foo{1}` in `result`.""" 80 for bench, data in result.iteritems(): 81 max_dup = _GetMaxDup(data) 82 # If there's nothing to expand, there's nothing to do. 83 if not max_dup: 84 continue 85 for i, runs in enumerate(data): 86 data[i] = _GetNonDupLabel(max_dup, runs) 87 _AdjustIteration(benchmarks, max_dup, bench) 88 89 90 def _ReadSummaryFile(filename): 91 """Reads the summary file at filename.""" 92 dirname, _ = misc.GetRoot(filename) 93 fullname = os.path.join(dirname, _TELEMETRY_RESULT_DEFAULTS_FILE) 94 try: 95 # Slurp the summary file into a dictionary. The keys in the dictionary are 96 # the benchmark names. The value for a key is a list containing the names 97 # of all the result fields that should be returned in a 'default' report. 98 with open(fullname) as in_file: 99 return json.load(in_file) 100 except IOError as e: 101 # ENOENT means "no such file or directory" 102 if e.errno == errno.ENOENT: 103 return {} 104 raise 105 106 107 def _MakeOrganizeResultOutline(benchmark_runs, labels): 108 """Creates the "outline" of the OrganizeResults result for a set of runs. 109 110 Report generation returns lists of different sizes, depending on the input 111 data. Depending on the order in which we iterate through said input data, we 112 may populate the Nth index of a list, then the N-1st, then the N+Mth, ... 113 114 It's cleaner to figure out the "skeleton"/"outline" ahead of time, so we don't 115 have to worry about resizing while computing results. 116 """ 117 # Count how many iterations exist for each benchmark run. 118 # We can't simply count up, since we may be given an incomplete set of 119 # iterations (e.g. [r.iteration for r in benchmark_runs] == [1, 3]) 120 iteration_count = {} 121 for run in benchmark_runs: 122 name = run.benchmark.name 123 old_iterations = iteration_count.get(name, -1) 124 # N.B. run.iteration starts at 1, not 0. 125 iteration_count[name] = max(old_iterations, run.iteration) 126 127 # Result structure: {benchmark_name: [[{key: val}]]} 128 result = {} 129 for run in benchmark_runs: 130 name = run.benchmark.name 131 num_iterations = iteration_count[name] 132 # default param makes cros lint be quiet about defining num_iterations in a 133 # loop. 134 make_dicts = lambda n=num_iterations: _Repeat(dict, n) 135 result[name] = _Repeat(make_dicts, len(labels)) 136 return result 137 138 def OrganizeResults(benchmark_runs, labels, benchmarks=None, json_report=False): 139 """Create a dict from benchmark_runs. 140 141 The structure of the output dict is as follows: 142 {"benchmark_1":[ 143 [{"key1":"v1", "key2":"v2"},{"key1":"v1", "key2","v2"}] 144 #one label 145 [] 146 #the other label 147 ] 148 "benchmark_2": 149 [ 150 ]}. 151 """ 152 result = _MakeOrganizeResultOutline(benchmark_runs, labels) 153 label_names = [label.name for label in labels] 154 label_indices = {name: i for i, name in enumerate(label_names)} 155 summary_file = _ReadSummaryFile(sys.argv[0]) 156 if benchmarks is None: 157 benchmarks = [] 158 159 for benchmark_run in benchmark_runs: 160 if not benchmark_run.result: 161 continue 162 benchmark = benchmark_run.benchmark 163 label_index = label_indices[benchmark_run.label.name] 164 cur_label_list = result[benchmark.name][label_index] 165 cur_dict = cur_label_list[benchmark_run.iteration - 1] 166 167 show_all_results = json_report or benchmark.show_all_results 168 if not show_all_results: 169 summary_list = summary_file.get(benchmark.test_name) 170 if summary_list: 171 summary_list.append('retval') 172 else: 173 # Did not find test_name in json file; show everything. 174 show_all_results = True 175 for test_key in benchmark_run.result.keyvals: 176 if show_all_results or test_key in summary_list: 177 cur_dict[test_key] = benchmark_run.result.keyvals[test_key] 178 # Occasionally Telemetry tests will not fail but they will not return a 179 # result, either. Look for those cases, and force them to be a fail. 180 # (This can happen if, for example, the test has been disabled.) 181 if len(cur_dict) == 1 and cur_dict['retval'] == 0: 182 cur_dict['retval'] = 1 183 # TODO: This output should be sent via logger. 184 print("WARNING: Test '%s' appears to have succeeded but returned" 185 ' no results.' % benchmark.name, 186 file=sys.stderr) 187 if json_report and benchmark_run.machine: 188 cur_dict['machine'] = benchmark_run.machine.name 189 cur_dict['machine_checksum'] = benchmark_run.machine.checksum 190 cur_dict['machine_string'] = benchmark_run.machine.checksum_string 191 _DuplicatePass(result, benchmarks) 192 return result 193
