1 # Copyright 2016 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import copy 6 import json 7 import sys 8 9 # These fields must appear in the test result output 10 REQUIRED = { 11 'interrupted', 12 'num_failures_by_type', 13 'seconds_since_epoch', 14 'tests', 15 } 16 17 # These fields are optional, but must have the same value on all shards 18 OPTIONAL_MATCHING = ( 19 'builder_name', 20 'build_number', 21 'chromium_revision', 22 'has_pretty_patch', 23 'has_wdiff', 24 'path_delimiter', 25 'pixel_tests_enabled', 26 'random_order_seed', 27 ) 28 29 OPTIONAL_IGNORED = ( 30 'layout_tests_dir', 31 ) 32 33 # These fields are optional and will be summed together 34 OPTIONAL_COUNTS = ( 35 'fixable', 36 'num_flaky', 37 'num_passes', 38 'num_regressions', 39 'skipped', 40 'skips', 41 ) 42 43 44 class MergeException(Exception): 45 pass 46 47 48 def merge_test_results(shard_results_list): 49 """ Merge list of results. 50 51 Args: 52 shard_results_list: list of results to merge. All the results must have the 53 same format. Supported format are simplified JSON format & Chromium JSON 54 test results format version 3 (see 55 https://www.chromium.org/developers/the-json-test-results-format) 56 57 Returns: 58 a dictionary that represent the merged results. Its format follow the same 59 format of all results in |shard_results_list|. 60 """ 61 if not shard_results_list: 62 return {} 63 64 if 'seconds_since_epoch' in shard_results_list[0]: 65 return _merge_json_test_result_format(shard_results_list) 66 else: 67 return _merge_simplified_json_format(shard_results_list) 68 69 70 def _merge_simplified_json_format(shard_results_list): 71 # This code is specialized to the "simplified" JSON format that used to be 72 # the standard for recipes. 73 74 # These are the only keys we pay attention to in the output JSON. 75 merged_results = { 76 'successes': [], 77 'failures': [], 78 'valid': True, 79 } 80 81 for result_json in shard_results_list: 82 successes = result_json.get('successes', []) 83 failures = result_json.get('failures', []) 84 valid = result_json.get('valid', True) 85 86 if (not isinstance(successes, list) or not isinstance(failures, list) or 87 not isinstance(valid, bool)): 88 raise MergeException( 89 'Unexpected value type in %s' % result_json) # pragma: no cover 90 91 merged_results['successes'].extend(successes) 92 merged_results['failures'].extend(failures) 93 merged_results['valid'] = merged_results['valid'] and valid 94 return merged_results 95 96 97 def _merge_json_test_result_format(shard_results_list): 98 # This code is specialized to the Chromium JSON test results format version 3: 99 # https://www.chromium.org/developers/the-json-test-results-format 100 101 # These are required fields for the JSON test result format version 3. 102 merged_results = { 103 'tests': {}, 104 'interrupted': False, 105 'version': 3, 106 'seconds_since_epoch': float('inf'), 107 'num_failures_by_type': { 108 } 109 } 110 111 # To make sure that we don't mutate existing shard_results_list. 112 shard_results_list = copy.deepcopy(shard_results_list) 113 for result_json in shard_results_list: 114 # TODO(tansell): check whether this deepcopy is actually neccessary. 115 result_json = copy.deepcopy(result_json) 116 117 # Check the version first 118 version = result_json.pop('version', -1) 119 if version != 3: 120 raise MergeException( # pragma: no cover (covered by 121 # results_merger_unittest). 122 'Unsupported version %s. Only version 3 is supported' % version) 123 124 # Check the results for each shard have the required keys 125 missing = REQUIRED - set(result_json) 126 if missing: 127 raise MergeException( # pragma: no cover (covered by 128 # results_merger_unittest). 129 'Invalid json test results (missing %s)' % missing) 130 131 # Curry merge_values for this result_json. 132 merge = lambda key, merge_func: merge_value( 133 result_json, merged_results, key, merge_func) 134 135 # Traverse the result_json's test trie & merged_results's test tries in 136 # DFS order & add the n to merged['tests']. 137 merge('tests', merge_tries) 138 139 # If any were interrupted, we are interrupted. 140 merge('interrupted', lambda x,y: x|y) 141 142 # Use the earliest seconds_since_epoch value 143 merge('seconds_since_epoch', min) 144 145 # Sum the number of failure types 146 merge('num_failures_by_type', sum_dicts) 147 148 # Optional values must match 149 for optional_key in OPTIONAL_MATCHING: 150 if optional_key not in result_json: 151 continue 152 153 if optional_key not in merged_results: 154 # Set this value to None, then blindly copy over it. 155 merged_results[optional_key] = None 156 merge(optional_key, lambda src, dst: src) 157 else: 158 merge(optional_key, ensure_match) 159 160 # Optional values ignored 161 for optional_key in OPTIONAL_IGNORED: 162 if optional_key in result_json: 163 merged_results[optional_key] = result_json.pop( 164 # pragma: no cover (covered by 165 # results_merger_unittest). 166 optional_key) 167 168 # Sum optional value counts 169 for count_key in OPTIONAL_COUNTS: 170 if count_key in result_json: # pragma: no cover 171 # TODO(mcgreevy): add coverage. 172 merged_results.setdefault(count_key, 0) 173 merge(count_key, lambda a, b: a+b) 174 175 if result_json: 176 raise MergeException( # pragma: no cover (covered by 177 # results_merger_unittest). 178 'Unmergable values %s' % result_json.keys()) 179 180 return merged_results 181 182 183 def merge_tries(source, dest): 184 """ Merges test tries. 185 186 This is intended for use as a merge_func parameter to merge_value. 187 188 Args: 189 source: A result json test trie. 190 dest: A json test trie merge destination. 191 """ 192 # merge_tries merges source into dest by performing a lock-step depth-first 193 # traversal of dest and source. 194 # pending_nodes contains a list of all sub-tries which have been reached but 195 # need further merging. 196 # Each element consists of a trie prefix, and a sub-trie from each of dest 197 # and source which is reached via that prefix. 198 pending_nodes = [('', dest, source)] 199 while pending_nodes: 200 prefix, dest_node, curr_node = pending_nodes.pop() 201 for k, v in curr_node.iteritems(): 202 if k in dest_node: 203 if not isinstance(v, dict): 204 raise MergeException( 205 "%s:%s: %r not mergable, curr_node: %r\ndest_node: %r" % ( 206 prefix, k, v, curr_node, dest_node)) 207 pending_nodes.append(("%s:%s" % (prefix, k), dest_node[k], v)) 208 else: 209 dest_node[k] = v 210 return dest 211 212 213 def ensure_match(source, dest): 214 """ Returns source if it matches dest. 215 216 This is intended for use as a merge_func parameter to merge_value. 217 218 Raises: 219 MergeException if source != dest 220 """ 221 if source != dest: 222 raise MergeException( # pragma: no cover (covered by 223 # results_merger_unittest). 224 "Values don't match: %s, %s" % (source, dest)) 225 return source 226 227 228 def sum_dicts(source, dest): 229 """ Adds values from source to corresponding values in dest. 230 231 This is intended for use as a merge_func parameter to merge_value. 232 """ 233 for k, v in source.iteritems(): 234 dest.setdefault(k, 0) 235 dest[k] += v 236 237 return dest 238 239 240 def merge_value(source, dest, key, merge_func): 241 """ Merges a value from source to dest. 242 243 The value is deleted from source. 244 245 Args: 246 source: A dictionary from which to pull a value, identified by key. 247 dest: The dictionary into to which the value is to be merged. 248 key: The key which identifies the value to be merged. 249 merge_func(src, dst): A function which merges its src into dst, 250 and returns the result. May modify dst. May raise a MergeException. 251 252 Raises: 253 MergeException if the values can not be merged. 254 """ 255 try: 256 dest[key] = merge_func(source[key], dest[key]) 257 except MergeException as e: 258 e.message = "MergeFailure for %s\n%s" % (key, e.message) 259 e.args = tuple([e.message] + list(e.args[1:])) 260 raise 261 del source[key] 262 263 264 def main(files): 265 if len(files) < 2: 266 sys.stderr.write("Not enough JSON files to merge.\n") 267 return 1 268 sys.stderr.write('Starting with %s\n' % files[0]) 269 result = json.load(open(files[0])) 270 for f in files[1:]: 271 sys.stderr.write('Merging %s\n' % f) 272 result = merge_test_results([result, json.load(open(f))]) 273 print json.dumps(result) 274 return 0 275 276 277 if __name__ == "__main__": 278 sys.exit(main(sys.argv[1:])) 279