1 #!/usr/bin/env python 2 # Copyright 2017 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 import argparse 7 import json 8 import os 9 import shutil 10 import sys 11 import tempfile 12 import traceback 13 14 from common import gtest_utils 15 from slave import annotation_utils 16 from slave import slave_utils 17 18 19 MISSING_SHARDS_MSG = r"""Missing results from the following shard(s): %s 20 21 This can happen in following cases: 22 * Test failed to start (missing *.dll/*.so dependency for example) 23 * Test crashed or hung 24 * Task expired because there are not enough bots available and are all used 25 * Swarming service experienced problems 26 27 Please examine logs to figure out what happened. 28 """ 29 30 31 def emit_warning(title, log=None): 32 print '@@@STEP_WARNINGS@@@' 33 print title 34 if log: 35 slave_utils.WriteLogLines(title, log.split('\n')) 36 37 38 def merge_shard_results(summary_json, jsons_to_merge): 39 """Reads JSON test output from all shards and combines them into one. 40 41 Returns dict with merged test output on success or None on failure. Emits 42 annotations. 43 """ 44 # summary.json is produced by swarming.py itself. We are mostly interested 45 # in the number of shards. 46 try: 47 with open(summary_json) as f: 48 summary = json.load(f) 49 except (IOError, ValueError): 50 emit_warning( 51 'summary.json is missing or can not be read', 52 'Something is seriously wrong with swarming_client/ or the bot.') 53 return None 54 55 # Merge all JSON files together. Keep track of missing shards. 56 merged = { 57 'all_tests': set(), 58 'disabled_tests': set(), 59 'global_tags': set(), 60 'missing_shards': [], 61 'per_iteration_data': [], 62 'swarming_summary': summary, 63 } 64 for index, result in enumerate(summary['shards']): 65 if result is not None: 66 # Author note: this code path doesn't trigger convert_to_old_format() in 67 # client/swarming.py, which means the state enum is saved in its string 68 # name form, not in the number form. 69 state = result.get('state') 70 if state == u'BOT_DIED': 71 emit_warning('Shard #%d had a Swarming internal failure' % index) 72 elif state == u'EXPIRED': 73 emit_warning('There wasn\'t enough capacity to run your test') 74 elif state == u'TIMED_OUT': 75 emit_warning( 76 'Test runtime exceeded allocated time', 77 'Either it ran for too long (hard timeout) or it didn\'t produce ' 78 'I/O for an extended period of time (I/O timeout)') 79 elif state == u'COMPLETED': 80 json_data, err_msg = load_shard_json(index, jsons_to_merge) 81 if json_data: 82 # Set-like fields. 83 for key in ('all_tests', 'disabled_tests', 'global_tags'): 84 merged[key].update(json_data.get(key), []) 85 86 # 'per_iteration_data' is a list of dicts. Dicts should be merged 87 # together, not the 'per_iteration_data' list itself. 88 merged['per_iteration_data'] = merge_list_of_dicts( 89 merged['per_iteration_data'], 90 json_data.get('per_iteration_data', [])) 91 continue 92 else: 93 emit_warning('Task ran but no result was found: %s' % err_msg) 94 else: 95 emit_warning('Invalid Swarming task state: %s' % state) 96 merged['missing_shards'].append(index) 97 98 # If some shards are missing, make it known. Continue parsing anyway. Step 99 # should be red anyway, since swarming.py return non-zero exit code in that 100 # case. 101 if merged['missing_shards']: 102 as_str = ', '.join(map(str, merged['missing_shards'])) 103 emit_warning( 104 'some shards did not complete: %s' % as_str, 105 MISSING_SHARDS_MSG % as_str) 106 # Not all tests run, combined JSON summary can not be trusted. 107 merged['global_tags'].add('UNRELIABLE_RESULTS') 108 109 # Convert to jsonish dict. 110 for key in ('all_tests', 'disabled_tests', 'global_tags'): 111 merged[key] = sorted(merged[key]) 112 return merged 113 114 115 OUTPUT_JSON_SIZE_LIMIT = 100 * 1024 * 1024 # 100 MB 116 117 118 def load_shard_json(index, jsons_to_merge): 119 """Reads JSON output of the specified shard. 120 121 Args: 122 output_dir: The directory in which to look for the JSON output to load. 123 index: The index of the shard to load data for. 124 125 Returns: A tuple containing: 126 * The contents of path, deserialized into a python object. 127 * An error string. 128 (exactly one of the tuple elements will be non-None). 129 """ 130 # 'output.json' is set in swarming/api.py, gtest_task method. 131 matching_json_files = [ 132 j for j in jsons_to_merge 133 if (os.path.basename(j) == 'output.json' 134 and os.path.basename(os.path.dirname(j)) == str(index))] 135 136 if not matching_json_files: 137 print >> sys.stderr, 'shard %s test output missing' % index 138 return (None, 'shard %s test output was missing' % index) 139 elif len(matching_json_files) > 1: 140 print >> sys.stderr, 'duplicate test output for shard %s' % index 141 return (None, 'shard %s test output was duplicated' % index) 142 143 path = matching_json_files[0] 144 145 try: 146 filesize = os.stat(path).st_size 147 if filesize > OUTPUT_JSON_SIZE_LIMIT: 148 print >> sys.stderr, 'output.json is %d bytes. Max size is %d' % ( 149 filesize, OUTPUT_JSON_SIZE_LIMIT) 150 return (None, 'shard %s test output exceeded the size limit' % index) 151 152 with open(path) as f: 153 return (json.load(f), None) 154 except (IOError, ValueError, OSError) as e: 155 print >> sys.stderr, 'Missing or invalid gtest JSON file: %s' % path 156 print >> sys.stderr, '%s: %s' % (type(e).__name__, e) 157 158 return (None, 'shard %s test output was missing or invalid' % index) 159 160 161 def merge_list_of_dicts(left, right): 162 """Merges dicts left[0] with right[0], left[1] with right[1], etc.""" 163 output = [] 164 for i in xrange(max(len(left), len(right))): 165 left_dict = left[i] if i < len(left) else {} 166 right_dict = right[i] if i < len(right) else {} 167 merged_dict = left_dict.copy() 168 merged_dict.update(right_dict) 169 output.append(merged_dict) 170 return output 171 172 173 def standard_gtest_merge( 174 output_json, summary_json, jsons_to_merge): 175 176 output = merge_shard_results(summary_json, jsons_to_merge) 177 with open(output_json, 'wb') as f: 178 json.dump(output, f) 179 180 return 0 181 182 183 def main(raw_args): 184 185 parser = argparse.ArgumentParser() 186 parser.add_argument('--build-properties') 187 parser.add_argument('--summary-json') 188 parser.add_argument('-o', '--output-json', required=True) 189 parser.add_argument('jsons_to_merge', nargs='*') 190 191 args = parser.parse_args(raw_args) 192 193 return standard_gtest_merge( 194 args.output_json, args.summary_json, args.jsons_to_merge) 195 196 197 if __name__ == '__main__': 198 sys.exit(main(sys.argv[1:])) 199
