1 #!/usr/bin/python 2 3 # Copyright 2017 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Load generator for devserver.""" 8 9 import argparse 10 import itertools 11 import json 12 import re 13 import sys 14 15 import common 16 from chromite.lib import commandline 17 18 19 # Default keys to skip displaying. 20 DEFAULT_SKIP = [ 21 'build_name', 22 'devserver', 23 'name', 24 'parent', 25 'quick_provision', 26 'trigger_response', 27 ] 28 29 # List of commandline arguments for easy filtering. 30 FILTER_ARGS = [ 31 'board', 32 'build_name', 33 'devserver', 34 'name', 35 'status', 36 ] 37 38 39 def get_parser(): 40 """Creates the argparse parser.""" 41 parser = commandline.ArgumentParser(description=__doc__) 42 parser.add_argument('infile', nargs='*', type=argparse.FileType('r'), 43 help='Path to JSON file to read.', 44 default=[sys.stdin]) 45 parser.add_argument('--boards', type=str, action='store', 46 help='Boards to show.') 47 parser.add_argument('--group', type=str, action='store', 48 help='Comma-spearated list of keys to group by.') 49 parser.add_argument('--dump', action='store_true', 50 help='Dump all filtered entries.') 51 parser.add_argument('--skip', type=str, action='store', 52 help='Comma-separated list of keys to skip displaying.', 53 default=','.join(DEFAULT_SKIP)) 54 parser.add_argument('--filter', type=str, action='store', 55 help='Filter expression to apply to each node.') 56 for arg in FILTER_ARGS: 57 parser.add_argument('--%s' % arg, type=str, action='store', 58 help='Comma-separated list of %s to filter by.' % 59 arg) 60 parser.add_argument('--no-summary', action='store_false', dest='summary', 61 help='Disable summary.') 62 63 return parser 64 65 def summarize_entries(entries, skip=set()): 66 """Summarize a list of entries.""" 67 TAG_KEYS = [ 68 'board', 'build_name', 'devserver', 'name', 69 'parent', 'quick_provision', 'status' 70 ] 71 VALUE_KEYS = [ 72 'avg_active', 'elapsed', 73 ] 74 summary = { 75 'COUNT': len(entries), 76 } 77 summary.update({key: summarize_tags(entries, key) for key in TAG_KEYS 78 if key not in skip}) 79 summary.update({key: summarize_values(entries, key) for key in VALUE_KEYS 80 if key not in skip}) 81 return summary 82 83 def summarize_tags(entries, key): 84 """Summarize all the different string values for a given key.""" 85 tags = {str(entry[key]) for entry in entries} 86 return list(tags) 87 88 def summarize_values(entries, key): 89 """Summarize the numeric values for a given key.""" 90 if entries is None or len(entries) == 0: 91 return None 92 93 values = [entry[key] for entry in entries if key in entry] 94 summary = {} 95 num_values = len(values) 96 if num_values: 97 summary['min'] = min(values) 98 summary['max'] = max(values) 99 summary['avg'] = sum(values) / num_values 100 num_skipped = len(entries) - num_values 101 if num_skipped: 102 summary['num'] = num_values 103 summary['skipped'] = num_skipped 104 return summary 105 106 def group_entries(keys, entries): 107 """Group entries based on different values of given keys. 108 109 @param keys: A list of keys to group by. 110 @param entries: A list of entries to split into groups. 111 112 @return A list of list of entries, where each list has a different key 113 value. 114 """ 115 if not keys: 116 return [entries] 117 118 # Divide the group based on the first key. 119 indexed = {} 120 for entry in entries: 121 value = str(entry[keys[0]]) 122 indexed.setdefault(value, []).append(entry) 123 groups = [indexed[value] for value in sorted(indexed.keys())] 124 125 # Recursively subdivide all the groups based on the rest of the keys. 126 subgroups = [] 127 for group in groups: 128 subgroups.extend(group_entries(keys[1:], group)) 129 return subgroups 130 131 def main(argv): 132 """Load generator for a devserver.""" 133 parser = get_parser() 134 options = parser.parse_args(argv) 135 136 # Read entries from the specified file. 137 all_entries = [] 138 for f in options.infile: 139 all_entries.extend([json.loads(line) for line in f]) 140 141 # Filter entries: 142 # - Ignore non-provisions. 143 # - Filter via the specified FILTER_ARGS arguments. 144 # - Filter via explicit filter request. 145 entries = filter(lambda x: x['name'] != 'Runner', all_entries) 146 for arg in FILTER_ARGS: 147 if options.__dict__.get(arg): 148 entries = filter(lambda x: x[arg] in 149 options.__dict__[arg].split(','), 150 entries) 151 if options.filter: 152 entries = filter(lambda x: eval(options.filter, {'re': re}, x), entries) 153 154 # Group the entries based on specified keys. 155 groups = group_entries(options.group.split(',') if options.group else None, 156 entries) 157 158 # Dump all filtered entries as groups, including their parents. 159 if options.dump: 160 dump_entries = itertools.chain(*groups) 161 # Dump all entries, tracking needed parents. 162 parents = [] 163 for entry in dump_entries: 164 print(json.dumps(entry)) 165 if 'parent' in entry and entry['parent'] not in parents: 166 parents.append(entry['parent']) 167 # Dump all parents. 168 for entry in all_entries: 169 if entry['id'] in parents: 170 print(json.dumps(entry)) 171 172 # Summarize the entries, group by group. 173 if options.summary: 174 skip = options.skip.split(',') if options.skip else set() 175 summaries = [summarize_entries(group, skip) for group in groups] 176 print(json.dumps(summaries, indent=2)) 177 178 if __name__ == '__main__': 179 sys.exit(main(sys.argv[1:])) 180
