1 #!/usr/bin/env python 2 3 """ 4 This script can generate XLS reports from OpenCV tests' XML output files. 5 6 To use it, first, create a directory for each machine you ran tests on. 7 Each such directory will become a sheet in the report. Put each XML file 8 into the corresponding directory. 9 10 Then, create your configuration file(s). You can have a global configuration 11 file (specified with the -c option), and per-sheet configuration files, which 12 must be called sheet.conf and placed in the directory corresponding to the sheet. 13 The settings in the per-sheet configuration file will override those in the 14 global configuration file, if both are present. 15 16 A configuration file must consist of a Python dictionary. The following keys 17 will be recognized: 18 19 * 'comparisons': [{'from': string, 'to': string}] 20 List of configurations to compare performance between. For each item, 21 the sheet will have a column showing speedup from configuration named 22 'from' to configuration named "to". 23 24 * 'configuration_matchers': [{'properties': {string: object}, 'name': string}] 25 Instructions for matching test run property sets to configuration names. 26 27 For each found XML file: 28 29 1) All attributes of the root element starting with the prefix 'cv_' are 30 placed in a dictionary, with the cv_ prefix stripped and the cv_module_name 31 element deleted. 32 33 2) The first matcher for which the XML's file property set contains the same 34 keys with equal values as its 'properties' dictionary is searched for. 35 A missing property can be matched by using None as the value. 36 37 Corollary 1: you should place more specific matchers before less specific 38 ones. 39 40 Corollary 2: an empty 'properties' dictionary matches every property set. 41 42 3) If a matching matcher is found, its 'name' string is presumed to be the name 43 of the configuration the XML file corresponds to. A warning is printed if 44 two different property sets match to the same configuration name. 45 46 4) If a such a matcher isn't found, if --include-unmatched was specified, the 47 configuration name is assumed to be the relative path from the sheet's 48 directory to the XML file's containing directory. If the XML file isinstance 49 directly inside the sheet's directory, the configuration name is instead 50 a dump of all its properties. If --include-unmatched wasn't specified, 51 the XML file is ignored and a warning is printed. 52 53 * 'configurations': [string] 54 List of names for compile-time and runtime configurations of OpenCV. 55 Each item will correspond to a column of the sheet. 56 57 * 'module_colors': {string: string} 58 Mapping from module name to color name. In the sheet, cells containing module 59 names from this mapping will be colored with the corresponding color. You can 60 find the list of available colors here: 61 <http://www.simplistix.co.uk/presentations/python-excel.pdf>. 62 63 * 'sheet_name': string 64 Name for the sheet. If this parameter is missing, the name of sheet's directory 65 will be used. 66 67 * 'sheet_properties': [(string, string)] 68 List of arbitrary (key, value) pairs that somehow describe the sheet. Will be 69 dumped into the first row of the sheet in string form. 70 71 Note that all keys are optional, although to get useful results, you'll want to 72 specify at least 'configurations' and 'configuration_matchers'. 73 74 Finally, run the script. Use the --help option for usage information. 75 """ 76 77 from __future__ import division 78 79 import ast 80 import errno 81 import fnmatch 82 import logging 83 import numbers 84 import os, os.path 85 import re 86 87 from argparse import ArgumentParser 88 from glob import glob 89 from itertools import ifilter 90 91 import xlwt 92 93 from testlog_parser import parseLogFile 94 95 re_image_size = re.compile(r'^ \d+ x \d+$', re.VERBOSE) 96 re_data_type = re.compile(r'^ (?: 8 | 16 | 32 | 64 ) [USF] C [1234] $', re.VERBOSE) 97 98 time_style = xlwt.easyxf(num_format_str='#0.00') 99 no_time_style = xlwt.easyxf('pattern: pattern solid, fore_color gray25') 100 failed_style = xlwt.easyxf('pattern: pattern solid, fore_color red') 101 noimpl_style = xlwt.easyxf('pattern: pattern solid, fore_color orange') 102 style_dict = {"failed": failed_style, "noimpl":noimpl_style} 103 104 speedup_style = time_style 105 good_speedup_style = xlwt.easyxf('font: color green', num_format_str='#0.00') 106 bad_speedup_style = xlwt.easyxf('font: color red', num_format_str='#0.00') 107 no_speedup_style = no_time_style 108 error_speedup_style = xlwt.easyxf('pattern: pattern solid, fore_color orange') 109 header_style = xlwt.easyxf('font: bold true; alignment: horizontal centre, vertical top, wrap True') 110 subheader_style = xlwt.easyxf('alignment: horizontal centre, vertical top') 111 112 class Collector(object): 113 def __init__(self, config_match_func, include_unmatched): 114 self.__config_cache = {} 115 self.config_match_func = config_match_func 116 self.include_unmatched = include_unmatched 117 self.tests = {} 118 self.extra_configurations = set() 119 120 # Format a sorted sequence of pairs as if it was a dictionary. 121 # We can't just use a dictionary instead, since we want to preserve the sorted order of the keys. 122 @staticmethod 123 def __format_config_cache_key(pairs, multiline=False): 124 return ( 125 ('{\n' if multiline else '{') + 126 (',\n' if multiline else ', ').join( 127 (' ' if multiline else '') + repr(k) + ': ' + repr(v) for (k, v) in pairs) + 128 ('\n}\n' if multiline else '}') 129 ) 130 131 def collect_from(self, xml_path, default_configuration): 132 run = parseLogFile(xml_path) 133 134 module = run.properties['module_name'] 135 136 properties = run.properties.copy() 137 del properties['module_name'] 138 139 props_key = tuple(sorted(properties.iteritems())) # dicts can't be keys 140 141 if props_key in self.__config_cache: 142 configuration = self.__config_cache[props_key] 143 else: 144 configuration = self.config_match_func(properties) 145 146 if configuration is None: 147 if self.include_unmatched: 148 if default_configuration is not None: 149 configuration = default_configuration 150 else: 151 configuration = Collector.__format_config_cache_key(props_key, multiline=True) 152 153 self.extra_configurations.add(configuration) 154 else: 155 logging.warning('failed to match properties to a configuration: %s', 156 Collector.__format_config_cache_key(props_key)) 157 158 else: 159 same_config_props = [it[0] for it in self.__config_cache.iteritems() if it[1] == configuration] 160 if len(same_config_props) > 0: 161 logging.warning('property set %s matches the same configuration %r as property set %s', 162 Collector.__format_config_cache_key(props_key), 163 configuration, 164 Collector.__format_config_cache_key(same_config_props[0])) 165 166 self.__config_cache[props_key] = configuration 167 168 if configuration is None: return 169 170 module_tests = self.tests.setdefault(module, {}) 171 172 for test in run.tests: 173 test_results = module_tests.setdefault((test.shortName(), test.param()), {}) 174 new_result = test.get("gmean") if test.status == 'run' else test.status 175 test_results[configuration] = min( 176 test_results.get(configuration), new_result, 177 key=lambda r: (1, r) if isinstance(r, numbers.Number) else 178 (2,) if r is not None else 179 (3,) 180 ) # prefer lower result; prefer numbers to errors and errors to nothing 181 182 def make_match_func(matchers): 183 def match_func(properties): 184 for matcher in matchers: 185 if all(properties.get(name) == value 186 for (name, value) in matcher['properties'].iteritems()): 187 return matcher['name'] 188 189 return None 190 191 return match_func 192 193 def main(): 194 arg_parser = ArgumentParser(description='Build an XLS performance report.') 195 arg_parser.add_argument('sheet_dirs', nargs='+', metavar='DIR', help='directory containing perf test logs') 196 arg_parser.add_argument('-o', '--output', metavar='XLS', default='report.xls', help='name of output file') 197 arg_parser.add_argument('-c', '--config', metavar='CONF', help='global configuration file') 198 arg_parser.add_argument('--include-unmatched', action='store_true', 199 help='include results from XML files that were not recognized by configuration matchers') 200 arg_parser.add_argument('--show-times-per-pixel', action='store_true', 201 help='for tests that have an image size parameter, show per-pixel time, as well as total time') 202 203 args = arg_parser.parse_args() 204 205 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG) 206 207 if args.config is not None: 208 with open(args.config) as global_conf_file: 209 global_conf = ast.literal_eval(global_conf_file.read()) 210 else: 211 global_conf = {} 212 213 wb = xlwt.Workbook() 214 215 for sheet_path in args.sheet_dirs: 216 try: 217 with open(os.path.join(sheet_path, 'sheet.conf')) as sheet_conf_file: 218 sheet_conf = ast.literal_eval(sheet_conf_file.read()) 219 except IOError as ioe: 220 if ioe.errno != errno.ENOENT: raise 221 sheet_conf = {} 222 logging.debug('no sheet.conf for %s', sheet_path) 223 224 sheet_conf = dict(global_conf.items() + sheet_conf.items()) 225 226 config_names = sheet_conf.get('configurations', []) 227 config_matchers = sheet_conf.get('configuration_matchers', []) 228 229 collector = Collector(make_match_func(config_matchers), args.include_unmatched) 230 231 for root, _, filenames in os.walk(sheet_path): 232 logging.info('looking in %s', root) 233 for filename in fnmatch.filter(filenames, '*.xml'): 234 if os.path.normpath(sheet_path) == os.path.normpath(root): 235 default_conf = None 236 else: 237 default_conf = os.path.relpath(root, sheet_path) 238 collector.collect_from(os.path.join(root, filename), default_conf) 239 240 config_names.extend(sorted(collector.extra_configurations - set(config_names))) 241 242 sheet = wb.add_sheet(sheet_conf.get('sheet_name', os.path.basename(os.path.abspath(sheet_path)))) 243 244 sheet_properties = sheet_conf.get('sheet_properties', []) 245 246 sheet.write(0, 0, 'Properties:') 247 248 sheet.write(0, 1, 249 'N/A' if len(sheet_properties) == 0 else 250 ' '.join(str(k) + '=' + repr(v) for (k, v) in sheet_properties)) 251 252 sheet.row(2).height = 800 253 sheet.panes_frozen = True 254 sheet.remove_splits = True 255 256 sheet_comparisons = sheet_conf.get('comparisons', []) 257 258 row = 2 259 260 col = 0 261 262 for (w, caption) in [ 263 (2500, 'Module'), 264 (10000, 'Test'), 265 (2000, 'Image\nwidth'), 266 (2000, 'Image\nheight'), 267 (2000, 'Data\ntype'), 268 (7500, 'Other parameters')]: 269 sheet.col(col).width = w 270 if args.show_times_per_pixel: 271 sheet.write_merge(row, row + 1, col, col, caption, header_style) 272 else: 273 sheet.write(row, col, caption, header_style) 274 col += 1 275 276 for config_name in config_names: 277 if args.show_times_per_pixel: 278 sheet.col(col).width = 3000 279 sheet.col(col + 1).width = 3000 280 sheet.write_merge(row, row, col, col + 1, config_name, header_style) 281 sheet.write(row + 1, col, 'total, ms', subheader_style) 282 sheet.write(row + 1, col + 1, 'per pixel, ns', subheader_style) 283 col += 2 284 else: 285 sheet.col(col).width = 4000 286 sheet.write(row, col, config_name, header_style) 287 col += 1 288 289 col += 1 # blank column between configurations and comparisons 290 291 for comp in sheet_comparisons: 292 sheet.col(col).width = 4000 293 caption = comp['to'] + '\nvs\n' + comp['from'] 294 if args.show_times_per_pixel: 295 sheet.write_merge(row, row + 1, col, col, caption, header_style) 296 else: 297 sheet.write(row, col, caption, header_style) 298 col += 1 299 300 row += 2 if args.show_times_per_pixel else 1 301 302 sheet.horz_split_pos = row 303 sheet.horz_split_first_visible = row 304 305 module_colors = sheet_conf.get('module_colors', {}) 306 module_styles = {module: xlwt.easyxf('pattern: pattern solid, fore_color {}'.format(color)) 307 for module, color in module_colors.iteritems()} 308 309 for module, tests in sorted(collector.tests.iteritems()): 310 for ((test, param), configs) in sorted(tests.iteritems()): 311 sheet.write(row, 0, module, module_styles.get(module, xlwt.Style.default_style)) 312 sheet.write(row, 1, test) 313 314 param_list = param[1:-1].split(', ') if param.startswith('(') and param.endswith(')') else [param] 315 316 image_size = next(ifilter(re_image_size.match, param_list), None) 317 if image_size is not None: 318 (image_width, image_height) = map(int, image_size.split('x', 1)) 319 sheet.write(row, 2, image_width) 320 sheet.write(row, 3, image_height) 321 del param_list[param_list.index(image_size)] 322 323 data_type = next(ifilter(re_data_type.match, param_list), None) 324 if data_type is not None: 325 sheet.write(row, 4, data_type) 326 del param_list[param_list.index(data_type)] 327 328 sheet.row(row).write(5, ' | '.join(param_list)) 329 330 col = 6 331 332 for c in config_names: 333 if c in configs: 334 sheet.write(row, col, configs[c], style_dict.get(configs[c], time_style)) 335 else: 336 sheet.write(row, col, None, no_time_style) 337 col += 1 338 if args.show_times_per_pixel: 339 sheet.write(row, col, 340 xlwt.Formula('{0} * 1000000 / ({1} * {2})'.format( 341 xlwt.Utils.rowcol_to_cell(row, col - 1), 342 xlwt.Utils.rowcol_to_cell(row, 2), 343 xlwt.Utils.rowcol_to_cell(row, 3) 344 )), 345 time_style 346 ) 347 col += 1 348 349 col += 1 # blank column 350 351 for comp in sheet_comparisons: 352 cmp_from = configs.get(comp["from"]) 353 cmp_to = configs.get(comp["to"]) 354 355 if isinstance(cmp_from, numbers.Number) and isinstance(cmp_to, numbers.Number): 356 try: 357 speedup = cmp_from / cmp_to 358 sheet.write(row, col, speedup, good_speedup_style if speedup > 1.1 else 359 bad_speedup_style if speedup < 0.9 else 360 speedup_style) 361 except ArithmeticError as e: 362 sheet.write(row, col, None, error_speedup_style) 363 else: 364 sheet.write(row, col, None, no_speedup_style) 365 366 col += 1 367 368 row += 1 369 if row % 1000 == 0: sheet.flush_row_data() 370 371 wb.save(args.output) 372 373 if __name__ == '__main__': 374 main() 375
