1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """A module providing the summary for multiple test results. 6 7 This firmware_summary module is used to collect the test results of 8 multiple rounds from the logs generated by different firmware versions. 9 The test results of the various validators of every gesture are displayed. 10 In addition, the test results of every validator across all gestures are 11 also summarized. 12 13 Usage: 14 $ python firmware_summary log_directory 15 16 17 A typical summary output looks like 18 19 Test Summary (by gesture) : fw_2.41 fw_2.42 count 20 --------------------------------------------------------------------- 21 one_finger_tracking 22 CountTrackingIDValidator : 1.00 0.90 12 23 LinearityBothEndsValidator : 0.97 0.89 12 24 LinearityMiddleValidator : 1.00 1.00 12 25 NoGapValidator : 0.74 0.24 12 26 NoReversedMotionBothEndsValidator : 0.68 0.34 12 27 NoReversedMotionMiddleValidator : 1.00 1.00 12 28 ReportRateValidator : 1.00 1.00 12 29 one_finger_to_edge 30 CountTrackingIDValidator : 1.00 1.00 4 31 LinearityBothEndsValidator : 0.88 0.89 4 32 LinearityMiddleValidator : 1.00 1.00 4 33 NoGapValidator : 0.50 0.00 4 34 NoReversedMotionMiddleValidator : 1.00 1.00 4 35 RangeValidator : 1.00 1.00 4 36 37 ... 38 39 40 Test Summary (by validator) : fw_2.4 fw_2.4.a count 41 --------------------------------------------------------------------- 42 CountPacketsValidator : 1.00 0.82 6 43 CountTrackingIDValidator : 0.92 0.88 84 44 45 ... 46 47 """ 48 49 50 import getopt 51 import os 52 import sys 53 54 import firmware_log 55 import test_conf as conf 56 57 from collections import defaultdict 58 59 from common_util import print_and_exit 60 from firmware_constants import OPTIONS 61 from test_conf import (log_root_dir, merged_validators, segment_weights, 62 validator_weights) 63 from validators import BaseValidator, get_parent_validators 64 65 66 class OptionsDisplayMetrics: 67 """The options of displaying metrics.""" 68 # Defining the options of displaying metrics 69 HIDE_SOME_METRICS_STATS = '0' 70 DISPLAY_ALL_METRICS_STATS = '1' 71 DISPLAY_ALL_METRICS_WITH_RAW_VALUES = '2' 72 DISPLAY_METRICS_OPTIONS = [HIDE_SOME_METRICS_STATS, 73 DISPLAY_ALL_METRICS_STATS, 74 DISPLAY_ALL_METRICS_WITH_RAW_VALUES] 75 DISPLAY_METRICS_DEFAULT = DISPLAY_ALL_METRICS_WITH_RAW_VALUES 76 77 def __init__(self, option): 78 """Initialize with the level value. 79 80 @param option: the option of display metrics 81 """ 82 if option not in self.DISPLAY_METRICS_OPTIONS: 83 option = self.DISPLAY_METRICS_DEFAULT 84 85 # To display all metrics statistics grouped by validators? 86 self.display_all_stats = ( 87 option == self.DISPLAY_ALL_METRICS_STATS or 88 option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES) 89 90 # To display the raw metrics values in details on file basis? 91 self.display_raw_values = ( 92 option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES) 93 94 95 class FirmwareSummary: 96 """Summary for touch device firmware tests.""" 97 98 def __init__(self, log_dir, display_metrics=False, debug_flag=False, 99 display_scores=False, individual_round_flag=False, 100 segment_weights=segment_weights, 101 validator_weights=validator_weights): 102 """ segment_weights and validator_weights are passed as arguments 103 so that it is possible to assign arbitrary weights in unit tests. 104 """ 105 if os.path.isdir(log_dir): 106 self.log_dir = log_dir 107 else: 108 error_msg = 'Error: The test result directory does not exist: %s' 109 print error_msg % log_dir 110 sys.exit(1) 111 112 self.display_metrics = display_metrics 113 self.display_scores = display_scores 114 self.slog = firmware_log.SummaryLog(log_dir, 115 segment_weights, 116 validator_weights, 117 individual_round_flag, 118 debug_flag) 119 120 def _print_summary_title(self, summary_title_str): 121 """Print the summary of the test results by gesture.""" 122 # Create a flexible column title format according to the number of 123 # firmware versions which could be 1, 2, or more. 124 # 125 # A typical summary title looks like 126 # Test Summary () : fw_11.26 fw_11.23 127 # mean ssd count mean ssd count 128 # ---------------------------------------------------------------------- 129 # 130 # The 1st line above is called title_fw. 131 # The 2nd line above is called title_statistics. 132 # 133 # As an example for 2 firmwares, title_fw_format looks like: 134 # '{0:<37}: {1:>12} {2:>21}' 135 title_fw_format_list = ['{0:<37}:',] 136 for i in range(len(self.slog.fws)): 137 format_space = 12 if i == 0 else (12 + 9) 138 title_fw_format_list.append('{%d:>%d}' % (i + 1, format_space)) 139 title_fw_format = ' '.join(title_fw_format_list) 140 141 # As an example for 2 firmwares, title_statistics_format looks like: 142 # '{0:>47} {1:>6} {2:>5} {3:>8} {4:>6} {5:>5}' 143 title_statistics_format_list = [] 144 for i in range(len(self.slog.fws)): 145 format_space = (12 + 35) if i == 0 else 8 146 title_statistics_format_list.append('{%d:>%d}' % (3 * i, 147 format_space)) 148 title_statistics_format_list.append('{%d:>%d}' % (3 * i + 1 , 6)) 149 title_statistics_format_list.append('{%d:>%d}' % (3 * i + 2 , 5)) 150 title_statistics_format = ' '.join(title_statistics_format_list) 151 152 # Create title_fw_list 153 # As an example for two firmware versions, it looks like 154 # ['Test Summary (by gesture)', 'fw_2.4', 'fw_2.5'] 155 title_fw_list = [summary_title_str,] + self.slog.fws 156 157 # Create title_statistics_list 158 # As an example for two firmware versions, it looks like 159 # ['mean', 'ssd', 'count', 'mean', 'ssd', 'count', ] 160 title_statistics_list = ['mean', 'ssd', 'count'] * len(self.slog.fws) 161 162 # Print the title. 163 title_fw = title_fw_format.format(*title_fw_list) 164 title_statistics = title_statistics_format.format( 165 *title_statistics_list) 166 print '\n\n', title_fw 167 print title_statistics 168 print '-' * len(title_statistics) 169 170 def _print_result_stats(self, gesture=None): 171 """Print the result statistics of validators.""" 172 for validator in self.slog.validators: 173 stat_scores_data = [] 174 statistics_format_list = [] 175 for fw in self.slog.fws: 176 result = self.slog.get_result(fw=fw, gesture=gesture, 177 validators=validator) 178 scores_data = result.stat_scores.all_data 179 if scores_data: 180 stat_scores_data += scores_data 181 statistics_format_list.append('{:>8.2f} {:>6.2f} {:>5}') 182 else: 183 stat_scores_data.append('') 184 statistics_format_list.append('{:>21}') 185 186 # Print the score statistics of all firmwares on the same row. 187 if any(stat_scores_data): 188 stat_scores_data.insert(0, validator) 189 statistics_format_list.insert(0,' {:<35}:') 190 statistics_format = ' '.join(statistics_format_list) 191 print statistics_format.format(*tuple(stat_scores_data)) 192 193 def _print_result_stats_by_gesture(self): 194 """Print the summary of the test results by gesture.""" 195 self._print_summary_title('Test Summary (by gesture)') 196 for gesture in self.slog.gestures: 197 print gesture 198 self._print_result_stats(gesture=gesture) 199 200 def _print_result_stats_by_validator(self): 201 """Print the summary of the test results by validator. The validator 202 results of all gestures are combined to compute the statistics. 203 """ 204 self._print_summary_title('Test Summary (by validator)') 205 self._print_result_stats() 206 207 def _get_metric_name_for_display(self, metric_name): 208 """Get the metric name for display. 209 We would like to shorten the metric name when displayed. 210 211 @param metric_name: a metric name 212 """ 213 return metric_name.split('--')[0] 214 215 def _get_merged_validators(self): 216 merged = defaultdict(list) 217 for validator_name in self.slog.validators: 218 parents = get_parent_validators(validator_name) 219 for parent in parents: 220 if parent in merged_validators: 221 merged[parent].append(validator_name) 222 break 223 else: 224 merged[validator_name] = [validator_name,] 225 return sorted(merged.values()) 226 227 def _print_statistics_of_metrics(self, detailed=True, gesture=None): 228 """Print the statistics of metrics by gesture or by validator. 229 230 @param gesture: print the statistics grouped by gesture 231 if this argument is specified; otherwise, by validator. 232 @param detailed: print statistics for all derived validators if True; 233 otherwise, print the merged statistics, e.g., 234 both StationaryFingerValidator and StationaryTapValidator 235 are merged into StationaryValidator. 236 """ 237 # Print the complete title which looks like: 238 # <title_str> <fw1> <fw2> ... <description> 239 fws = self.slog.fws 240 num_fws = len(fws) 241 fws_str_max_width = max(map(len, fws)) 242 fws_str_width = max(fws_str_max_width + 1, 10) 243 table_name = ('Detailed table (for debugging)' if detailed else 244 'Summary table') 245 title_str = ('Metrics statistics by gesture: ' + gesture if gesture else 246 'Metrics statistics by validator') 247 description_str = 'description (lower is better)' 248 fw_format = '{:>%d}' % fws_str_width 249 complete_title = ('{:<37}: '.format(title_str) + 250 (fw_format * num_fws).format(*fws) + 251 ' {:<40}'.format(description_str)) 252 print '\n' * 2 253 print table_name 254 print complete_title 255 print '-' * len(complete_title) 256 257 # Print the metric name and the metric stats values of every firmwares 258 name_format = ' ' * 6 + '{:<31}:' 259 description_format = ' {:<40}' 260 float_format = '{:>%d.2f}' % fws_str_width 261 blank_format = '{:>%d}' % fws_str_width 262 263 validators = (self.slog.validators if detailed else 264 self._get_merged_validators()) 265 266 for validator in validators: 267 fw_stats_values = defaultdict(dict) 268 for fw in fws: 269 result = self.slog.get_result(fw=fw, gesture=gesture, 270 validators=validator) 271 stat_metrics = result.stat_metrics 272 273 for metric_name in stat_metrics.metrics_values: 274 fw_stats_values[metric_name][fw] = \ 275 stat_metrics.stats_values[metric_name] 276 277 fw_stats_values_printed = False 278 for metric_name, fw_values_dict in sorted(fw_stats_values.items()): 279 values = [] 280 values_format = '' 281 for fw in fws: 282 value = fw_values_dict.get(fw, '') 283 values.append(value) 284 values_format += float_format if value else blank_format 285 286 # The metrics of some special validators will not be shown 287 # unless the display_all_stats flag is True or any stats values 288 # are non-zero. 289 if (validator not in conf.validators_hidden_when_no_failures or 290 self.display_metrics.display_all_stats or any(values)): 291 if not fw_stats_values_printed: 292 fw_stats_values_printed = True 293 if isinstance(validator, list): 294 print (' ' + ' {}' * len(validator)).format(*validator) 295 else: 296 print ' ' + validator 297 disp_name = self._get_metric_name_for_display(metric_name) 298 print name_format.format(disp_name), 299 print values_format.format(*values), 300 print description_format.format( 301 stat_metrics.metrics_props[metric_name].description) 302 303 def _print_raw_metrics_values(self): 304 """Print the raw metrics values.""" 305 # The subkey() below extracts (gesture, variation, round) from 306 # metric.key which is (fw, round, gesture, variation, validator) 307 subkey = lambda key: (key[2], key[3], key[1]) 308 309 # The sum_len() below is used to calculate the sum of the length 310 # of the elements in the subkey. 311 sum_len = lambda lst: sum([len(str(l)) if l else 0 for l in lst]) 312 313 mnprops = firmware_log.MetricNameProps() 314 print '\n\nRaw metrics values' 315 print '-' * 80 316 for fw in self.slog.fws: 317 print '\n', fw 318 for validator in self.slog.validators: 319 result = self.slog.get_result(fw=fw, validators=validator) 320 metrics_dict = result.stat_metrics.metrics_dict 321 if metrics_dict: 322 print '\n' + ' ' * 3 + validator 323 for metric_name, metrics in sorted(metrics_dict.items()): 324 disp_name = self._get_metric_name_for_display(metric_name) 325 print ' ' * 6 + disp_name 326 327 metric_note = mnprops.metrics_props[metric_name].note 328 if metric_note: 329 msg = '** Note: value below represents ' 330 print ' ' * 9 + msg + metric_note 331 332 # Make a metric value list sorted by 333 # (gesture, variation, round) 334 value_list = sorted([(subkey(metric.key), metric.value) 335 for metric in metrics]) 336 337 max_len = max([sum_len(value[0]) for value in value_list]) 338 template_prefix = ' ' * 9 + '{:<%d}: ' % (max_len + 5) 339 for (gesture, variation, round), value in value_list: 340 template = template_prefix + ( 341 '{}' if isinstance(value, tuple) else '{:.2f}') 342 gvr_str = '%s.%s (%s)' % (gesture, variation, round) 343 print template.format(gvr_str, value) 344 345 def _print_final_weighted_averages(self): 346 """Print the final weighted averages of all validators.""" 347 title_str = 'Test Summary (final weighted averages)' 348 print '\n\n' + title_str 349 print '-' * len(title_str) 350 weighted_average = self.slog.get_final_weighted_average() 351 for fw in self.slog.fws: 352 print '%s: %4.3f' % (fw, weighted_average[fw]) 353 354 def print_result_summary(self): 355 """Print the summary of the test results.""" 356 print self.slog.test_version 357 if self.display_metrics: 358 self._print_statistics_of_metrics(detailed=False) 359 self._print_statistics_of_metrics(detailed=True) 360 if self.display_metrics.display_raw_values: 361 self._print_raw_metrics_values() 362 if self.display_scores: 363 self._print_result_stats_by_gesture() 364 self._print_result_stats_by_validator() 365 self._print_final_weighted_averages() 366 367 368 def _usage_and_exit(): 369 """Print the usage message and exit.""" 370 prog = sys.argv[0] 371 print 'Usage: $ python %s [options]\n' % prog 372 print 'options:' 373 print ' -D, --%s' % OPTIONS.DEBUG 374 print ' enable debug flag' 375 print ' -d, --%s <directory>' % OPTIONS.DIR 376 print ' specify which log directory to derive the summary' 377 print ' -h, --%s' % OPTIONS.HELP 378 print ' show this help' 379 print ' -i, --%s' % OPTIONS.INDIVIDUAL 380 print ' Calculate statistics of every individual round separately' 381 print ' -m, --%s <verbose_level>' % OPTIONS.METRICS 382 print ' display the summary metrics.' 383 print ' verbose_level:' 384 print ' 0: hide some metrics statistics if they passed' 385 print ' 1: display all metrics statistics' 386 print ' 2: display all metrics statistics and ' \ 387 'the detailed raw metrics values (default)' 388 print ' -s, --%s' % OPTIONS.SCORES 389 print ' display the scores (0.0 ~ 1.0)' 390 print 391 print 'Examples:' 392 print ' Specify the log root directory.' 393 print ' $ python %s -d /tmp' % prog 394 print ' Hide some metrics statistics.' 395 print ' $ python %s -m 0' % prog 396 print ' Display all metrics statistics.' 397 print ' $ python %s -m 1' % prog 398 print ' Display all metrics statistics with detailed raw metrics values.' 399 print ' $ python %s # or' % prog 400 print ' $ python %s -m 2' % prog 401 sys.exit(1) 402 403 404 def _parsing_error(msg): 405 """Print the usage and exit when encountering parsing error.""" 406 print 'Error: %s' % msg 407 _usage_and_exit() 408 409 410 def _parse_options(): 411 """Parse the options.""" 412 # Set the default values of options. 413 options = {OPTIONS.DEBUG: False, 414 OPTIONS.DIR: log_root_dir, 415 OPTIONS.INDIVIDUAL: False, 416 OPTIONS.METRICS: OptionsDisplayMetrics(None), 417 OPTIONS.SCORES: False, 418 } 419 420 try: 421 short_opt = 'Dd:him:s' 422 long_opt = [OPTIONS.DEBUG, 423 OPTIONS.DIR + '=', 424 OPTIONS.HELP, 425 OPTIONS.INDIVIDUAL, 426 OPTIONS.METRICS + '=', 427 OPTIONS.SCORES, 428 ] 429 opts, args = getopt.getopt(sys.argv[1:], short_opt, long_opt) 430 except getopt.GetoptError, err: 431 _parsing_error(str(err)) 432 433 for opt, arg in opts: 434 if opt in ('-h', '--%s' % OPTIONS.HELP): 435 _usage_and_exit() 436 elif opt in ('-D', '--%s' % OPTIONS.DEBUG): 437 options[OPTIONS.DEBUG] = True 438 elif opt in ('-d', '--%s' % OPTIONS.DIR): 439 options[OPTIONS.DIR] = arg 440 if not os.path.isdir(arg): 441 print 'Error: the log directory %s does not exist.' % arg 442 _usage_and_exit() 443 elif opt in ('-i', '--%s' % OPTIONS.INDIVIDUAL): 444 options[OPTIONS.INDIVIDUAL] = True 445 elif opt in ('-m', '--%s' % OPTIONS.METRICS): 446 options[OPTIONS.METRICS] = OptionsDisplayMetrics(arg) 447 elif opt in ('-s', '--%s' % OPTIONS.SCORES): 448 options[OPTIONS.SCORES] = True 449 else: 450 msg = 'This option "%s" is not supported.' % opt 451 _parsing_error(opt) 452 453 return options 454 455 456 if __name__ == '__main__': 457 options = _parse_options() 458 summary = FirmwareSummary(options[OPTIONS.DIR], 459 display_metrics=options[OPTIONS.METRICS], 460 individual_round_flag=options[OPTIONS.INDIVIDUAL], 461 display_scores=options[OPTIONS.SCORES], 462 debug_flag=options[OPTIONS.DEBUG]) 463 summary.print_result_summary() 464
