# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """A module providing the summary for multiple test results. This firmware_summary module is used to collect the test results of multiple rounds from the logs generated by different firmware versions. The test results of the various validators of every gesture are displayed. In addition, the test results of every validator across all gestures are also summarized. Usage: $ python firmware_summary log_directory A typical summary output looks like Test Summary (by gesture) : fw_2.41 fw_2.42 count --------------------------------------------------------------------- one_finger_tracking CountTrackingIDValidator : 1.00 0.90 12 LinearityBothEndsValidator : 0.97 0.89 12 LinearityMiddleValidator : 1.00 1.00 12 NoGapValidator : 0.74 0.24 12 NoReversedMotionBothEndsValidator : 0.68 0.34 12 NoReversedMotionMiddleValidator : 1.00 1.00 12 ReportRateValidator : 1.00 1.00 12 one_finger_to_edge CountTrackingIDValidator : 1.00 1.00 4 LinearityBothEndsValidator : 0.88 0.89 4 LinearityMiddleValidator : 1.00 1.00 4 NoGapValidator : 0.50 0.00 4 NoReversedMotionMiddleValidator : 1.00 1.00 4 RangeValidator : 1.00 1.00 4 ... Test Summary (by validator) : fw_2.4 fw_2.4.a count --------------------------------------------------------------------- CountPacketsValidator : 1.00 0.82 6 CountTrackingIDValidator : 0.92 0.88 84 ... """ import getopt import os import sys import firmware_log import test_conf as conf from collections import defaultdict from common_util import print_and_exit from firmware_constants import OPTIONS from test_conf import (log_root_dir, merged_validators, segment_weights, validator_weights) from validators import BaseValidator, get_parent_validators class OptionsDisplayMetrics: """The options of displaying metrics.""" # Defining the options of displaying metrics HIDE_SOME_METRICS_STATS = '0' DISPLAY_ALL_METRICS_STATS = '1' DISPLAY_ALL_METRICS_WITH_RAW_VALUES = '2' DISPLAY_METRICS_OPTIONS = [HIDE_SOME_METRICS_STATS, DISPLAY_ALL_METRICS_STATS, DISPLAY_ALL_METRICS_WITH_RAW_VALUES] DISPLAY_METRICS_DEFAULT = DISPLAY_ALL_METRICS_WITH_RAW_VALUES def __init__(self, option): """Initialize with the level value. @param option: the option of display metrics """ if option not in self.DISPLAY_METRICS_OPTIONS: option = self.DISPLAY_METRICS_DEFAULT # To display all metrics statistics grouped by validators? self.display_all_stats = ( option == self.DISPLAY_ALL_METRICS_STATS or option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES) # To display the raw metrics values in details on file basis? self.display_raw_values = ( option == self.DISPLAY_ALL_METRICS_WITH_RAW_VALUES) class FirmwareSummary: """Summary for touch device firmware tests.""" def __init__(self, log_dir, display_metrics=False, debug_flag=False, display_scores=False, individual_round_flag=False, segment_weights=segment_weights, validator_weights=validator_weights): """ segment_weights and validator_weights are passed as arguments so that it is possible to assign arbitrary weights in unit tests. """ if os.path.isdir(log_dir): self.log_dir = log_dir else: error_msg = 'Error: The test result directory does not exist: %s' print error_msg % log_dir sys.exit(1) self.display_metrics = display_metrics self.display_scores = display_scores self.slog = firmware_log.SummaryLog(log_dir, segment_weights, validator_weights, individual_round_flag, debug_flag) def _print_summary_title(self, summary_title_str): """Print the summary of the test results by gesture.""" # Create a flexible column title format according to the number of # firmware versions which could be 1, 2, or more. # # A typical summary title looks like # Test Summary () : fw_11.26 fw_11.23 # mean ssd count mean ssd count # ---------------------------------------------------------------------- # # The 1st line above is called title_fw. # The 2nd line above is called title_statistics. # # As an example for 2 firmwares, title_fw_format looks like: # '{0:<37}: {1:>12} {2:>21}' title_fw_format_list = ['{0:<37}:',] for i in range(len(self.slog.fws)): format_space = 12 if i == 0 else (12 + 9) title_fw_format_list.append('{%d:>%d}' % (i + 1, format_space)) title_fw_format = ' '.join(title_fw_format_list) # As an example for 2 firmwares, title_statistics_format looks like: # '{0:>47} {1:>6} {2:>5} {3:>8} {4:>6} {5:>5}' title_statistics_format_list = [] for i in range(len(self.slog.fws)): format_space = (12 + 35) if i == 0 else 8 title_statistics_format_list.append('{%d:>%d}' % (3 * i, format_space)) title_statistics_format_list.append('{%d:>%d}' % (3 * i + 1 , 6)) title_statistics_format_list.append('{%d:>%d}' % (3 * i + 2 , 5)) title_statistics_format = ' '.join(title_statistics_format_list) # Create title_fw_list # As an example for two firmware versions, it looks like # ['Test Summary (by gesture)', 'fw_2.4', 'fw_2.5'] title_fw_list = [summary_title_str,] + self.slog.fws # Create title_statistics_list # As an example for two firmware versions, it looks like # ['mean', 'ssd', 'count', 'mean', 'ssd', 'count', ] title_statistics_list = ['mean', 'ssd', 'count'] * len(self.slog.fws) # Print the title. title_fw = title_fw_format.format(*title_fw_list) title_statistics = title_statistics_format.format( *title_statistics_list) print '\n\n', title_fw print title_statistics print '-' * len(title_statistics) def _print_result_stats(self, gesture=None): """Print the result statistics of validators.""" for validator in self.slog.validators: stat_scores_data = [] statistics_format_list = [] for fw in self.slog.fws: result = self.slog.get_result(fw=fw, gesture=gesture, validators=validator) scores_data = result.stat_scores.all_data if scores_data: stat_scores_data += scores_data statistics_format_list.append('{:>8.2f} {:>6.2f} {:>5}') else: stat_scores_data.append('') statistics_format_list.append('{:>21}') # Print the score statistics of all firmwares on the same row. if any(stat_scores_data): stat_scores_data.insert(0, validator) statistics_format_list.insert(0,' {:<35}:') statistics_format = ' '.join(statistics_format_list) print statistics_format.format(*tuple(stat_scores_data)) def _print_result_stats_by_gesture(self): """Print the summary of the test results by gesture.""" self._print_summary_title('Test Summary (by gesture)') for gesture in self.slog.gestures: print gesture self._print_result_stats(gesture=gesture) def _print_result_stats_by_validator(self): """Print the summary of the test results by validator. The validator results of all gestures are combined to compute the statistics. """ self._print_summary_title('Test Summary (by validator)') self._print_result_stats() def _get_metric_name_for_display(self, metric_name): """Get the metric name for display. We would like to shorten the metric name when displayed. @param metric_name: a metric name """ return metric_name.split('--')[0] def _get_merged_validators(self): merged = defaultdict(list) for validator_name in self.slog.validators: parents = get_parent_validators(validator_name) for parent in parents: if parent in merged_validators: merged[parent].append(validator_name) break else: merged[validator_name] = [validator_name,] return sorted(merged.values()) def _print_statistics_of_metrics(self, detailed=True, gesture=None): """Print the statistics of metrics by gesture or by validator. @param gesture: print the statistics grouped by gesture if this argument is specified; otherwise, by validator. @param detailed: print statistics for all derived validators if True; otherwise, print the merged statistics, e.g., both StationaryFingerValidator and StationaryTapValidator are merged into StationaryValidator. """ # Print the complete title which looks like: # ... fws = self.slog.fws num_fws = len(fws) fws_str_max_width = max(map(len, fws)) fws_str_width = max(fws_str_max_width + 1, 10) table_name = ('Detailed table (for debugging)' if detailed else 'Summary table') title_str = ('Metrics statistics by gesture: ' + gesture if gesture else 'Metrics statistics by validator') description_str = 'description (lower is better)' fw_format = '{:>%d}' % fws_str_width complete_title = ('{:<37}: '.format(title_str) + (fw_format * num_fws).format(*fws) + ' {:<40}'.format(description_str)) print '\n' * 2 print table_name print complete_title print '-' * len(complete_title) # Print the metric name and the metric stats values of every firmwares name_format = ' ' * 6 + '{:<31}:' description_format = ' {:<40}' float_format = '{:>%d.2f}' % fws_str_width blank_format = '{:>%d}' % fws_str_width validators = (self.slog.validators if detailed else self._get_merged_validators()) for validator in validators: fw_stats_values = defaultdict(dict) for fw in fws: result = self.slog.get_result(fw=fw, gesture=gesture, validators=validator) stat_metrics = result.stat_metrics for metric_name in stat_metrics.metrics_values: fw_stats_values[metric_name][fw] = \ stat_metrics.stats_values[metric_name] fw_stats_values_printed = False for metric_name, fw_values_dict in sorted(fw_stats_values.items()): values = [] values_format = '' for fw in fws: value = fw_values_dict.get(fw, '') values.append(value) values_format += float_format if value else blank_format # The metrics of some special validators will not be shown # unless the display_all_stats flag is True or any stats values # are non-zero. if (validator not in conf.validators_hidden_when_no_failures or self.display_metrics.display_all_stats or any(values)): if not fw_stats_values_printed: fw_stats_values_printed = True if isinstance(validator, list): print (' ' + ' {}' * len(validator)).format(*validator) else: print ' ' + validator disp_name = self._get_metric_name_for_display(metric_name) print name_format.format(disp_name), print values_format.format(*values), print description_format.format( stat_metrics.metrics_props[metric_name].description) def _print_raw_metrics_values(self): """Print the raw metrics values.""" # The subkey() below extracts (gesture, variation, round) from # metric.key which is (fw, round, gesture, variation, validator) subkey = lambda key: (key[2], key[3], key[1]) # The sum_len() below is used to calculate the sum of the length # of the elements in the subkey. sum_len = lambda lst: sum([len(str(l)) if l else 0 for l in lst]) mnprops = firmware_log.MetricNameProps() print '\n\nRaw metrics values' print '-' * 80 for fw in self.slog.fws: print '\n', fw for validator in self.slog.validators: result = self.slog.get_result(fw=fw, validators=validator) metrics_dict = result.stat_metrics.metrics_dict if metrics_dict: print '\n' + ' ' * 3 + validator for metric_name, metrics in sorted(metrics_dict.items()): disp_name = self._get_metric_name_for_display(metric_name) print ' ' * 6 + disp_name metric_note = mnprops.metrics_props[metric_name].note if metric_note: msg = '** Note: value below represents ' print ' ' * 9 + msg + metric_note # Make a metric value list sorted by # (gesture, variation, round) value_list = sorted([(subkey(metric.key), metric.value) for metric in metrics]) max_len = max([sum_len(value[0]) for value in value_list]) template_prefix = ' ' * 9 + '{:<%d}: ' % (max_len + 5) for (gesture, variation, round), value in value_list: template = template_prefix + ( '{}' if isinstance(value, tuple) else '{:.2f}') gvr_str = '%s.%s (%s)' % (gesture, variation, round) print template.format(gvr_str, value) def _print_final_weighted_averages(self): """Print the final weighted averages of all validators.""" title_str = 'Test Summary (final weighted averages)' print '\n\n' + title_str print '-' * len(title_str) weighted_average = self.slog.get_final_weighted_average() for fw in self.slog.fws: print '%s: %4.3f' % (fw, weighted_average[fw]) def print_result_summary(self): """Print the summary of the test results.""" print self.slog.test_version if self.display_metrics: self._print_statistics_of_metrics(detailed=False) self._print_statistics_of_metrics(detailed=True) if self.display_metrics.display_raw_values: self._print_raw_metrics_values() if self.display_scores: self._print_result_stats_by_gesture() self._print_result_stats_by_validator() self._print_final_weighted_averages() def _usage_and_exit(): """Print the usage message and exit.""" prog = sys.argv[0] print 'Usage: $ python %s [options]\n' % prog print 'options:' print ' -D, --%s' % OPTIONS.DEBUG print ' enable debug flag' print ' -d, --%s ' % OPTIONS.DIR print ' specify which log directory to derive the summary' print ' -h, --%s' % OPTIONS.HELP print ' show this help' print ' -i, --%s' % OPTIONS.INDIVIDUAL print ' Calculate statistics of every individual round separately' print ' -m, --%s ' % OPTIONS.METRICS print ' display the summary metrics.' print ' verbose_level:' print ' 0: hide some metrics statistics if they passed' print ' 1: display all metrics statistics' print ' 2: display all metrics statistics and ' \ 'the detailed raw metrics values (default)' print ' -s, --%s' % OPTIONS.SCORES print ' display the scores (0.0 ~ 1.0)' print print 'Examples:' print ' Specify the log root directory.' print ' $ python %s -d /tmp' % prog print ' Hide some metrics statistics.' print ' $ python %s -m 0' % prog print ' Display all metrics statistics.' print ' $ python %s -m 1' % prog print ' Display all metrics statistics with detailed raw metrics values.' print ' $ python %s # or' % prog print ' $ python %s -m 2' % prog sys.exit(1) def _parsing_error(msg): """Print the usage and exit when encountering parsing error.""" print 'Error: %s' % msg _usage_and_exit() def _parse_options(): """Parse the options.""" # Set the default values of options. options = {OPTIONS.DEBUG: False, OPTIONS.DIR: log_root_dir, OPTIONS.INDIVIDUAL: False, OPTIONS.METRICS: OptionsDisplayMetrics(None), OPTIONS.SCORES: False, } try: short_opt = 'Dd:him:s' long_opt = [OPTIONS.DEBUG, OPTIONS.DIR + '=', OPTIONS.HELP, OPTIONS.INDIVIDUAL, OPTIONS.METRICS + '=', OPTIONS.SCORES, ] opts, args = getopt.getopt(sys.argv[1:], short_opt, long_opt) except getopt.GetoptError, err: _parsing_error(str(err)) for opt, arg in opts: if opt in ('-h', '--%s' % OPTIONS.HELP): _usage_and_exit() elif opt in ('-D', '--%s' % OPTIONS.DEBUG): options[OPTIONS.DEBUG] = True elif opt in ('-d', '--%s' % OPTIONS.DIR): options[OPTIONS.DIR] = arg if not os.path.isdir(arg): print 'Error: the log directory %s does not exist.' % arg _usage_and_exit() elif opt in ('-i', '--%s' % OPTIONS.INDIVIDUAL): options[OPTIONS.INDIVIDUAL] = True elif opt in ('-m', '--%s' % OPTIONS.METRICS): options[OPTIONS.METRICS] = OptionsDisplayMetrics(arg) elif opt in ('-s', '--%s' % OPTIONS.SCORES): options[OPTIONS.SCORES] = True else: msg = 'This option "%s" is not supported.' % opt _parsing_error(opt) return options if __name__ == '__main__': options = _parse_options() summary = FirmwareSummary(options[OPTIONS.DIR], display_metrics=options[OPTIONS.METRICS], individual_round_flag=options[OPTIONS.INDIVIDUAL], display_scores=options[OPTIONS.SCORES], debug_flag=options[OPTIONS.DEBUG]) summary.print_result_summary()