1 #!/usr/bin/python 2 3 """ 4 Copyright 2014 Google Inc. 5 6 Use of this source code is governed by a BSD-style license that can be 7 found in the LICENSE file. 8 9 Compare GM results for two configs, across all builders. 10 """ 11 12 # System-level imports 13 import argparse 14 import fnmatch 15 import json 16 import logging 17 import re 18 import time 19 20 # Imports from within Skia 21 import fix_pythonpath # must do this first 22 from pyutils import url_utils 23 import gm_json 24 import imagediffdb 25 import imagepair 26 import imagepairset 27 import results 28 29 30 class ConfigComparisons(results.BaseComparisons): 31 """Loads results from two different configurations into an ImagePairSet. 32 33 Loads actual and expected results from all builders, except for those skipped 34 by _ignore_builder(). 35 """ 36 37 def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR, 38 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, 39 diff_base_url=None, builder_regex_list=None): 40 """ 41 Args: 42 configs: (string, string) tuple; pair of configs to compare 43 actuals_root: root directory containing all actual-results.json files 44 generated_images_root: directory within which to create all pixel diffs; 45 if this directory does not yet exist, it will be created 46 diff_base_url: base URL within which the client should look for diff 47 images; if not specified, defaults to a "file:///" URL representation 48 of generated_images_root 49 builder_regex_list: List of regular expressions specifying which builders 50 we will process. If None, process all builders. 51 """ 52 time_start = int(time.time()) 53 if builder_regex_list != None: 54 self.set_match_builders_pattern_list(builder_regex_list) 55 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) 56 self._diff_base_url = ( 57 diff_base_url or 58 url_utils.create_filepath_url(generated_images_root)) 59 self._actuals_root = actuals_root 60 self._load_config_pairs(configs) 61 self._timestamp = int(time.time()) 62 logging.info('Results complete; took %d seconds.' % 63 (self._timestamp - time_start)) 64 65 def _load_config_pairs(self, configs): 66 """Loads the results of all tests, across all builders (based on the 67 files within self._actuals_root), compares them across two configs, 68 and stores the summary in self._results. 69 70 Args: 71 configs: tuple of strings; pair of configs to compare 72 """ 73 logging.info('Reading actual-results JSON files from %s...' % 74 self._actuals_root) 75 actual_builder_dicts = self._read_builder_dicts_from_root( 76 self._actuals_root) 77 configA, configB = configs 78 logging.info('Comparing configs %s and %s...' % (configA, configB)) 79 80 all_image_pairs = imagepairset.ImagePairSet( 81 descriptions=configs, 82 diff_base_url=self._diff_base_url) 83 failing_image_pairs = imagepairset.ImagePairSet( 84 descriptions=configs, 85 diff_base_url=self._diff_base_url) 86 87 all_image_pairs.ensure_extra_column_values_in_summary( 88 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 89 results.KEY__RESULT_TYPE__FAILED, 90 results.KEY__RESULT_TYPE__NOCOMPARISON, 91 results.KEY__RESULT_TYPE__SUCCEEDED, 92 ]) 93 failing_image_pairs.ensure_extra_column_values_in_summary( 94 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 95 results.KEY__RESULT_TYPE__FAILED, 96 results.KEY__RESULT_TYPE__NOCOMPARISON, 97 ]) 98 99 builders = sorted(actual_builder_dicts.keys()) 100 num_builders = len(builders) 101 builder_num = 0 102 for builder in builders: 103 builder_num += 1 104 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % 105 (builder_num, num_builders, builder)) 106 actual_results_for_this_builder = ( 107 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) 108 for result_type in sorted(actual_results_for_this_builder.keys()): 109 results_of_this_type = actual_results_for_this_builder[result_type] 110 if not results_of_this_type: 111 continue 112 113 tests_found = set() 114 for image_name in sorted(results_of_this_type.keys()): 115 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() 116 tests_found.add(test) 117 118 for test in tests_found: 119 # Get image_relative_url (or None) for each of configA, configB 120 image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA) 121 configA_image_relative_url = ConfigComparisons._create_relative_url( 122 hashtype_and_digest=results_of_this_type.get(image_name_A), 123 test_name=test) 124 image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB) 125 configB_image_relative_url = ConfigComparisons._create_relative_url( 126 hashtype_and_digest=results_of_this_type.get(image_name_B), 127 test_name=test) 128 129 # If we have images for at least one of these two configs, 130 # add them to our list. 131 if configA_image_relative_url or configB_image_relative_url: 132 if configA_image_relative_url == configB_image_relative_url: 133 result_type = results.KEY__RESULT_TYPE__SUCCEEDED 134 elif not configA_image_relative_url: 135 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON 136 elif not configB_image_relative_url: 137 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON 138 else: 139 result_type = results.KEY__RESULT_TYPE__FAILED 140 141 extra_columns_dict = { 142 results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, 143 results.KEY__EXTRACOLUMNS__BUILDER: builder, 144 results.KEY__EXTRACOLUMNS__TEST: test, 145 # TODO(epoger): Right now, the client UI crashes if it receives 146 # results that do not include a 'config' column. 147 # Until we fix that, keep the client happy. 148 results.KEY__EXTRACOLUMNS__CONFIG: 'TODO', 149 } 150 151 try: 152 image_pair = imagepair.ImagePair( 153 image_diff_db=self._image_diff_db, 154 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, 155 imageA_relative_url=configA_image_relative_url, 156 imageB_relative_url=configB_image_relative_url, 157 extra_columns=extra_columns_dict) 158 all_image_pairs.add_image_pair(image_pair) 159 if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: 160 failing_image_pairs.add_image_pair(image_pair) 161 except (KeyError, TypeError): 162 logging.exception( 163 'got exception while creating ImagePair for image_name ' 164 '"%s", builder "%s"' % (image_name, builder)) 165 166 self._results = { 167 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), 168 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), 169 } 170 171 172 def main(): 173 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', 174 datefmt='%m/%d/%Y %H:%M:%S', 175 level=logging.INFO) 176 parser = argparse.ArgumentParser() 177 parser.add_argument( 178 '--actuals', default=results.DEFAULT_ACTUALS_DIR, 179 help='Directory containing all actual-result JSON files; defaults to ' 180 '\'%(default)s\' .') 181 parser.add_argument( 182 'config', nargs=2, 183 help='Two configurations to compare (8888, gpu, etc.).') 184 parser.add_argument( 185 '--outfile', required=True, 186 help='File to write result summary into, in JSON format.') 187 parser.add_argument( 188 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, 189 help='Which result types to include. Defaults to \'%(default)s\'; ' 190 'must be one of ' + 191 str([results.KEY__HEADER__RESULTS_FAILURES, 192 results.KEY__HEADER__RESULTS_ALL])) 193 parser.add_argument( 194 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, 195 help='Directory within which to download images and generate diffs; ' 196 'defaults to \'%(default)s\' .') 197 args = parser.parse_args() 198 results_obj = ConfigComparisons(configs=args.config, 199 actuals_root=args.actuals, 200 generated_images_root=args.workdir) 201 gm_json.WriteToFile( 202 results_obj.get_packaged_results_of_type(results_type=args.results), 203 args.outfile) 204 205 206 if __name__ == '__main__': 207 main() 208
