1 #!/usr/bin/python 2 3 """ 4 Copyright 2014 Google Inc. 5 6 Use of this source code is governed by a BSD-style license that can be 7 found in the LICENSE file. 8 9 Compare results of two render_pictures runs. 10 """ 11 12 # System-level imports 13 import logging 14 import os 15 import re 16 import time 17 18 # Imports from within Skia 19 import fix_pythonpath # must do this first 20 from pyutils import url_utils 21 import gm_json 22 import imagediffdb 23 import imagepair 24 import imagepairset 25 import results 26 27 # URL under which all render_pictures images can be found in Google Storage. 28 # TODO(epoger): Move this default value into 29 # https://skia.googlesource.com/buildbot/+/master/site_config/global_variables.json 30 DEFAULT_IMAGE_BASE_URL = 'http://chromium-skia-gm.commondatastorage.googleapis.com/render_pictures/images' 31 32 33 class RenderedPicturesComparisons(results.BaseComparisons): 34 """Loads results from two different render_pictures runs into an ImagePairSet. 35 """ 36 37 def __init__(self, subdirs, actuals_root, 38 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, 39 image_base_url=DEFAULT_IMAGE_BASE_URL, 40 diff_base_url=None): 41 """ 42 Args: 43 actuals_root: root directory containing all render_pictures-generated 44 JSON files 45 subdirs: (string, string) tuple; pair of subdirectories within 46 actuals_root to compare 47 generated_images_root: directory within which to create all pixel diffs; 48 if this directory does not yet exist, it will be created 49 image_base_url: URL under which all render_pictures result images can 50 be found; this will be used to read images for comparison within 51 this code, and included in the ImagePairSet so its consumers know 52 where to download the images from 53 diff_base_url: base URL within which the client should look for diff 54 images; if not specified, defaults to a "file:///" URL representation 55 of generated_images_root 56 """ 57 time_start = int(time.time()) 58 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) 59 self._image_base_url = image_base_url 60 self._diff_base_url = ( 61 diff_base_url or 62 url_utils.create_filepath_url(generated_images_root)) 63 self._load_result_pairs(actuals_root, subdirs) 64 self._timestamp = int(time.time()) 65 logging.info('Results complete; took %d seconds.' % 66 (self._timestamp - time_start)) 67 68 def _load_result_pairs(self, actuals_root, subdirs): 69 """Loads all JSON files found within two subdirs in actuals_root, 70 compares across those two subdirs, and stores the summary in self._results. 71 72 Args: 73 actuals_root: root directory containing all render_pictures-generated 74 JSON files 75 subdirs: (string, string) tuple; pair of subdirectories within 76 actuals_root to compare 77 """ 78 logging.info( 79 'Reading actual-results JSON files from %s subdirs within %s...' % ( 80 subdirs, actuals_root)) 81 subdirA, subdirB = subdirs 82 subdirA_dicts = self._read_dicts_from_root( 83 os.path.join(actuals_root, subdirA)) 84 subdirB_dicts = self._read_dicts_from_root( 85 os.path.join(actuals_root, subdirB)) 86 logging.info('Comparing subdirs %s and %s...' % (subdirA, subdirB)) 87 88 all_image_pairs = imagepairset.ImagePairSet( 89 descriptions=subdirs, 90 diff_base_url=self._diff_base_url) 91 failing_image_pairs = imagepairset.ImagePairSet( 92 descriptions=subdirs, 93 diff_base_url=self._diff_base_url) 94 95 all_image_pairs.ensure_extra_column_values_in_summary( 96 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 97 results.KEY__RESULT_TYPE__FAILED, 98 results.KEY__RESULT_TYPE__NOCOMPARISON, 99 results.KEY__RESULT_TYPE__SUCCEEDED, 100 ]) 101 failing_image_pairs.ensure_extra_column_values_in_summary( 102 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 103 results.KEY__RESULT_TYPE__FAILED, 104 results.KEY__RESULT_TYPE__NOCOMPARISON, 105 ]) 106 107 common_dict_paths = sorted(set(subdirA_dicts.keys() + subdirB_dicts.keys())) 108 num_common_dict_paths = len(common_dict_paths) 109 dict_num = 0 110 for dict_path in common_dict_paths: 111 dict_num += 1 112 logging.info('Generating pixel diffs for dict #%d of %d, "%s"...' % 113 (dict_num, num_common_dict_paths, dict_path)) 114 dictA = subdirA_dicts[dict_path] 115 dictB = subdirB_dicts[dict_path] 116 self._validate_dict_version(dictA) 117 self._validate_dict_version(dictB) 118 dictA_results = dictA[gm_json.JSONKEY_ACTUALRESULTS] 119 dictB_results = dictB[gm_json.JSONKEY_ACTUALRESULTS] 120 skp_names = sorted(set(dictA_results.keys() + dictB_results.keys())) 121 for skp_name in skp_names: 122 imagepairs_for_this_skp = [] 123 124 whole_image_A = RenderedPicturesComparisons.get_multilevel( 125 dictA_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) 126 whole_image_B = RenderedPicturesComparisons.get_multilevel( 127 dictB_results, skp_name, gm_json.JSONKEY_SOURCE_WHOLEIMAGE) 128 imagepairs_for_this_skp.append(self._create_image_pair( 129 test=skp_name, config=gm_json.JSONKEY_SOURCE_WHOLEIMAGE, 130 image_dict_A=whole_image_A, image_dict_B=whole_image_B)) 131 132 tiled_images_A = RenderedPicturesComparisons.get_multilevel( 133 dictA_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) 134 tiled_images_B = RenderedPicturesComparisons.get_multilevel( 135 dictB_results, skp_name, gm_json.JSONKEY_SOURCE_TILEDIMAGES) 136 # TODO(epoger): Report an error if we find tiles for A but not B? 137 if tiled_images_A and tiled_images_B: 138 # TODO(epoger): Report an error if we find a different number of tiles 139 # for A and B? 140 num_tiles = len(tiled_images_A) 141 for tile_num in range(num_tiles): 142 imagepairs_for_this_skp.append(self._create_image_pair( 143 test=skp_name, 144 config='%s-%d' % (gm_json.JSONKEY_SOURCE_TILEDIMAGES, tile_num), 145 image_dict_A=tiled_images_A[tile_num], 146 image_dict_B=tiled_images_B[tile_num])) 147 148 for imagepair in imagepairs_for_this_skp: 149 if imagepair: 150 all_image_pairs.add_image_pair(imagepair) 151 result_type = imagepair.extra_columns_dict\ 152 [results.KEY__EXTRACOLUMNS__RESULT_TYPE] 153 if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: 154 failing_image_pairs.add_image_pair(imagepair) 155 156 self._results = { 157 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), 158 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), 159 } 160 161 def _validate_dict_version(self, result_dict): 162 """Raises Exception if the dict is not the type/version we know how to read. 163 164 Args: 165 result_dict: dictionary holding output of render_pictures 166 """ 167 expected_header_type = 'ChecksummedImages' 168 expected_header_revision = 1 169 170 header = result_dict[gm_json.JSONKEY_HEADER] 171 header_type = header[gm_json.JSONKEY_HEADER_TYPE] 172 if header_type != expected_header_type: 173 raise Exception('expected header_type "%s", but got "%s"' % ( 174 expected_header_type, header_type)) 175 header_revision = header[gm_json.JSONKEY_HEADER_REVISION] 176 if header_revision != expected_header_revision: 177 raise Exception('expected header_revision %d, but got %d' % ( 178 expected_header_revision, header_revision)) 179 180 def _create_image_pair(self, test, config, image_dict_A, image_dict_B): 181 """Creates an ImagePair object for this pair of images. 182 183 Args: 184 test: string; name of the test 185 config: string; name of the config 186 image_dict_A: dict with JSONKEY_IMAGE_* keys, or None if no image 187 image_dict_B: dict with JSONKEY_IMAGE_* keys, or None if no image 188 189 Returns: 190 An ImagePair object, or None if both image_dict_A and image_dict_B are 191 None. 192 """ 193 if (not image_dict_A) and (not image_dict_B): 194 return None 195 196 def _checksum_and_relative_url(dic): 197 if dic: 198 return ((dic[gm_json.JSONKEY_IMAGE_CHECKSUMALGORITHM], 199 dic[gm_json.JSONKEY_IMAGE_CHECKSUMVALUE]), 200 dic[gm_json.JSONKEY_IMAGE_FILEPATH]) 201 else: 202 return None, None 203 204 imageA_checksum, imageA_relative_url = _checksum_and_relative_url( 205 image_dict_A) 206 imageB_checksum, imageB_relative_url = _checksum_and_relative_url( 207 image_dict_B) 208 209 if not imageA_checksum: 210 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON 211 elif not imageB_checksum: 212 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON 213 elif imageA_checksum == imageB_checksum: 214 result_type = results.KEY__RESULT_TYPE__SUCCEEDED 215 else: 216 result_type = results.KEY__RESULT_TYPE__FAILED 217 218 extra_columns_dict = { 219 results.KEY__EXTRACOLUMNS__CONFIG: config, 220 results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, 221 results.KEY__EXTRACOLUMNS__TEST: test, 222 # TODO(epoger): Right now, the client UI crashes if it receives 223 # results that do not include this column. 224 # Until we fix that, keep the client happy. 225 results.KEY__EXTRACOLUMNS__BUILDER: 'TODO', 226 } 227 228 try: 229 return imagepair.ImagePair( 230 image_diff_db=self._image_diff_db, 231 base_url=self._image_base_url, 232 imageA_relative_url=imageA_relative_url, 233 imageB_relative_url=imageB_relative_url, 234 extra_columns=extra_columns_dict) 235 except (KeyError, TypeError): 236 logging.exception( 237 'got exception while creating ImagePair for' 238 ' test="%s", config="%s", urlPair=("%s","%s")' % ( 239 test, config, imageA_relative_url, imageB_relative_url)) 240 return None 241 242 243 # TODO(epoger): Add main() so this can be called by vm_run_skia_try.sh 244
