1 #!/usr/bin/python 2 3 """ 4 Copyright 2013 Google Inc. 5 6 Use of this source code is governed by a BSD-style license that can be 7 found in the LICENSE file. 8 9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. 10 """ 11 12 # System-level imports 13 import fnmatch 14 import os 15 import re 16 17 # Imports from within Skia 18 import fix_pythonpath # must do this first 19 import gm_json 20 import imagepairset 21 22 # Keys used to link an image to a particular GM test. 23 # NOTE: Keep these in sync with static/constants.js 24 VALUE__HEADER__SCHEMA_VERSION = 3 25 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS 26 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE 27 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED 28 KEY__EXTRACOLUMNS__BUILDER = 'builder' 29 KEY__EXTRACOLUMNS__CONFIG = 'config' 30 KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType' 31 KEY__EXTRACOLUMNS__TEST = 'test' 32 KEY__HEADER__DATAHASH = 'dataHash' 33 KEY__HEADER__IS_EDITABLE = 'isEditable' 34 KEY__HEADER__IS_EXPORTED = 'isExported' 35 KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' 36 KEY__HEADER__RESULTS_ALL = 'all' 37 KEY__HEADER__RESULTS_FAILURES = 'failures' 38 KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' 39 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' 40 KEY__HEADER__TIME_UPDATED = 'timeUpdated' 41 KEY__HEADER__TYPE = 'type' 42 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED 43 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED 44 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON 45 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED 46 47 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 48 IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) 49 50 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) 51 DEFAULT_ACTUALS_DIR = '.gm-actuals' 52 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( 53 PARENT_DIRECTORY, '.generated-images') 54 55 # Define the default set of builders we will process expectations/actuals for. 56 # This allows us to ignore builders for which we don't maintain expectations 57 # (trybots, Valgrind, ASAN, TSAN), and avoid problems like 58 # https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server 59 # produces error when trying to add baselines for ASAN/TSAN builders') 60 DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*'] 61 DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [ 62 '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*'] 63 64 65 class BaseComparisons(object): 66 """Base class for generating summary of comparisons between two image sets. 67 """ 68 69 def get_results_of_type(self, results_type): 70 """Return results of some/all tests (depending on 'results_type' parameter). 71 72 Args: 73 results_type: string describing which types of results to include; must 74 be one of the RESULTS_* constants 75 76 Results are returned in a dictionary as output by ImagePairSet.as_dict(). 77 """ 78 return self._results[results_type] 79 80 def get_packaged_results_of_type(self, results_type, reload_seconds=None, 81 is_editable=False, is_exported=True): 82 """Package the results of some/all tests as a complete response_dict. 83 84 Args: 85 results_type: string indicating which set of results to return; 86 must be one of the RESULTS_* constants 87 reload_seconds: if specified, note that new results may be available once 88 these results are reload_seconds old 89 is_editable: whether clients are allowed to submit new baselines 90 is_exported: whether these results are being made available to other 91 network hosts 92 """ 93 response_dict = self._results[results_type] 94 time_updated = self.get_timestamp() 95 response_dict[imagepairset.KEY__ROOT__HEADER] = { 96 KEY__HEADER__SCHEMA_VERSION: ( 97 VALUE__HEADER__SCHEMA_VERSION), 98 99 # Timestamps: 100 # 1. when this data was last updated 101 # 2. when the caller should check back for new data (if ever) 102 KEY__HEADER__TIME_UPDATED: time_updated, 103 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( 104 (time_updated+reload_seconds) if reload_seconds else None), 105 106 # The type we passed to get_results_of_type() 107 KEY__HEADER__TYPE: results_type, 108 109 # Hash of dataset, which the client must return with any edits-- 110 # this ensures that the edits were made to a particular dataset. 111 KEY__HEADER__DATAHASH: str(hash(repr( 112 response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))), 113 114 # Whether the server will accept edits back. 115 KEY__HEADER__IS_EDITABLE: is_editable, 116 117 # Whether the service is accessible from other hosts. 118 KEY__HEADER__IS_EXPORTED: is_exported, 119 } 120 return response_dict 121 122 def get_timestamp(self): 123 """Return the time at which this object was created, in seconds past epoch 124 (UTC). 125 """ 126 return self._timestamp 127 128 _match_builders_pattern_list = [ 129 re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST] 130 _skip_builders_pattern_list = [ 131 re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST] 132 133 def set_match_builders_pattern_list(self, pattern_list): 134 """Override the default set of builders we should process. 135 136 The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST . 137 138 Note that skip_builders_pattern_list overrides this; regardless of whether a 139 builder is in the "match" list, if it's in the "skip" list, we will skip it. 140 141 Args: 142 pattern_list: list of regex patterns; process builders that match any 143 entry within this list 144 """ 145 if pattern_list == None: 146 pattern_list = [] 147 self._match_builders_pattern_list = [re.compile(p) for p in pattern_list] 148 149 def set_skip_builders_pattern_list(self, pattern_list): 150 """Override the default set of builders we should skip while processing. 151 152 The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST . 153 154 This overrides match_builders_pattern_list; regardless of whether a 155 builder is in the "match" list, if it's in the "skip" list, we will skip it. 156 157 Args: 158 pattern_list: list of regex patterns; skip builders that match any 159 entry within this list 160 """ 161 if pattern_list == None: 162 pattern_list = [] 163 self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list] 164 165 def _ignore_builder(self, builder): 166 """Returns True if we should skip processing this builder. 167 168 Args: 169 builder: name of this builder, as a string 170 171 Returns: 172 True if we should ignore expectations and actuals for this builder. 173 """ 174 for pattern in self._skip_builders_pattern_list: 175 if pattern.match(builder): 176 return True 177 for pattern in self._match_builders_pattern_list: 178 if pattern.match(builder): 179 return False 180 return True 181 182 def _read_builder_dicts_from_root(self, root, pattern='*.json'): 183 """Read all JSON dictionaries within a directory tree. 184 185 Skips any dictionaries belonging to a builder we have chosen to ignore. 186 187 Args: 188 root: path to root of directory tree 189 pattern: which files to read within root (fnmatch-style pattern) 190 191 Returns: 192 A meta-dictionary containing all the JSON dictionaries found within 193 the directory tree, keyed by builder name (the basename of the directory 194 where each JSON dictionary was found). 195 196 Raises: 197 IOError if root does not refer to an existing directory 198 """ 199 # I considered making this call _read_dicts_from_root(), but I decided 200 # it was better to prune out the ignored builders within the os.walk(). 201 if not os.path.isdir(root): 202 raise IOError('no directory found at path %s' % root) 203 meta_dict = {} 204 for dirpath, dirnames, filenames in os.walk(root): 205 for matching_filename in fnmatch.filter(filenames, pattern): 206 builder = os.path.basename(dirpath) 207 if self._ignore_builder(builder): 208 continue 209 full_path = os.path.join(dirpath, matching_filename) 210 meta_dict[builder] = gm_json.LoadFromFile(full_path) 211 return meta_dict 212 213 def _read_dicts_from_root(self, root, pattern='*.json'): 214 """Read all JSON dictionaries within a directory tree. 215 216 Args: 217 root: path to root of directory tree 218 pattern: which files to read within root (fnmatch-style pattern) 219 220 Returns: 221 A meta-dictionary containing all the JSON dictionaries found within 222 the directory tree, keyed by the pathname (relative to root) of each JSON 223 dictionary. 224 225 Raises: 226 IOError if root does not refer to an existing directory 227 """ 228 if not os.path.isdir(root): 229 raise IOError('no directory found at path %s' % root) 230 meta_dict = {} 231 for abs_dirpath, dirnames, filenames in os.walk(root): 232 rel_dirpath = os.path.relpath(abs_dirpath, root) 233 for matching_filename in fnmatch.filter(filenames, pattern): 234 abs_path = os.path.join(abs_dirpath, matching_filename) 235 rel_path = os.path.join(rel_dirpath, matching_filename) 236 meta_dict[rel_path] = gm_json.LoadFromFile(abs_path) 237 return meta_dict 238 239 @staticmethod 240 def _read_noncomment_lines(path): 241 """Return a list of all noncomment lines within a file. 242 243 (A "noncomment" line is one that does not start with a '#'.) 244 245 Args: 246 path: path to file 247 """ 248 lines = [] 249 with open(path, 'r') as fh: 250 for line in fh: 251 if not line.startswith('#'): 252 lines.append(line.strip()) 253 return lines 254 255 @staticmethod 256 def _create_relative_url(hashtype_and_digest, test_name): 257 """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL. 258 259 If we don't have a record of this image, returns None. 260 261 Args: 262 hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we 263 don't have a record of this image 264 test_name: string; name of the GM test that created this image 265 """ 266 if not hashtype_and_digest: 267 return None 268 return gm_json.CreateGmRelativeUrl( 269 test_name=test_name, 270 hash_type=hashtype_and_digest[0], 271 hash_digest=hashtype_and_digest[1]) 272 273 @staticmethod 274 def combine_subdicts(input_dict): 275 """ Flatten out a dictionary structure by one level. 276 277 Input: 278 { 279 KEY_A1 : { 280 KEY_B1 : VALUE_B1, 281 }, 282 KEY_A2 : { 283 KEY_B2 : VALUE_B2, 284 } 285 } 286 287 Output: 288 { 289 KEY_B1 : VALUE_B1, 290 KEY_B2 : VALUE_B2, 291 } 292 293 If this would result in any repeated keys, it will raise an Exception. 294 """ 295 output_dict = {} 296 for key, subdict in input_dict.iteritems(): 297 for subdict_key, subdict_value in subdict.iteritems(): 298 if subdict_key in output_dict: 299 raise Exception('duplicate key %s in combine_subdicts' % subdict_key) 300 output_dict[subdict_key] = subdict_value 301 return output_dict 302 303 @staticmethod 304 def get_multilevel(input_dict, *keys): 305 """ Returns input_dict[key1][key2][...], or None if any key is not found. 306 """ 307 for key in keys: 308 if input_dict == None: 309 return None 310 input_dict = input_dict.get(key, None) 311 return input_dict 312
