1 #!/usr/bin/python 2 3 """ 4 Copyright 2013 Google Inc. 5 6 Use of this source code is governed by a BSD-style license that can be 7 found in the LICENSE file. 8 9 Repackage expected/actual GM results as needed by our HTML rebaseline viewer. 10 """ 11 12 # System-level imports 13 import fnmatch 14 import os 15 import re 16 17 # Must fix up PYTHONPATH before importing from within Skia 18 import rs_fixpypath # pylint: disable=W0611 19 20 # Imports from within Skia 21 import gm_json 22 import imagepairset 23 24 # Keys used to link an image to a particular GM test. 25 # NOTE: Keep these in sync with static/constants.js 26 VALUE__HEADER__SCHEMA_VERSION = 5 27 KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS 28 KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE 29 KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED 30 KEY__EXTRACOLUMNS__BUILDER = 'builder' 31 KEY__EXTRACOLUMNS__CONFIG = 'config' 32 KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType' 33 KEY__EXTRACOLUMNS__TEST = 'test' 34 KEY__HEADER__DATAHASH = 'dataHash' 35 KEY__HEADER__IS_EDITABLE = 'isEditable' 36 KEY__HEADER__IS_EXPORTED = 'isExported' 37 KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' 38 KEY__HEADER__RESULTS_ALL = 'all' 39 KEY__HEADER__RESULTS_FAILURES = 'failures' 40 KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' 41 KEY__HEADER__SET_A_DESCRIPTIONS = 'setA' 42 KEY__HEADER__SET_B_DESCRIPTIONS = 'setB' 43 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' 44 KEY__HEADER__TIME_UPDATED = 'timeUpdated' 45 KEY__HEADER__TYPE = 'type' 46 KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED 47 KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED 48 KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON 49 KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED 50 KEY__SET_DESCRIPTIONS__DIR = 'dir' 51 KEY__SET_DESCRIPTIONS__REPO_REVISION = 'repoRevision' 52 KEY__SET_DESCRIPTIONS__SECTION = 'section' 53 54 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 55 IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) 56 57 PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) 58 DEFAULT_ACTUALS_DIR = '.gm-actuals' 59 DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( 60 PARENT_DIRECTORY, '.generated-images') 61 62 # Define the default set of builders we will process expectations/actuals for. 63 # This allows us to ignore builders for which we don't maintain expectations 64 # (trybots, Valgrind, ASAN, TSAN), and avoid problems like 65 # https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server 66 # produces error when trying to add baselines for ASAN/TSAN builders') 67 DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*'] 68 DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [ 69 '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*'] 70 71 72 class BaseComparisons(object): 73 """Base class for generating summary of comparisons between two image sets. 74 """ 75 76 def __init__(self): 77 """Base constructor; most subclasses will override.""" 78 self._setA_descriptions = None 79 self._setB_descriptions = None 80 81 def get_results_of_type(self, results_type): 82 """Return results of some/all tests (depending on 'results_type' parameter). 83 84 Args: 85 results_type: string describing which types of results to include; must 86 be one of the RESULTS_* constants 87 88 Results are returned in a dictionary as output by ImagePairSet.as_dict(). 89 """ 90 return self._results[results_type] 91 92 def get_packaged_results_of_type(self, results_type, reload_seconds=None, 93 is_editable=False, is_exported=True): 94 """Package the results of some/all tests as a complete response_dict. 95 96 Args: 97 results_type: string indicating which set of results to return; 98 must be one of the RESULTS_* constants 99 reload_seconds: if specified, note that new results may be available once 100 these results are reload_seconds old 101 is_editable: whether clients are allowed to submit new baselines 102 is_exported: whether these results are being made available to other 103 network hosts 104 """ 105 response_dict = self._results[results_type] 106 time_updated = self.get_timestamp() 107 header_dict = { 108 KEY__HEADER__SCHEMA_VERSION: ( 109 VALUE__HEADER__SCHEMA_VERSION), 110 111 # Timestamps: 112 # 1. when this data was last updated 113 # 2. when the caller should check back for new data (if ever) 114 KEY__HEADER__TIME_UPDATED: time_updated, 115 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( 116 (time_updated+reload_seconds) if reload_seconds else None), 117 118 # The type we passed to get_results_of_type() 119 KEY__HEADER__TYPE: results_type, 120 121 # Hash of dataset, which the client must return with any edits-- 122 # this ensures that the edits were made to a particular dataset. 123 KEY__HEADER__DATAHASH: str(hash(repr( 124 response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))), 125 126 # Whether the server will accept edits back. 127 KEY__HEADER__IS_EDITABLE: is_editable, 128 129 # Whether the service is accessible from other hosts. 130 KEY__HEADER__IS_EXPORTED: is_exported, 131 } 132 if self._setA_descriptions: 133 header_dict[KEY__HEADER__SET_A_DESCRIPTIONS] = self._setA_descriptions 134 if self._setB_descriptions: 135 header_dict[KEY__HEADER__SET_B_DESCRIPTIONS] = self._setB_descriptions 136 response_dict[imagepairset.KEY__ROOT__HEADER] = header_dict 137 return response_dict 138 139 def get_timestamp(self): 140 """Return the time at which this object was created, in seconds past epoch 141 (UTC). 142 """ 143 return self._timestamp 144 145 _match_builders_pattern_list = [ 146 re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST] 147 _skip_builders_pattern_list = [ 148 re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST] 149 150 def set_match_builders_pattern_list(self, pattern_list): 151 """Override the default set of builders we should process. 152 153 The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST . 154 155 Note that skip_builders_pattern_list overrides this; regardless of whether a 156 builder is in the "match" list, if it's in the "skip" list, we will skip it. 157 158 Args: 159 pattern_list: list of regex patterns; process builders that match any 160 entry within this list 161 """ 162 if pattern_list == None: 163 pattern_list = [] 164 self._match_builders_pattern_list = [re.compile(p) for p in pattern_list] 165 166 def set_skip_builders_pattern_list(self, pattern_list): 167 """Override the default set of builders we should skip while processing. 168 169 The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST . 170 171 This overrides match_builders_pattern_list; regardless of whether a 172 builder is in the "match" list, if it's in the "skip" list, we will skip it. 173 174 Args: 175 pattern_list: list of regex patterns; skip builders that match any 176 entry within this list 177 """ 178 if pattern_list == None: 179 pattern_list = [] 180 self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list] 181 182 def _ignore_builder(self, builder): 183 """Returns True if we should skip processing this builder. 184 185 Args: 186 builder: name of this builder, as a string 187 188 Returns: 189 True if we should ignore expectations and actuals for this builder. 190 """ 191 for pattern in self._skip_builders_pattern_list: 192 if pattern.match(builder): 193 return True 194 for pattern in self._match_builders_pattern_list: 195 if pattern.match(builder): 196 return False 197 return True 198 199 def _read_builder_dicts_from_root(self, root, pattern='*.json'): 200 """Read all JSON dictionaries within a directory tree. 201 202 Skips any dictionaries belonging to a builder we have chosen to ignore. 203 204 Args: 205 root: path to root of directory tree 206 pattern: which files to read within root (fnmatch-style pattern) 207 208 Returns: 209 A meta-dictionary containing all the JSON dictionaries found within 210 the directory tree, keyed by builder name (the basename of the directory 211 where each JSON dictionary was found). 212 213 Raises: 214 IOError if root does not refer to an existing directory 215 """ 216 # I considered making this call read_dicts_from_root(), but I decided 217 # it was better to prune out the ignored builders within the os.walk(). 218 if not os.path.isdir(root): 219 raise IOError('no directory found at path %s' % root) 220 meta_dict = {} 221 for dirpath, _, filenames in os.walk(root): 222 for matching_filename in fnmatch.filter(filenames, pattern): 223 builder = os.path.basename(dirpath) 224 if self._ignore_builder(builder): 225 continue 226 full_path = os.path.join(dirpath, matching_filename) 227 meta_dict[builder] = gm_json.LoadFromFile(full_path) 228 return meta_dict 229 230 @staticmethod 231 def read_dicts_from_root(root, pattern='*.json'): 232 """Read all JSON dictionaries within a directory tree. 233 234 TODO(stephana): Factor this out into a utility module, as a standalone 235 function (not part of a class). 236 237 Args: 238 root: path to root of directory tree 239 pattern: which files to read within root (fnmatch-style pattern) 240 241 Returns: 242 A meta-dictionary containing all the JSON dictionaries found within 243 the directory tree, keyed by the pathname (relative to root) of each JSON 244 dictionary. 245 246 Raises: 247 IOError if root does not refer to an existing directory 248 """ 249 if not os.path.isdir(root): 250 raise IOError('no directory found at path %s' % root) 251 meta_dict = {} 252 for abs_dirpath, _, filenames in os.walk(root): 253 rel_dirpath = os.path.relpath(abs_dirpath, root) 254 for matching_filename in fnmatch.filter(filenames, pattern): 255 abs_path = os.path.join(abs_dirpath, matching_filename) 256 rel_path = os.path.join(rel_dirpath, matching_filename) 257 meta_dict[rel_path] = gm_json.LoadFromFile(abs_path) 258 return meta_dict 259 260 @staticmethod 261 def _read_noncomment_lines(path): 262 """Return a list of all noncomment lines within a file. 263 264 (A "noncomment" line is one that does not start with a '#'.) 265 266 Args: 267 path: path to file 268 """ 269 lines = [] 270 with open(path, 'r') as fh: 271 for line in fh: 272 if not line.startswith('#'): 273 lines.append(line.strip()) 274 return lines 275 276 @staticmethod 277 def _create_relative_url(hashtype_and_digest, test_name): 278 """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL. 279 280 If we don't have a record of this image, returns None. 281 282 Args: 283 hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we 284 don't have a record of this image 285 test_name: string; name of the GM test that created this image 286 """ 287 if not hashtype_and_digest: 288 return None 289 return gm_json.CreateGmRelativeUrl( 290 test_name=test_name, 291 hash_type=hashtype_and_digest[0], 292 hash_digest=hashtype_and_digest[1]) 293 294 @staticmethod 295 def combine_subdicts(input_dict): 296 """ Flatten out a dictionary structure by one level. 297 298 Input: 299 { 300 KEY_A1 : { 301 KEY_B1 : VALUE_B1, 302 }, 303 KEY_A2 : { 304 KEY_B2 : VALUE_B2, 305 } 306 } 307 308 Output: 309 { 310 KEY_B1 : VALUE_B1, 311 KEY_B2 : VALUE_B2, 312 } 313 314 If this would result in any repeated keys, it will raise an Exception. 315 """ 316 output_dict = {} 317 for subdict in input_dict.values(): 318 for subdict_key, subdict_value in subdict.iteritems(): 319 if subdict_key in output_dict: 320 raise Exception('duplicate key %s in combine_subdicts' % subdict_key) 321 output_dict[subdict_key] = subdict_value 322 return output_dict 323 324 @staticmethod 325 def get_default(input_dict, default_value, *keys): 326 """Returns input_dict[key1][key2][...], or default_value. 327 328 If input_dict is None, or any key is missing along the way, this returns 329 default_value. 330 331 Args: 332 input_dict: dictionary to look within 333 key: key indicating which value to return from input_dict 334 default_value: value to return if input_dict is None or any key cannot 335 be found along the way 336 """ 337 if input_dict == None: 338 return default_value 339 for key in keys: 340 input_dict = input_dict.get(key, None) 341 if input_dict == None: 342 return default_value 343 return input_dict 344
