1 #!/usr/bin/python 2 3 ''' 4 Copyright 2012 Google Inc. 5 6 Use of this source code is governed by a BSD-style license that can be 7 found in the LICENSE file. 8 ''' 9 10 ''' 11 Rebaselines the given GM tests, on all bots and all configurations. 12 ''' 13 14 # System-level imports 15 import argparse 16 import os 17 import re 18 import subprocess 19 import sys 20 import urllib2 21 22 # Imports from within Skia 23 # 24 # We need to add the 'gm' directory, so that we can import gm_json.py within 25 # that directory. That script allows us to parse the actual-results.json file 26 # written out by the GM tool. 27 # Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 28 # so any dirs that are already in the PYTHONPATH will be preferred. 29 # 30 # This assumes that the 'gm' directory has been checked out as a sibling of 31 # the 'tools' directory containing this script, which will be the case if 32 # 'trunk' was checked out as a single unit. 33 GM_DIRECTORY = os.path.realpath( 34 os.path.join(os.path.dirname(os.path.dirname(__file__)), 'gm')) 35 if GM_DIRECTORY not in sys.path: 36 sys.path.append(GM_DIRECTORY) 37 import gm_json 38 39 # Mapping of expectations/gm subdir (under 40 # https://skia.googlecode.com/svn/trunk/expectations/gm/ ) 41 # to builder name (see list at http://108.170.217.252:10117/builders ) 42 SUBDIR_MAPPING = { 43 'base-shuttle-win7-intel-float': 44 'Test-Win7-ShuttleA-HD2000-x86-Release', 45 'base-shuttle-win7-intel-angle': 46 'Test-Win7-ShuttleA-HD2000-x86-Release-ANGLE', 47 'base-shuttle-win7-intel-directwrite': 48 'Test-Win7-ShuttleA-HD2000-x86-Release-DirectWrite', 49 'base-shuttle_ubuntu12_ati5770': 50 'Test-Ubuntu12-ShuttleA-ATI5770-x86_64-Release', 51 'base-macmini': 52 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Release', 53 'base-macmini-lion-float': 54 'Test-Mac10.7-MacMini4.1-GeForce320M-x86-Release', 55 'base-android-galaxy-nexus': 56 'Test-Android-GalaxyNexus-SGX540-Arm7-Debug', 57 'base-android-nexus-7': 58 'Test-Android-Nexus7-Tegra3-Arm7-Release', 59 'base-android-nexus-s': 60 'Test-Android-NexusS-SGX540-Arm7-Release', 61 'base-android-xoom': 62 'Test-Android-Xoom-Tegra2-Arm7-Release', 63 'base-android-nexus-10': 64 'Test-Android-Nexus10-MaliT604-Arm7-Release', 65 'base-android-nexus-4': 66 'Test-Android-Nexus4-Adreno320-Arm7-Release', 67 } 68 69 70 class _InternalException(Exception): 71 pass 72 73 # Object that handles exceptions, either raising them immediately or collecting 74 # them to display later on. 75 class ExceptionHandler(object): 76 77 # params: 78 # keep_going_on_failure: if False, report failures and quit right away; 79 # if True, collect failures until 80 # ReportAllFailures() is called 81 def __init__(self, keep_going_on_failure=False): 82 self._keep_going_on_failure = keep_going_on_failure 83 self._failures_encountered = [] 84 self._exiting = False 85 86 # Exit the program with the given status value. 87 def _Exit(self, status=1): 88 self._exiting = True 89 sys.exit(status) 90 91 # We have encountered an exception; either collect the info and keep going, 92 # or exit the program right away. 93 def RaiseExceptionOrContinue(self, e): 94 # If we are already quitting the program, propagate any exceptions 95 # so that the proper exit status will be communicated to the shell. 96 if self._exiting: 97 raise e 98 99 if self._keep_going_on_failure: 100 print >> sys.stderr, 'WARNING: swallowing exception %s' % e 101 self._failures_encountered.append(e) 102 else: 103 print >> sys.stderr, e 104 print >> sys.stderr, ( 105 'Halting at first exception; to keep going, re-run ' + 106 'with the --keep-going-on-failure option set.') 107 self._Exit() 108 109 def ReportAllFailures(self): 110 if self._failures_encountered: 111 print >> sys.stderr, ('Encountered %d failures (see above).' % 112 len(self._failures_encountered)) 113 self._Exit() 114 115 116 # Object that rebaselines a JSON expectations file (not individual image files). 117 class JsonRebaseliner(object): 118 119 # params: 120 # expectations_root: root directory of all expectations JSON files 121 # expectations_input_filename: filename (under expectations_root) of JSON 122 # expectations file to read; typically 123 # "expected-results.json" 124 # expectations_output_filename: filename (under expectations_root) to 125 # which updated expectations should be 126 # written; typically the same as 127 # expectations_input_filename, to overwrite 128 # the old content 129 # actuals_base_url: base URL from which to read actual-result JSON files 130 # actuals_filename: filename (under actuals_base_url) from which to read a 131 # summary of results; typically "actual-results.json" 132 # exception_handler: reference to rebaseline.ExceptionHandler object 133 # tests: list of tests to rebaseline, or None if we should rebaseline 134 # whatever files the JSON results summary file tells us to 135 # configs: which configs to run for each test, or None if we should 136 # rebaseline whatever configs the JSON results summary file tells 137 # us to 138 # add_new: if True, add expectations for tests which don't have any yet 139 def __init__(self, expectations_root, expectations_input_filename, 140 expectations_output_filename, actuals_base_url, 141 actuals_filename, exception_handler, 142 tests=None, configs=None, add_new=False): 143 self._expectations_root = expectations_root 144 self._expectations_input_filename = expectations_input_filename 145 self._expectations_output_filename = expectations_output_filename 146 self._tests = tests 147 self._configs = configs 148 self._actuals_base_url = actuals_base_url 149 self._actuals_filename = actuals_filename 150 self._exception_handler = exception_handler 151 self._add_new = add_new 152 self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 153 self._using_svn = os.path.isdir(os.path.join(expectations_root, '.svn')) 154 155 # Executes subprocess.call(cmd). 156 # Raises an Exception if the command fails. 157 def _Call(self, cmd): 158 if subprocess.call(cmd) != 0: 159 raise _InternalException('error running command: ' + ' '.join(cmd)) 160 161 # Returns the full contents of filepath, as a single string. 162 # If filepath looks like a URL, try to read it that way instead of as 163 # a path on local storage. 164 # 165 # Raises _InternalException if there is a problem. 166 def _GetFileContents(self, filepath): 167 if filepath.startswith('http:') or filepath.startswith('https:'): 168 try: 169 return urllib2.urlopen(filepath).read() 170 except urllib2.HTTPError as e: 171 raise _InternalException('unable to read URL %s: %s' % ( 172 filepath, e)) 173 else: 174 return open(filepath, 'r').read() 175 176 # Returns a dictionary of actual results from actual-results.json file. 177 # 178 # The dictionary returned has this format: 179 # { 180 # u'imageblur_565.png': [u'bitmap-64bitMD5', 3359963596899141322], 181 # u'imageblur_8888.png': [u'bitmap-64bitMD5', 4217923806027861152], 182 # u'shadertext3_8888.png': [u'bitmap-64bitMD5', 3713708307125704716] 183 # } 184 # 185 # If the JSON actual result summary file cannot be loaded, logs a warning 186 # message and returns None. 187 # If the JSON actual result summary file can be loaded, but we have 188 # trouble parsing it, raises an Exception. 189 # 190 # params: 191 # json_url: URL pointing to a JSON actual result summary file 192 # sections: a list of section names to include in the results, e.g. 193 # [gm_json.JSONKEY_ACTUALRESULTS_FAILED, 194 # gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON] ; 195 # if None, then include ALL sections. 196 def _GetActualResults(self, json_url, sections=None): 197 try: 198 json_contents = self._GetFileContents(json_url) 199 except _InternalException: 200 print >> sys.stderr, ( 201 'could not read json_url %s ; skipping this platform.' % 202 json_url) 203 return None 204 json_dict = gm_json.LoadFromString(json_contents) 205 results_to_return = {} 206 actual_results = json_dict[gm_json.JSONKEY_ACTUALRESULTS] 207 if not sections: 208 sections = actual_results.keys() 209 for section in sections: 210 section_results = actual_results[section] 211 if section_results: 212 results_to_return.update(section_results) 213 return results_to_return 214 215 # Rebaseline all tests/types we specified in the constructor, 216 # within this expectations/gm subdir. 217 # 218 # params: 219 # subdir : e.g. 'base-shuttle-win7-intel-float' 220 # builder : e.g. 'Test-Win7-ShuttleA-HD2000-x86-Release' 221 def RebaselineSubdir(self, subdir, builder): 222 # Read in the actual result summary, and extract all the tests whose 223 # results we need to update. 224 actuals_url = '/'.join([self._actuals_base_url, 225 subdir, builder, subdir, 226 self._actuals_filename]) 227 # In most cases, we won't need to re-record results that are already 228 # succeeding, but including the SUCCEEDED results will allow us to 229 # re-record expectations if they somehow get out of sync. 230 sections = [gm_json.JSONKEY_ACTUALRESULTS_FAILED, 231 gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED] 232 if self._add_new: 233 sections.append(gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON) 234 results_to_update = self._GetActualResults(json_url=actuals_url, 235 sections=sections) 236 237 # Read in current expectations. 238 expectations_input_filepath = os.path.join( 239 self._expectations_root, subdir, self._expectations_input_filename) 240 expectations_dict = gm_json.LoadFromFile(expectations_input_filepath) 241 expected_results = expectations_dict[gm_json.JSONKEY_EXPECTEDRESULTS] 242 243 # Update the expectations in memory, skipping any tests/configs that 244 # the caller asked to exclude. 245 skipped_images = [] 246 if results_to_update: 247 for (image_name, image_results) in results_to_update.iteritems(): 248 (test, config) = self._image_filename_re.match(image_name).groups() 249 if self._tests: 250 if test not in self._tests: 251 skipped_images.append(image_name) 252 continue 253 if self._configs: 254 if config not in self._configs: 255 skipped_images.append(image_name) 256 continue 257 if not expected_results.get(image_name): 258 expected_results[image_name] = {} 259 expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS] = \ 260 [image_results] 261 262 # Write out updated expectations. 263 expectations_output_filepath = os.path.join( 264 self._expectations_root, subdir, self._expectations_output_filename) 265 gm_json.WriteToFile(expectations_dict, expectations_output_filepath) 266 267 # Mark the JSON file as plaintext, so text-style diffs can be applied. 268 # Fixes https://code.google.com/p/skia/issues/detail?id=1442 269 if self._using_svn: 270 self._Call(['svn', 'propset', '--quiet', 'svn:mime-type', 271 'text/x-json', expectations_output_filepath]) 272 273 # main... 274 275 parser = argparse.ArgumentParser() 276 parser.add_argument('--actuals-base-url', 277 help='base URL from which to read files containing JSON ' + 278 'summaries of actual GM results; defaults to %(default)s', 279 default='http://skia-autogen.googlecode.com/svn/gm-actual') 280 parser.add_argument('--actuals-filename', 281 help='filename (within platform-specific subdirectories ' + 282 'of ACTUALS_BASE_URL) to read a summary of results from; ' + 283 'defaults to %(default)s', 284 default='actual-results.json') 285 # TODO(epoger): Add test that exercises --add-new argument. 286 parser.add_argument('--add-new', action='store_true', 287 help='in addition to the standard behavior of ' + 288 'updating expectations for failing tests, add ' + 289 'expectations for tests which don\'t have expectations ' + 290 'yet.') 291 # TODO(epoger): Add test that exercises --configs argument. 292 parser.add_argument('--configs', metavar='CONFIG', nargs='+', 293 help='which configurations to rebaseline, e.g. ' + 294 '"--configs 565 8888", as a filter over the full set of ' + 295 'results in ACTUALS_FILENAME; if unspecified, rebaseline ' + 296 '*all* configs that are available.') 297 parser.add_argument('--expectations-filename', 298 help='filename (under EXPECTATIONS_ROOT) to read ' + 299 'current expectations from, and to write new ' + 300 'expectations into (unless a separate ' + 301 'EXPECTATIONS_FILENAME_OUTPUT has been specified); ' + 302 'defaults to %(default)s', 303 default='expected-results.json') 304 parser.add_argument('--expectations-filename-output', 305 help='filename (under EXPECTATIONS_ROOT) to write ' + 306 'updated expectations into; by default, overwrites the ' + 307 'input file (EXPECTATIONS_FILENAME)', 308 default='') 309 parser.add_argument('--expectations-root', 310 help='root of expectations directory to update-- should ' + 311 'contain one or more base-* subdirectories. Defaults to ' + 312 '%(default)s', 313 default=os.path.join('expectations', 'gm')) 314 parser.add_argument('--keep-going-on-failure', action='store_true', 315 help='instead of halting at the first error encountered, ' + 316 'keep going and rebaseline as many tests as possible, ' + 317 'and then report the full set of errors at the end') 318 parser.add_argument('--subdirs', metavar='SUBDIR', nargs='+', 319 help='which platform subdirectories to rebaseline; ' + 320 'if unspecified, rebaseline all subdirs, same as ' + 321 '"--subdirs %s"' % ' '.join(sorted(SUBDIR_MAPPING.keys()))) 322 # TODO(epoger): Add test that exercises --tests argument. 323 parser.add_argument('--tests', metavar='TEST', nargs='+', 324 help='which tests to rebaseline, e.g. ' + 325 '"--tests aaclip bigmatrix", as a filter over the full ' + 326 'set of results in ACTUALS_FILENAME; if unspecified, ' + 327 'rebaseline *all* tests that are available.') 328 args = parser.parse_args() 329 exception_handler = ExceptionHandler( 330 keep_going_on_failure=args.keep_going_on_failure) 331 if args.subdirs: 332 subdirs = args.subdirs 333 missing_json_is_fatal = True 334 else: 335 subdirs = sorted(SUBDIR_MAPPING.keys()) 336 missing_json_is_fatal = False 337 for subdir in subdirs: 338 if not subdir in SUBDIR_MAPPING.keys(): 339 raise Exception(('unrecognized platform subdir "%s"; ' + 340 'should be one of %s') % ( 341 subdir, SUBDIR_MAPPING.keys())) 342 builder = SUBDIR_MAPPING[subdir] 343 344 # We instantiate different Rebaseliner objects depending 345 # on whether we are rebaselining an expected-results.json file, or 346 # individual image files. Different expectations/gm subdirectories may move 347 # from individual image files to JSON-format expectations at different 348 # times, so we need to make this determination per subdirectory. 349 # 350 # See https://goto.google.com/ChecksumTransitionDetail 351 expectations_json_file = os.path.join(args.expectations_root, subdir, 352 args.expectations_filename) 353 if os.path.isfile(expectations_json_file): 354 rebaseliner = JsonRebaseliner( 355 expectations_root=args.expectations_root, 356 expectations_input_filename=args.expectations_filename, 357 expectations_output_filename=(args.expectations_filename_output or 358 args.expectations_filename), 359 tests=args.tests, configs=args.configs, 360 actuals_base_url=args.actuals_base_url, 361 actuals_filename=args.actuals_filename, 362 exception_handler=exception_handler, 363 add_new=args.add_new) 364 try: 365 rebaseliner.RebaselineSubdir(subdir=subdir, builder=builder) 366 except BaseException as e: 367 exception_handler.RaiseExceptionOrContinue(e) 368 else: 369 exception_handler.RaiseExceptionOrContinue(_InternalException( 370 'expectations_json_file %s not found' % expectations_json_file)) 371 372 exception_handler.ReportAllFailures() 373
