1 #!/usr/bin/python 2 # -*- coding: utf-8 -*- 3 4 from __future__ import print_function 5 import argparse 6 import BaseHTTPServer 7 import json 8 import os 9 import os.path 10 import re 11 import subprocess 12 import sys 13 import tempfile 14 import urllib2 15 16 # Grab the script path because that is where all the static assets are 17 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 18 19 # Find the tools directory for python imports 20 TOOLS_DIR = os.path.dirname(SCRIPT_DIR) 21 22 # Find the root of the skia trunk for finding skpdiff binary 23 SKIA_ROOT_DIR = os.path.dirname(TOOLS_DIR) 24 25 # Find the default location of gm expectations 26 DEFAULT_GM_EXPECTATIONS_DIR = os.path.join(SKIA_ROOT_DIR, 'expectations', 'gm') 27 28 # Imports from within Skia 29 if TOOLS_DIR not in sys.path: 30 sys.path.append(TOOLS_DIR) 31 GM_DIR = os.path.join(SKIA_ROOT_DIR, 'gm') 32 if GM_DIR not in sys.path: 33 sys.path.append(GM_DIR) 34 import gm_json 35 import jsondiff 36 37 # A simple dictionary of file name extensions to MIME types. The empty string 38 # entry is used as the default when no extension was given or if the extension 39 # has no entry in this dictionary. 40 MIME_TYPE_MAP = {'': 'application/octet-stream', 41 'html': 'text/html', 42 'css': 'text/css', 43 'png': 'image/png', 44 'js': 'application/javascript', 45 'json': 'application/json' 46 } 47 48 49 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 50 51 SKPDIFF_INVOKE_FORMAT = '{} --jsonp=false -o {} -f {} {}' 52 53 54 def get_skpdiff_path(user_path=None): 55 """Find the skpdiff binary. 56 57 @param user_path If none, searches in Release and Debug out directories of 58 the skia root. If set, checks that the path is a real file and 59 returns it. 60 """ 61 skpdiff_path = None 62 possible_paths = [] 63 64 # Use the user given path, or try out some good default paths. 65 if user_path: 66 possible_paths.append(user_path) 67 else: 68 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', 69 'Release', 'skpdiff')) 70 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', 71 'Debug', 'skpdiff')) 72 # Use the first path that actually points to the binary 73 for possible_path in possible_paths: 74 if os.path.isfile(possible_path): 75 skpdiff_path = possible_path 76 break 77 78 # If skpdiff was not found, print out diagnostic info for the user. 79 if skpdiff_path is None: 80 print('Could not find skpdiff binary. Either build it into the ' + 81 'default directory, or specify the path on the command line.') 82 print('skpdiff paths tried:') 83 for possible_path in possible_paths: 84 print(' ', possible_path) 85 return skpdiff_path 86 87 88 def download_file(url, output_path): 89 """Download the file at url and place it in output_path""" 90 reader = urllib2.urlopen(url) 91 with open(output_path, 'wb') as writer: 92 writer.write(reader.read()) 93 94 95 def download_gm_image(image_name, image_path, hash_val): 96 """Download the gm result into the given path. 97 98 @param image_name The GM file name, for example imageblur_gpu.png. 99 @param image_path Path to place the image. 100 @param hash_val The hash value of the image. 101 """ 102 if hash_val is None: 103 return 104 105 # Separate the test name from a image name 106 image_match = IMAGE_FILENAME_RE.match(image_name) 107 test_name = image_match.group(1) 108 109 # Calculate the URL of the requested image 110 image_url = gm_json.CreateGmActualUrl( 111 test_name, gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, hash_val) 112 113 # Download the image as requested 114 download_file(image_url, image_path) 115 116 117 def get_image_set_from_skpdiff(skpdiff_records): 118 """Get the set of all images references in the given records. 119 120 @param skpdiff_records An array of records, which are dictionary objects. 121 """ 122 expected_set = frozenset([r['baselinePath'] for r in skpdiff_records]) 123 actual_set = frozenset([r['testPath'] for r in skpdiff_records]) 124 return expected_set | actual_set 125 126 127 def set_expected_hash_in_json(expected_results_json, image_name, hash_value): 128 """Set the expected hash for the object extracted from 129 expected-results.json. Note that this only work with bitmap-64bitMD5 hash 130 types. 131 132 @param expected_results_json The Python dictionary with the results to 133 modify. 134 @param image_name The name of the image to set the hash of. 135 @param hash_value The hash to set for the image. 136 """ 137 expected_results = expected_results_json[gm_json.JSONKEY_EXPECTEDRESULTS] 138 139 if image_name in expected_results: 140 expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0][1] = hash_value 141 else: 142 expected_results[image_name] = { 143 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: 144 [ 145 [ 146 gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 147 hash_value 148 ] 149 ] 150 } 151 152 153 def get_head_version(path): 154 """Get the version of the file at the given path stored inside the HEAD of 155 the git repository. It is returned as a string. 156 157 @param path The path of the file whose HEAD is returned. It is assumed the 158 path is inside a git repo rooted at SKIA_ROOT_DIR. 159 """ 160 161 # git-show will not work with absolute paths. This ensures we give it a path 162 # relative to the skia root. 163 git_path = os.path.relpath(path, SKIA_ROOT_DIR) 164 git_show_proc = subprocess.Popen(['git', 'show', 'HEAD:' + git_path], 165 stdout=subprocess.PIPE) 166 167 # When invoked outside a shell, git will output the last committed version 168 # of the file directly to stdout. 169 git_version_content, _ = git_show_proc.communicate() 170 return git_version_content 171 172 173 class GMInstance: 174 """Information about a GM test result on a specific device: 175 - device_name = the name of the device that rendered it 176 - image_name = the GM test name and config 177 - expected_hash = the current expected hash value 178 - actual_hash = the actual hash value 179 - is_rebaselined = True if actual_hash is what is currently in the expected 180 results file, False otherwise. 181 """ 182 def __init__(self, 183 device_name, image_name, 184 expected_hash, actual_hash, 185 is_rebaselined): 186 self.device_name = device_name 187 self.image_name = image_name 188 self.expected_hash = expected_hash 189 self.actual_hash = actual_hash 190 self.is_rebaselined = is_rebaselined 191 192 193 class ExpectationsManager: 194 def __init__(self, expectations_dir, expected_name, updated_name, 195 skpdiff_path): 196 """ 197 @param expectations_dir The directory to traverse for results files. 198 This should resemble expectations/gm in the Skia trunk. 199 @param expected_name The name of the expected result files. These 200 are in the format of expected-results.json. 201 @param updated_name The name of the updated expected result files. 202 Normally this matches --expectations-filename-output for the 203 rebaseline.py tool. 204 @param skpdiff_path The path used to execute the skpdiff command. 205 """ 206 self._expectations_dir = expectations_dir 207 self._expected_name = expected_name 208 self._updated_name = updated_name 209 self._skpdiff_path = skpdiff_path 210 self._generate_gm_comparison() 211 212 def _generate_gm_comparison(self): 213 """Generate all the data needed to compare GMs: 214 - determine which GMs changed 215 - download the changed images 216 - compare them with skpdiff 217 """ 218 219 # Get the expectations and compare them with actual hashes 220 self._get_expectations() 221 222 223 # Create a temporary file tree that makes sense for skpdiff to operate 224 # on. 225 image_output_dir = tempfile.mkdtemp('skpdiff') 226 expected_image_dir = os.path.join(image_output_dir, 'expected') 227 actual_image_dir = os.path.join(image_output_dir, 'actual') 228 os.mkdir(expected_image_dir) 229 os.mkdir(actual_image_dir) 230 231 # Download expected and actual images that differed into the temporary 232 # file tree. 233 self._download_expectation_images(expected_image_dir, actual_image_dir) 234 235 # Invoke skpdiff with our downloaded images and place its results in the 236 # temporary directory. 237 self._skpdiff_output_path = os.path.join(image_output_dir, 238 'skpdiff_output.json') 239 skpdiff_cmd = SKPDIFF_INVOKE_FORMAT.format(self._skpdiff_path, 240 self._skpdiff_output_path, 241 expected_image_dir, 242 actual_image_dir) 243 os.system(skpdiff_cmd) 244 self._load_skpdiff_output() 245 246 247 def _get_expectations(self): 248 """Fills self._expectations with GMInstance objects for each test whose 249 expectation is different between the following two files: 250 - the local filesystem's updated results file 251 - git's head version of the expected results file 252 """ 253 differ = jsondiff.GMDiffer() 254 self._expectations = [] 255 for root, dirs, files in os.walk(self._expectations_dir): 256 for expectation_file in files: 257 # There are many files in the expectations directory. We only 258 # care about expected results. 259 if expectation_file != self._expected_name: 260 continue 261 262 # Get the name of the results file, and be sure there is an 263 # updated result to compare against. If there is not, there is 264 # no point in diffing this device. 265 expected_file_path = os.path.join(root, self._expected_name) 266 updated_file_path = os.path.join(root, self._updated_name) 267 if not os.path.isfile(updated_file_path): 268 continue 269 270 # Always get the expected results from git because we may have 271 # changed them in a previous instance of the server. 272 expected_contents = get_head_version(expected_file_path) 273 updated_contents = None 274 with open(updated_file_path, 'rb') as updated_file: 275 updated_contents = updated_file.read() 276 277 # Read the expected results on disk to determine what we've 278 # already rebaselined. 279 commited_contents = None 280 with open(expected_file_path, 'rb') as expected_file: 281 commited_contents = expected_file.read() 282 283 # Find all expectations that did not match. 284 expected_diff = differ.GenerateDiffDictFromStrings( 285 expected_contents, 286 updated_contents) 287 288 # Generate a set of images that have already been rebaselined 289 # onto disk. 290 rebaselined_diff = differ.GenerateDiffDictFromStrings( 291 expected_contents, 292 commited_contents) 293 294 rebaselined_set = set(rebaselined_diff.keys()) 295 296 # The name of the device corresponds to the name of the folder 297 # we are in. 298 device_name = os.path.basename(root) 299 300 # Store old and new versions of the expectation for each GM 301 for image_name, hashes in expected_diff.iteritems(): 302 self._expectations.append( 303 GMInstance(device_name, image_name, 304 hashes['old'], hashes['new'], 305 image_name in rebaselined_set)) 306 307 def _load_skpdiff_output(self): 308 """Loads the results of skpdiff and annotates them with whether they 309 have already been rebaselined or not. The resulting data is store in 310 self.skpdiff_records.""" 311 self.skpdiff_records = None 312 with open(self._skpdiff_output_path, 'rb') as skpdiff_output_file: 313 self.skpdiff_records = json.load(skpdiff_output_file)['records'] 314 for record in self.skpdiff_records: 315 record['isRebaselined'] = self.image_map[record['baselinePath']][1].is_rebaselined 316 317 318 def _download_expectation_images(self, expected_image_dir, actual_image_dir): 319 """Download the expected and actual images for the _expectations array. 320 321 @param expected_image_dir The directory to download expected images 322 into. 323 @param actual_image_dir The directory to download actual images into. 324 """ 325 image_map = {} 326 327 # Look through expectations and download their images. 328 for expectation in self._expectations: 329 # Build appropriate paths to download the images into. 330 expected_image_path = os.path.join(expected_image_dir, 331 expectation.device_name + '-' + 332 expectation.image_name) 333 334 actual_image_path = os.path.join(actual_image_dir, 335 expectation.device_name + '-' + 336 expectation.image_name) 337 338 print('Downloading %s for device %s' % ( 339 expectation.image_name, expectation.device_name)) 340 341 # Download images 342 download_gm_image(expectation.image_name, 343 expected_image_path, 344 expectation.expected_hash) 345 346 download_gm_image(expectation.image_name, 347 actual_image_path, 348 expectation.actual_hash) 349 350 # Annotate the expectations with where the images were downloaded 351 # to. 352 expectation.expected_image_path = expected_image_path 353 expectation.actual_image_path = actual_image_path 354 355 # Map the image paths back to the expectations. 356 image_map[expected_image_path] = (False, expectation) 357 image_map[actual_image_path] = (True, expectation) 358 359 self.image_map = image_map 360 361 def _set_expected_hash(self, device_name, image_name, hash_value): 362 """Set the expected hash for the image of the given device. This always 363 writes directly to the expected results file of the given device 364 365 @param device_name The name of the device to write the hash to. 366 @param image_name The name of the image whose hash to set. 367 @param hash_value The value of the hash to set. 368 """ 369 370 # Retrieve the expected results file as it is in the working tree 371 json_path = os.path.join(self._expectations_dir, device_name, 372 self._expected_name) 373 expectations = gm_json.LoadFromFile(json_path) 374 375 # Set the specified hash. 376 set_expected_hash_in_json(expectations, image_name, hash_value) 377 378 # Write it out to disk using gm_json to keep the formatting consistent. 379 gm_json.WriteToFile(expectations, json_path) 380 381 def commit_rebaselines(self, rebaselines): 382 """Sets the expected results file to use the hashes of the images in 383 the rebaselines list. If a expected result image is not in rebaselines 384 at all, the old hash will be used. 385 386 @param rebaselines A list of image paths to use the hash of. 387 """ 388 # Reset all expectations to their old hashes because some of them may 389 # have been set to the new hash by a previous call to this function. 390 for expectation in self._expectations: 391 expectation.is_rebaselined = False 392 self._set_expected_hash(expectation.device_name, 393 expectation.image_name, 394 expectation.expected_hash) 395 396 # Take all the images to rebaseline 397 for image_path in rebaselines: 398 # Get the metadata about the image at the path. 399 is_actual, expectation = self.image_map[image_path] 400 401 expectation.is_rebaselined = is_actual 402 expectation_hash = expectation.actual_hash if is_actual else\ 403 expectation.expected_hash 404 405 # Write out that image's hash directly to the expected results file. 406 self._set_expected_hash(expectation.device_name, 407 expectation.image_name, 408 expectation_hash) 409 410 self._load_skpdiff_output() 411 412 413 class SkPDiffHandler(BaseHTTPServer.BaseHTTPRequestHandler): 414 def send_file(self, file_path): 415 # Grab the extension if there is one 416 extension = os.path.splitext(file_path)[1] 417 if len(extension) >= 1: 418 extension = extension[1:] 419 420 # Determine the MIME type of the file from its extension 421 mime_type = MIME_TYPE_MAP.get(extension, MIME_TYPE_MAP['']) 422 423 # Open the file and send it over HTTP 424 if os.path.isfile(file_path): 425 with open(file_path, 'rb') as sending_file: 426 self.send_response(200) 427 self.send_header('Content-type', mime_type) 428 self.end_headers() 429 self.wfile.write(sending_file.read()) 430 else: 431 self.send_error(404) 432 433 def serve_if_in_dir(self, dir_path, file_path): 434 # Determine if the file exists relative to the given dir_path AND exists 435 # under the dir_path. This is to prevent accidentally serving files 436 # outside the directory intended using symlinks, or '../'. 437 real_path = os.path.normpath(os.path.join(dir_path, file_path)) 438 if os.path.commonprefix([real_path, dir_path]) == dir_path: 439 if os.path.isfile(real_path): 440 self.send_file(real_path) 441 return True 442 return False 443 444 def do_GET(self): 445 # Simple rewrite rule of the root path to 'viewer.html' 446 if self.path == '' or self.path == '/': 447 self.path = '/viewer.html' 448 449 # The [1:] chops off the leading '/' 450 file_path = self.path[1:] 451 452 # Handle skpdiff_output.json manually because it is was processed by the 453 # server when it was started and does not exist as a file. 454 if file_path == 'skpdiff_output.json': 455 self.send_response(200) 456 self.send_header('Content-type', MIME_TYPE_MAP['json']) 457 self.end_headers() 458 459 # Add JSONP padding to the JSON because the web page expects it. It 460 # expects it because it was designed to run with or without a web 461 # server. Without a web server, the only way to load JSON is with 462 # JSONP. 463 skpdiff_records = self.server.expectations_manager.skpdiff_records 464 self.wfile.write('var SkPDiffRecords = ') 465 json.dump({'records': skpdiff_records}, self.wfile) 466 self.wfile.write(';') 467 return 468 469 # Attempt to send static asset files first. 470 if self.serve_if_in_dir(SCRIPT_DIR, file_path): 471 return 472 473 # WARNING: Serving any file the user wants is incredibly insecure. Its 474 # redeeming quality is that we only serve gm files on a white list. 475 if self.path in self.server.image_set: 476 self.send_file(self.path) 477 return 478 479 # If no file to send was found, just give the standard 404 480 self.send_error(404) 481 482 def do_POST(self): 483 if self.path == '/commit_rebaselines': 484 content_length = int(self.headers['Content-length']) 485 request_data = json.loads(self.rfile.read(content_length)) 486 rebaselines = request_data['rebaselines'] 487 self.server.expectations_manager.commit_rebaselines(rebaselines) 488 self.send_response(200) 489 self.send_header('Content-type', 'application/json') 490 self.end_headers() 491 self.wfile.write('{"success":true}') 492 return 493 494 # If the we have no handler for this path, give em' the 404 495 self.send_error(404) 496 497 498 def run_server(expectations_manager, port=8080): 499 # It's important to parse the results file so that we can make a set of 500 # images that the web page might request. 501 skpdiff_records = expectations_manager.skpdiff_records 502 image_set = get_image_set_from_skpdiff(skpdiff_records) 503 504 # Do not bind to interfaces other than localhost because the server will 505 # attempt to serve files relative to the root directory as a last resort 506 # before 404ing. This means all of your files can be accessed from this 507 # server, so DO NOT let this server listen to anything but localhost. 508 server_address = ('127.0.0.1', port) 509 http_server = BaseHTTPServer.HTTPServer(server_address, SkPDiffHandler) 510 http_server.image_set = image_set 511 http_server.expectations_manager = expectations_manager 512 print('Navigate thine browser to: http://{}:{}/'.format(*server_address)) 513 http_server.serve_forever() 514 515 516 def main(): 517 parser = argparse.ArgumentParser() 518 parser.add_argument('--port', '-p', metavar='PORT', 519 type=int, 520 default=8080, 521 help='port to bind the server to; ' + 522 'defaults to %(default)s', 523 ) 524 525 parser.add_argument('--expectations-dir', metavar='EXPECTATIONS_DIR', 526 default=DEFAULT_GM_EXPECTATIONS_DIR, 527 help='path to the gm expectations; ' + 528 'defaults to %(default)s' 529 ) 530 531 parser.add_argument('--expected', 532 metavar='EXPECTATIONS_FILE_NAME', 533 default='expected-results.json', 534 help='the file name of the expectations JSON; ' + 535 'defaults to %(default)s' 536 ) 537 538 parser.add_argument('--updated', 539 metavar='UPDATED_FILE_NAME', 540 default='updated-results.json', 541 help='the file name of the updated expectations JSON;' + 542 ' defaults to %(default)s' 543 ) 544 545 parser.add_argument('--skpdiff-path', metavar='SKPDIFF_PATH', 546 default=None, 547 help='the path to the skpdiff binary to use; ' + 548 'defaults to out/Release/skpdiff or out/Default/skpdiff' 549 ) 550 551 args = vars(parser.parse_args()) # Convert args into a python dict 552 553 # Make sure we have access to an skpdiff binary 554 skpdiff_path = get_skpdiff_path(args['skpdiff_path']) 555 if skpdiff_path is None: 556 sys.exit(1) 557 558 # Print out the paths of things for easier debugging 559 print('script dir :', SCRIPT_DIR) 560 print('tools dir :', TOOLS_DIR) 561 print('root dir :', SKIA_ROOT_DIR) 562 print('expectations dir :', args['expectations_dir']) 563 print('skpdiff path :', skpdiff_path) 564 565 expectations_manager = ExpectationsManager(args['expectations_dir'], 566 args['expected'], 567 args['updated'], 568 skpdiff_path) 569 570 run_server(expectations_manager, port=args['port']) 571 572 if __name__ == '__main__': 573 main() 574
