1 #!/usr/bin/python 2 # -*- coding: utf-8 -*- 3 4 from __future__ import print_function 5 import argparse 6 import BaseHTTPServer 7 import json 8 import os 9 import os.path 10 import re 11 import subprocess 12 import sys 13 import tempfile 14 import urllib2 15 16 # Grab the script path because that is where all the static assets are 17 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 18 19 # Find the tools directory for python imports 20 TOOLS_DIR = os.path.dirname(SCRIPT_DIR) 21 22 # Find the root of the skia trunk for finding skpdiff binary 23 SKIA_ROOT_DIR = os.path.dirname(TOOLS_DIR) 24 25 # Find the default location of gm expectations 26 DEFAULT_GM_EXPECTATIONS_DIR = os.path.join(SKIA_ROOT_DIR, 'expectations', 'gm') 27 28 # Imports from within Skia 29 if TOOLS_DIR not in sys.path: 30 sys.path.append(TOOLS_DIR) 31 GM_DIR = os.path.join(SKIA_ROOT_DIR, 'gm') 32 if GM_DIR not in sys.path: 33 sys.path.append(GM_DIR) 34 import gm_json 35 import jsondiff 36 37 # A simple dictionary of file name extensions to MIME types. The empty string 38 # entry is used as the default when no extension was given or if the extension 39 # has no entry in this dictionary. 40 MIME_TYPE_MAP = {'': 'application/octet-stream', 41 'html': 'text/html', 42 'css': 'text/css', 43 'png': 'image/png', 44 'js': 'application/javascript', 45 'json': 'application/json' 46 } 47 48 49 IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 50 51 SKPDIFF_INVOKE_FORMAT = '{} --jsonp=false -o {} -f {} {}' 52 53 54 def get_skpdiff_path(user_path=None): 55 """Find the skpdiff binary. 56 57 @param user_path If none, searches in Release and Debug out directories of 58 the skia root. If set, checks that the path is a real file and 59 returns it. 60 """ 61 skpdiff_path = None 62 possible_paths = [] 63 64 # Use the user given path, or try out some good default paths. 65 if user_path: 66 possible_paths.append(user_path) 67 else: 68 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', 69 'Release', 'skpdiff')) 70 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', 71 'Release', 'skpdiff.exe')) 72 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', 73 'Debug', 'skpdiff')) 74 possible_paths.append(os.path.join(SKIA_ROOT_DIR, 'out', 75 'Debug', 'skpdiff.exe')) 76 # Use the first path that actually points to the binary 77 for possible_path in possible_paths: 78 if os.path.isfile(possible_path): 79 skpdiff_path = possible_path 80 break 81 82 # If skpdiff was not found, print out diagnostic info for the user. 83 if skpdiff_path is None: 84 print('Could not find skpdiff binary. Either build it into the ' + 85 'default directory, or specify the path on the command line.') 86 print('skpdiff paths tried:') 87 for possible_path in possible_paths: 88 print(' ', possible_path) 89 return skpdiff_path 90 91 92 def download_file(url, output_path): 93 """Download the file at url and place it in output_path""" 94 reader = urllib2.urlopen(url) 95 with open(output_path, 'wb') as writer: 96 writer.write(reader.read()) 97 98 99 def download_gm_image(image_name, image_path, hash_val): 100 """Download the gm result into the given path. 101 102 @param image_name The GM file name, for example imageblur_gpu.png. 103 @param image_path Path to place the image. 104 @param hash_val The hash value of the image. 105 """ 106 if hash_val is None: 107 return 108 109 # Separate the test name from a image name 110 image_match = IMAGE_FILENAME_RE.match(image_name) 111 test_name = image_match.group(1) 112 113 # Calculate the URL of the requested image 114 image_url = gm_json.CreateGmActualUrl( 115 test_name, gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, hash_val) 116 117 # Download the image as requested 118 download_file(image_url, image_path) 119 120 121 def get_image_set_from_skpdiff(skpdiff_records): 122 """Get the set of all images references in the given records. 123 124 @param skpdiff_records An array of records, which are dictionary objects. 125 """ 126 expected_set = frozenset([r['baselinePath'] for r in skpdiff_records]) 127 actual_set = frozenset([r['testPath'] for r in skpdiff_records]) 128 return expected_set | actual_set 129 130 131 def set_expected_hash_in_json(expected_results_json, image_name, hash_value): 132 """Set the expected hash for the object extracted from 133 expected-results.json. Note that this only work with bitmap-64bitMD5 hash 134 types. 135 136 @param expected_results_json The Python dictionary with the results to 137 modify. 138 @param image_name The name of the image to set the hash of. 139 @param hash_value The hash to set for the image. 140 """ 141 expected_results = expected_results_json[gm_json.JSONKEY_EXPECTEDRESULTS] 142 143 if image_name in expected_results: 144 expected_results[image_name][gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0][1] = hash_value 145 else: 146 expected_results[image_name] = { 147 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: 148 [ 149 [ 150 gm_json.JSONKEY_HASHTYPE_BITMAP_64BITMD5, 151 hash_value 152 ] 153 ] 154 } 155 156 157 def get_head_version(path): 158 """Get the version of the file at the given path stored inside the HEAD of 159 the git repository. It is returned as a string. 160 161 @param path The path of the file whose HEAD is returned. It is assumed the 162 path is inside a git repo rooted at SKIA_ROOT_DIR. 163 """ 164 165 # git-show will not work with absolute paths. This ensures we give it a path 166 # relative to the skia root. This path also has to use forward slashes, even 167 # on windows. 168 git_path = os.path.relpath(path, SKIA_ROOT_DIR).replace('\\', '/') 169 git_show_proc = subprocess.Popen(['git', 'show', 'HEAD:' + git_path], 170 stdout=subprocess.PIPE) 171 172 # When invoked outside a shell, git will output the last committed version 173 # of the file directly to stdout. 174 git_version_content, _ = git_show_proc.communicate() 175 return git_version_content 176 177 178 class GMInstance: 179 """Information about a GM test result on a specific device: 180 - device_name = the name of the device that rendered it 181 - image_name = the GM test name and config 182 - expected_hash = the current expected hash value 183 - actual_hash = the actual hash value 184 - is_rebaselined = True if actual_hash is what is currently in the expected 185 results file, False otherwise. 186 """ 187 def __init__(self, 188 device_name, image_name, 189 expected_hash, actual_hash, 190 is_rebaselined): 191 self.device_name = device_name 192 self.image_name = image_name 193 self.expected_hash = expected_hash 194 self.actual_hash = actual_hash 195 self.is_rebaselined = is_rebaselined 196 197 198 class ExpectationsManager: 199 def __init__(self, expectations_dir, expected_name, updated_name, 200 skpdiff_path): 201 """ 202 @param expectations_dir The directory to traverse for results files. 203 This should resemble expectations/gm in the Skia trunk. 204 @param expected_name The name of the expected result files. These 205 are in the format of expected-results.json. 206 @param updated_name The name of the updated expected result files. 207 Normally this matches --expectations-filename-output for the 208 rebaseline.py tool. 209 @param skpdiff_path The path used to execute the skpdiff command. 210 """ 211 self._expectations_dir = expectations_dir 212 self._expected_name = expected_name 213 self._updated_name = updated_name 214 self._skpdiff_path = skpdiff_path 215 self._generate_gm_comparison() 216 217 def _generate_gm_comparison(self): 218 """Generate all the data needed to compare GMs: 219 - determine which GMs changed 220 - download the changed images 221 - compare them with skpdiff 222 """ 223 224 # Get the expectations and compare them with actual hashes 225 self._get_expectations() 226 227 228 # Create a temporary file tree that makes sense for skpdiff to operate 229 # on. We take the realpath of the new temp directory because some OSs 230 # (*cough* osx) put the temp directory behind a symlink that gets 231 # resolved later down the pipeline and breaks the image map. 232 image_output_dir = os.path.realpath(tempfile.mkdtemp('skpdiff')) 233 expected_image_dir = os.path.join(image_output_dir, 'expected') 234 actual_image_dir = os.path.join(image_output_dir, 'actual') 235 os.mkdir(expected_image_dir) 236 os.mkdir(actual_image_dir) 237 238 # Download expected and actual images that differed into the temporary 239 # file tree. 240 self._download_expectation_images(expected_image_dir, actual_image_dir) 241 242 # Invoke skpdiff with our downloaded images and place its results in the 243 # temporary directory. 244 self._skpdiff_output_path = os.path.join(image_output_dir, 245 'skpdiff_output.json') 246 skpdiff_cmd = SKPDIFF_INVOKE_FORMAT.format(self._skpdiff_path, 247 self._skpdiff_output_path, 248 expected_image_dir, 249 actual_image_dir) 250 os.system(skpdiff_cmd) 251 self._load_skpdiff_output() 252 253 254 def _get_expectations(self): 255 """Fills self._expectations with GMInstance objects for each test whose 256 expectation is different between the following two files: 257 - the local filesystem's updated results file 258 - git's head version of the expected results file 259 """ 260 differ = jsondiff.GMDiffer() 261 self._expectations = [] 262 for root, dirs, files in os.walk(self._expectations_dir): 263 for expectation_file in files: 264 # There are many files in the expectations directory. We only 265 # care about expected results. 266 if expectation_file != self._expected_name: 267 continue 268 269 # Get the name of the results file, and be sure there is an 270 # updated result to compare against. If there is not, there is 271 # no point in diffing this device. 272 expected_file_path = os.path.join(root, self._expected_name) 273 updated_file_path = os.path.join(root, self._updated_name) 274 if not os.path.isfile(updated_file_path): 275 continue 276 277 # Always get the expected results from git because we may have 278 # changed them in a previous instance of the server. 279 expected_contents = get_head_version(expected_file_path) 280 updated_contents = None 281 with open(updated_file_path, 'rb') as updated_file: 282 updated_contents = updated_file.read() 283 284 # Read the expected results on disk to determine what we've 285 # already rebaselined. 286 commited_contents = None 287 with open(expected_file_path, 'rb') as expected_file: 288 commited_contents = expected_file.read() 289 290 # Find all expectations that did not match. 291 expected_diff = differ.GenerateDiffDictFromStrings( 292 expected_contents, 293 updated_contents) 294 295 # Generate a set of images that have already been rebaselined 296 # onto disk. 297 rebaselined_diff = differ.GenerateDiffDictFromStrings( 298 expected_contents, 299 commited_contents) 300 301 rebaselined_set = set(rebaselined_diff.keys()) 302 303 # The name of the device corresponds to the name of the folder 304 # we are in. 305 device_name = os.path.basename(root) 306 307 # Store old and new versions of the expectation for each GM 308 for image_name, hashes in expected_diff.iteritems(): 309 self._expectations.append( 310 GMInstance(device_name, image_name, 311 hashes['old'], hashes['new'], 312 image_name in rebaselined_set)) 313 314 def _load_skpdiff_output(self): 315 """Loads the results of skpdiff and annotates them with whether they 316 have already been rebaselined or not. The resulting data is store in 317 self.skpdiff_records.""" 318 self.skpdiff_records = None 319 with open(self._skpdiff_output_path, 'rb') as skpdiff_output_file: 320 self.skpdiff_records = json.load(skpdiff_output_file)['records'] 321 for record in self.skpdiff_records: 322 record['isRebaselined'] = self.image_map[record['baselinePath']][1].is_rebaselined 323 324 325 def _download_expectation_images(self, expected_image_dir, actual_image_dir): 326 """Download the expected and actual images for the _expectations array. 327 328 @param expected_image_dir The directory to download expected images 329 into. 330 @param actual_image_dir The directory to download actual images into. 331 """ 332 image_map = {} 333 334 # Look through expectations and download their images. 335 for expectation in self._expectations: 336 # Build appropriate paths to download the images into. 337 expected_image_path = os.path.join(expected_image_dir, 338 expectation.device_name + '-' + 339 expectation.image_name) 340 341 actual_image_path = os.path.join(actual_image_dir, 342 expectation.device_name + '-' + 343 expectation.image_name) 344 345 print('Downloading %s for device %s' % ( 346 expectation.image_name, expectation.device_name)) 347 348 # Download images 349 download_gm_image(expectation.image_name, 350 expected_image_path, 351 expectation.expected_hash) 352 353 download_gm_image(expectation.image_name, 354 actual_image_path, 355 expectation.actual_hash) 356 357 # Annotate the expectations with where the images were downloaded 358 # to. 359 expectation.expected_image_path = expected_image_path 360 expectation.actual_image_path = actual_image_path 361 362 # Map the image paths back to the expectations. 363 image_map[expected_image_path] = (False, expectation) 364 image_map[actual_image_path] = (True, expectation) 365 366 self.image_map = image_map 367 368 def _set_expected_hash(self, device_name, image_name, hash_value): 369 """Set the expected hash for the image of the given device. This always 370 writes directly to the expected results file of the given device 371 372 @param device_name The name of the device to write the hash to. 373 @param image_name The name of the image whose hash to set. 374 @param hash_value The value of the hash to set. 375 """ 376 377 # Retrieve the expected results file as it is in the working tree 378 json_path = os.path.join(self._expectations_dir, device_name, 379 self._expected_name) 380 expectations = gm_json.LoadFromFile(json_path) 381 382 # Set the specified hash. 383 set_expected_hash_in_json(expectations, image_name, hash_value) 384 385 # Write it out to disk using gm_json to keep the formatting consistent. 386 gm_json.WriteToFile(expectations, json_path) 387 388 def commit_rebaselines(self, rebaselines): 389 """Sets the expected results file to use the hashes of the images in 390 the rebaselines list. If a expected result image is not in rebaselines 391 at all, the old hash will be used. 392 393 @param rebaselines A list of image paths to use the hash of. 394 """ 395 # Reset all expectations to their old hashes because some of them may 396 # have been set to the new hash by a previous call to this function. 397 for expectation in self._expectations: 398 expectation.is_rebaselined = False 399 self._set_expected_hash(expectation.device_name, 400 expectation.image_name, 401 expectation.expected_hash) 402 403 # Take all the images to rebaseline 404 for image_path in rebaselines: 405 # Get the metadata about the image at the path. 406 is_actual, expectation = self.image_map[image_path] 407 408 expectation.is_rebaselined = is_actual 409 expectation_hash = expectation.actual_hash if is_actual else\ 410 expectation.expected_hash 411 412 # Write out that image's hash directly to the expected results file. 413 self._set_expected_hash(expectation.device_name, 414 expectation.image_name, 415 expectation_hash) 416 417 self._load_skpdiff_output() 418 419 420 class SkPDiffHandler(BaseHTTPServer.BaseHTTPRequestHandler): 421 def send_file(self, file_path): 422 # Grab the extension if there is one 423 extension = os.path.splitext(file_path)[1] 424 if len(extension) >= 1: 425 extension = extension[1:] 426 427 # Determine the MIME type of the file from its extension 428 mime_type = MIME_TYPE_MAP.get(extension, MIME_TYPE_MAP['']) 429 430 # Open the file and send it over HTTP 431 if os.path.isfile(file_path): 432 with open(file_path, 'rb') as sending_file: 433 self.send_response(200) 434 self.send_header('Content-type', mime_type) 435 self.end_headers() 436 self.wfile.write(sending_file.read()) 437 else: 438 self.send_error(404) 439 440 def serve_if_in_dir(self, dir_path, file_path): 441 # Determine if the file exists relative to the given dir_path AND exists 442 # under the dir_path. This is to prevent accidentally serving files 443 # outside the directory intended using symlinks, or '../'. 444 real_path = os.path.normpath(os.path.join(dir_path, file_path)) 445 if os.path.commonprefix([real_path, dir_path]) == dir_path: 446 if os.path.isfile(real_path): 447 self.send_file(real_path) 448 return True 449 return False 450 451 def do_GET(self): 452 # Simple rewrite rule of the root path to 'viewer.html' 453 if self.path == '' or self.path == '/': 454 self.path = '/viewer.html' 455 456 # The [1:] chops off the leading '/' 457 file_path = self.path[1:] 458 459 # Handle skpdiff_output.json manually because it is was processed by the 460 # server when it was started and does not exist as a file. 461 if file_path == 'skpdiff_output.json': 462 self.send_response(200) 463 self.send_header('Content-type', MIME_TYPE_MAP['json']) 464 self.end_headers() 465 466 # Add JSONP padding to the JSON because the web page expects it. It 467 # expects it because it was designed to run with or without a web 468 # server. Without a web server, the only way to load JSON is with 469 # JSONP. 470 skpdiff_records = self.server.expectations_manager.skpdiff_records 471 self.wfile.write('var SkPDiffRecords = ') 472 json.dump({'records': skpdiff_records}, self.wfile) 473 self.wfile.write(';') 474 return 475 476 # Attempt to send static asset files first. 477 if self.serve_if_in_dir(SCRIPT_DIR, file_path): 478 return 479 480 # WARNING: Serving any file the user wants is incredibly insecure. Its 481 # redeeming quality is that we only serve gm files on a white list. 482 if self.path in self.server.image_set: 483 self.send_file(self.path) 484 return 485 486 # If no file to send was found, just give the standard 404 487 self.send_error(404) 488 489 def do_POST(self): 490 if self.path == '/commit_rebaselines': 491 content_length = int(self.headers['Content-length']) 492 request_data = json.loads(self.rfile.read(content_length)) 493 rebaselines = request_data['rebaselines'] 494 self.server.expectations_manager.commit_rebaselines(rebaselines) 495 self.send_response(200) 496 self.send_header('Content-type', 'application/json') 497 self.end_headers() 498 self.wfile.write('{"success":true}') 499 return 500 501 # If the we have no handler for this path, give em' the 404 502 self.send_error(404) 503 504 505 def run_server(expectations_manager, port=8080): 506 # It's important to parse the results file so that we can make a set of 507 # images that the web page might request. 508 skpdiff_records = expectations_manager.skpdiff_records 509 image_set = get_image_set_from_skpdiff(skpdiff_records) 510 511 # Do not bind to interfaces other than localhost because the server will 512 # attempt to serve files relative to the root directory as a last resort 513 # before 404ing. This means all of your files can be accessed from this 514 # server, so DO NOT let this server listen to anything but localhost. 515 server_address = ('127.0.0.1', port) 516 http_server = BaseHTTPServer.HTTPServer(server_address, SkPDiffHandler) 517 http_server.image_set = image_set 518 http_server.expectations_manager = expectations_manager 519 print('Navigate thine browser to: http://{}:{}/'.format(*server_address)) 520 http_server.serve_forever() 521 522 523 def main(): 524 parser = argparse.ArgumentParser() 525 parser.add_argument('--port', '-p', metavar='PORT', 526 type=int, 527 default=8080, 528 help='port to bind the server to; ' + 529 'defaults to %(default)s', 530 ) 531 532 parser.add_argument('--expectations-dir', metavar='EXPECTATIONS_DIR', 533 default=DEFAULT_GM_EXPECTATIONS_DIR, 534 help='path to the gm expectations; ' + 535 'defaults to %(default)s' 536 ) 537 538 parser.add_argument('--expected', 539 metavar='EXPECTATIONS_FILE_NAME', 540 default='expected-results.json', 541 help='the file name of the expectations JSON; ' + 542 'defaults to %(default)s' 543 ) 544 545 parser.add_argument('--updated', 546 metavar='UPDATED_FILE_NAME', 547 default='updated-results.json', 548 help='the file name of the updated expectations JSON;' + 549 ' defaults to %(default)s' 550 ) 551 552 parser.add_argument('--skpdiff-path', metavar='SKPDIFF_PATH', 553 default=None, 554 help='the path to the skpdiff binary to use; ' + 555 'defaults to out/Release/skpdiff or out/Default/skpdiff' 556 ) 557 558 args = vars(parser.parse_args()) # Convert args into a python dict 559 560 # Make sure we have access to an skpdiff binary 561 skpdiff_path = get_skpdiff_path(args['skpdiff_path']) 562 if skpdiff_path is None: 563 sys.exit(1) 564 565 # Print out the paths of things for easier debugging 566 print('script dir :', SCRIPT_DIR) 567 print('tools dir :', TOOLS_DIR) 568 print('root dir :', SKIA_ROOT_DIR) 569 print('expectations dir :', args['expectations_dir']) 570 print('skpdiff path :', skpdiff_path) 571 572 expectations_manager = ExpectationsManager(args['expectations_dir'], 573 args['expected'], 574 args['updated'], 575 skpdiff_path) 576 577 run_server(expectations_manager, port=args['port']) 578 579 if __name__ == '__main__': 580 main() 581
