1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 """Starts a local HTTP server which displays layout test failures (given a test 30 results directory), provides comparisons of expected and actual results (both 31 images and text) and allows one-click rebaselining of tests.""" 32 from __future__ import with_statement 33 34 import codecs 35 import datetime 36 import fnmatch 37 import mimetypes 38 import os 39 import os.path 40 import shutil 41 import threading 42 import time 43 import urlparse 44 import BaseHTTPServer 45 46 from optparse import make_option 47 from wsgiref.handlers import format_date_time 48 49 from webkitpy.common import system 50 from webkitpy.layout_tests.layout_package import json_results_generator 51 from webkitpy.layout_tests.port import factory 52 from webkitpy.layout_tests.port.webkit import WebKitPort 53 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand 54 from webkitpy.thirdparty import simplejson 55 56 STATE_NEEDS_REBASELINE = 'needs_rebaseline' 57 STATE_REBASELINE_FAILED = 'rebaseline_failed' 58 STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded' 59 60 class RebaselineHTTPServer(BaseHTTPServer.HTTPServer): 61 def __init__(self, httpd_port, test_config, results_json, platforms_json): 62 BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler) 63 self.test_config = test_config 64 self.results_json = results_json 65 self.platforms_json = platforms_json 66 67 68 class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): 69 STATIC_FILE_NAMES = frozenset([ 70 "index.html", 71 "loupe.js", 72 "main.js", 73 "main.css", 74 "queue.js", 75 "util.js", 76 ]) 77 78 STATIC_FILE_DIRECTORY = os.path.join( 79 os.path.dirname(__file__), "data", "rebaselineserver") 80 81 def do_GET(self): 82 self._handle_request() 83 84 def do_POST(self): 85 self._handle_request() 86 87 def _handle_request(self): 88 # Parse input. 89 if "?" in self.path: 90 path, query_string = self.path.split("?", 1) 91 self.query = urlparse.parse_qs(query_string) 92 else: 93 path = self.path 94 self.query = {} 95 function_or_file_name = path[1:] or "index.html" 96 97 # See if a static file matches. 98 if function_or_file_name in RebaselineHTTPRequestHandler.STATIC_FILE_NAMES: 99 self._serve_static_file(function_or_file_name) 100 return 101 102 # See if a class method matches. 103 function_name = function_or_file_name.replace(".", "_") 104 if not hasattr(self, function_name): 105 self.send_error(404, "Unknown function %s" % function_name) 106 return 107 if function_name[0] == "_": 108 self.send_error( 109 401, "Not allowed to invoke private or protected methods") 110 return 111 function = getattr(self, function_name) 112 function() 113 114 def _serve_static_file(self, static_path): 115 self._serve_file(os.path.join( 116 RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path)) 117 118 def rebaseline(self): 119 test = self.query['test'][0] 120 baseline_target = self.query['baseline-target'][0] 121 baseline_move_to = self.query['baseline-move-to'][0] 122 test_json = self.server.results_json['tests'][test] 123 124 if test_json['state'] != STATE_NEEDS_REBASELINE: 125 self.send_error(400, "Test %s is in unexpected state: %s" % 126 (test, test_json["state"])) 127 return 128 129 log = [] 130 success = _rebaseline_test( 131 test, 132 baseline_target, 133 baseline_move_to, 134 self.server.test_config, 135 log=lambda l: log.append(l)) 136 137 if success: 138 test_json['state'] = STATE_REBASELINE_SUCCEEDED 139 self.send_response(200) 140 else: 141 test_json['state'] = STATE_REBASELINE_FAILED 142 self.send_response(500) 143 144 self.send_header('Content-type', 'text/plain') 145 self.end_headers() 146 self.wfile.write('\n'.join(log)) 147 148 def quitquitquit(self): 149 self.send_response(200) 150 self.send_header("Content-type", "text/plain") 151 self.end_headers() 152 self.wfile.write("Quit.\n") 153 154 # Shutdown has to happen on another thread from the server's thread, 155 # otherwise there's a deadlock 156 threading.Thread(target=lambda: self.server.shutdown()).start() 157 158 def test_result(self): 159 test_name, _ = os.path.splitext(self.query['test'][0]) 160 mode = self.query['mode'][0] 161 if mode == 'expected-image': 162 file_name = test_name + '-expected.png' 163 elif mode == 'actual-image': 164 file_name = test_name + '-actual.png' 165 if mode == 'expected-checksum': 166 file_name = test_name + '-expected.checksum' 167 elif mode == 'actual-checksum': 168 file_name = test_name + '-actual.checksum' 169 elif mode == 'diff-image': 170 file_name = test_name + '-diff.png' 171 if mode == 'expected-text': 172 file_name = test_name + '-expected.txt' 173 elif mode == 'actual-text': 174 file_name = test_name + '-actual.txt' 175 elif mode == 'diff-text': 176 file_name = test_name + '-diff.txt' 177 elif mode == 'diff-text-pretty': 178 file_name = test_name + '-pretty-diff.html' 179 180 file_path = os.path.join(self.server.test_config.results_directory, file_name) 181 182 # Let results be cached for 60 seconds, so that they can be pre-fetched 183 # by the UI 184 self._serve_file(file_path, cacheable_seconds=60) 185 186 def results_json(self): 187 self._serve_json(self.server.results_json) 188 189 def platforms_json(self): 190 self._serve_json(self.server.platforms_json) 191 192 def _serve_json(self, json): 193 self.send_response(200) 194 self.send_header('Content-type', 'application/json') 195 self.end_headers() 196 simplejson.dump(json, self.wfile) 197 198 def _serve_file(self, file_path, cacheable_seconds=0): 199 if not os.path.exists(file_path): 200 self.send_error(404, "File not found") 201 return 202 with codecs.open(file_path, "rb") as static_file: 203 self.send_response(200) 204 self.send_header("Content-Length", os.path.getsize(file_path)) 205 mime_type, encoding = mimetypes.guess_type(file_path) 206 if mime_type: 207 self.send_header("Content-type", mime_type) 208 209 if cacheable_seconds: 210 expires_time = (datetime.datetime.now() + 211 datetime.timedelta(0, cacheable_seconds)) 212 expires_formatted = format_date_time( 213 time.mktime(expires_time.timetuple())) 214 self.send_header("Expires", expires_formatted) 215 self.end_headers() 216 217 shutil.copyfileobj(static_file, self.wfile) 218 219 220 class TestConfig(object): 221 def __init__(self, test_port, layout_tests_directory, results_directory, platforms, filesystem, scm): 222 self.test_port = test_port 223 self.layout_tests_directory = layout_tests_directory 224 self.results_directory = results_directory 225 self.platforms = platforms 226 self.filesystem = filesystem 227 self.scm = scm 228 229 230 def _get_actual_result_files(test_file, test_config): 231 test_name, _ = os.path.splitext(test_file) 232 test_directory = os.path.dirname(test_file) 233 234 test_results_directory = test_config.filesystem.join( 235 test_config.results_directory, test_directory) 236 actual_pattern = os.path.basename(test_name) + '-actual.*' 237 actual_files = [] 238 for filename in test_config.filesystem.listdir(test_results_directory): 239 if fnmatch.fnmatch(filename, actual_pattern): 240 actual_files.append(filename) 241 actual_files.sort() 242 return tuple(actual_files) 243 244 245 def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log): 246 test_name, _ = os.path.splitext(test_file) 247 test_directory = os.path.dirname(test_name) 248 249 log('Rebaselining %s...' % test_name) 250 251 actual_result_files = _get_actual_result_files(test_file, test_config) 252 filesystem = test_config.filesystem 253 scm = test_config.scm 254 layout_tests_directory = test_config.layout_tests_directory 255 results_directory = test_config.results_directory 256 target_expectations_directory = filesystem.join( 257 layout_tests_directory, 'platform', baseline_target, test_directory) 258 test_results_directory = test_config.filesystem.join( 259 test_config.results_directory, test_directory) 260 261 # If requested, move current baselines out 262 current_baselines = _get_test_baselines(test_file, test_config) 263 if baseline_target in current_baselines and baseline_move_to != 'none': 264 log(' Moving current %s baselines to %s' % 265 (baseline_target, baseline_move_to)) 266 267 # See which ones we need to move (only those that are about to be 268 # updated), and make sure we're not clobbering any files in the 269 # destination. 270 current_extensions = set(current_baselines[baseline_target].keys()) 271 actual_result_extensions = [ 272 os.path.splitext(f)[1] for f in actual_result_files] 273 extensions_to_move = current_extensions.intersection( 274 actual_result_extensions) 275 276 if extensions_to_move.intersection( 277 current_baselines.get(baseline_move_to, {}).keys()): 278 log(' Already had baselines in %s, could not move existing ' 279 '%s ones' % (baseline_move_to, baseline_target)) 280 return False 281 282 # Do the actual move. 283 if extensions_to_move: 284 if not _move_test_baselines( 285 test_file, 286 list(extensions_to_move), 287 baseline_target, 288 baseline_move_to, 289 test_config, 290 log): 291 return False 292 else: 293 log(' No current baselines to move') 294 295 log(' Updating baselines for %s' % baseline_target) 296 filesystem.maybe_make_directory(target_expectations_directory) 297 for source_file in actual_result_files: 298 source_path = filesystem.join(test_results_directory, source_file) 299 destination_file = source_file.replace('-actual', '-expected') 300 destination_path = filesystem.join( 301 target_expectations_directory, destination_file) 302 filesystem.copyfile(source_path, destination_path) 303 exit_code = scm.add(destination_path, return_exit_code=True) 304 if exit_code: 305 log(' Could not update %s in SCM, exit code %d' % 306 (destination_file, exit_code)) 307 return False 308 else: 309 log(' Updated %s' % destination_file) 310 311 return True 312 313 314 def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log): 315 test_file_name = os.path.splitext(os.path.basename(test_file))[0] 316 test_directory = os.path.dirname(test_file) 317 filesystem = test_config.filesystem 318 319 # Want predictable output order for unit tests. 320 extensions_to_move.sort() 321 322 source_directory = os.path.join( 323 test_config.layout_tests_directory, 324 'platform', 325 source_platform, 326 test_directory) 327 destination_directory = os.path.join( 328 test_config.layout_tests_directory, 329 'platform', 330 destination_platform, 331 test_directory) 332 filesystem.maybe_make_directory(destination_directory) 333 334 for extension in extensions_to_move: 335 file_name = test_file_name + '-expected' + extension 336 source_path = filesystem.join(source_directory, file_name) 337 destination_path = filesystem.join(destination_directory, file_name) 338 filesystem.copyfile(source_path, destination_path) 339 exit_code = test_config.scm.add(destination_path, return_exit_code=True) 340 if exit_code: 341 log(' Could not update %s in SCM, exit code %d' % 342 (file_name, exit_code)) 343 return False 344 else: 345 log(' Moved %s' % file_name) 346 347 return True 348 349 def _get_test_baselines(test_file, test_config): 350 class AllPlatformsPort(WebKitPort): 351 def __init__(self): 352 WebKitPort.__init__(self, filesystem=test_config.filesystem) 353 self._platforms_by_directory = dict( 354 [(self._webkit_baseline_path(p), p) for p in test_config.platforms]) 355 356 def baseline_search_path(self): 357 return self._platforms_by_directory.keys() 358 359 def platform_from_directory(self, directory): 360 return self._platforms_by_directory[directory] 361 362 test_path = test_config.filesystem.join( 363 test_config.layout_tests_directory, test_file) 364 365 all_platforms_port = AllPlatformsPort() 366 367 all_test_baselines = {} 368 for baseline_extension in ('.txt', '.checksum', '.png'): 369 test_baselines = test_config.test_port.expected_baselines( 370 test_path, baseline_extension) 371 baselines = all_platforms_port.expected_baselines( 372 test_path, baseline_extension, all_baselines=True) 373 for platform_directory, expected_filename in baselines: 374 if not platform_directory: 375 continue 376 if platform_directory == test_config.layout_tests_directory: 377 platform = 'base' 378 else: 379 platform = all_platforms_port.platform_from_directory( 380 platform_directory) 381 platform_baselines = all_test_baselines.setdefault(platform, {}) 382 was_used_for_test = ( 383 platform_directory, expected_filename) in test_baselines 384 platform_baselines[baseline_extension] = was_used_for_test 385 386 return all_test_baselines 387 388 389 class RebaselineServer(AbstractDeclarativeCommand): 390 name = "rebaseline-server" 391 help_text = __doc__ 392 argument_names = "/path/to/results/directory" 393 394 def __init__(self): 395 options = [ 396 make_option("--httpd-port", action="store", type="int", default=8127, help="Port to use for the the rebaseline HTTP server"), 397 ] 398 AbstractDeclarativeCommand.__init__(self, options=options) 399 400 def execute(self, options, args, tool): 401 results_directory = args[0] 402 filesystem = system.filesystem.FileSystem() 403 scm = self._tool.scm() 404 405 if options.dry_run: 406 407 def no_op_copyfile(src, dest): 408 pass 409 410 def no_op_add(path, return_exit_code=False): 411 if return_exit_code: 412 return 0 413 414 filesystem.copyfile = no_op_copyfile 415 scm.add = no_op_add 416 417 print 'Parsing unexpected_results.json...' 418 results_json_path = filesystem.join(results_directory, 'unexpected_results.json') 419 results_json = json_results_generator.load_json(filesystem, results_json_path) 420 421 port = factory.get() 422 layout_tests_directory = port.layout_tests_dir() 423 platforms = filesystem.listdir( 424 filesystem.join(layout_tests_directory, 'platform')) 425 test_config = TestConfig( 426 port, 427 layout_tests_directory, 428 results_directory, 429 platforms, 430 filesystem, 431 scm) 432 433 print 'Gathering current baselines...' 434 for test_file, test_json in results_json['tests'].items(): 435 test_json['state'] = STATE_NEEDS_REBASELINE 436 test_path = filesystem.join(layout_tests_directory, test_file) 437 test_json['baselines'] = _get_test_baselines(test_file, test_config) 438 439 server_url = "http://localhost:%d/" % options.httpd_port 440 print "Starting server at %s" % server_url 441 print ("Use the 'Exit' link in the UI, %squitquitquit " 442 "or Ctrl-C to stop") % server_url 443 444 threading.Timer( 445 .1, lambda: self._tool.user.open_url(server_url)).start() 446 447 httpd = RebaselineHTTPServer( 448 httpd_port=options.httpd_port, 449 test_config=test_config, 450 results_json=results_json, 451 platforms_json={ 452 'platforms': platforms, 453 'defaultPlatform': port.name(), 454 }) 455 httpd.serve_forever() 456
