1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import fnmatch 30 import os 31 import os.path 32 import BaseHTTPServer 33 34 from webkitpy.layout_tests.port.base import Port 35 from webkitpy.tool.servers.reflectionhandler import ReflectionHandler 36 37 38 STATE_NEEDS_REBASELINE = 'needs_rebaseline' 39 STATE_REBASELINE_FAILED = 'rebaseline_failed' 40 STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded' 41 42 43 def _get_actual_result_files(test_file, test_config): 44 test_name, _ = os.path.splitext(test_file) 45 test_directory = os.path.dirname(test_file) 46 47 test_results_directory = test_config.filesystem.join( 48 test_config.results_directory, test_directory) 49 actual_pattern = os.path.basename(test_name) + '-actual.*' 50 actual_files = [] 51 for filename in test_config.filesystem.listdir(test_results_directory): 52 if fnmatch.fnmatch(filename, actual_pattern): 53 actual_files.append(filename) 54 actual_files.sort() 55 return tuple(actual_files) 56 57 58 def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log): 59 test_name, _ = os.path.splitext(test_file) 60 test_directory = os.path.dirname(test_name) 61 62 log('Rebaselining %s...' % test_name) 63 64 actual_result_files = _get_actual_result_files(test_file, test_config) 65 filesystem = test_config.filesystem 66 scm = test_config.scm 67 layout_tests_directory = test_config.layout_tests_directory 68 results_directory = test_config.results_directory 69 target_expectations_directory = filesystem.join( 70 layout_tests_directory, 'platform', baseline_target, test_directory) 71 test_results_directory = test_config.filesystem.join( 72 test_config.results_directory, test_directory) 73 74 # If requested, move current baselines out 75 current_baselines = get_test_baselines(test_file, test_config) 76 if baseline_target in current_baselines and baseline_move_to != 'none': 77 log(' Moving current %s baselines to %s' % 78 (baseline_target, baseline_move_to)) 79 80 # See which ones we need to move (only those that are about to be 81 # updated), and make sure we're not clobbering any files in the 82 # destination. 83 current_extensions = set(current_baselines[baseline_target].keys()) 84 actual_result_extensions = [ 85 os.path.splitext(f)[1] for f in actual_result_files] 86 extensions_to_move = current_extensions.intersection( 87 actual_result_extensions) 88 89 if extensions_to_move.intersection( 90 current_baselines.get(baseline_move_to, {}).keys()): 91 log(' Already had baselines in %s, could not move existing ' 92 '%s ones' % (baseline_move_to, baseline_target)) 93 return False 94 95 # Do the actual move. 96 if extensions_to_move: 97 if not _move_test_baselines( 98 test_file, 99 list(extensions_to_move), 100 baseline_target, 101 baseline_move_to, 102 test_config, 103 log): 104 return False 105 else: 106 log(' No current baselines to move') 107 108 log(' Updating baselines for %s' % baseline_target) 109 filesystem.maybe_make_directory(target_expectations_directory) 110 for source_file in actual_result_files: 111 source_path = filesystem.join(test_results_directory, source_file) 112 destination_file = source_file.replace('-actual', '-expected') 113 destination_path = filesystem.join( 114 target_expectations_directory, destination_file) 115 filesystem.copyfile(source_path, destination_path) 116 exit_code = scm.add(destination_path, return_exit_code=True) 117 if exit_code: 118 log(' Could not update %s in SCM, exit code %d' % 119 (destination_file, exit_code)) 120 return False 121 else: 122 log(' Updated %s' % destination_file) 123 124 return True 125 126 127 def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log): 128 test_file_name = os.path.splitext(os.path.basename(test_file))[0] 129 test_directory = os.path.dirname(test_file) 130 filesystem = test_config.filesystem 131 132 # Want predictable output order for unit tests. 133 extensions_to_move.sort() 134 135 source_directory = os.path.join( 136 test_config.layout_tests_directory, 137 'platform', 138 source_platform, 139 test_directory) 140 destination_directory = os.path.join( 141 test_config.layout_tests_directory, 142 'platform', 143 destination_platform, 144 test_directory) 145 filesystem.maybe_make_directory(destination_directory) 146 147 for extension in extensions_to_move: 148 file_name = test_file_name + '-expected' + extension 149 source_path = filesystem.join(source_directory, file_name) 150 destination_path = filesystem.join(destination_directory, file_name) 151 filesystem.copyfile(source_path, destination_path) 152 exit_code = test_config.scm.add(destination_path, return_exit_code=True) 153 if exit_code: 154 log(' Could not update %s in SCM, exit code %d' % 155 (file_name, exit_code)) 156 return False 157 else: 158 log(' Moved %s' % file_name) 159 160 return True 161 162 163 def get_test_baselines(test_file, test_config): 164 # FIXME: This seems like a hack. This only seems used to access the Port.expected_baselines logic. 165 class AllPlatformsPort(Port): 166 def __init__(self, host): 167 super(AllPlatformsPort, self).__init__(host, 'mac') 168 self._platforms_by_directory = dict([(self._webkit_baseline_path(p), p) for p in test_config.platforms]) 169 170 def baseline_search_path(self): 171 return self._platforms_by_directory.keys() 172 173 def platform_from_directory(self, directory): 174 return self._platforms_by_directory[directory] 175 176 test_path = test_config.filesystem.join(test_config.layout_tests_directory, test_file) 177 host = test_config.host 178 all_platforms_port = AllPlatformsPort(host) 179 180 all_test_baselines = {} 181 for baseline_extension in ('.txt', '.checksum', '.png'): 182 test_baselines = test_config.test_port.expected_baselines(test_file, baseline_extension) 183 baselines = all_platforms_port.expected_baselines(test_file, baseline_extension, all_baselines=True) 184 for platform_directory, expected_filename in baselines: 185 if not platform_directory: 186 continue 187 if platform_directory == test_config.layout_tests_directory: 188 platform = 'base' 189 else: 190 platform = all_platforms_port.platform_from_directory(platform_directory) 191 platform_baselines = all_test_baselines.setdefault(platform, {}) 192 was_used_for_test = (platform_directory, expected_filename) in test_baselines 193 platform_baselines[baseline_extension] = was_used_for_test 194 195 return all_test_baselines 196 197 198 class RebaselineHTTPServer(BaseHTTPServer.HTTPServer): 199 def __init__(self, httpd_port, config): 200 server_name = "" 201 BaseHTTPServer.HTTPServer.__init__(self, (server_name, httpd_port), RebaselineHTTPRequestHandler) 202 self.test_config = config['test_config'] 203 self.results_json = config['results_json'] 204 self.platforms_json = config['platforms_json'] 205 206 207 class RebaselineHTTPRequestHandler(ReflectionHandler): 208 STATIC_FILE_NAMES = frozenset([ 209 "index.html", 210 "loupe.js", 211 "main.js", 212 "main.css", 213 "queue.js", 214 "util.js", 215 ]) 216 217 STATIC_FILE_DIRECTORY = os.path.join(os.path.dirname(__file__), "data", "rebaselineserver") 218 219 def results_json(self): 220 self._serve_json(self.server.results_json) 221 222 def test_config(self): 223 self._serve_json(self.server.test_config) 224 225 def platforms_json(self): 226 self._serve_json(self.server.platforms_json) 227 228 def rebaseline(self): 229 test = self.query['test'][0] 230 baseline_target = self.query['baseline-target'][0] 231 baseline_move_to = self.query['baseline-move-to'][0] 232 test_json = self.server.results_json['tests'][test] 233 234 if test_json['state'] != STATE_NEEDS_REBASELINE: 235 self.send_error(400, "Test %s is in unexpected state: %s" % (test, test_json["state"])) 236 return 237 238 log = [] 239 success = _rebaseline_test( 240 test, 241 baseline_target, 242 baseline_move_to, 243 self.server.test_config, 244 log=lambda l: log.append(l)) 245 246 if success: 247 test_json['state'] = STATE_REBASELINE_SUCCEEDED 248 self.send_response(200) 249 else: 250 test_json['state'] = STATE_REBASELINE_FAILED 251 self.send_response(500) 252 253 self.send_header('Content-type', 'text/plain') 254 self.end_headers() 255 self.wfile.write('\n'.join(log)) 256 257 def test_result(self): 258 test_name, _ = os.path.splitext(self.query['test'][0]) 259 mode = self.query['mode'][0] 260 if mode == 'expected-image': 261 file_name = test_name + '-expected.png' 262 elif mode == 'actual-image': 263 file_name = test_name + '-actual.png' 264 if mode == 'expected-checksum': 265 file_name = test_name + '-expected.checksum' 266 elif mode == 'actual-checksum': 267 file_name = test_name + '-actual.checksum' 268 elif mode == 'diff-image': 269 file_name = test_name + '-diff.png' 270 if mode == 'expected-text': 271 file_name = test_name + '-expected.txt' 272 elif mode == 'actual-text': 273 file_name = test_name + '-actual.txt' 274 elif mode == 'diff-text': 275 file_name = test_name + '-diff.txt' 276 elif mode == 'diff-text-pretty': 277 file_name = test_name + '-pretty-diff.html' 278 279 file_path = os.path.join(self.server.test_config.results_directory, file_name) 280 281 # Let results be cached for 60 seconds, so that they can be pre-fetched 282 # by the UI 283 self._serve_file(file_path, cacheable_seconds=60) 284
