1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the Google name nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Dummy Port implementation used for testing.""" 31 from __future__ import with_statement 32 33 import base64 34 import time 35 36 from webkitpy.common.system import filesystem_mock 37 from webkitpy.tool import mocktool 38 39 import base 40 41 42 # This sets basic expectations for a test. Each individual expectation 43 # can be overridden by a keyword argument in TestList.add(). 44 class TestInstance: 45 def __init__(self, name): 46 self.name = name 47 self.base = name[(name.rfind("/") + 1):name.rfind(".html")] 48 self.crash = False 49 self.exception = False 50 self.hang = False 51 self.keyboard = False 52 self.error = '' 53 self.timeout = False 54 self.is_reftest = False 55 56 # The values of each field are treated as raw byte strings. They 57 # will be converted to unicode strings where appropriate using 58 # MockFileSystem.read_text_file(). 59 self.actual_text = self.base + '-txt' 60 self.actual_checksum = self.base + '-checksum' 61 62 # We add the '\x8a' for the image file to prevent the value from 63 # being treated as UTF-8 (the character is invalid) 64 self.actual_image = self.base + '\x8a' + '-png' 65 66 self.expected_text = self.actual_text 67 self.expected_checksum = self.actual_checksum 68 self.expected_image = self.actual_image 69 70 self.actual_audio = None 71 self.expected_audio = None 72 73 # This is an in-memory list of tests, what we want them to produce, and 74 # what we want to claim are the expected results. 75 class TestList: 76 def __init__(self): 77 self.tests = {} 78 79 def add(self, name, **kwargs): 80 test = TestInstance(name) 81 for key, value in kwargs.items(): 82 test.__dict__[key] = value 83 self.tests[name] = test 84 85 def add_reftest(self, name, reference_name, same_image): 86 self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True) 87 if same_image: 88 self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True) 89 else: 90 self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True) 91 92 def keys(self): 93 return self.tests.keys() 94 95 def __contains__(self, item): 96 return item in self.tests 97 98 def __getitem__(self, item): 99 return self.tests[item] 100 101 102 def unit_test_list(): 103 tests = TestList() 104 tests.add('failures/expected/checksum.html', 105 actual_checksum='checksum_fail-checksum') 106 tests.add('failures/expected/crash.html', crash=True) 107 tests.add('failures/expected/exception.html', exception=True) 108 tests.add('failures/expected/timeout.html', timeout=True) 109 tests.add('failures/expected/hang.html', hang=True) 110 tests.add('failures/expected/missing_text.html', expected_text=None) 111 tests.add('failures/expected/image.html', 112 actual_image='image_fail-png', 113 expected_image='image-png') 114 tests.add('failures/expected/image_checksum.html', 115 actual_checksum='image_checksum_fail-checksum', 116 actual_image='image_checksum_fail-png') 117 tests.add('failures/expected/audio.html', 118 actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav', 119 actual_text=None, expected_text=None, 120 actual_image=None, expected_image=None, 121 actual_checksum=None, expected_checksum=None) 122 tests.add('failures/expected/keyboard.html', keyboard=True) 123 tests.add('failures/expected/missing_check.html', 124 expected_checksum=None, 125 expected_image=None) 126 tests.add('failures/expected/missing_image.html', expected_image=None) 127 tests.add('failures/expected/missing_audio.html', expected_audio=None, 128 actual_text=None, expected_text=None, 129 actual_image=None, expected_image=None, 130 actual_checksum=None, expected_checksum=None) 131 tests.add('failures/expected/missing_text.html', expected_text=None) 132 tests.add('failures/expected/newlines_leading.html', 133 expected_text="\nfoo\n", actual_text="foo\n") 134 tests.add('failures/expected/newlines_trailing.html', 135 expected_text="foo\n\n", actual_text="foo\n") 136 tests.add('failures/expected/newlines_with_excess_CR.html', 137 expected_text="foo\r\r\r\n", actual_text="foo\n") 138 tests.add('failures/expected/text.html', actual_text='text_fail-png') 139 tests.add('failures/unexpected/crash.html', crash=True) 140 tests.add('failures/unexpected/text-image-checksum.html', 141 actual_text='text-image-checksum_fail-txt', 142 actual_checksum='text-image-checksum_fail-checksum') 143 tests.add('failures/unexpected/timeout.html', timeout=True) 144 tests.add('http/tests/passes/text.html') 145 tests.add('http/tests/passes/image.html') 146 tests.add('http/tests/ssl/text.html') 147 tests.add('passes/error.html', error='stuff going to stderr') 148 tests.add('passes/image.html') 149 tests.add('passes/audio.html', 150 actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav', 151 actual_text=None, expected_text=None, 152 actual_image=None, expected_image=None, 153 actual_checksum=None, expected_checksum=None) 154 tests.add('passes/platform_image.html') 155 tests.add('passes/checksum_in_image.html', 156 expected_checksum=None, 157 expected_image='tEXtchecksum\x00checksum_in_image-checksum') 158 159 # Text output files contain "\r\n" on Windows. This may be 160 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. 161 tests.add('passes/text.html', 162 expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') 163 164 # For reftests. 165 tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True) 166 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False) 167 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False) 168 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True) 169 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False) 170 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True) 171 # FIXME: Add a reftest which crashes. 172 173 tests.add('websocket/tests/passes/text.html') 174 return tests 175 176 177 # Here we use a non-standard location for the layout tests, to ensure that 178 # this works. The path contains a '.' in the name because we've seen bugs 179 # related to this before. 180 181 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' 182 183 184 # Here we synthesize an in-memory filesystem from the test list 185 # in order to fully control the test output and to demonstrate that 186 # we don't need a real filesystem to run the tests. 187 188 def unit_test_filesystem(files=None): 189 """Return the FileSystem object used by the unit tests.""" 190 test_list = unit_test_list() 191 files = files or {} 192 193 def add_file(files, test, suffix, contents): 194 dirname = test.name[0:test.name.rfind('/')] 195 base = test.base 196 path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix 197 files[path] = contents 198 199 # Add each test and the expected output, if any. 200 for test in test_list.tests.values(): 201 add_file(files, test, '.html', '') 202 if test.is_reftest: 203 continue 204 if test.actual_audio: 205 add_file(files, test, '-expected.wav', test.expected_audio) 206 continue 207 208 add_file(files, test, '-expected.txt', test.expected_text) 209 add_file(files, test, '-expected.checksum', test.expected_checksum) 210 add_file(files, test, '-expected.png', test.expected_image) 211 212 213 # Add the test_expectations file. 214 files[LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'] = """ 215 WONTFIX : failures/expected/checksum.html = IMAGE 216 WONTFIX : failures/expected/crash.html = CRASH 217 // This one actually passes because the checksums will match. 218 WONTFIX : failures/expected/image.html = PASS 219 WONTFIX : failures/expected/audio.html = AUDIO 220 WONTFIX : failures/expected/image_checksum.html = IMAGE 221 WONTFIX : failures/expected/mismatch.html = IMAGE 222 WONTFIX : failures/expected/missing_check.html = MISSING PASS 223 WONTFIX : failures/expected/missing_image.html = MISSING PASS 224 WONTFIX : failures/expected/missing_audio.html = MISSING PASS 225 WONTFIX : failures/expected/missing_text.html = MISSING PASS 226 WONTFIX : failures/expected/newlines_leading.html = TEXT 227 WONTFIX : failures/expected/newlines_trailing.html = TEXT 228 WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT 229 WONTFIX : failures/expected/reftest.html = IMAGE 230 WONTFIX : failures/expected/text.html = TEXT 231 WONTFIX : failures/expected/timeout.html = TIMEOUT 232 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT 233 WONTFIX SKIP : failures/expected/keyboard.html = CRASH 234 WONTFIX SKIP : failures/expected/exception.html = CRASH 235 """ 236 237 # Add in a file should be ignored by test_files.find(). 238 files[LAYOUT_TEST_DIR + 'userscripts/resources/iframe.html'] = 'iframe' 239 240 fs = filesystem_mock.MockFileSystem(files) 241 fs._tests = test_list 242 return fs 243 244 245 class TestPort(base.Port): 246 """Test implementation of the Port interface.""" 247 ALL_BASELINE_VARIANTS = ( 248 'test-mac-snowleopard', 'test-mac-leopard', 249 'test-win-win7', 'test-win-vista', 'test-win-xp', 250 'test-linux-x86', 251 ) 252 253 def __init__(self, port_name=None, user=None, filesystem=None, **kwargs): 254 if not port_name or port_name == 'test': 255 port_name = 'test-mac-leopard' 256 user = user or mocktool.MockUser() 257 filesystem = filesystem or unit_test_filesystem() 258 base.Port.__init__(self, port_name=port_name, filesystem=filesystem, user=user, 259 **kwargs) 260 self._results_directory = None 261 262 assert filesystem._tests 263 self._tests = filesystem._tests 264 265 self._operating_system = 'mac' 266 if port_name.startswith('test-win'): 267 self._operating_system = 'win' 268 elif port_name.startswith('test-linux'): 269 self._operating_system = 'linux' 270 271 version_map = { 272 'test-win-xp': 'xp', 273 'test-win-win7': 'win7', 274 'test-win-vista': 'vista', 275 'test-mac-leopard': 'leopard', 276 'test-mac-snowleopard': 'snowleopard', 277 'test-linux-x86': '', 278 } 279 self._version = version_map[port_name] 280 281 self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt' 282 283 def _path_to_driver(self): 284 # This routine shouldn't normally be called, but it is called by 285 # the mock_drt Driver. We return something, but make sure it's useless. 286 return 'junk' 287 288 def baseline_path(self): 289 # We don't bother with a fallback path. 290 return self._filesystem.join(self.layout_tests_dir(), 'platform', self.name()) 291 292 def baseline_search_path(self): 293 search_paths = { 294 'test-mac-snowleopard': ['test-mac-snowleopard'], 295 'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'], 296 'test-win-win7': ['test-win-win7'], 297 'test-win-vista': ['test-win-vista', 'test-win-win7'], 298 'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'], 299 'test-linux-x86': ['test-linux', 'test-win-win7'], 300 } 301 return [self._webkit_baseline_path(d) for d in search_paths[self.name()]] 302 303 def default_child_processes(self): 304 return 1 305 306 def default_worker_model(self): 307 return 'inline' 308 309 def check_build(self, needs_http): 310 return True 311 312 def default_configuration(self): 313 return 'Release' 314 315 def diff_image(self, expected_contents, actual_contents, 316 diff_filename=None): 317 diffed = actual_contents != expected_contents 318 if diffed and diff_filename: 319 self._filesystem.write_binary_file(diff_filename, 320 "< %s\n---\n> %s\n" % (expected_contents, actual_contents)) 321 return diffed 322 323 def layout_tests_dir(self): 324 return LAYOUT_TEST_DIR 325 326 def name(self): 327 return self._name 328 329 def _path_to_wdiff(self): 330 return None 331 332 def default_results_directory(self): 333 return '/tmp/layout-test-results' 334 335 def setup_test_run(self): 336 pass 337 338 def create_driver(self, worker_number): 339 return TestDriver(self, worker_number) 340 341 def start_http_server(self): 342 pass 343 344 def start_websocket_server(self): 345 pass 346 347 def stop_http_server(self): 348 pass 349 350 def stop_websocket_server(self): 351 pass 352 353 def path_to_test_expectations_file(self): 354 return self._expectations_path 355 356 def all_baseline_variants(self): 357 return self.ALL_BASELINE_VARIANTS 358 359 # FIXME: These next two routines are copied from base.py with 360 # the calls to path.abspath_to_uri() removed. We shouldn't have 361 # to do this. 362 def filename_to_uri(self, filename): 363 """Convert a test file (which is an absolute path) to a URI.""" 364 LAYOUTTEST_HTTP_DIR = "http/tests/" 365 LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/" 366 367 relative_path = self.relative_test_filename(filename) 368 port = None 369 use_ssl = False 370 371 if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR) 372 or relative_path.startswith(LAYOUTTEST_HTTP_DIR)): 373 relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):] 374 port = 8000 375 376 # Make http/tests/local run as local files. This is to mimic the 377 # logic in run-webkit-tests. 378 # 379 # TODO(dpranke): remove the media reference and the SSL reference? 380 if (port and not relative_path.startswith("local/") and 381 not relative_path.startswith("media/")): 382 if relative_path.startswith("ssl/"): 383 port += 443 384 protocol = "https" 385 else: 386 protocol = "http" 387 return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path) 388 389 return "file://" + self._filesystem.abspath(filename) 390 391 def uri_to_test_name(self, uri): 392 """Return the base layout test name for a given URI. 393 394 This returns the test name for a given URI, e.g., if you passed in 395 "file:///src/LayoutTests/fast/html/keygen.html" it would return 396 "fast/html/keygen.html". 397 398 """ 399 test = uri 400 if uri.startswith("file:///"): 401 prefix = "file://" + self.layout_tests_dir() + "/" 402 return test[len(prefix):] 403 404 if uri.startswith("http://127.0.0.1:8880/"): 405 # websocket tests 406 return test.replace('http://127.0.0.1:8880/', '') 407 408 if uri.startswith("http://"): 409 # regular HTTP test 410 return test.replace('http://127.0.0.1:8000/', 'http/tests/') 411 412 if uri.startswith("https://"): 413 return test.replace('https://127.0.0.1:8443/', 'http/tests/') 414 415 raise NotImplementedError('unknown url type: %s' % uri) 416 417 418 class TestDriver(base.Driver): 419 """Test/Dummy implementation of the DumpRenderTree interface.""" 420 421 def __init__(self, port, worker_number): 422 self._port = port 423 424 def cmd_line(self): 425 return [self._port._path_to_driver()] + self._port.get_option('additional_drt_flag', []) 426 427 def poll(self): 428 return True 429 430 def run_test(self, test_input): 431 start_time = time.time() 432 test_name = self._port.relative_test_filename(test_input.filename) 433 test = self._port._tests[test_name] 434 if test.keyboard: 435 raise KeyboardInterrupt 436 if test.exception: 437 raise ValueError('exception from ' + test_name) 438 if test.hang: 439 time.sleep((float(test_input.timeout) * 4) / 1000.0) 440 441 audio = None 442 if test.actual_audio: 443 audio = base64.b64decode(test.actual_audio) 444 return base.DriverOutput(test.actual_text, test.actual_image, 445 test.actual_checksum, audio, crash=test.crash, 446 test_time=time.time() - start_time, timeout=test.timeout, error=test.error) 447 448 def start(self): 449 pass 450 451 def stop(self): 452 pass 453
