1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the Google name nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import base64 30 import copy 31 import sys 32 import time 33 34 from webkitpy.layout_tests.port import Port, Driver, DriverOutput 35 from webkitpy.layout_tests.port.base import VirtualTestSuite 36 from webkitpy.layout_tests.models.test_configuration import TestConfiguration 37 from webkitpy.common.system.filesystem_mock import MockFileSystem 38 from webkitpy.common.system.crashlogs import CrashLogs 39 40 41 # This sets basic expectations for a test. Each individual expectation 42 # can be overridden by a keyword argument in TestList.add(). 43 class TestInstance(object): 44 def __init__(self, name): 45 self.name = name 46 self.base = name[(name.rfind("/") + 1):name.rfind(".")] 47 self.crash = False 48 self.web_process_crash = False 49 self.exception = False 50 self.hang = False 51 self.keyboard = False 52 self.error = '' 53 self.timeout = False 54 self.is_reftest = False 55 56 # The values of each field are treated as raw byte strings. They 57 # will be converted to unicode strings where appropriate using 58 # FileSystem.read_text_file(). 59 self.actual_text = self.base + '-txt' 60 self.actual_checksum = self.base + '-checksum' 61 62 # We add the '\x8a' for the image file to prevent the value from 63 # being treated as UTF-8 (the character is invalid) 64 self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum 65 66 self.expected_text = self.actual_text 67 self.expected_image = self.actual_image 68 69 self.actual_audio = None 70 self.expected_audio = None 71 72 73 # This is an in-memory list of tests, what we want them to produce, and 74 # what we want to claim are the expected results. 75 class TestList(object): 76 def __init__(self): 77 self.tests = {} 78 79 def add(self, name, **kwargs): 80 test = TestInstance(name) 81 for key, value in kwargs.items(): 82 test.__dict__[key] = value 83 self.tests[name] = test 84 85 def add_reftest(self, name, reference_name, same_image): 86 self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True) 87 if same_image: 88 self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True) 89 else: 90 self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True) 91 92 def keys(self): 93 return self.tests.keys() 94 95 def __contains__(self, item): 96 return item in self.tests 97 98 def __getitem__(self, item): 99 return self.tests[item] 100 101 # 102 # These numbers may need to be updated whenever we add or delete tests. 103 # 104 TOTAL_TESTS = 106 105 TOTAL_SKIPS = 27 106 107 UNEXPECTED_PASSES = 1 108 UNEXPECTED_FAILURES = 22 109 110 def unit_test_list(): 111 tests = TestList() 112 tests.add('failures/expected/crash.html', crash=True) 113 tests.add('failures/expected/exception.html', exception=True) 114 tests.add('failures/expected/timeout.html', timeout=True) 115 tests.add('failures/expected/hang.html', hang=True) 116 tests.add('failures/expected/missing_text.html', expected_text=None) 117 tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebaseline text') 118 tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needsmanualrebaseline text') 119 tests.add('failures/expected/image.html', 120 actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', 121 expected_image='image-pngtEXtchecksum\x00checksum-png') 122 tests.add('failures/expected/image_checksum.html', 123 actual_checksum='image_checksum_fail-checksum', 124 actual_image='image_checksum_fail-png') 125 tests.add('failures/expected/audio.html', 126 actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav', 127 actual_text=None, expected_text=None, 128 actual_image=None, expected_image=None, 129 actual_checksum=None) 130 tests.add('failures/expected/keyboard.html', keyboard=True) 131 tests.add('failures/expected/missing_check.html', 132 expected_image='missing_check-png') 133 tests.add('failures/expected/missing_image.html', expected_image=None) 134 tests.add('failures/expected/missing_audio.html', expected_audio=None, 135 actual_text=None, expected_text=None, 136 actual_image=None, expected_image=None, 137 actual_checksum=None) 138 tests.add('failures/expected/missing_text.html', expected_text=None) 139 tests.add('failures/expected/newlines_leading.html', 140 expected_text="\nfoo\n", actual_text="foo\n") 141 tests.add('failures/expected/newlines_trailing.html', 142 expected_text="foo\n\n", actual_text="foo\n") 143 tests.add('failures/expected/newlines_with_excess_CR.html', 144 expected_text="foo\r\r\r\n", actual_text="foo\n") 145 tests.add('failures/expected/text.html', actual_text='text_fail-png') 146 tests.add('failures/expected/skip_text.html', actual_text='text diff') 147 tests.add('failures/flaky/text.html') 148 tests.add('failures/unexpected/missing_text.html', expected_text=None) 149 tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png') 150 tests.add('failures/unexpected/missing_image.html', expected_image=None) 151 tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600 152 RenderView at (0,0) size 800x600 153 layer at (0,0) size 800x34 154 RenderBlock {HTML} at (0,0) size 800x34 155 RenderBody {BODY} at (8,8) size 784x18 156 RenderText {#text} at (0,0) size 133x18 157 text run at (0,0) width 133: "This is an image test!" 158 """, expected_text=None) 159 tests.add('failures/unexpected/crash.html', crash=True) 160 tests.add('failures/unexpected/crash-with-stderr.html', crash=True, 161 error="mock-std-error-output") 162 tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True, 163 error="mock-std-error-output") 164 tests.add('failures/unexpected/pass.html') 165 tests.add('failures/unexpected/text-checksum.html', 166 actual_text='text-checksum_fail-txt', 167 actual_checksum='text-checksum_fail-checksum') 168 tests.add('failures/unexpected/text-image-checksum.html', 169 actual_text='text-image-checksum_fail-txt', 170 actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail', 171 actual_checksum='text-image-checksum_fail-checksum') 172 tests.add('failures/unexpected/checksum-with-matching-image.html', 173 actual_checksum='text-image-checksum_fail-checksum') 174 tests.add('failures/unexpected/skip_pass.html') 175 tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') 176 tests.add('failures/unexpected/timeout.html', timeout=True) 177 tests.add('http/tests/passes/text.html') 178 tests.add('http/tests/passes/image.html') 179 tests.add('http/tests/ssl/text.html') 180 tests.add('passes/args.html') 181 tests.add('passes/error.html', error='stuff going to stderr') 182 tests.add('passes/image.html') 183 tests.add('passes/audio.html', 184 actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav', 185 actual_text=None, expected_text=None, 186 actual_image=None, expected_image=None, 187 actual_checksum=None) 188 tests.add('passes/platform_image.html') 189 tests.add('passes/checksum_in_image.html', 190 expected_image='tEXtchecksum\x00checksum_in_image-checksum') 191 tests.add('passes/skipped/skip.html') 192 193 # Note that here the checksums don't match but the images do, so this test passes "unexpectedly". 194 # See https://bugs.webkit.org/show_bug.cgi?id=69444 . 195 tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum') 196 197 # Text output files contain "\r\n" on Windows. This may be 198 # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. 199 tests.add('passes/text.html', 200 expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') 201 202 # For reftests. 203 tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True) 204 tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False) 205 tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True) 206 tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True) 207 tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False) 208 tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False) 209 tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True) 210 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False) 211 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True) 212 tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True) 213 tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True) 214 # FIXME: Add a reftest which crashes. 215 tests.add('reftests/foo/test.html') 216 tests.add('reftests/foo/test-ref.html') 217 218 tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc') 219 tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc') 220 tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc') 221 tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc') 222 tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc') 223 tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc') 224 225 tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc') 226 tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def') 227 tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi') 228 229 # The following files shouldn't be treated as reftests 230 tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True) 231 tests.add('reftests/foo/reference/bar/common.html') 232 tests.add('reftests/foo/reftest/bar/shared.html') 233 234 tests.add('websocket/tests/passes/text.html') 235 236 # For testing that we don't run tests under platform/. Note that these don't contribute to TOTAL_TESTS. 237 tests.add('platform/test-mac-leopard/http/test.html') 238 tests.add('platform/test-win-win7/http/test.html') 239 240 # For testing if perf tests are running in a locked shard. 241 tests.add('perf/foo/test.html') 242 tests.add('perf/foo/test-ref.html') 243 244 # For testing --pixel-test-directories. 245 tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', 246 actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', 247 expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png') 248 tests.add('failures/unexpected/image_not_in_pixeldir.html', 249 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', 250 expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') 251 252 # For testing that virtual test suites don't expand names containing themselves 253 # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). 254 tests.add('passes/test-virtual-passes.html') 255 tests.add('passes/passes/test-virtual-passes.html') 256 257 return tests 258 259 260 # Here we use a non-standard location for the layout tests, to ensure that 261 # this works. The path contains a '.' in the name because we've seen bugs 262 # related to this before. 263 264 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' 265 PERF_TEST_DIR = '/test.checkout/PerformanceTests' 266 267 268 # Here we synthesize an in-memory filesystem from the test list 269 # in order to fully control the test output and to demonstrate that 270 # we don't need a real filesystem to run the tests. 271 def add_unit_tests_to_mock_filesystem(filesystem): 272 # Add the test_expectations file. 273 filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test') 274 if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'): 275 filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """ 276 Bug(test) failures/expected/crash.html [ Crash ] 277 Bug(test) failures/expected/image.html [ ImageOnlyFailure ] 278 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] 279 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] 280 Bug(test) failures/expected/audio.html [ Failure ] 281 Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ] 282 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ] 283 Bug(test) failures/expected/missing_check.html [ Missing Pass ] 284 Bug(test) failures/expected/missing_image.html [ Missing Pass ] 285 Bug(test) failures/expected/missing_audio.html [ Missing Pass ] 286 Bug(test) failures/expected/missing_text.html [ Missing Pass ] 287 Bug(test) failures/expected/newlines_leading.html [ Failure ] 288 Bug(test) failures/expected/newlines_trailing.html [ Failure ] 289 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ] 290 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ] 291 Bug(test) failures/expected/text.html [ Failure ] 292 Bug(test) failures/expected/timeout.html [ Timeout ] 293 Bug(test) failures/expected/hang.html [ WontFix ] 294 Bug(test) failures/expected/keyboard.html [ WontFix ] 295 Bug(test) failures/expected/exception.html [ WontFix ] 296 Bug(test) failures/unexpected/pass.html [ Failure ] 297 Bug(test) passes/skipped/skip.html [ Skip ] 298 Bug(test) passes/text.html [ Pass ] 299 """) 300 301 filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo') 302 filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """ 303 == test.html test-ref.html 304 305 == multiple-match-success.html mismatching-ref.html 306 == multiple-match-success.html matching-ref.html 307 == multiple-match-failure.html mismatching-ref.html 308 == multiple-match-failure.html second-mismatching-ref.html 309 != multiple-mismatch-success.html mismatching-ref.html 310 != multiple-mismatch-success.html second-mismatching-ref.html 311 != multiple-mismatch-failure.html mismatching-ref.html 312 != multiple-mismatch-failure.html matching-ref.html 313 == multiple-both-success.html matching-ref.html 314 == multiple-both-success.html mismatching-ref.html 315 != multiple-both-success.html second-mismatching-ref.html 316 == multiple-both-failure.html matching-ref.html 317 != multiple-both-failure.html second-mismatching-ref.html 318 != multiple-both-failure.html matching-ref.html 319 """) 320 321 # FIXME: This test was only being ignored because of missing a leading '/'. 322 # Fixing the typo causes several tests to assert, so disabling the test entirely. 323 # Add in a file should be ignored by port.find_test_files(). 324 #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe' 325 326 def add_file(test, suffix, contents): 327 dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')]) 328 base = test.base 329 filesystem.maybe_make_directory(dirname) 330 filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents) 331 332 # Add each test and the expected output, if any. 333 test_list = unit_test_list() 334 for test in test_list.tests.values(): 335 add_file(test, test.name[test.name.rfind('.'):], '') 336 if test.is_reftest: 337 continue 338 if test.actual_audio: 339 add_file(test, '-expected.wav', test.expected_audio) 340 continue 341 add_file(test, '-expected.txt', test.expected_text) 342 add_file(test, '-expected.png', test.expected_image) 343 344 filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg') 345 # Clear the list of written files so that we can watch what happens during testing. 346 filesystem.clear_written_files() 347 348 349 class TestPort(Port): 350 port_name = 'test' 351 default_port_name = 'test-mac-leopard' 352 353 """Test implementation of the Port interface.""" 354 ALL_BASELINE_VARIANTS = ( 355 'test-linux-x86_64', 356 'test-mac-snowleopard', 'test-mac-leopard', 357 'test-win-win7', 'test-win-xp', 358 ) 359 360 @classmethod 361 def determine_full_port_name(cls, host, options, port_name): 362 if port_name == 'test': 363 return TestPort.default_port_name 364 return port_name 365 366 def __init__(self, host, port_name=None, **kwargs): 367 Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs) 368 self._tests = unit_test_list() 369 self._flakes = set() 370 self._generic_expectations_path = LAYOUT_TEST_DIR + '/TestExpectations' 371 self._results_directory = None 372 373 self._operating_system = 'mac' 374 if self._name.startswith('test-win'): 375 self._operating_system = 'win' 376 elif self._name.startswith('test-linux'): 377 self._operating_system = 'linux' 378 379 version_map = { 380 'test-win-xp': 'xp', 381 'test-win-win7': 'win7', 382 'test-mac-leopard': 'leopard', 383 'test-mac-snowleopard': 'snowleopard', 384 'test-linux-x86_64': 'lucid', 385 } 386 self._version = version_map[self._name] 387 388 def buildbot_archives_baselines(self): 389 return self._name != 'test-win-xp' 390 391 def default_pixel_tests(self): 392 return True 393 394 def _path_to_driver(self): 395 # This routine shouldn't normally be called, but it is called by 396 # the mock_drt Driver. We return something, but make sure it's useless. 397 return 'MOCK _path_to_driver' 398 399 def baseline_search_path(self): 400 search_paths = { 401 'test-mac-snowleopard': ['test-mac-snowleopard'], 402 'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'], 403 'test-win-win7': ['test-win-win7'], 404 'test-win-xp': ['test-win-xp', 'test-win-win7'], 405 'test-linux-x86_64': ['test-linux-x86_64', 'test-win-win7'], 406 } 407 return [self._webkit_baseline_path(d) for d in search_paths[self.name()]] 408 409 def default_child_processes(self): 410 return 1 411 412 def worker_startup_delay_secs(self): 413 return 0 414 415 def check_build(self, needs_http): 416 return True 417 418 def check_sys_deps(self, needs_http): 419 return True 420 421 def default_configuration(self): 422 return 'Release' 423 424 def diff_image(self, expected_contents, actual_contents): 425 diffed = actual_contents != expected_contents 426 if not actual_contents and not expected_contents: 427 return (None, None) 428 if not actual_contents or not expected_contents: 429 return (True, None) 430 if diffed: 431 return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), None) 432 return (None, None) 433 434 def layout_tests_dir(self): 435 return LAYOUT_TEST_DIR 436 437 def perf_tests_dir(self): 438 return PERF_TEST_DIR 439 440 def webkit_base(self): 441 return '/test.checkout' 442 443 def _skipped_tests_for_unsupported_features(self, test_list): 444 return set(['failures/expected/skip_text.html', 445 'failures/unexpected/skip_pass.html', 446 'virtual/skipped']) 447 448 def name(self): 449 return self._name 450 451 def operating_system(self): 452 return self._operating_system 453 454 def _path_to_wdiff(self): 455 return None 456 457 def default_results_directory(self): 458 return '/tmp/layout-test-results' 459 460 def setup_test_run(self): 461 pass 462 463 def _driver_class(self): 464 return TestDriver 465 466 def start_http_server(self, additional_dirs=None, number_of_servers=None): 467 pass 468 469 def start_websocket_server(self): 470 pass 471 472 def acquire_http_lock(self): 473 pass 474 475 def stop_http_server(self): 476 pass 477 478 def stop_websocket_server(self): 479 pass 480 481 def release_http_lock(self): 482 pass 483 484 def _path_to_lighttpd(self): 485 return "/usr/sbin/lighttpd" 486 487 def _path_to_lighttpd_modules(self): 488 return "/usr/lib/lighttpd" 489 490 def _path_to_lighttpd_php(self): 491 return "/usr/bin/php-cgi" 492 493 def _path_to_apache(self): 494 return "/usr/sbin/httpd" 495 496 def _path_to_apache_config_file(self): 497 return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf') 498 499 def path_to_generic_test_expectations_file(self): 500 return self._generic_expectations_path 501 502 def all_test_configurations(self): 503 """Returns a sequence of the TestConfigurations the port supports.""" 504 # By default, we assume we want to test every graphics type in 505 # every configuration on every system. 506 test_configurations = [] 507 for version, architecture in self._all_systems(): 508 for build_type in self._all_build_types(): 509 test_configurations.append(TestConfiguration( 510 version=version, 511 architecture=architecture, 512 build_type=build_type)) 513 return test_configurations 514 515 def _all_systems(self): 516 return (('leopard', 'x86'), 517 ('snowleopard', 'x86'), 518 ('xp', 'x86'), 519 ('win7', 'x86'), 520 ('lucid', 'x86'), 521 ('lucid', 'x86_64')) 522 523 def _all_build_types(self): 524 return ('debug', 'release') 525 526 def configuration_specifier_macros(self): 527 """To avoid surprises when introducing new macros, these are intentionally fixed in time.""" 528 return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'win7'], 'linux': ['lucid']} 529 530 def all_baseline_variants(self): 531 return self.ALL_BASELINE_VARIANTS 532 533 def virtual_test_suites(self): 534 return [ 535 VirtualTestSuite('virtual/passes', 'passes', ['--virtual-arg']), 536 VirtualTestSuite('virtual/skipped', 'failures/expected', ['--virtual-arg2']), 537 ] 538 539 540 class TestDriver(Driver): 541 """Test/Dummy implementation of the driver interface.""" 542 next_pid = 1 543 544 def __init__(self, *args, **kwargs): 545 super(TestDriver, self).__init__(*args, **kwargs) 546 self.started = False 547 self.pid = 0 548 549 def cmd_line(self, pixel_tests, per_test_args): 550 pixel_tests_flag = '-p' if pixel_tests else '' 551 return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args 552 553 def run_test(self, driver_input, stop_when_done): 554 base = self._port.lookup_virtual_test_base(driver_input.test_name) 555 if base: 556 virtual_driver_input = copy.copy(driver_input) 557 virtual_driver_input.test_name = base 558 virtual_driver_input.args = self._port.lookup_virtual_test_args(driver_input.test_name) 559 return self.run_test(virtual_driver_input, stop_when_done) 560 561 if not self.started: 562 self.started = True 563 self.pid = TestDriver.next_pid 564 TestDriver.next_pid += 1 565 566 start_time = time.time() 567 test_name = driver_input.test_name 568 test_args = driver_input.args or [] 569 test = self._port._tests[test_name] 570 if test.keyboard: 571 raise KeyboardInterrupt 572 if test.exception: 573 raise ValueError('exception from ' + test_name) 574 if test.hang: 575 time.sleep((float(driver_input.timeout) * 4) / 1000.0 + 1.0) # The 1.0 comes from thread_padding_sec in layout_test_runnery. 576 577 audio = None 578 actual_text = test.actual_text 579 580 if 'flaky' in test_name and not test_name in self._port._flakes: 581 self._port._flakes.add(test_name) 582 actual_text = 'flaky text failure' 583 584 if actual_text and test_args and test_name == 'passes/args.html': 585 actual_text = actual_text + ' ' + ' '.join(test_args) 586 587 if test.actual_audio: 588 audio = base64.b64decode(test.actual_audio) 589 crashed_process_name = None 590 crashed_pid = None 591 if test.crash: 592 crashed_process_name = self._port.driver_name() 593 crashed_pid = 1 594 elif test.web_process_crash: 595 crashed_process_name = 'WebProcess' 596 crashed_pid = 2 597 598 crash_log = '' 599 if crashed_process_name: 600 crash_logs = CrashLogs(self._port.host) 601 crash_log = crash_logs.find_newest_log(crashed_process_name, None) or '' 602 603 if stop_when_done: 604 self.stop() 605 606 if test.actual_checksum == driver_input.image_hash: 607 image = None 608 else: 609 image = test.actual_image 610 return DriverOutput(actual_text, image, test.actual_checksum, audio, 611 crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name, 612 crashed_pid=crashed_pid, crash_log=crash_log, 613 test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid) 614 615 def stop(self): 616 self.started = False 617
