1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor (at] inf.u-szeged.hu), University of Szeged 3 # Copyright (C) 2011 Apple Inc. All rights reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 import Queue 32 import StringIO 33 import codecs 34 import json 35 import logging 36 import os 37 import platform 38 import re 39 import sys 40 import thread 41 import time 42 import threading 43 import unittest 44 45 from webkitpy.common.system import outputcapture, path 46 from webkitpy.common.system.crashlogs_unittest import make_mock_crash_report_darwin 47 from webkitpy.common.system.systemhost import SystemHost 48 from webkitpy.common.host import Host 49 from webkitpy.common.host_mock import MockHost 50 51 from webkitpy.layout_tests import port 52 from webkitpy.layout_tests import run_webkit_tests 53 from webkitpy.layout_tests.models import test_run_results 54 from webkitpy.layout_tests.port import Port 55 from webkitpy.layout_tests.port import test 56 from webkitpy.test.skip import skip_if 57 from webkitpy.tool import grammar 58 from webkitpy.tool.mocktool import MockOptions 59 60 61 def parse_args(extra_args=None, tests_included=False, new_results=False, print_nothing=True): 62 extra_args = extra_args or [] 63 args = [] 64 if not '--platform' in extra_args: 65 args.extend(['--platform', 'test']) 66 if not new_results: 67 args.append('--no-new-test-results') 68 69 if not '--child-processes' in extra_args: 70 args.extend(['--child-processes', 1]) 71 args.extend(extra_args) 72 if not tests_included: 73 # We use the glob to test that globbing works. 74 args.extend(['passes', 75 'http/tests', 76 'websocket/tests', 77 'failures/expected/*']) 78 return run_webkit_tests.parse_args(args) 79 80 81 def passing_run(extra_args=None, port_obj=None, tests_included=False, host=None, shared_port=True): 82 options, parsed_args = parse_args(extra_args, tests_included) 83 if not port_obj: 84 host = host or MockHost() 85 port_obj = host.port_factory.get(port_name=options.platform, options=options) 86 87 if shared_port: 88 port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj 89 90 logging_stream = StringIO.StringIO() 91 run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) 92 return run_details.exit_code == 0 93 94 95 def logging_run(extra_args=None, port_obj=None, tests_included=False, host=None, new_results=False, shared_port=True): 96 options, parsed_args = parse_args(extra_args=extra_args, 97 tests_included=tests_included, 98 print_nothing=False, new_results=new_results) 99 host = host or MockHost() 100 if not port_obj: 101 port_obj = host.port_factory.get(port_name=options.platform, options=options) 102 103 run_details, output = run_and_capture(port_obj, options, parsed_args, shared_port) 104 return (run_details, output, host.user) 105 106 107 def run_and_capture(port_obj, options, parsed_args, shared_port=True): 108 if shared_port: 109 port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj 110 oc = outputcapture.OutputCapture() 111 try: 112 oc.capture_output() 113 logging_stream = StringIO.StringIO() 114 run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) 115 finally: 116 oc.restore_output() 117 return (run_details, logging_stream) 118 119 120 def get_tests_run(args, host=None, port_obj=None): 121 results = get_test_results(args, host=host, port_obj=port_obj) 122 return [result.test_name for result in results] 123 124 125 def get_test_batches(args, host=None): 126 results = get_test_results(args, host) 127 batches = [] 128 batch = [] 129 current_pid = None 130 for result in results: 131 if batch and result.pid != current_pid: 132 batches.append(batch) 133 batch = [] 134 batch.append(result.test_name) 135 if batch: 136 batches.append(batch) 137 return batches 138 139 140 def get_test_results(args, host=None, port_obj=None): 141 options, parsed_args = parse_args(args, tests_included=True) 142 143 host = host or MockHost() 144 port_obj = port_obj or host.port_factory.get(port_name=options.platform, options=options) 145 146 oc = outputcapture.OutputCapture() 147 oc.capture_output() 148 logging_stream = StringIO.StringIO() 149 try: 150 run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) 151 finally: 152 oc.restore_output() 153 154 all_results = [] 155 if run_details.initial_results: 156 all_results.extend(run_details.initial_results.all_results) 157 158 if run_details.retry_results: 159 all_results.extend(run_details.retry_results.all_results) 160 return all_results 161 162 163 def parse_full_results(full_results_text): 164 json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "") 165 compressed_results = json.loads(json_to_eval) 166 return compressed_results 167 168 169 class StreamTestingMixin(object): 170 def assertContains(self, stream, string): 171 self.assertTrue(string in stream.getvalue()) 172 173 def assertEmpty(self, stream): 174 self.assertFalse(stream.getvalue()) 175 176 def assertNotEmpty(self, stream): 177 self.assertTrue(stream.getvalue()) 178 179 180 class RunTest(unittest.TestCase, StreamTestingMixin): 181 def setUp(self): 182 # A real PlatformInfo object is used here instead of a 183 # MockPlatformInfo because we need to actually check for 184 # Windows and Mac to skip some tests. 185 self._platform = SystemHost().platform 186 187 # FIXME: Remove this when we fix test-webkitpy to work 188 # properly on cygwin (bug 63846). 189 self.should_test_processes = not self._platform.is_win() 190 191 def test_basic(self): 192 options, args = parse_args(tests_included=True) 193 logging_stream = StringIO.StringIO() 194 host = MockHost() 195 port_obj = host.port_factory.get(options.platform, options) 196 details = run_webkit_tests.run(port_obj, options, args, logging_stream) 197 198 # These numbers will need to be updated whenever we add new tests. 199 self.assertEqual(details.initial_results.total, test.TOTAL_TESTS) 200 self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS) 201 self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES) 202 self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES) 203 self.assertEqual(details.retry_results.total, test.UNEXPECTED_FAILURES) 204 205 expected_tests = details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name) 206 expected_summary_str = '' 207 if details.initial_results.expected_failures > 0: 208 expected_summary_str = " (%d passed, %d didn't)" % (expected_tests - details.initial_results.expected_failures, details.initial_results.expected_failures) 209 one_line_summary = "%d tests ran as expected%s, %d didn't:\n" % ( 210 expected_tests, 211 expected_summary_str, 212 len(details.initial_results.unexpected_results_by_name)) 213 self.assertTrue(one_line_summary in logging_stream.buflist) 214 215 # Ensure the results were summarized properly. 216 self.assertEqual(details.summarized_failing_results['num_regressions'], details.exit_code) 217 218 # Ensure the results were written out and displayed. 219 failing_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/failing_results.json') 220 json_to_eval = failing_results_text.replace("ADD_RESULTS(", "").replace(");", "") 221 self.assertEqual(json.loads(json_to_eval), details.summarized_failing_results) 222 223 full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') 224 self.assertEqual(json.loads(full_results_text), details.summarized_full_results) 225 226 self.assertEqual(host.user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')]) 227 228 def test_batch_size(self): 229 batch_tests_run = get_test_batches(['--batch-size', '2']) 230 for batch in batch_tests_run: 231 self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch)) 232 233 def test_max_locked_shards(self): 234 # Tests for the default of using one locked shard even in the case of more than one child process. 235 if not self.should_test_processes: 236 return 237 save_env_webkit_test_max_locked_shards = None 238 if "WEBKIT_TEST_MAX_LOCKED_SHARDS" in os.environ: 239 save_env_webkit_test_max_locked_shards = os.environ["WEBKIT_TEST_MAX_LOCKED_SHARDS"] 240 del os.environ["WEBKIT_TEST_MAX_LOCKED_SHARDS"] 241 _, regular_output, _ = logging_run(['--debug-rwt-logging', '--child-processes', '2'], shared_port=False) 242 try: 243 self.assertTrue(any(['1 locked' in line for line in regular_output.buflist])) 244 finally: 245 if save_env_webkit_test_max_locked_shards: 246 os.environ["WEBKIT_TEST_MAX_LOCKED_SHARDS"] = save_env_webkit_test_max_locked_shards 247 248 def test_child_processes_2(self): 249 if self.should_test_processes: 250 _, regular_output, _ = logging_run( 251 ['--debug-rwt-logging', '--child-processes', '2'], shared_port=False) 252 self.assertTrue(any(['Running 2 ' in line for line in regular_output.buflist])) 253 254 def test_child_processes_min(self): 255 if self.should_test_processes: 256 _, regular_output, _ = logging_run( 257 ['--debug-rwt-logging', '--child-processes', '2', '-i', 'passes/virtual_passes', 'passes'], 258 tests_included=True, shared_port=False) 259 self.assertTrue(any(['Running 1 ' in line for line in regular_output.buflist])) 260 261 def test_dryrun(self): 262 tests_run = get_tests_run(['--dry-run']) 263 self.assertEqual(tests_run, []) 264 265 tests_run = get_tests_run(['-n']) 266 self.assertEqual(tests_run, []) 267 268 def test_enable_sanitizer(self): 269 self.assertTrue(passing_run(['--enable-sanitizer', 'failures/expected/text.html'])) 270 271 def test_exception_raised(self): 272 # Exceptions raised by a worker are treated differently depending on 273 # whether they are in-process or out. inline exceptions work as normal, 274 # which allows us to get the full stack trace and traceback from the 275 # worker. The downside to this is that it could be any error, but this 276 # is actually useful in testing. 277 # 278 # Exceptions raised in a separate process are re-packaged into 279 # WorkerExceptions (a subclass of BaseException), which have a string capture of the stack which can 280 # be printed, but don't display properly in the unit test exception handlers. 281 self.assertRaises(BaseException, logging_run, 282 ['failures/expected/exception.html', '--child-processes', '1'], tests_included=True) 283 284 if self.should_test_processes: 285 self.assertRaises(BaseException, logging_run, 286 ['--child-processes', '2', '--skipped=ignore', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True, shared_port=False) 287 288 def test_device_failure(self): 289 # Test that we handle a device going offline during a test properly. 290 details, regular_output, _ = logging_run(['failures/expected/device_failure.html'], tests_included=True) 291 self.assertEqual(details.exit_code, 0) 292 self.assertTrue('worker/0 has failed' in regular_output.getvalue()) 293 294 def test_full_results_html(self): 295 host = MockHost() 296 details, _, _ = logging_run(['--full-results-html'], host=host) 297 self.assertEqual(details.exit_code, 0) 298 self.assertEqual(len(host.user.opened_urls), 1) 299 300 def test_keyboard_interrupt(self): 301 # Note that this also tests running a test marked as SKIP if 302 # you specify it explicitly. 303 details, _, _ = logging_run(['failures/expected/keyboard.html', '--child-processes', '1'], tests_included=True) 304 self.assertEqual(details.exit_code, test_run_results.INTERRUPTED_EXIT_STATUS) 305 306 if self.should_test_processes: 307 _, regular_output, _ = logging_run(['failures/expected/keyboard.html', 'passes/text.html', '--child-processes', '2', '--skipped=ignore'], tests_included=True, shared_port=False) 308 self.assertTrue(any(['Interrupted, exiting' in line for line in regular_output.buflist])) 309 310 def test_no_tests_found(self): 311 details, err, _ = logging_run(['resources'], tests_included=True) 312 self.assertEqual(details.exit_code, test_run_results.NO_TESTS_EXIT_STATUS) 313 self.assertContains(err, 'No tests to run.\n') 314 315 def test_no_tests_found_2(self): 316 details, err, _ = logging_run(['foo'], tests_included=True) 317 self.assertEqual(details.exit_code, test_run_results.NO_TESTS_EXIT_STATUS) 318 self.assertContains(err, 'No tests to run.\n') 319 320 def test_no_tests_found_3(self): 321 details, err, _ = logging_run(['--run-chunk', '5:400', 'foo/bar.html'], tests_included=True) 322 self.assertEqual(details.exit_code, test_run_results.NO_TESTS_EXIT_STATUS) 323 self.assertContains(err, 'No tests to run.\n') 324 325 def test_natural_order(self): 326 tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] 327 tests_run = get_tests_run(['--order=natural'] + tests_to_run) 328 self.assertEqual(['failures/expected/missing_text.html', 'failures/expected/text.html', 'passes/args.html', 'passes/audio.html'], tests_run) 329 330 def test_natural_order_test_specified_multiple_times(self): 331 tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html'] 332 tests_run = get_tests_run(['--order=natural'] + tests_to_run) 333 self.assertEqual(['passes/args.html', 'passes/args.html', 'passes/audio.html', 'passes/audio.html'], tests_run) 334 335 def test_random_order(self): 336 tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] 337 tests_run = get_tests_run(['--order=random'] + tests_to_run) 338 self.assertEqual(sorted(tests_to_run), sorted(tests_run)) 339 340 def test_random_daily_seed_order(self): 341 tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] 342 tests_run = get_tests_run(['--order=random-seeded'] + tests_to_run) 343 self.assertEqual(sorted(tests_to_run), sorted(tests_run)) 344 345 def test_random_order_test_specified_multiple_times(self): 346 tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html'] 347 tests_run = get_tests_run(['--order=random'] + tests_to_run) 348 self.assertEqual(tests_run.count('passes/audio.html'), 2) 349 self.assertEqual(tests_run.count('passes/args.html'), 2) 350 351 def test_no_order(self): 352 tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html'] 353 tests_run = get_tests_run(['--order=none'] + tests_to_run) 354 self.assertEqual(tests_to_run, tests_run) 355 356 def test_no_order_test_specified_multiple_times(self): 357 tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html'] 358 tests_run = get_tests_run(['--order=none'] + tests_to_run) 359 self.assertEqual(tests_to_run, tests_run) 360 361 def test_no_order_with_directory_entries_in_natural_order(self): 362 tests_to_run = ['http/tests/ssl', 'perf/foo', 'http/tests/passes'] 363 tests_run = get_tests_run(['--order=none'] + tests_to_run) 364 self.assertEqual(tests_run, ['http/tests/ssl/text.html', 'perf/foo/test.html', 'http/tests/passes/image.html', 'http/tests/passes/text.html']) 365 366 def test_repeat_each(self): 367 tests_to_run = ['passes/image.html', 'passes/text.html'] 368 tests_run = get_tests_run(['--repeat-each', '2'] + tests_to_run) 369 self.assertEqual(tests_run, ['passes/image.html', 'passes/image.html', 'passes/text.html', 'passes/text.html']) 370 371 def test_ignore_flag(self): 372 # Note that passes/image.html is expected to be run since we specified it directly. 373 tests_run = get_tests_run(['-i', 'passes', 'passes/image.html']) 374 self.assertFalse('passes/text.html' in tests_run) 375 self.assertTrue('passes/image.html' in tests_run) 376 377 def test_skipped_flag(self): 378 tests_run = get_tests_run(['passes']) 379 self.assertFalse('passes/skipped/skip.html' in tests_run) 380 num_tests_run_by_default = len(tests_run) 381 382 # Check that nothing changes when we specify skipped=default. 383 self.assertEqual(len(get_tests_run(['--skipped=default', 'passes'])), 384 num_tests_run_by_default) 385 386 # Now check that we run one more test (the skipped one). 387 tests_run = get_tests_run(['--skipped=ignore', 'passes']) 388 self.assertTrue('passes/skipped/skip.html' in tests_run) 389 self.assertEqual(len(tests_run), num_tests_run_by_default + 1) 390 391 # Now check that we only run the skipped test. 392 self.assertEqual(get_tests_run(['--skipped=only', 'passes']), ['passes/skipped/skip.html']) 393 394 # Now check that we don't run anything. 395 self.assertEqual(get_tests_run(['--skipped=always', 'passes/skipped/skip.html']), []) 396 397 def test_iterations(self): 398 tests_to_run = ['passes/image.html', 'passes/text.html'] 399 tests_run = get_tests_run(['--iterations', '2'] + tests_to_run) 400 self.assertEqual(tests_run, ['passes/image.html', 'passes/text.html', 'passes/image.html', 'passes/text.html']) 401 402 def test_repeat_each_iterations_num_tests(self): 403 # The total number of tests should be: number_of_tests * 404 # repeat_each * iterations 405 host = MockHost() 406 _, err, _ = logging_run( 407 ['--iterations', '2', '--repeat-each', '4', '--debug-rwt-logging', 'passes/text.html', 'failures/expected/text.html'], 408 tests_included=True, host=host) 409 self.assertContains(err, "All 16 tests ran as expected (8 passed, 8 didn't).\n") 410 411 def test_run_chunk(self): 412 # Test that we actually select the right chunk 413 all_tests_run = get_tests_run(['passes', 'failures']) 414 chunk_tests_run = get_tests_run(['--run-chunk', '1:4', 'passes', 'failures']) 415 self.assertEqual(all_tests_run[4:8], chunk_tests_run) 416 417 # Test that we wrap around if the number of tests is not evenly divisible by the chunk size 418 tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html'] 419 chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run) 420 self.assertEqual(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run) 421 422 def test_run_part(self): 423 # Test that we actually select the right part 424 tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html'] 425 tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run) 426 self.assertEqual(['passes/error.html', 'passes/image.html'], tests_run) 427 428 # Test that we wrap around if the number of tests is not evenly divisible by the chunk size 429 # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the 430 # last part repeats the first two tests). 431 chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run) 432 self.assertEqual(['passes/error.html', 'passes/image.html'], chunk_tests_run) 433 434 def test_run_singly(self): 435 batch_tests_run = get_test_batches(['--run-singly']) 436 for batch in batch_tests_run: 437 self.assertEqual(len(batch), 1, '%s had too many tests' % ', '.join(batch)) 438 439 def test_skip_failing_tests(self): 440 # This tests that we skip both known failing and known flaky tests. Because there are 441 # no known flaky tests in the default test_expectations, we add additional expectations. 442 host = MockHost() 443 host.filesystem.write_text_file('/tmp/overrides.txt', 'Bug(x) passes/image.html [ ImageOnlyFailure Pass ]\n') 444 445 batches = get_test_batches(['--skip-failing-tests', '--additional-expectations', '/tmp/overrides.txt'], host=host) 446 has_passes_text = False 447 for batch in batches: 448 self.assertFalse('failures/expected/text.html' in batch) 449 self.assertFalse('passes/image.html' in batch) 450 has_passes_text = has_passes_text or ('passes/text.html' in batch) 451 self.assertTrue(has_passes_text) 452 453 def test_single_file(self): 454 tests_run = get_tests_run(['passes/text.html']) 455 self.assertEqual(tests_run, ['passes/text.html']) 456 457 def test_single_file_with_prefix(self): 458 tests_run = get_tests_run(['LayoutTests/passes/text.html']) 459 self.assertEqual(['passes/text.html'], tests_run) 460 461 def test_single_skipped_file(self): 462 tests_run = get_tests_run(['failures/expected/keybaord.html']) 463 self.assertEqual([], tests_run) 464 465 def test_stderr_is_saved(self): 466 host = MockHost() 467 self.assertTrue(passing_run(host=host)) 468 self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'), 469 'stuff going to stderr') 470 471 def test_test_list(self): 472 host = MockHost() 473 filename = '/tmp/foo.txt' 474 host.filesystem.write_text_file(filename, 'passes/text.html') 475 tests_run = get_tests_run(['--test-list=%s' % filename], host=host) 476 self.assertEqual(['passes/text.html'], tests_run) 477 host.filesystem.remove(filename) 478 details, err, user = logging_run(['--test-list=%s' % filename], tests_included=True, host=host) 479 self.assertEqual(details.exit_code, test_run_results.NO_TESTS_EXIT_STATUS) 480 self.assertNotEmpty(err) 481 482 def test_test_list_with_prefix(self): 483 host = MockHost() 484 filename = '/tmp/foo.txt' 485 host.filesystem.write_text_file(filename, 'LayoutTests/passes/text.html') 486 tests_run = get_tests_run(['--test-list=%s' % filename], host=host) 487 self.assertEqual(['passes/text.html'], tests_run) 488 489 def test_smoke_test(self): 490 host = MockHost() 491 smoke_test_filename = test.LAYOUT_TEST_DIR + '/SmokeTests' 492 host.filesystem.write_text_file(smoke_test_filename, 'passes/text.html\n') 493 494 # Test the default smoke testing. 495 tests_run = get_tests_run(['--smoke'], host=host) 496 self.assertEqual(['passes/text.html'], tests_run) 497 498 # Test running the smoke tests plus some manually-specified tests. 499 tests_run = get_tests_run(['--smoke', 'passes/image.html'], host=host) 500 self.assertEqual(['passes/image.html', 'passes/text.html'], tests_run) 501 502 # Test running the smoke tests plus some manually-specified tests. 503 tests_run = get_tests_run(['--no-smoke', 'passes/image.html'], host=host) 504 self.assertEqual(['passes/image.html'], tests_run) 505 506 # Test that we don't run just the smoke tests by default on a normal test port. 507 tests_run = get_tests_run([], host=host) 508 self.assertNotEqual(['passes/text.html'], tests_run) 509 510 # Create a port that does run only the smoke tests by default, and verify that works as expected. 511 port_obj = host.port_factory.get('test') 512 port_obj.default_smoke_test_only = lambda: True 513 tests_run = get_tests_run([], host=host, port_obj=port_obj) 514 self.assertEqual(['passes/text.html'], tests_run) 515 516 # Verify that --no-smoke continues to work on a smoke-by-default port. 517 tests_run = get_tests_run(['--no-smoke'], host=host, port_obj=port_obj) 518 self.assertNotEqual(['passes/text.html'], tests_run) 519 520 def test_missing_and_unexpected_results(self): 521 # Test that we update expectations in place. If the expectation 522 # is missing, update the expected generic location. 523 host = MockHost() 524 details, err, _ = logging_run(['--no-show-results', '--retry-failures', 525 'failures/expected/missing_image.html', 526 'failures/unexpected/missing_text.html', 527 'failures/unexpected/text-image-checksum.html'], 528 tests_included=True, host=host) 529 file_list = host.filesystem.written_files.keys() 530 self.assertEqual(details.exit_code, 2) 531 json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') 532 self.assertTrue(json_string.find('"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","is_unexpected":true') != -1) 533 self.assertTrue(json_string.find('"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_unexpected":true') != -1) 534 self.assertTrue(json_string.find('"num_regressions":2') != -1) 535 self.assertTrue(json_string.find('"num_flaky":0') != -1) 536 537 def test_different_failure_on_retry(self): 538 # This tests that if a test fails two different ways -- both unexpected 539 # -- we treat it as a failure rather than a flaky result. We use the 540 # initial failure for simplicity and consistency w/ the flakiness 541 # dashboard, even if the second failure is worse. 542 543 details, err, _ = logging_run(['--retry-failures', 'failures/unexpected/text_then_crash.html'], tests_included=True) 544 self.assertEqual(details.exit_code, 1) 545 self.assertEqual(details.summarized_failing_results['tests']['failures']['unexpected']['text_then_crash.html']['actual'], 546 'TEXT CRASH') 547 548 # If we get a test that fails two different ways -- but the second one is expected -- 549 # we should treat it as a flaky result and report the initial unexpected failure type 550 # to the dashboard. However, the test should be considered passing. 551 details, err, _ = logging_run(['--retry-failures', 'failures/expected/crash_then_text.html'], tests_included=True) 552 self.assertEqual(details.exit_code, 0) 553 self.assertEqual(details.summarized_failing_results['tests']['failures']['expected']['crash_then_text.html']['actual'], 554 'CRASH FAIL') 555 556 def test_pixel_test_directories(self): 557 host = MockHost() 558 559 """Both tests have failing checksum. We include only the first in pixel tests so only that should fail.""" 560 args = ['--pixel-tests', '--retry-failures', '--pixel-test-directory', 'failures/unexpected/pixeldir', 561 'failures/unexpected/pixeldir/image_in_pixeldir.html', 562 'failures/unexpected/image_not_in_pixeldir.html'] 563 details, err, _ = logging_run(extra_args=args, host=host, tests_included=True) 564 565 self.assertEqual(details.exit_code, 1) 566 expected_token = '"pixeldir":{"image_in_pixeldir.html":{"expected":"PASS","actual":"IMAGE","is_unexpected":true' 567 json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') 568 self.assertTrue(json_string.find(expected_token) != -1) 569 570 def test_crash_with_stderr(self): 571 host = MockHost() 572 _, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html'], tests_included=True, host=host) 573 self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true,"is_unexpected":true') != -1) 574 575 def test_no_image_failure_with_image_diff(self): 576 host = MockHost() 577 _, regular_output, _ = logging_run(['failures/unexpected/checksum-with-matching-image.html'], tests_included=True, host=host) 578 self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1) 579 580 def test_exit_after_n_failures_upload(self): 581 host = MockHost() 582 details, regular_output, user = logging_run( 583 ['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1'], 584 tests_included=True, host=host) 585 586 # By returning False, we know that the incremental results were generated and then deleted. 587 self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/incremental_results.json')) 588 589 self.assertEqual(details.exit_code, test_run_results.EARLY_EXIT_STATUS) 590 591 # This checks that passes/text.html is considered SKIPped. 592 self.assertTrue('"skipped":1' in host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')) 593 594 # This checks that we told the user we bailed out. 595 self.assertTrue('Exiting early after 1 failures. 1 tests run.\n' in regular_output.getvalue()) 596 597 # This checks that neither test ran as expected. 598 # FIXME: This log message is confusing; tests that were skipped should be called out separately. 599 self.assertTrue('0 tests ran as expected, 2 didn\'t:\n' in regular_output.getvalue()) 600 601 def test_exit_after_n_failures(self): 602 # Unexpected failures should result in tests stopping. 603 tests_run = get_tests_run(['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1']) 604 self.assertEqual(['failures/unexpected/text-image-checksum.html'], tests_run) 605 606 # But we'll keep going for expected ones. 607 tests_run = get_tests_run(['failures/expected/text.html', 'passes/text.html', '--exit-after-n-failures', '1']) 608 self.assertEqual(['failures/expected/text.html', 'passes/text.html'], tests_run) 609 610 def test_exit_after_n_crashes(self): 611 # Unexpected crashes should result in tests stopping. 612 tests_run = get_tests_run(['failures/unexpected/crash.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1']) 613 self.assertEqual(['failures/unexpected/crash.html'], tests_run) 614 615 # Same with timeouts. 616 tests_run = get_tests_run(['failures/unexpected/timeout.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1']) 617 self.assertEqual(['failures/unexpected/timeout.html'], tests_run) 618 619 # But we'll keep going for expected ones. 620 tests_run = get_tests_run(['failures/expected/crash.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1']) 621 self.assertEqual(['failures/expected/crash.html', 'passes/text.html'], tests_run) 622 623 def test_results_directory_absolute(self): 624 # We run a configuration that should fail, to generate output, then 625 # look for what the output results url was. 626 627 host = MockHost() 628 with host.filesystem.mkdtemp() as tmpdir: 629 _, _, user = logging_run(['--results-directory=' + str(tmpdir)], tests_included=True, host=host) 630 self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, host.filesystem.join(tmpdir, 'results.html'))]) 631 632 def test_results_directory_default(self): 633 # We run a configuration that should fail, to generate output, then 634 # look for what the output results url was. 635 636 # This is the default location. 637 _, _, user = logging_run(tests_included=True) 638 self.assertEqual(user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')]) 639 640 def test_results_directory_relative(self): 641 # We run a configuration that should fail, to generate output, then 642 # look for what the output results url was. 643 host = MockHost() 644 host.filesystem.maybe_make_directory('/tmp/cwd') 645 host.filesystem.chdir('/tmp/cwd') 646 _, _, user = logging_run(['--results-directory=foo'], tests_included=True, host=host) 647 self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, '/tmp/cwd/foo/results.html')]) 648 649 def test_retrying_default_value(self): 650 host = MockHost() 651 details, err, _ = logging_run(['--debug-rwt-logging', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) 652 self.assertEqual(details.exit_code, 1) 653 self.assertFalse('Retrying' in err.getvalue()) 654 655 host = MockHost() 656 details, err, _ = logging_run(['--debug-rwt-logging', 'failures/unexpected'], tests_included=True, host=host) 657 self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES - 7) # FIXME: This should be a constant in test.py . 658 self.assertTrue('Retrying' in err.getvalue()) 659 660 def test_retrying_default_value_test_list(self): 661 host = MockHost() 662 filename = '/tmp/foo.txt' 663 host.filesystem.write_text_file(filename, 'failures/unexpected/text-image-checksum.html\nfailures/unexpected/crash.html') 664 details, err, _ = logging_run(['--debug-rwt-logging', '--test-list=%s' % filename], tests_included=True, host=host) 665 self.assertEqual(details.exit_code, 2) 666 self.assertFalse('Retrying' in err.getvalue()) 667 668 host = MockHost() 669 filename = '/tmp/foo.txt' 670 host.filesystem.write_text_file(filename, 'failures') 671 details, err, _ = logging_run(['--debug-rwt-logging', '--test-list=%s' % filename], tests_included=True, host=host) 672 self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES - 7) 673 self.assertTrue('Retrying' in err.getvalue()) 674 675 def test_retrying_and_flaky_tests(self): 676 host = MockHost() 677 details, err, _ = logging_run(['--debug-rwt-logging', '--retry-failures', 'failures/flaky'], tests_included=True, host=host) 678 self.assertEqual(details.exit_code, 0) 679 self.assertTrue('Retrying' in err.getvalue()) 680 self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt')) 681 self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/retries/failures/flaky/text-actual.txt')) 682 self.assertEqual(len(host.user.opened_urls), 0) 683 684 # Now we test that --clobber-old-results does remove the old entries and the old retries, 685 # and that we don't retry again. 686 host = MockHost() 687 details, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host) 688 self.assertEqual(details.exit_code, 1) 689 self.assertTrue('Clobbering old results' in err.getvalue()) 690 self.assertTrue('flaky/text.html' in err.getvalue()) 691 self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt')) 692 self.assertFalse(host.filesystem.exists('retries')) 693 self.assertEqual(len(host.user.opened_urls), 1) 694 695 def test_retrying_crashed_tests(self): 696 host = MockHost() 697 details, err, _ = logging_run(['--retry-failures', 'failures/unexpected/crash.html'], tests_included=True, host=host) 698 self.assertEqual(details.exit_code, 1) 699 self.assertTrue('Retrying' in err.getvalue()) 700 701 def test_retrying_leak_tests(self): 702 host = MockHost() 703 details, err, _ = logging_run(['--retry-failures', 'failures/unexpected/leak.html'], tests_included=True, host=host) 704 self.assertEqual(details.exit_code, 1) 705 self.assertTrue('Retrying' in err.getvalue()) 706 707 def test_retrying_force_pixel_tests(self): 708 host = MockHost() 709 details, err, _ = logging_run(['--no-pixel-tests', '--retry-failures', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) 710 self.assertEqual(details.exit_code, 1) 711 self.assertTrue('Retrying' in err.getvalue()) 712 self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt')) 713 self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.png')) 714 self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.txt')) 715 self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.png')) 716 json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') 717 json = parse_full_results(json_string) 718 self.assertEqual(json["tests"]["failures"]["unexpected"]["text-image-checksum.html"], 719 {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "is_unexpected": True}) 720 self.assertFalse(json["pixel_tests_enabled"]) 721 self.assertEqual(details.enabled_pixel_tests_in_retry, True) 722 723 def test_retrying_uses_retries_directory(self): 724 host = MockHost() 725 details, err, _ = logging_run(['--debug-rwt-logging', '--retry-failures', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) 726 self.assertEqual(details.exit_code, 1) 727 self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt')) 728 self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.txt')) 729 730 def test_run_order__inline(self): 731 # These next tests test that we run the tests in ascending alphabetical 732 # order per directory. HTTP tests are sharded separately from other tests, 733 # so we have to test both. 734 tests_run = get_tests_run(['-i', 'passes/virtual_passes', 'passes']) 735 self.assertEqual(tests_run, sorted(tests_run)) 736 737 tests_run = get_tests_run(['http/tests/passes']) 738 self.assertEqual(tests_run, sorted(tests_run)) 739 740 def test_virtual(self): 741 self.assertTrue(passing_run(['passes/text.html', 'passes/args.html', 742 'virtual/passes/text.html', 'virtual/passes/args.html'])) 743 744 def test_reftest_run(self): 745 tests_run = get_tests_run(['passes/reftest.html']) 746 self.assertEqual(['passes/reftest.html'], tests_run) 747 748 def test_reftest_run_reftests_if_pixel_tests_are_disabled(self): 749 tests_run = get_tests_run(['--no-pixel-tests', 'passes/reftest.html']) 750 self.assertEqual(['passes/reftest.html'], tests_run) 751 752 def test_reftest_expected_html_should_be_ignored(self): 753 tests_run = get_tests_run(['passes/reftest-expected.html']) 754 self.assertEqual([], tests_run) 755 756 def test_reftest_driver_should_run_expected_html(self): 757 tests_run = get_test_results(['passes/reftest.html']) 758 self.assertEqual(tests_run[0].references, ['passes/reftest-expected.html']) 759 760 def test_reftest_driver_should_run_expected_mismatch_html(self): 761 tests_run = get_test_results(['passes/mismatch.html']) 762 self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.html']) 763 764 def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self): 765 host = MockHost() 766 _, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host) 767 results = parse_full_results(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')) 768 769 self.assertEqual(results["tests"]["reftests"]["foo"]["unlistedtest.html"]["actual"], "MISSING"), 770 self.assertEqual(results["num_regressions"], 5) 771 self.assertEqual(results["num_flaky"], 0) 772 773 def test_reftest_crash(self): 774 test_results = get_test_results(['failures/unexpected/crash-reftest.html']) 775 # The list of references should be empty since the test crashed and we didn't run any references. 776 self.assertEqual(test_results[0].references, []) 777 778 def test_reftest_with_virtual_reference(self): 779 _, err, _ = logging_run(['--details', 'virtual/virtual_passes/passes/reftest.html'], tests_included=True) 780 self.assertTrue('ref: virtual/virtual_passes/passes/reftest-expected.html' in err.getvalue()) 781 782 def test_additional_platform_directory(self): 783 self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo'])) 784 self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo'])) 785 self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', '--additional-platform-directory', '/tmp/bar'])) 786 self.assertTrue(passing_run(['--additional-platform-directory', 'foo'])) 787 788 def test_additional_expectations(self): 789 host = MockHost() 790 host.filesystem.write_text_file('/tmp/overrides.txt', 'Bug(x) failures/unexpected/mismatch.html [ ImageOnlyFailure ]\n') 791 self.assertTrue(passing_run(['--additional-expectations', '/tmp/overrides.txt', 'failures/unexpected/mismatch.html'], 792 tests_included=True, host=host)) 793 794 @staticmethod 795 def has_test_of_type(tests, type): 796 return [test for test in tests if type in test] 797 798 def test_platform_directories_ignored_when_searching_for_tests(self): 799 tests_run = get_tests_run(['--platform', 'test-mac-leopard']) 800 self.assertFalse('platform/test-mac-leopard/http/test.html' in tests_run) 801 self.assertFalse('platform/test-win-win7/http/test.html' in tests_run) 802 803 def test_platform_directories_not_searched_for_additional_tests(self): 804 tests_run = get_tests_run(['--platform', 'test-mac-leopard', 'http']) 805 self.assertFalse('platform/test-mac-leopard/http/test.html' in tests_run) 806 self.assertFalse('platform/test-win-win7/http/test.html' in tests_run) 807 808 def test_output_diffs(self): 809 # Test to ensure that we don't generate -wdiff.html or -pretty.html if wdiff and PrettyPatch 810 # aren't available. 811 host = MockHost() 812 _, err, _ = logging_run(['--pixel-tests', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host) 813 written_files = host.filesystem.written_files 814 self.assertTrue(any(path.endswith('-diff.txt') for path in written_files.keys())) 815 self.assertFalse(any(path.endswith('-wdiff.html') for path in written_files.keys())) 816 self.assertFalse(any(path.endswith('-pretty-diff.html') for path in written_files.keys())) 817 818 full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json') 819 full_results = json.loads(full_results_text.replace("ADD_RESULTS(", "").replace(");", "")) 820 self.assertEqual(full_results['has_wdiff'], False) 821 self.assertEqual(full_results['has_pretty_patch'], False) 822 823 def test_unsupported_platform(self): 824 stdout = StringIO.StringIO() 825 stderr = StringIO.StringIO() 826 res = run_webkit_tests.main(['--platform', 'foo'], stdout, stderr) 827 828 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 829 self.assertEqual(stdout.getvalue(), '') 830 self.assertTrue('unsupported platform' in stderr.getvalue()) 831 832 def test_build_check(self): 833 # By using a port_name for a different platform than the one we're running on, the build check should always fail. 834 if sys.platform == 'darwin': 835 port_name = 'linux-x86' 836 else: 837 port_name = 'mac-lion' 838 out = StringIO.StringIO() 839 err = StringIO.StringIO() 840 self.assertEqual(run_webkit_tests.main(['--platform', port_name, 'fast/harness/results.html'], out, err), test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 841 842 def test_verbose_in_child_processes(self): 843 # When we actually run multiple processes, we may have to reconfigure logging in the 844 # child process (e.g., on win32) and we need to make sure that works and we still 845 # see the verbose log output. However, we can't use logging_run() because using 846 # outputcapture to capture stdout and stderr latter results in a nonpicklable host. 847 848 # Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559 849 if not self.should_test_processes: 850 return 851 852 options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False) 853 host = MockHost() 854 port_obj = host.port_factory.get(port_name=options.platform, options=options) 855 logging_stream = StringIO.StringIO() 856 run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream) 857 self.assertTrue('text.html passed' in logging_stream.getvalue()) 858 self.assertTrue('image.html passed' in logging_stream.getvalue()) 859 860 def disabled_test_driver_logging(self): 861 # FIXME: Figure out how to either use a mock-test port to 862 # get output or mack mock ports work again. 863 host = Host() 864 _, err, _ = logging_run(['--platform', 'mock-win', '--driver-logging', 'fast/harness/results.html'], 865 tests_included=True, host=host) 866 self.assertTrue('OUT:' in err.getvalue()) 867 868 def test_write_full_results_to(self): 869 host = MockHost() 870 details, _, _ = logging_run(['--write-full-results-to', '/tmp/full_results.json'], host=host) 871 self.assertEqual(details.exit_code, 0) 872 self.assertTrue(host.filesystem.exists('/tmp/full_results.json')) 873 874 875 class EndToEndTest(unittest.TestCase): 876 def test_reftest_with_two_notrefs(self): 877 # Test that we update expectations in place. If the expectation 878 # is missing, update the expected generic location. 879 host = MockHost() 880 _, _, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host) 881 file_list = host.filesystem.written_files.keys() 882 883 json_string = host.filesystem.read_text_file('/tmp/layout-test-results/failing_results.json') 884 json = parse_full_results(json_string) 885 self.assertTrue("multiple-match-success.html" not in json["tests"]["reftests"]["foo"]) 886 self.assertTrue("multiple-mismatch-success.html" not in json["tests"]["reftests"]["foo"]) 887 self.assertTrue("multiple-both-success.html" not in json["tests"]["reftests"]["foo"]) 888 889 self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-match-failure.html"], 890 {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "is_unexpected": True}) 891 self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-mismatch-failure.html"], 892 {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="], "is_unexpected": True}) 893 self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-both-failure.html"], 894 {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["==", "!="], "is_unexpected": True}) 895 896 897 class RebaselineTest(unittest.TestCase, StreamTestingMixin): 898 def assertBaselines(self, file_list, file, extensions, err): 899 "assert that the file_list contains the baselines.""" 900 for ext in extensions: 901 baseline = file + "-expected" + ext 902 baseline_msg = 'Writing new expected result "%s"\n' % baseline 903 self.assertTrue(any(f.find(baseline) != -1 for f in file_list)) 904 self.assertContains(err, baseline_msg) 905 906 # FIXME: Add tests to ensure that we're *not* writing baselines when we're not 907 # supposed to be. 908 909 def test_reset_results(self): 910 # Test that we update expectations in place. If the expectation 911 # is missing, update the expected generic location. 912 host = MockHost() 913 details, err, _ = logging_run( 914 ['--pixel-tests', '--reset-results', 'passes/image.html', 'failures/expected/missing_image.html'], 915 tests_included=True, host=host, new_results=True) 916 file_list = host.filesystem.written_files.keys() 917 self.assertEqual(details.exit_code, 0) 918 self.assertEqual(len(file_list), 8) 919 self.assertBaselines(file_list, "passes/image", [".txt", ".png"], err) 920 self.assertBaselines(file_list, "failures/expected/missing_image", [".txt", ".png"], err) 921 922 def test_missing_results(self): 923 # Test that we update expectations in place. If the expectation 924 # is missing, update the expected generic location. 925 host = MockHost() 926 details, err, _ = logging_run(['--no-show-results', 927 'failures/unexpected/missing_text.html', 928 'failures/unexpected/missing_image.html', 929 'failures/unexpected/missing_render_tree_dump.html'], 930 tests_included=True, host=host, new_results=True) 931 file_list = host.filesystem.written_files.keys() 932 self.assertEqual(details.exit_code, 3) 933 self.assertEqual(len(file_list), 10) 934 self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err) 935 self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err) 936 self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_render_tree_dump", [".txt"], err) 937 938 def test_missing_results_not_added_if_expected_missing(self): 939 # Test that we update expectations in place. If the expectation 940 # is missing, update the expected generic location. 941 host = MockHost() 942 options, parsed_args = run_webkit_tests.parse_args([]) 943 944 port = test.TestPort(host, options=options) 945 host.filesystem.write_text_file(port.path_to_generic_test_expectations_file(), """ 946 Bug(foo) failures/unexpected/missing_text.html [ Missing ] 947 Bug(foo) failures/unexpected/missing_image.html [ NeedsRebaseline ] 948 Bug(foo) failures/unexpected/missing_audio.html [ NeedsManualRebaseline ] 949 Bug(foo) failures/unexpected/missing_render_tree_dump.html [ Missing ] 950 """) 951 details, err, _ = logging_run(['--no-show-results', 952 'failures/unexpected/missing_text.html', 953 'failures/unexpected/missing_image.html', 954 'failures/unexpected/missing_audio.html', 955 'failures/unexpected/missing_render_tree_dump.html'], 956 tests_included=True, host=host, new_results=True, port_obj=port) 957 file_list = host.filesystem.written_files.keys() 958 self.assertEqual(details.exit_code, 0) 959 self.assertEqual(len(file_list), 7) 960 self.assertFalse(any('failures/unexpected/missing_text-expected' in file for file in file_list)) 961 self.assertFalse(any('failures/unexpected/missing_image-expected' in file for file in file_list)) 962 self.assertFalse(any('failures/unexpected/missing_render_tree_dump-expected' in file for file in file_list)) 963 964 def test_missing_results_not_added_if_expected_missing_and_reset_results(self): 965 # Test that we update expectations in place. If the expectation 966 # is missing, update the expected generic location. 967 host = MockHost() 968 options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--reset-results']) 969 970 port = test.TestPort(host, options=options) 971 host.filesystem.write_text_file(port.path_to_generic_test_expectations_file(), """ 972 Bug(foo) failures/unexpected/missing_text.html [ Missing ] 973 Bug(foo) failures/unexpected/missing_image.html [ NeedsRebaseline ] 974 Bug(foo) failures/unexpected/missing_audio.html [ NeedsManualRebaseline ] 975 Bug(foo) failures/unexpected/missing_render_tree_dump.html [ Missing ] 976 """) 977 details, err, _ = logging_run(['--pixel-tests', '--reset-results', 978 'failures/unexpected/missing_text.html', 979 'failures/unexpected/missing_image.html', 980 'failures/unexpected/missing_audio.html', 981 'failures/unexpected/missing_render_tree_dump.html'], 982 tests_included=True, host=host, new_results=True, port_obj=port) 983 file_list = host.filesystem.written_files.keys() 984 self.assertEqual(details.exit_code, 0) 985 self.assertEqual(len(file_list), 11) 986 self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err) 987 self.assertBaselines(file_list, "failures/unexpected/missing_image", [".png"], err) 988 self.assertBaselines(file_list, "failures/unexpected/missing_render_tree_dump", [".txt"], err) 989 990 def test_new_baseline(self): 991 # Test that we update the platform expectations in the version-specific directories 992 # for both existing and new baselines. 993 host = MockHost() 994 details, err, _ = logging_run( 995 ['--pixel-tests', '--new-baseline', 'passes/image.html', 'failures/expected/missing_image.html'], 996 tests_included=True, host=host, new_results=True) 997 file_list = host.filesystem.written_files.keys() 998 self.assertEqual(details.exit_code, 0) 999 self.assertEqual(len(file_list), 8) 1000 self.assertBaselines(file_list, 1001 "platform/test-mac-leopard/passes/image", [".txt", ".png"], err) 1002 self.assertBaselines(file_list, 1003 "platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err) 1004 1005 1006 class PortTest(unittest.TestCase): 1007 def assert_mock_port_works(self, port_name, args=[]): 1008 self.assertTrue(passing_run(args + ['--platform', 'mock-' + port_name, 'fast/harness/results.html'], tests_included=True, host=Host())) 1009 1010 def disabled_test_mac_lion(self): 1011 self.assert_mock_port_works('mac-lion') 1012 1013 1014 class MainTest(unittest.TestCase): 1015 def test_exception_handling(self): 1016 orig_run_fn = run_webkit_tests.run 1017 1018 # unused args pylint: disable=W0613 1019 def interrupting_run(port, options, args, stderr): 1020 raise KeyboardInterrupt 1021 1022 def successful_run(port, options, args, stderr): 1023 1024 class FakeRunDetails(object): 1025 exit_code = test_run_results.UNEXPECTED_ERROR_EXIT_STATUS 1026 1027 return FakeRunDetails() 1028 1029 def exception_raising_run(port, options, args, stderr): 1030 assert False 1031 1032 stdout = StringIO.StringIO() 1033 stderr = StringIO.StringIO() 1034 try: 1035 run_webkit_tests.run = interrupting_run 1036 res = run_webkit_tests.main([], stdout, stderr) 1037 self.assertEqual(res, test_run_results.INTERRUPTED_EXIT_STATUS) 1038 1039 run_webkit_tests.run = successful_run 1040 res = run_webkit_tests.main(['--platform', 'test'], stdout, stderr) 1041 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 1042 1043 run_webkit_tests.run = exception_raising_run 1044 res = run_webkit_tests.main([], stdout, stderr) 1045 self.assertEqual(res, test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 1046 finally: 1047 run_webkit_tests.run = orig_run_fn 1048 1049 def test_buildbot_results_are_printed_on_early_exit(self): 1050 # unused args pylint: disable=W0613 1051 stdout = StringIO.StringIO() 1052 stderr = StringIO.StringIO() 1053 res = run_webkit_tests.main(['--platform', 'test', '--exit-after-n-failures', '1', 1054 'failures/unexpected/missing_text.html', 1055 'failures/unexpected/missing_image.html'], 1056 stdout, stderr) 1057 self.assertEqual(res, test_run_results.EARLY_EXIT_STATUS) 1058 self.assertEqual(stdout.getvalue(), 1059 ('\n' 1060 'Regressions: Unexpected missing results (1)\n' 1061 ' failures/unexpected/missing_image.html [ Missing ]\n\n')) 1062
