1 # Copyright (C) 2012 Google Inc. All rights reserved. 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor (at] inf.u-szeged.hu), University of Szeged 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 import unittest 31 32 from webkitpy.common.host_mock import MockHost 33 from webkitpy.common.system.systemhost_mock import MockSystemHost 34 from webkitpy.layout_tests import run_webkit_tests 35 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException 36 from webkitpy.layout_tests.models import test_expectations 37 from webkitpy.layout_tests.models import test_failures 38 from webkitpy.layout_tests.models.test_run_results import TestRunResults 39 from webkitpy.layout_tests.models.test_input import TestInput 40 from webkitpy.layout_tests.models.test_results import TestResult 41 from webkitpy.layout_tests.port.test import TestPort 42 43 44 TestExpectations = test_expectations.TestExpectations 45 46 47 class FakePrinter(object): 48 num_completed = 0 49 num_tests = 0 50 51 def print_expected(self, run_results, get_tests_with_result_type): 52 pass 53 54 def print_workers_and_shards(self, num_workers, num_shards, num_locked_shards): 55 pass 56 57 def print_started_test(self, test_name): 58 pass 59 60 def print_finished_test(self, result, expected, exp_str, got_str): 61 pass 62 63 def write(self, msg): 64 pass 65 66 def write_update(self, msg): 67 pass 68 69 def flush(self): 70 pass 71 72 73 class LockCheckingRunner(LayoutTestRunner): 74 def __init__(self, port, options, printer, tester, http_lock): 75 super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False) 76 self._finished_list_called = False 77 self._tester = tester 78 self._should_have_http_lock = http_lock 79 80 def handle_finished_list(self, source, list_name, num_tests, elapsed_time): 81 if not self._finished_list_called: 82 self._tester.assertEqual(list_name, 'locked_tests') 83 self._tester.assertTrue(self._remaining_locked_shards) 84 self._tester.assertTrue(self._has_http_lock is self._should_have_http_lock) 85 86 super(LockCheckingRunner, self).handle_finished_list(source, list_name, num_tests, elapsed_time) 87 88 if not self._finished_list_called: 89 self._tester.assertEqual(self._remaining_locked_shards, []) 90 self._tester.assertFalse(self._has_http_lock) 91 self._finished_list_called = True 92 93 94 class LayoutTestRunnerTests(unittest.TestCase): 95 def _runner(self, port=None): 96 # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need. 97 options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0] 98 options.child_processes = '1' 99 100 host = MockHost() 101 port = port or host.port_factory.get(options.platform, options=options) 102 return LockCheckingRunner(port, options, FakePrinter(), self, True) 103 104 def _run_tests(self, runner, tests): 105 test_inputs = [TestInput(test, 6000) for test in tests] 106 expectations = TestExpectations(runner._port, tests) 107 runner.run_tests(expectations, test_inputs, set(), num_workers=1, retrying=False) 108 109 def test_interrupt_if_at_failure_limits(self): 110 runner = self._runner() 111 runner._options.exit_after_n_failures = None 112 runner._options.exit_after_n_crashes_or_times = None 113 test_names = ['passes/text.html', 'passes/image.html'] 114 runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names] 115 116 run_results = TestRunResults(TestExpectations(runner._port, test_names), len(test_names)) 117 run_results.unexpected_failures = 100 118 run_results.unexpected_crashes = 50 119 run_results.unexpected_timeouts = 50 120 # No exception when the exit_after* options are None. 121 runner._interrupt_if_at_failure_limits(run_results) 122 123 # No exception when we haven't hit the limit yet. 124 runner._options.exit_after_n_failures = 101 125 runner._options.exit_after_n_crashes_or_timeouts = 101 126 runner._interrupt_if_at_failure_limits(run_results) 127 128 # Interrupt if we've exceeded either limit: 129 runner._options.exit_after_n_crashes_or_timeouts = 10 130 self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results) 131 self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP) 132 self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP) 133 134 runner._options.exit_after_n_crashes_or_timeouts = None 135 runner._options.exit_after_n_failures = 10 136 exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results) 137 138 def test_update_summary_with_result(self): 139 # Reftests expected to be image mismatch should be respected when pixel_tests=False. 140 runner = self._runner() 141 runner._options.pixel_tests = False 142 test = 'failures/expected/reftest.html' 143 expectations = TestExpectations(runner._port, tests=[test]) 144 runner._expectations = expectations 145 146 run_results = TestRunResults(expectations, 1) 147 result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!=']) 148 runner._update_summary_with_result(run_results, result) 149 self.assertEqual(1, run_results.expected) 150 self.assertEqual(0, run_results.unexpected) 151 152 run_results = TestRunResults(expectations, 1) 153 result = TestResult(test_name=test, failures=[], reftest_type=['==']) 154 runner._update_summary_with_result(run_results, result) 155 self.assertEqual(0, run_results.expected) 156 self.assertEqual(1, run_results.unexpected) 157 158 159 class SharderTests(unittest.TestCase): 160 161 test_list = [ 162 "http/tests/websocket/tests/unicode.htm", 163 "animations/keyframes.html", 164 "http/tests/security/view-source-no-refresh.html", 165 "http/tests/websocket/tests/websocket-protocol-ignored.html", 166 "fast/css/display-none-inline-style-change-crash.html", 167 "http/tests/xmlhttprequest/supported-xml-content-types.html", 168 "dom/html/level2/html/HTMLAnchorElement03.html", 169 "ietestcenter/Javascript/11.1.5_4-4-c-1.html", 170 "dom/html/level2/html/HTMLAnchorElement06.html", 171 "perf/object-keys.html", 172 "virtual/threaded/dir/test.html", 173 "virtual/threaded/fast/foo/test.html", 174 ] 175 176 def get_test_input(self, test_file): 177 return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf'))) 178 179 def get_shards(self, num_workers, fully_parallel, run_singly, test_list=None, max_locked_shards=1): 180 port = TestPort(MockSystemHost()) 181 self.sharder = Sharder(port.split_test, max_locked_shards) 182 test_list = test_list or self.test_list 183 return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], 184 num_workers, fully_parallel, run_singly) 185 186 def assert_shards(self, actual_shards, expected_shard_names): 187 self.assertEqual(len(actual_shards), len(expected_shard_names)) 188 for i, shard in enumerate(actual_shards): 189 expected_shard_name, expected_test_names = expected_shard_names[i] 190 self.assertEqual(shard.name, expected_shard_name) 191 self.assertEqual([test_input.test_name for test_input in shard.test_inputs], 192 expected_test_names) 193 194 def test_shard_by_dir(self): 195 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False, run_singly=False) 196 197 # Note that although there are tests in multiple dirs that need locks, 198 # they are crammed into a single shard in order to reduce the # of 199 # workers hitting the server at once. 200 self.assert_shards(locked, 201 [('locked_shard_1', 202 ['http/tests/security/view-source-no-refresh.html', 203 'http/tests/websocket/tests/unicode.htm', 204 'http/tests/websocket/tests/websocket-protocol-ignored.html', 205 'http/tests/xmlhttprequest/supported-xml-content-types.html', 206 'perf/object-keys.html'])]) 207 self.assert_shards(unlocked, 208 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), 209 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), 210 ('animations', ['animations/keyframes.html']), 211 ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 212 'dom/html/level2/html/HTMLAnchorElement06.html']), 213 ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), 214 ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) 215 216 def test_shard_every_file(self): 217 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False) 218 self.assert_shards(locked, 219 [('locked_shard_1', 220 ['http/tests/websocket/tests/unicode.htm', 221 'http/tests/security/view-source-no-refresh.html', 222 'http/tests/websocket/tests/websocket-protocol-ignored.html']), 223 ('locked_shard_2', 224 ['http/tests/xmlhttprequest/supported-xml-content-types.html', 225 'perf/object-keys.html'])]), 226 self.assert_shards(unlocked, 227 [('virtual/threaded/dir', ['virtual/threaded/dir/test.html']), 228 ('virtual/threaded/fast/foo', ['virtual/threaded/fast/foo/test.html']), 229 ('.', ['animations/keyframes.html']), 230 ('.', ['fast/css/display-none-inline-style-change-crash.html']), 231 ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), 232 ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), 233 ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])]) 234 235 def test_shard_in_two(self): 236 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False) 237 self.assert_shards(locked, 238 [('locked_tests', 239 ['http/tests/websocket/tests/unicode.htm', 240 'http/tests/security/view-source-no-refresh.html', 241 'http/tests/websocket/tests/websocket-protocol-ignored.html', 242 'http/tests/xmlhttprequest/supported-xml-content-types.html', 243 'perf/object-keys.html'])]) 244 self.assert_shards(unlocked, 245 [('unlocked_tests', 246 ['animations/keyframes.html', 247 'fast/css/display-none-inline-style-change-crash.html', 248 'dom/html/level2/html/HTMLAnchorElement03.html', 249 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', 250 'dom/html/level2/html/HTMLAnchorElement06.html', 251 'virtual/threaded/dir/test.html', 252 'virtual/threaded/fast/foo/test.html'])]) 253 254 def test_shard_in_two_has_no_locked_shards(self): 255 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, 256 test_list=['animations/keyframe.html']) 257 self.assertEqual(len(locked), 0) 258 self.assertEqual(len(unlocked), 1) 259 260 def test_shard_in_two_has_no_unlocked_shards(self): 261 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, run_singly=False, 262 test_list=['http/tests/websocket/tests/unicode.htm']) 263 self.assertEqual(len(locked), 1) 264 self.assertEqual(len(unlocked), 0) 265 266 def test_multiple_locked_shards(self): 267 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2, run_singly=False) 268 self.assert_shards(locked, 269 [('locked_shard_1', 270 ['http/tests/security/view-source-no-refresh.html', 271 'http/tests/websocket/tests/unicode.htm', 272 'http/tests/websocket/tests/websocket-protocol-ignored.html']), 273 ('locked_shard_2', 274 ['http/tests/xmlhttprequest/supported-xml-content-types.html', 275 'perf/object-keys.html'])]) 276 277 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, run_singly=False) 278 self.assert_shards(locked, 279 [('locked_shard_1', 280 ['http/tests/security/view-source-no-refresh.html', 281 'http/tests/websocket/tests/unicode.htm', 282 'http/tests/websocket/tests/websocket-protocol-ignored.html', 283 'http/tests/xmlhttprequest/supported-xml-content-types.html', 284 'perf/object-keys.html'])]) 285 286 def test_virtual_shards(self): 287 # With run_singly=False, we try to keep all of the tests in a virtual suite together even 288 # when fully_parallel=True, so that we don't restart every time the command line args change. 289 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=False, 290 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) 291 self.assert_shards(unlocked, 292 [('virtual/foo', ['virtual/foo/bar1.html', 'virtual/foo/bar2.html'])]) 293 294 # But, with run_singly=True, we have to restart every time anyway, so we want full parallelism. 295 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True, max_locked_shards=2, run_singly=True, 296 test_list=['virtual/foo/bar1.html', 'virtual/foo/bar2.html']) 297 self.assert_shards(unlocked, 298 [('.', ['virtual/foo/bar1.html']), 299 ('.', ['virtual/foo/bar2.html'])]) 300
