1 # Copyright (C) 2012 Google Inc. All rights reserved. 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor (at] inf.u-szeged.hu), University of Szeged 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 import webkitpy.thirdparty.unittest2 as unittest 31 32 from webkitpy.common.host_mock import MockHost 33 from webkitpy.common.system.systemhost_mock import MockSystemHost 34 from webkitpy.layout_tests import run_webkit_tests 35 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException 36 from webkitpy.layout_tests.models import test_expectations 37 from webkitpy.layout_tests.models import test_failures 38 from webkitpy.layout_tests.models.test_run_results import TestRunResults 39 from webkitpy.layout_tests.models.test_input import TestInput 40 from webkitpy.layout_tests.models.test_results import TestResult 41 from webkitpy.layout_tests.port.test import TestPort 42 43 44 TestExpectations = test_expectations.TestExpectations 45 46 47 class FakePrinter(object): 48 num_completed = 0 49 num_tests = 0 50 51 def print_expected(self, run_results, get_tests_with_result_type): 52 pass 53 54 def print_workers_and_shards(self, num_workers, num_shards, num_locked_shards): 55 pass 56 57 def print_started_test(self, test_name): 58 pass 59 60 def print_finished_test(self, result, expected, exp_str, got_str): 61 pass 62 63 def write(self, msg): 64 pass 65 66 def write_update(self, msg): 67 pass 68 69 def flush(self): 70 pass 71 72 73 class LockCheckingRunner(LayoutTestRunner): 74 def __init__(self, port, options, printer, tester, http_lock): 75 super(LockCheckingRunner, self).__init__(options, port, printer, port.results_directory(), lambda test_name: False) 76 self._finished_list_called = False 77 self._tester = tester 78 self._should_have_http_lock = http_lock 79 80 def handle_finished_list(self, source, list_name, num_tests, elapsed_time): 81 if not self._finished_list_called: 82 self._tester.assertEqual(list_name, 'locked_tests') 83 self._tester.assertTrue(self._remaining_locked_shards) 84 self._tester.assertTrue(self._has_http_lock is self._should_have_http_lock) 85 86 super(LockCheckingRunner, self).handle_finished_list(source, list_name, num_tests, elapsed_time) 87 88 if not self._finished_list_called: 89 self._tester.assertEqual(self._remaining_locked_shards, []) 90 self._tester.assertFalse(self._has_http_lock) 91 self._finished_list_called = True 92 93 94 class LayoutTestRunnerTests(unittest.TestCase): 95 def _runner(self, port=None): 96 # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need. 97 options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0] 98 options.child_processes = '1' 99 100 host = MockHost() 101 port = port or host.port_factory.get(options.platform, options=options) 102 return LockCheckingRunner(port, options, FakePrinter(), self, True) 103 104 def _run_tests(self, runner, tests): 105 test_inputs = [TestInput(test, 6000) for test in tests] 106 expectations = TestExpectations(runner._port, tests) 107 runner.run_tests(expectations, test_inputs, set(), num_workers=1, retrying=False) 108 109 def test_interrupt_if_at_failure_limits(self): 110 runner = self._runner() 111 runner._options.exit_after_n_failures = None 112 runner._options.exit_after_n_crashes_or_times = None 113 test_names = ['passes/text.html', 'passes/image.html'] 114 runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names] 115 116 run_results = TestRunResults(TestExpectations(runner._port, test_names), len(test_names)) 117 run_results.unexpected_failures = 100 118 run_results.unexpected_crashes = 50 119 run_results.unexpected_timeouts = 50 120 # No exception when the exit_after* options are None. 121 runner._interrupt_if_at_failure_limits(run_results) 122 123 # No exception when we haven't hit the limit yet. 124 runner._options.exit_after_n_failures = 101 125 runner._options.exit_after_n_crashes_or_timeouts = 101 126 runner._interrupt_if_at_failure_limits(run_results) 127 128 # Interrupt if we've exceeded either limit: 129 runner._options.exit_after_n_crashes_or_timeouts = 10 130 self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results) 131 self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP) 132 self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP) 133 134 runner._options.exit_after_n_crashes_or_timeouts = None 135 runner._options.exit_after_n_failures = 10 136 exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results) 137 138 def test_update_summary_with_result(self): 139 # Reftests expected to be image mismatch should be respected when pixel_tests=False. 140 runner = self._runner() 141 runner._options.pixel_tests = False 142 test = 'failures/expected/reftest.html' 143 expectations = TestExpectations(runner._port, tests=[test]) 144 runner._expectations = expectations 145 146 run_results = TestRunResults(expectations, 1) 147 result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!=']) 148 runner._update_summary_with_result(run_results, result) 149 self.assertEqual(1, run_results.expected) 150 self.assertEqual(0, run_results.unexpected) 151 152 run_results = TestRunResults(expectations, 1) 153 result = TestResult(test_name=test, failures=[], reftest_type=['==']) 154 runner._update_summary_with_result(run_results, result) 155 self.assertEqual(0, run_results.expected) 156 self.assertEqual(1, run_results.unexpected) 157 158 159 class SharderTests(unittest.TestCase): 160 161 test_list = [ 162 "http/tests/websocket/tests/unicode.htm", 163 "animations/keyframes.html", 164 "http/tests/security/view-source-no-refresh.html", 165 "http/tests/websocket/tests/websocket-protocol-ignored.html", 166 "fast/css/display-none-inline-style-change-crash.html", 167 "http/tests/xmlhttprequest/supported-xml-content-types.html", 168 "dom/html/level2/html/HTMLAnchorElement03.html", 169 "ietestcenter/Javascript/11.1.5_4-4-c-1.html", 170 "dom/html/level2/html/HTMLAnchorElement06.html", 171 "perf/object-keys.html", 172 "virtual/threaded/test.html", 173 ] 174 175 def get_test_input(self, test_file): 176 return TestInput(test_file, requires_lock=(test_file.startswith('http') or test_file.startswith('perf'))) 177 178 def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=1): 179 port = TestPort(MockSystemHost()) 180 self.sharder = Sharder(port.split_test, max_locked_shards) 181 test_list = test_list or self.test_list 182 return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel) 183 184 def assert_shards(self, actual_shards, expected_shard_names): 185 self.assertEqual(len(actual_shards), len(expected_shard_names)) 186 for i, shard in enumerate(actual_shards): 187 expected_shard_name, expected_test_names = expected_shard_names[i] 188 self.assertEqual(shard.name, expected_shard_name) 189 self.assertEqual([test_input.test_name for test_input in shard.test_inputs], 190 expected_test_names) 191 192 def test_shard_by_dir(self): 193 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False) 194 195 # Note that although there are tests in multiple dirs that need locks, 196 # they are crammed into a single shard in order to reduce the # of 197 # workers hitting the server at once. 198 self.assert_shards(locked, 199 [('locked_shard_1', 200 ['http/tests/security/view-source-no-refresh.html', 201 'http/tests/websocket/tests/unicode.htm', 202 'http/tests/websocket/tests/websocket-protocol-ignored.html', 203 'http/tests/xmlhttprequest/supported-xml-content-types.html', 204 'perf/object-keys.html'])]) 205 self.assert_shards(unlocked, 206 [('virtual/threaded', ['virtual/threaded/test.html']), 207 ('animations', ['animations/keyframes.html']), 208 ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html', 209 'dom/html/level2/html/HTMLAnchorElement06.html']), 210 ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']), 211 ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])]) 212 213 def test_shard_every_file(self): 214 locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True) 215 self.assert_shards(locked, 216 [('.', ['http/tests/websocket/tests/unicode.htm']), 217 ('.', ['http/tests/security/view-source-no-refresh.html']), 218 ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']), 219 ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']), 220 ('.', ['perf/object-keys.html'])]), 221 self.assert_shards(unlocked, 222 [('.', ['animations/keyframes.html']), 223 ('.', ['fast/css/display-none-inline-style-change-crash.html']), 224 ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']), 225 ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']), 226 ('.', ['dom/html/level2/html/HTMLAnchorElement06.html']), 227 ('.', ['virtual/threaded/test.html'])]) 228 229 def test_shard_in_two(self): 230 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False) 231 self.assert_shards(locked, 232 [('locked_tests', 233 ['http/tests/websocket/tests/unicode.htm', 234 'http/tests/security/view-source-no-refresh.html', 235 'http/tests/websocket/tests/websocket-protocol-ignored.html', 236 'http/tests/xmlhttprequest/supported-xml-content-types.html', 237 'perf/object-keys.html'])]) 238 self.assert_shards(unlocked, 239 [('unlocked_tests', 240 ['animations/keyframes.html', 241 'fast/css/display-none-inline-style-change-crash.html', 242 'dom/html/level2/html/HTMLAnchorElement03.html', 243 'ietestcenter/Javascript/11.1.5_4-4-c-1.html', 244 'dom/html/level2/html/HTMLAnchorElement06.html', 245 'virtual/threaded/test.html'])]) 246 247 def test_shard_in_two_has_no_locked_shards(self): 248 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, 249 test_list=['animations/keyframe.html']) 250 self.assertEqual(len(locked), 0) 251 self.assertEqual(len(unlocked), 1) 252 253 def test_shard_in_two_has_no_unlocked_shards(self): 254 locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False, 255 test_list=['http/tests/websocket/tests/unicode.htm']) 256 self.assertEqual(len(locked), 1) 257 self.assertEqual(len(unlocked), 0) 258 259 def test_multiple_locked_shards(self): 260 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2) 261 self.assert_shards(locked, 262 [('locked_shard_1', 263 ['http/tests/security/view-source-no-refresh.html', 264 'http/tests/websocket/tests/unicode.htm', 265 'http/tests/websocket/tests/websocket-protocol-ignored.html']), 266 ('locked_shard_2', 267 ['http/tests/xmlhttprequest/supported-xml-content-types.html', 268 'perf/object-keys.html'])]) 269 270 locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False) 271 self.assert_shards(locked, 272 [('locked_shard_1', 273 ['http/tests/security/view-source-no-refresh.html', 274 'http/tests/websocket/tests/unicode.htm', 275 'http/tests/websocket/tests/websocket-protocol-ignored.html', 276 'http/tests/xmlhttprequest/supported-xml-content-types.html', 277 'perf/object-keys.html'])]) 278
