1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import webkitpy.thirdparty.unittest2 as unittest 30 31 from webkitpy.common.system.outputcapture import OutputCapture 32 from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer 33 from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder 34 from webkitpy.common.net.layouttestresults import LayoutTestResults 35 from webkitpy.common.system.executive_mock import MockExecutive2 36 from webkitpy.thirdparty.mock import Mock 37 from webkitpy.tool.commands.rebaseline import * 38 from webkitpy.tool.mocktool import MockTool, MockOptions 39 40 41 class _BaseTestCase(unittest.TestCase): 42 MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' 43 WEB_PREFIX = 'http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results' 44 45 command_constructor = None 46 47 def setUp(self): 48 self.tool = MockTool() 49 self.command = self.command_constructor() # lint warns that command_constructor might not be set, but this is intentional; pylint: disable=E1102 50 self.command.bind_to_tool(self.tool) 51 self.lion_port = self.tool.port_factory.get_from_builder_name("WebKit Mac10.7") 52 self.lion_expectations_path = self.lion_port.path_to_generic_test_expectations_file() 53 54 # FIXME: we should override builders._exact_matches here to point to a set 55 # of test ports and restore the value in tearDown(), and that way the 56 # individual tests wouldn't have to worry about it. 57 58 def _expand(self, path): 59 if self.tool.filesystem.isabs(path): 60 return path 61 return self.tool.filesystem.join(self.lion_port.layout_tests_dir(), path) 62 63 def _read(self, path): 64 return self.tool.filesystem.read_text_file(self._expand(path)) 65 66 def _write(self, path, contents): 67 self.tool.filesystem.write_text_file(self._expand(path), contents) 68 69 def _zero_out_test_expectations(self): 70 for port_name in self.tool.port_factory.all_port_names(): 71 port = self.tool.port_factory.get(port_name) 72 for path in port.expectations_files(): 73 self._write(path, '') 74 self.tool.filesystem.written_files = {} 75 76 def _setup_mock_builder_data(self): 77 data = LayoutTestResults.results_from_string("""ADD_RESULTS({ 78 "tests": { 79 "userscripts": { 80 "first-test.html": { 81 "expected": "PASS", 82 "actual": "IMAGE+TEXT" 83 }, 84 "second-test.html": { 85 "expected": "FAIL", 86 "actual": "IMAGE+TEXT" 87 } 88 } 89 } 90 });""") 91 for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7']: 92 self.command._builder_data[builder] = data 93 94 95 class TestCopyExistingBaselinesInternal(_BaseTestCase): 96 command_constructor = CopyExistingBaselinesInternal # AKA webkit-patch rebaseline-test-internal 97 98 def setUp(self): 99 super(TestCopyExistingBaselinesInternal, self).setUp() 100 101 def test_copying_overwritten_baseline(self): 102 self.tool.executive = MockExecutive2() 103 104 # FIXME: it's confusing that this is the test- port, and not the regular lion port. Really all of the tests should be using the test ports. 105 port = self.tool.port_factory.get('test-mac-snowleopard') 106 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-mac-snowleopard/failures/expected/image-expected.txt'), 'original snowleopard result') 107 108 old_exact_matches = builders._exact_matches 109 oc = OutputCapture() 110 try: 111 builders._exact_matches = { 112 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 113 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 114 } 115 116 options = MockOptions(builder="MOCK SnowLeopard", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 117 118 oc.capture_output() 119 self.command.execute(options, [], self.tool) 120 finally: 121 out, _, _ = oc.restore_output() 122 builders._exact_matches = old_exact_matches 123 124 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-mac-leopard/failures/expected/image-expected.txt')), 'original snowleopard result') 125 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": []}\n') 126 127 def test_copying_overwritten_baseline_to_multiple_locations(self): 128 self.tool.executive = MockExecutive2() 129 130 # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 131 port = self.tool.port_factory.get('test-win-win7') 132 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 133 134 old_exact_matches = builders._exact_matches 135 oc = OutputCapture() 136 try: 137 builders._exact_matches = { 138 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 139 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 140 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 141 } 142 143 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 144 145 oc.capture_output() 146 self.command.execute(options, [], self.tool) 147 finally: 148 out, _, _ = oc.restore_output() 149 builders._exact_matches = old_exact_matches 150 151 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') 152 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt'))) 153 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": []}\n') 154 155 def test_no_copy_existing_baseline(self): 156 self.tool.executive = MockExecutive2() 157 158 # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 159 port = self.tool.port_factory.get('test-win-win7') 160 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 161 162 old_exact_matches = builders._exact_matches 163 oc = OutputCapture() 164 try: 165 builders._exact_matches = { 166 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 167 "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 168 "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 169 } 170 171 options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 172 173 oc.capture_output() 174 self.command.execute(options, [], self.tool) 175 finally: 176 out, _, _ = oc.restore_output() 177 builders._exact_matches = old_exact_matches 178 179 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') 180 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'original win7 result') 181 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt'))) 182 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": []}\n') 183 184 185 class TestRebaselineTest(_BaseTestCase): 186 command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-internal 187 188 def setUp(self): 189 super(TestRebaselineTest, self).setUp() 190 self.options = MockOptions(builder="WebKit Mac10.7", test="userscripts/another-test.html", suffixes="txt", results_directory=None) 191 192 def test_baseline_directory(self): 193 command = self.command 194 self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/LayoutTests/platform/mac-lion") 195 self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/LayoutTests/platform/mac-snowleopard") 196 197 def test_rebaseline_updates_expectations_file_noop(self): 198 self._zero_out_test_expectations() 199 self._write(self.lion_expectations_path, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] 200 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] 201 """) 202 self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.html", "Dummy test contents") 203 self._write("fast/css/large-list-of-rules-crash.html", "Dummy test contents") 204 self._write("userscripts/another-test.html", "Dummy test contents") 205 206 self.options.suffixes = "png,wav,txt" 207 self.command._rebaseline_test_and_update_expectations(self.options) 208 209 self.assertItemsEqual(self.tool.web.urls_fetched, 210 [self.WEB_PREFIX + '/userscripts/another-test-actual.png', 211 self.WEB_PREFIX + '/userscripts/another-test-actual.wav', 212 self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) 213 new_expectations = self._read(self.lion_expectations_path) 214 self.assertMultiLineEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] 215 Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] 216 """) 217 218 def test_rebaseline_test(self): 219 self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", self.WEB_PREFIX) 220 self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) 221 222 def test_rebaseline_test_with_results_directory(self): 223 self._write("userscripts/another-test.html", "test data") 224 self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") 225 self.options.results_directory = '/tmp' 226 self.command._rebaseline_test_and_update_expectations(self.options) 227 self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscripts/another-test-actual.txt']) 228 229 def test_rebaseline_reftest(self): 230 self._write("userscripts/another-test.html", "test data") 231 self._write("userscripts/another-test-expected.html", "generic result") 232 OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_update_expectations, args=[self.options], 233 expected_logs="Cannot rebaseline reftest: userscripts/another-test.html\n") 234 self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-lines': []}) 235 236 def test_rebaseline_test_and_print_scm_changes(self): 237 self.command._print_scm_changes = True 238 self.command._scm_changes = {'add': [], 'delete': []} 239 self.tool._scm.exists = lambda x: False 240 241 self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", None) 242 243 self.assertDictEqual(self.command._scm_changes, {'add': ['/mock-checkout/LayoutTests/platform/linux/userscripts/another-test-expected.txt'], 'delete': []}) 244 245 def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): 246 self.tool.executive = MockExecutive2() 247 248 # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 249 port = self.tool.port_factory.get('test-win-win7') 250 self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 251 252 old_exact_matches = builders._exact_matches 253 oc = OutputCapture() 254 try: 255 builders._exact_matches = { 256 "MOCK XP": {"port_name": "test-win-xp"}, 257 "MOCK Win7": {"port_name": "test-win-win7"}, 258 } 259 260 options = MockOptions(optimize=True, builder="MOCK Win7", suffixes="txt", 261 verbose=True, test="failures/expected/image.html", results_directory=None) 262 263 oc.capture_output() 264 self.command.execute(options, [], self.tool) 265 finally: 266 out, _, _ = oc.restore_output() 267 builders._exact_matches = old_exact_matches 268 269 self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True') 270 self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-xp/failures/expected/image-expected.txt'))) 271 self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win7"}]}\n') 272 273 274 class TestAbstractParallelRebaselineCommand(_BaseTestCase): 275 command_constructor = AbstractParallelRebaselineCommand 276 277 def test_builders_to_fetch_from(self): 278 old_exact_matches = builders._exact_matches 279 try: 280 builders._exact_matches = { 281 "MOCK XP": {"port_name": "test-win-xp"}, 282 "MOCK Win7": {"port_name": "test-win-win7"}, 283 "MOCK Win7 (dbg)(1)": {"port_name": "test-win-win7"}, 284 "MOCK Win7 (dbg)(2)": {"port_name": "test-win-win7"}, 285 } 286 287 builders_to_fetch = self.command._builders_to_fetch_from(["MOCK XP", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win7"]) 288 self.assertEqual(builders_to_fetch, ["MOCK XP", "MOCK Win7"]) 289 finally: 290 builders._exact_matches = old_exact_matches 291 292 293 class TestRebaselineJson(_BaseTestCase): 294 command_constructor = RebaselineJson 295 296 def setUp(self): 297 super(TestRebaselineJson, self).setUp() 298 self.tool.executive = MockExecutive2() 299 self.old_exact_matches = builders._exact_matches 300 builders._exact_matches = { 301 "MOCK builder": {"port_name": "test-mac-snowleopard"}, 302 "MOCK builder (Debug)": {"port_name": "test-mac-snowleopard"}, 303 } 304 305 def tearDown(self): 306 builders._exact_matches = self.old_exact_matches 307 super(TestRebaselineJson, self).tearDown() 308 309 def test_rebaseline_all(self): 310 self._setup_mock_builder_data() 311 312 options = MockOptions(optimize=True, verbose=True, results_directory=None) 313 self._write("userscripts/first-test.html", "Dummy test contents") 314 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 315 316 # Note that we have one run_in_parallel() call followed by a run_command() 317 self.assertEqual(self.tool.executive.calls, 318 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 319 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 320 ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'userscripts/first-test.html']]) 321 322 def test_rebaseline_debug(self): 323 self._setup_mock_builder_data() 324 325 options = MockOptions(optimize=True, verbose=True, results_directory=None) 326 self._write("userscripts/first-test.html", "Dummy test contents") 327 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) 328 329 # Note that we have one run_in_parallel() call followed by a run_command() 330 self.assertEqual(self.tool.executive.calls, 331 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 332 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 333 ['echo', '--verbose', 'optimize-baselines', '--suffixes', 'txt,png', 'userscripts/first-test.html']]) 334 335 def test_no_optimize(self): 336 self._setup_mock_builder_data() 337 338 options = MockOptions(optimize=False, verbose=True, results_directory=None) 339 self._write("userscripts/first-test.html", "Dummy test contents") 340 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) 341 342 # Note that we have only one run_in_parallel() call 343 self.assertEqual(self.tool.executive.calls, 344 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 345 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']]]) 346 347 def test_results_directory(self): 348 self._setup_mock_builder_data() 349 350 options = MockOptions(optimize=False, verbose=True, results_directory='/tmp') 351 self._write("userscripts/first-test.html", "Dummy test contents") 352 self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 353 354 # Note that we have only one run_in_parallel() call 355 self.assertEqual(self.tool.executive.calls, 356 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']], 357 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']]]) 358 359 class TestRebaselineJsonUpdatesExpectationsFiles(_BaseTestCase): 360 command_constructor = RebaselineJson 361 362 def setUp(self): 363 super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp() 364 self.tool.executive = MockExecutive2() 365 366 def mock_run_command(args, 367 cwd=None, 368 input=None, 369 error_handler=None, 370 return_exit_code=False, 371 return_stderr=True, 372 decode_output=False, 373 env=None): 374 return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "builder": "WebKit Mac10.7"}]}\n' 375 self.tool.executive.run_command = mock_run_command 376 377 def test_rebaseline_updates_expectations_file(self): 378 options = MockOptions(optimize=False, verbose=True, results_directory=None) 379 380 self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 381 self._write("userscripts/first-test.html", "Dummy test contents") 382 self._setup_mock_builder_data() 383 384 self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 385 386 new_expectations = self._read(self.lion_expectations_path) 387 self.assertMultiLineEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 388 389 def test_rebaseline_updates_expectations_file_all_platforms(self): 390 options = MockOptions(optimize=False, verbose=True, results_directory=None) 391 392 self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 393 self._write("userscripts/first-test.html", "Dummy test contents") 394 self._setup_mock_builder_data() 395 396 self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 397 398 new_expectations = self._read(self.lion_expectations_path) 399 self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux MountainLion SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 400 401 402 class TestRebaseline(_BaseTestCase): 403 # This command shares most of its logic with RebaselineJson, so these tests just test what is different. 404 405 command_constructor = Rebaseline # AKA webkit-patch rebaseline 406 407 def test_rebaseline(self): 408 self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] 409 410 self._write("userscripts/first-test.html", "test data") 411 412 self._zero_out_test_expectations() 413 self._setup_mock_builder_data() 414 415 old_exact_matches = builders._exact_matches 416 try: 417 builders._exact_matches = { 418 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 419 } 420 self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts/first-test.html'], self.tool) 421 finally: 422 builders._exact_matches = old_exact_matches 423 424 calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) 425 self.assertEqual(calls, 426 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 427 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']]]) 428 429 def test_rebaseline_directory(self): 430 self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] 431 432 self._write("userscripts/first-test.html", "test data") 433 self._write("userscripts/second-test.html", "test data") 434 435 self._setup_mock_builder_data() 436 437 old_exact_matches = builders._exact_matches 438 try: 439 builders._exact_matches = { 440 "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 441 } 442 self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts'], self.tool) 443 finally: 444 builders._exact_matches = old_exact_matches 445 446 calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) 447 self.assertEqual(calls, 448 [[['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'], 449 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']], 450 [['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'], 451 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']]]) 452 453 454 class TestRebaselineExpectations(_BaseTestCase): 455 command_constructor = RebaselineExpectations 456 457 def setUp(self): 458 super(TestRebaselineExpectations, self).setUp() 459 self.options = MockOptions(optimize=False, builders=None, suffixes=['txt'], verbose=False, platform=None, results_directory=None) 460 461 def test_rebaseline_expectations(self): 462 self._zero_out_test_expectations() 463 464 self.tool.executive = MockExecutive2() 465 466 def builder_data(): 467 self.command._builder_data['MOCK SnowLeopard'] = self.command._builder_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 468 "tests": { 469 "userscripts": { 470 "another-test.html": { 471 "expected": "PASS", 472 "actual": "PASS TEXT" 473 }, 474 "images.svg": { 475 "expected": "FAIL", 476 "actual": "IMAGE+TEXT" 477 } 478 } 479 } 480 });""") 481 return self.command._builder_data 482 483 self.command.builder_data = builder_data 484 485 self._write("userscripts/another-test.html", "Dummy test contents") 486 self._write("userscripts/images.svg", "Dummy test contents") 487 self.command._tests_to_rebaseline = lambda port: { 488 'userscripts/another-test.html': set(['txt']), 489 'userscripts/images.svg': set(['png']), 490 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']), 491 } 492 493 old_exact_matches = builders._exact_matches 494 try: 495 builders._exact_matches = { 496 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 497 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 498 } 499 self.command.execute(self.options, [], self.tool) 500 finally: 501 builders._exact_matches = old_exact_matches 502 503 # FIXME: change this to use the test- ports. 504 calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls) 505 self.assertEqual(self.tool.executive.calls, [ 506 [ 507 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'], 508 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'], 509 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'], 510 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'] 511 ], 512 [ 513 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'], 514 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'], 515 ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'], 516 ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'] 517 ] 518 ]) 519 520 def test_rebaseline_expectations_noop(self): 521 self._zero_out_test_expectations() 522 523 oc = OutputCapture() 524 try: 525 oc.capture_output() 526 self.command.execute(self.options, [], self.tool) 527 finally: 528 _, _, logs = oc.restore_output() 529 self.assertEqual(self.tool.filesystem.written_files, {}) 530 self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n') 531 532 def disabled_test_overrides_are_included_correctly(self): 533 # This tests that the any tests marked as REBASELINE in the overrides are found, but 534 # that the overrides do not get written into the main file. 535 self._zero_out_test_expectations() 536 537 self._write(self.lion_expectations_path, '') 538 self.lion_port.expectations_dict = lambda: { 539 self.lion_expectations_path: '', 540 'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n' 541 'Bug(y) userscripts/test.html [ Crash ]\n')} 542 self._write('/userscripts/another-test.html', '') 543 544 self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': set(['png', 'txt', 'wav'])}) 545 self.assertEqual(self._read(self.lion_expectations_path), '') 546 547 def test_rebaseline_without_other_expectations(self): 548 self._write("userscripts/another-test.html", "Dummy test contents") 549 self._write(self.lion_expectations_path, "Bug(x) userscripts/another-test.html [ Rebaseline ]\n") 550 self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': ('png', 'wav', 'txt')}) 551 552 553 class _FakeOptimizer(BaselineOptimizer): 554 def read_results_by_directory(self, baseline_name): 555 if baseline_name.endswith('txt'): 556 return {'LayoutTests/passes/text.html': '123456'} 557 return {} 558 559 560 class TestAnalyzeBaselines(_BaseTestCase): 561 command_constructor = AnalyzeBaselines 562 563 def setUp(self): 564 super(TestAnalyzeBaselines, self).setUp() 565 self.port = self.tool.port_factory.get('test') 566 self.tool.port_factory.get = (lambda port_name=None, options=None: self.port) 567 self.lines = [] 568 self.command._optimizer_class = _FakeOptimizer 569 self.command._write = (lambda msg: self.lines.append(msg)) # pylint bug warning about unnecessary lambda? pylint: disable=W0108 570 571 def test_default(self): 572 self.command.execute(MockOptions(suffixes='txt', missing=False, platform=None), ['passes/text.html'], self.tool) 573 self.assertEqual(self.lines, 574 ['passes/text-expected.txt:', 575 ' (generic): 123456']) 576 577 def test_missing_baselines(self): 578 self.command.execute(MockOptions(suffixes='png,txt', missing=True, platform=None), ['passes/text.html'], self.tool) 579 self.assertEqual(self.lines, 580 ['passes/text-expected.png: (no baselines found)', 581 'passes/text-expected.txt:', 582 ' (generic): 123456']) 583 584 585 class TestAutoRebaseline(_BaseTestCase): 586 command_constructor = AutoRebaseline 587 588 def _write_test_file(self, port, path, contents): 589 abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) 590 self.tool.filesystem.write_text_file(abs_path, contents) 591 592 def setUp(self): 593 super(TestAutoRebaseline, self).setUp() 594 self.command.latest_revision_processed_on_all_bots = lambda: 9000 595 596 def test_tests_to_rebaseline(self): 597 def blame(path): 598 return """ 599 624c3081c0 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 600 624c3081c0 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) path/to/rebaseline-without-bug-number.html [ NeedsRebaseline ] 601 624c3081c0 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/rebaseline-with-modifiers.html [ NeedsRebaseline ] 602 624c3081c0 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 crbug.com/234 path/to/rebaseline-without-modifiers.html [ NeedsRebaseline ] 603 6469e754a1 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/rebaseline-new-revision.html [ NeedsRebaseline ] 604 624caaaaaa path/to/TestExpectations (foo (at] chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 605 0000000000 path/to/TestExpectations (foo (at] chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 606 """ 607 self.tool.scm().blame = blame 608 609 min_revision = 9000 610 self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), ( 611 set(['path/to/rebaseline-without-bug-number.html', 'path/to/rebaseline-with-modifiers.html', 'path/to/rebaseline-without-modifiers.html']), 612 5678, 613 'foobarbaz1 (at] chromium.org', 614 set(['24182', '234']))) 615 616 def test_tests_to_rebaseline_over_limit(self): 617 def blame(path): 618 result = "" 619 for i in range(0, self.command.MAX_LINES_TO_REBASELINE + 1): 620 result += "624c3081c0 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-04-28 04:52:41 +0000 13) crbug.com/24182 path/to/rebaseline-%s.html [ NeedsRebaseline ]\n" % i 621 return result 622 self.tool.scm().blame = blame 623 624 expected_list_of_tests = [] 625 for i in range(0, self.command.MAX_LINES_TO_REBASELINE): 626 expected_list_of_tests.append("path/to/rebaseline-%s.html" % i) 627 628 min_revision = 9000 629 self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), ( 630 set(expected_list_of_tests), 631 5678, 632 'foobarbaz1 (at] chromium.org', 633 set(['24182']))) 634 635 def test_commit_message(self): 636 author = "foo (at] chromium.org" 637 revision = 1234 638 bugs = set() 639 self.assertEqual(self.command.commit_message(author, revision, bugs), 640 """Auto-rebaseline for r1234 641 642 http://src.chromium.org/viewvc/blink?view=revision&revision=1234 643 644 TBR=foo (at] chromium.org 645 """) 646 647 bugs = set(["234", "345"]) 648 self.assertEqual(self.command.commit_message(author, revision, bugs), 649 """Auto-rebaseline for r1234 650 651 http://src.chromium.org/viewvc/blink?view=revision&revision=1234 652 653 BUG=234,345 654 TBR=foo (at] chromium.org 655 """) 656 657 def test_no_needs_rebaseline_lines(self): 658 def blame(path): 659 return """ 660 6469e754a1 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 661 """ 662 self.tool.scm().blame = blame 663 664 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 665 self.assertEqual(self.tool.executive.calls, []) 666 667 def test_execute(self): 668 def blame(path): 669 return """ 670 6469e754a1 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 671 6469e754a1 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 672 6469e754a1 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] 673 6469e754a1 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 674 624caaaaaa path/to/TestExpectations (foo (at] chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 675 0000000000 path/to/TestExpectations (foo (at] chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 676 """ 677 self.tool.scm().blame = blame 678 679 test_port = self.tool.port_factory.get('test') 680 original_get = self.tool.port_factory.get 681 682 def get_test_port(port_name=None, options=None, **kwargs): 683 if not port_name: 684 return test_port 685 return original_get(port_name, options, **kwargs) 686 # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path. 687 self.tool.port_factory.get = get_test_port 688 689 old_builder_data = self.command.builder_data 690 691 def builder_data(): 692 old_builder_data() 693 # have prototype-chocolate only fail on "MOCK Leopard". 694 self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 695 "tests": { 696 "fast": { 697 "dom": { 698 "prototype-taco.html": { 699 "expected": "PASS", 700 "actual": "PASS TEXT", 701 "is_unexpected": true 702 }, 703 "prototype-chocolate.html": { 704 "expected": "FAIL", 705 "actual": "PASS" 706 }, 707 "prototype-strawberry.html": { 708 "expected": "PASS", 709 "actual": "IMAGE PASS", 710 "is_unexpected": true 711 } 712 } 713 } 714 } 715 });""") 716 return self.command._builder_data 717 718 self.command.builder_data = builder_data 719 720 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 721 crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] 722 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 723 crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] 724 crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 725 crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 726 crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 727 """) 728 729 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 730 self._write_test_file(test_port, 'fast/dom/prototype-strawberry.html', "Dummy test contents") 731 self._write_test_file(test_port, 'fast/dom/prototype-chocolate.html', "Dummy test contents") 732 733 old_exact_matches = builders._exact_matches 734 try: 735 builders._exact_matches = { 736 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 737 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 738 } 739 740 self.command.tree_status = lambda: 'closed' 741 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 742 self.assertEqual(self.tool.executive.calls, []) 743 744 self.command.tree_status = lambda: 'open' 745 746 self.tool.executive.calls = [] 747 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 748 self.assertEqual(self.tool.executive.calls, [ 749 [ 750 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'], 751 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'], 752 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 753 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 754 ], 755 [ 756 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'], 757 ['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'], 758 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 759 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 760 ], 761 ['echo', 'optimize-baselines', '--suffixes', 'txt,png', 'fast/dom/prototype-chocolate.html'], 762 ['echo', 'optimize-baselines', '--suffixes', 'png', 'fast/dom/prototype-strawberry.html'], 763 ['echo', 'optimize-baselines', '--suffixes', 'txt', 'fast/dom/prototype-taco.html'], 764 ['git', 'pull'], 765 ]) 766 767 # The mac ports should both be removed since they're the only ones in builders._exact_matches. 768 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 769 crbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] 770 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] 771 crbug.com/24182 [ Linux Win ] fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 772 crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 773 crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 774 """) 775 finally: 776 builders._exact_matches = old_exact_matches 777 778 def test_execute_test_passes_everywhere(self): 779 def blame(path): 780 return """ 781 6469e754a1 path/to/TestExpectations (foobarbaz1 (at] chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 782 """ 783 self.tool.scm().blame = blame 784 785 test_port = self.tool.port_factory.get('test') 786 original_get = self.tool.port_factory.get 787 788 def get_test_port(port_name=None, options=None, **kwargs): 789 if not port_name: 790 return test_port 791 return original_get(port_name, options, **kwargs) 792 # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path. 793 self.tool.port_factory.get = get_test_port 794 795 old_builder_data = self.command.builder_data 796 797 def builder_data(): 798 self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 799 "tests": { 800 "fast": { 801 "dom": { 802 "prototype-taco.html": { 803 "expected": "PASS", 804 "actual": "PASS TEXT", 805 "is_unexpected": true 806 } 807 } 808 } 809 } 810 });""") 811 return self.command._builder_data 812 813 self.command.builder_data = builder_data 814 815 self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 816 Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 817 """) 818 819 self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 820 821 old_exact_matches = builders._exact_matches 822 try: 823 builders._exact_matches = { 824 "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 825 "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 826 } 827 828 self.command.tree_status = lambda: 'open' 829 self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 830 self.assertEqual(self.tool.executive.calls, [ 831 [ 832 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 833 ['echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 834 ], 835 [ 836 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 837 ['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 838 ], 839 ['echo', 'optimize-baselines', '--suffixes', 'txt', 'fast/dom/prototype-taco.html'], 840 ['git', 'pull'], 841 ]) 842 843 # The mac ports should both be removed since they're the only ones in builders._exact_matches. 844 self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 845 Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] 846 """) 847 finally: 848 builders._exact_matches = old_exact_matches 849
