1 # Copyright (C) 2009 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import os 30 import StringIO 31 32 from webkitpy.common.checkout.scm import CheckoutNeedsUpdate 33 from webkitpy.common.net.bugzilla import Attachment 34 from webkitpy.common.system.filesystem_mock import MockFileSystem 35 from webkitpy.common.system.outputcapture import OutputCapture 36 from webkitpy.layout_tests.layout_package import test_results 37 from webkitpy.layout_tests.layout_package import test_failures 38 from webkitpy.thirdparty.mock import Mock 39 from webkitpy.tool.commands.commandtest import CommandsTest 40 from webkitpy.tool.commands.queues import * 41 from webkitpy.tool.commands.queuestest import QueuesTest 42 from webkitpy.tool.commands.stepsequence import StepSequence 43 from webkitpy.tool.mocktool import MockTool, MockSCM, MockStatusServer 44 45 46 class TestQueue(AbstractPatchQueue): 47 name = "test-queue" 48 49 50 class TestReviewQueue(AbstractReviewQueue): 51 name = "test-review-queue" 52 53 54 class TestFeederQueue(FeederQueue): 55 _sleep_duration = 0 56 57 58 class AbstractQueueTest(CommandsTest): 59 def test_log_directory(self): 60 self.assertEquals(TestQueue()._log_directory(), os.path.join("..", "test-queue-logs")) 61 62 def _assert_run_webkit_patch(self, run_args, port=None): 63 queue = TestQueue() 64 tool = MockTool() 65 tool.status_server.bot_id = "gort" 66 tool.executive = Mock() 67 queue.bind_to_tool(tool) 68 queue._options = Mock() 69 queue._options.port = port 70 71 queue.run_webkit_patch(run_args) 72 expected_run_args = ["echo", "--status-host=example.com", "--bot-id=gort"] 73 if port: 74 expected_run_args.append("--port=%s" % port) 75 expected_run_args.extend(run_args) 76 tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args) 77 78 def test_run_webkit_patch(self): 79 self._assert_run_webkit_patch([1]) 80 self._assert_run_webkit_patch(["one", 2]) 81 self._assert_run_webkit_patch([1], port="mockport") 82 83 def test_iteration_count(self): 84 queue = TestQueue() 85 queue._options = Mock() 86 queue._options.iterations = 3 87 self.assertTrue(queue.should_continue_work_queue()) 88 self.assertTrue(queue.should_continue_work_queue()) 89 self.assertTrue(queue.should_continue_work_queue()) 90 self.assertFalse(queue.should_continue_work_queue()) 91 92 def test_no_iteration_count(self): 93 queue = TestQueue() 94 queue._options = Mock() 95 self.assertTrue(queue.should_continue_work_queue()) 96 self.assertTrue(queue.should_continue_work_queue()) 97 self.assertTrue(queue.should_continue_work_queue()) 98 self.assertTrue(queue.should_continue_work_queue()) 99 100 def _assert_log_message(self, script_error, log_message): 101 failure_log = AbstractQueue._log_from_script_error_for_upload(script_error, output_limit=10) 102 self.assertTrue(failure_log.read(), log_message) 103 104 def test_log_from_script_error_for_upload(self): 105 self._assert_log_message(ScriptError("test"), "test") 106 # In python 2.5 unicode(Exception) is busted. See: 107 # http://bugs.python.org/issue2517 108 # With no good workaround, we just ignore these tests. 109 if not hasattr(Exception, "__unicode__"): 110 return 111 112 unicode_tor = u"WebKit \u2661 Tor Arne Vestb\u00F8!" 113 utf8_tor = unicode_tor.encode("utf-8") 114 self._assert_log_message(ScriptError(unicode_tor), utf8_tor) 115 script_error = ScriptError(unicode_tor, output=unicode_tor) 116 expected_output = "%s\nLast %s characters of output:\n%s" % (utf8_tor, 10, utf8_tor[-10:]) 117 self._assert_log_message(script_error, expected_output) 118 119 120 class FeederQueueTest(QueuesTest): 121 def test_feeder_queue(self): 122 queue = TestFeederQueue() 123 tool = MockTool(log_executive=True) 124 expected_stderr = { 125 "begin_work_queue": self._default_begin_work_queue_stderr("feeder-queue", MockSCM.fake_checkout_root), 126 "should_proceed_with_work_item": "", 127 "next_work_item": "", 128 "process_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer (at] example.com) 129 Warning, attachment 128 on bug 42 has invalid committer (non-committer (at] example.com) 130 MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting attachment 128 from commit-queue.' and additional comment 'non-committer (at] example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/Tools/Scripts/webkitpy/common/config/committers.py. 131 132 - If you do not have committer rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags. 133 134 - If you have committer rights please correct the error in Tools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed). The commit-queue restarts itself every 2 hours. After restart the commit-queue will correctly respect your committer rights.' 135 MOCK: update_work_items: commit-queue [106, 197] 136 Feeding commit-queue items [106, 197] 137 Feeding EWS (1 r? patch, 1 new) 138 MOCK: submit_to_ews: 103 139 """, 140 "handle_unexpected_error": "Mock error message\n", 141 } 142 self.assert_queue_outputs(queue, tool=tool, expected_stderr=expected_stderr) 143 144 145 class AbstractPatchQueueTest(CommandsTest): 146 def test_next_patch(self): 147 queue = AbstractPatchQueue() 148 tool = MockTool() 149 queue.bind_to_tool(tool) 150 queue._options = Mock() 151 queue._options.port = None 152 self.assertEquals(queue._next_patch(), None) 153 tool.status_server = MockStatusServer(work_items=[2, 197]) 154 expected_stdout = "MOCK: fetch_attachment: 2 is not a known attachment id\n" # A mock-only message to prevent us from making mistakes. 155 expected_stderr = "MOCK: release_work_item: None 2\n" 156 patch_id = OutputCapture().assert_outputs(self, queue._next_patch, [], expected_stdout=expected_stdout, expected_stderr=expected_stderr) 157 self.assertEquals(patch_id, None) # 2 is an invalid patch id 158 self.assertEquals(queue._next_patch().id(), 197) 159 160 161 class NeedsUpdateSequence(StepSequence): 162 def _run(self, tool, options, state): 163 raise CheckoutNeedsUpdate([], 1, "", None) 164 165 166 class AlwaysCommitQueueTool(object): 167 def __init__(self): 168 self.status_server = MockStatusServer() 169 170 def command_by_name(self, name): 171 return CommitQueue 172 173 174 class SecondThoughtsCommitQueue(CommitQueue): 175 def __init__(self): 176 self._reject_patch = False 177 CommitQueue.__init__(self) 178 179 def run_command(self, command): 180 # We want to reject the patch after the first validation, 181 # so wait to reject it until after some other command has run. 182 self._reject_patch = True 183 return CommitQueue.run_command(self, command) 184 185 def refetch_patch(self, patch): 186 if not self._reject_patch: 187 return self._tool.bugs.fetch_attachment(patch.id()) 188 189 attachment_dictionary = { 190 "id": patch.id(), 191 "bug_id": patch.bug_id(), 192 "name": "Rejected", 193 "is_obsolete": True, 194 "is_patch": False, 195 "review": "-", 196 "reviewer_email": "foo (at] bar.com", 197 "commit-queue": "-", 198 "committer_email": "foo (at] bar.com", 199 "attacher_email": "Contributer1", 200 } 201 return Attachment(attachment_dictionary, None) 202 203 204 class CommitQueueTest(QueuesTest): 205 def _mock_test_result(self, testname): 206 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) 207 208 def test_commit_queue(self): 209 expected_stderr = { 210 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 211 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n", 212 "next_work_item": "", 213 "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory 214 MOCK: update_status: commit-queue Updated working directory 215 MOCK: update_status: commit-queue Applied patch 216 MOCK: update_status: commit-queue Built patch 217 MOCK: update_status: commit-queue Passed tests 218 MOCK: update_status: commit-queue Landed patch 219 MOCK: update_status: commit-queue Pass 220 MOCK: release_work_item: commit-queue 197 221 """, 222 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n", 223 "handle_script_error": "ScriptError error message\n", 224 } 225 self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr) 226 227 def test_commit_queue_failure(self): 228 expected_stderr = { 229 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 230 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n", 231 "next_work_item": "", 232 "process_work_item": """MOCK: update_status: commit-queue Cleaned working directory 233 MOCK: update_status: commit-queue Updated working directory 234 MOCK: update_status: commit-queue Patch does not apply 235 MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'MOCK script error' 236 MOCK: update_status: commit-queue Fail 237 MOCK: release_work_item: commit-queue 197 238 """, 239 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n", 240 "handle_script_error": "ScriptError error message\n", 241 } 242 queue = CommitQueue() 243 244 def mock_run_webkit_patch(command): 245 if command == ['clean'] or command == ['update']: 246 # We want cleaning to succeed so we can error out on a step 247 # that causes the commit-queue to reject the patch. 248 return 249 raise ScriptError('MOCK script error') 250 251 queue.run_webkit_patch = mock_run_webkit_patch 252 self.assert_queue_outputs(queue, expected_stderr=expected_stderr) 253 254 def test_rollout(self): 255 tool = MockTool(log_executive=True) 256 tool.filesystem.write_text_file('/mock/results.html', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem. 257 tool.buildbot.light_tree_on_fire() 258 expected_stderr = { 259 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 260 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing patch\n", 261 "next_work_item": "", 262 "process_work_item": """MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'clean'] 263 MOCK: update_status: commit-queue Cleaned working directory 264 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'update'] 265 MOCK: update_status: commit-queue Updated working directory 266 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'apply-attachment', '--no-update', '--non-interactive', 197] 267 MOCK: update_status: commit-queue Applied patch 268 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build', '--no-clean', '--no-update', '--build-style=both'] 269 MOCK: update_status: commit-queue Built patch 270 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 271 MOCK: update_status: commit-queue Passed tests 272 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 273 MOCK: update_status: commit-queue Landed patch 274 MOCK: update_status: commit-queue Pass 275 MOCK: release_work_item: commit-queue 197 276 """, 277 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting attachment 197 from commit-queue.' and additional comment 'Mock error message'\n", 278 "handle_script_error": "ScriptError error message\n", 279 } 280 self.assert_queue_outputs(CommitQueue(), tool=tool, expected_stderr=expected_stderr) 281 282 def test_rollout_lands(self): 283 tool = MockTool(log_executive=True) 284 tool.buildbot.light_tree_on_fire() 285 rollout_patch = tool.bugs.fetch_attachment(106) # _patch6, a rollout patch. 286 assert(rollout_patch.is_rollout()) 287 expected_stderr = { 288 "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), 289 "should_proceed_with_work_item": "MOCK: update_status: commit-queue Processing rollout patch\n", 290 "next_work_item": "", 291 "process_work_item": """MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'clean'] 292 MOCK: update_status: commit-queue Cleaned working directory 293 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'update'] 294 MOCK: update_status: commit-queue Updated working directory 295 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'apply-attachment', '--no-update', '--non-interactive', 106] 296 MOCK: update_status: commit-queue Applied patch 297 MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 106] 298 MOCK: update_status: commit-queue Landed patch 299 MOCK: update_status: commit-queue Pass 300 MOCK: release_work_item: commit-queue 106 301 """, 302 "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '106' with comment 'Rejecting attachment 106 from commit-queue.' and additional comment 'Mock error message'\n", 303 "handle_script_error": "ScriptError error message\n", 304 } 305 self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=rollout_patch, expected_stderr=expected_stderr) 306 307 def test_auto_retry(self): 308 queue = CommitQueue() 309 options = Mock() 310 options.parent_command = "commit-queue" 311 tool = AlwaysCommitQueueTool() 312 sequence = NeedsUpdateSequence(None) 313 314 expected_stderr = "Commit failed because the checkout is out of date. Please update and try again.\nMOCK: update_status: commit-queue Tests passed, but commit failed (checkout out of date). Updating, then landing without building or re-running tests.\n" 315 state = {'patch': None} 316 OutputCapture().assert_outputs(self, sequence.run_and_handle_errors, [tool, options, state], expected_exception=TryAgain, expected_stderr=expected_stderr) 317 318 self.assertEquals(options.update, True) 319 self.assertEquals(options.build, False) 320 self.assertEquals(options.test, False) 321 322 def test_manual_reject_during_processing(self): 323 queue = SecondThoughtsCommitQueue() 324 queue.bind_to_tool(MockTool()) 325 queue._tool.filesystem.write_text_file('/mock/results.html', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem. 326 queue._options = Mock() 327 queue._options.port = None 328 expected_stderr = """MOCK: update_status: commit-queue Cleaned working directory 329 MOCK: update_status: commit-queue Updated working directory 330 MOCK: update_status: commit-queue Applied patch 331 MOCK: update_status: commit-queue Built patch 332 MOCK: update_status: commit-queue Passed tests 333 MOCK: update_status: commit-queue Retry 334 MOCK: release_work_item: commit-queue 197 335 """ 336 OutputCapture().assert_outputs(self, queue.process_work_item, [QueuesTest.mock_work_item], expected_stderr=expected_stderr) 337 338 def test_report_flaky_tests(self): 339 queue = CommitQueue() 340 queue.bind_to_tool(MockTool()) 341 expected_stderr = """MOCK bug comment: bug_id=76, cc=None 342 --- Begin comment --- 343 The commit-queue just saw foo/bar.html flake (Text diff mismatch) while processing attachment 197 on bug 42. 344 Port: MockPort Platform: MockPlatform 1.0 345 --- End comment --- 346 347 MOCK add_attachment_to_bug: bug_id=76, description=Failure diff from bot filename=failure.diff 348 MOCK bug comment: bug_id=76, cc=None 349 --- Begin comment --- 350 The commit-queue just saw bar/baz.html flake (Text diff mismatch) while processing attachment 197 on bug 42. 351 Port: MockPort Platform: MockPlatform 1.0 352 --- End comment --- 353 354 MOCK add_attachment_to_bug: bug_id=76, description=Archive of layout-test-results from bot filename=layout-test-results.zip 355 MOCK bug comment: bug_id=42, cc=None 356 --- Begin comment --- 357 The commit-queue encountered the following flaky tests while processing attachment 197: 358 359 foo/bar.html bug 76 (author: abarth (at] webkit.org) 360 bar/baz.html bug 76 (author: abarth (at] webkit.org) 361 The commit-queue is continuing to process your patch. 362 --- End comment --- 363 364 """ 365 test_names = ["foo/bar.html", "bar/baz.html"] 366 test_results = [self._mock_test_result(name) for name in test_names] 367 368 class MockZipFile(object): 369 def __init__(self): 370 self.fp = StringIO() 371 372 def read(self, path): 373 return "" 374 375 def namelist(self): 376 # This is intentionally missing one diffs.txt to exercise the "upload the whole zip" codepath. 377 return ['foo/bar-diffs.txt'] 378 379 OutputCapture().assert_outputs(self, queue.report_flaky_tests, [QueuesTest.mock_work_item, test_results, MockZipFile()], expected_stderr=expected_stderr) 380 381 def test_missing_layout_test_results(self): 382 queue = CommitQueue() 383 tool = MockTool() 384 results_path = '/mock/results.html' 385 tool.filesystem = MockFileSystem({results_path: None}) 386 queue.bind_to_tool(tool) 387 # Make sure that our filesystem mock functions as we expect. 388 self.assertRaises(IOError, tool.filesystem.read_text_file, results_path) 389 # layout_test_results shouldn't raise even if the results.html file is missing. 390 self.assertEquals(queue.layout_test_results(), None) 391 392 def test_layout_test_results(self): 393 queue = CommitQueue() 394 queue.bind_to_tool(MockTool()) 395 queue._read_file_contents = lambda path: None 396 self.assertEquals(queue.layout_test_results(), None) 397 queue._read_file_contents = lambda path: "" 398 self.assertEquals(queue.layout_test_results(), None) 399 queue._create_layout_test_results = lambda: LayoutTestResults([]) 400 results = queue.layout_test_results() 401 self.assertNotEquals(results, None) 402 self.assertEquals(results.failure_limit_count(), 10) # This value matches RunTests.NON_INTERACTIVE_FAILURE_LIMIT_COUNT 403 404 def test_archive_last_layout_test_results(self): 405 queue = CommitQueue() 406 queue.bind_to_tool(MockTool()) 407 patch = queue._tool.bugs.fetch_attachment(128) 408 # This is just to test that the method doesn't raise. 409 queue.archive_last_layout_test_results(patch) 410 411 def test_upload_results_archive_for_patch(self): 412 queue = CommitQueue() 413 queue.bind_to_tool(MockTool()) 414 patch = queue._tool.bugs.fetch_attachment(128) 415 expected_stderr = """MOCK add_attachment_to_bug: bug_id=42, description=Archive of layout-test-results from bot filename=layout-test-results.zip 416 -- Begin comment -- 417 The attached test failures were seen while running run-webkit-tests on the commit-queue. 418 Port: MockPort Platform: MockPlatform 1.0 419 -- End comment -- 420 """ 421 OutputCapture().assert_outputs(self, queue._upload_results_archive_for_patch, [patch, Mock()], expected_stderr=expected_stderr) 422 423 424 class StyleQueueTest(QueuesTest): 425 def test_style_queue(self): 426 expected_stderr = { 427 "begin_work_queue": self._default_begin_work_queue_stderr("style-queue", MockSCM.fake_checkout_root), 428 "next_work_item": "", 429 "should_proceed_with_work_item": "MOCK: update_status: style-queue Checking style\n", 430 "process_work_item": "MOCK: update_status: style-queue Pass\nMOCK: release_work_item: style-queue 197\n", 431 "handle_unexpected_error": "Mock error message\n", 432 "handle_script_error": "MOCK: update_status: style-queue ScriptError error message\nMOCK bug comment: bug_id=42, cc=[]\n--- Begin comment ---\nAttachment 197 did not pass style-queue:\n\nScriptError error message\n\nIf any of these errors are false positives, please file a bug against check-webkit-style.\n--- End comment ---\n\n", 433 } 434 expected_exceptions = { 435 "handle_script_error": SystemExit, 436 } 437 self.assert_queue_outputs(StyleQueue(), expected_stderr=expected_stderr, expected_exceptions=expected_exceptions) 438
