1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 from datetime import datetime 30 import unittest 31 32 from webkitpy.common.net import bugzilla 33 from webkitpy.common.net.layouttestresults import LayoutTestResults 34 from webkitpy.common.system.deprecated_logging import error, log 35 from webkitpy.common.system.outputcapture import OutputCapture 36 from webkitpy.layout_tests.layout_package import test_results 37 from webkitpy.layout_tests.layout_package import test_failures 38 from webkitpy.thirdparty.mock import Mock 39 from webkitpy.tool.bot.commitqueuetask import * 40 from webkitpy.tool.mocktool import MockTool 41 42 43 class MockCommitQueue(CommitQueueTaskDelegate): 44 def __init__(self, error_plan): 45 self._error_plan = error_plan 46 47 def run_command(self, command): 48 log("run_webkit_patch: %s" % command) 49 if self._error_plan: 50 error = self._error_plan.pop(0) 51 if error: 52 raise error 53 54 def command_passed(self, success_message, patch): 55 log("command_passed: success_message='%s' patch='%s'" % ( 56 success_message, patch.id())) 57 58 def command_failed(self, failure_message, script_error, patch): 59 log("command_failed: failure_message='%s' script_error='%s' patch='%s'" % ( 60 failure_message, script_error, patch.id())) 61 return 3947 62 63 def refetch_patch(self, patch): 64 return patch 65 66 def layout_test_results(self): 67 return None 68 69 def report_flaky_tests(self, patch, flaky_results, results_archive): 70 flaky_tests = [result.filename for result in flaky_results] 71 log("report_flaky_tests: patch='%s' flaky_tests='%s' archive='%s'" % (patch.id(), flaky_tests, results_archive.filename)) 72 73 def archive_last_layout_test_results(self, patch): 74 log("archive_last_layout_test_results: patch='%s'" % patch.id()) 75 archive = Mock() 76 archive.filename = "mock-archive-%s.zip" % patch.id() 77 return archive 78 79 80 class CommitQueueTaskTest(unittest.TestCase): 81 def _run_through_task(self, commit_queue, expected_stderr, expected_exception=None, expect_retry=False): 82 tool = MockTool(log_executive=True) 83 patch = tool.bugs.fetch_attachment(197) 84 task = CommitQueueTask(commit_queue, patch) 85 success = OutputCapture().assert_outputs(self, task.run, expected_stderr=expected_stderr, expected_exception=expected_exception) 86 if not expected_exception: 87 self.assertEqual(success, not expect_retry) 88 89 def test_success_case(self): 90 commit_queue = MockCommitQueue([]) 91 expected_stderr = """run_webkit_patch: ['clean'] 92 command_passed: success_message='Cleaned working directory' patch='197' 93 run_webkit_patch: ['update'] 94 command_passed: success_message='Updated working directory' patch='197' 95 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 96 command_passed: success_message='Applied patch' patch='197' 97 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 98 command_passed: success_message='Built patch' patch='197' 99 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 100 command_passed: success_message='Passed tests' patch='197' 101 run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 102 command_passed: success_message='Landed patch' patch='197' 103 """ 104 self._run_through_task(commit_queue, expected_stderr) 105 106 def test_clean_failure(self): 107 commit_queue = MockCommitQueue([ 108 ScriptError("MOCK clean failure"), 109 ]) 110 expected_stderr = """run_webkit_patch: ['clean'] 111 command_failed: failure_message='Unable to clean working directory' script_error='MOCK clean failure' patch='197' 112 """ 113 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 114 115 def test_update_failure(self): 116 commit_queue = MockCommitQueue([ 117 None, 118 ScriptError("MOCK update failure"), 119 ]) 120 expected_stderr = """run_webkit_patch: ['clean'] 121 command_passed: success_message='Cleaned working directory' patch='197' 122 run_webkit_patch: ['update'] 123 command_failed: failure_message='Unable to update working directory' script_error='MOCK update failure' patch='197' 124 """ 125 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 126 127 def test_apply_failure(self): 128 commit_queue = MockCommitQueue([ 129 None, 130 None, 131 ScriptError("MOCK apply failure"), 132 ]) 133 expected_stderr = """run_webkit_patch: ['clean'] 134 command_passed: success_message='Cleaned working directory' patch='197' 135 run_webkit_patch: ['update'] 136 command_passed: success_message='Updated working directory' patch='197' 137 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 138 command_failed: failure_message='Patch does not apply' script_error='MOCK apply failure' patch='197' 139 """ 140 self._run_through_task(commit_queue, expected_stderr, ScriptError) 141 142 def test_build_failure(self): 143 commit_queue = MockCommitQueue([ 144 None, 145 None, 146 None, 147 ScriptError("MOCK build failure"), 148 ]) 149 expected_stderr = """run_webkit_patch: ['clean'] 150 command_passed: success_message='Cleaned working directory' patch='197' 151 run_webkit_patch: ['update'] 152 command_passed: success_message='Updated working directory' patch='197' 153 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 154 command_passed: success_message='Applied patch' patch='197' 155 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 156 command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='197' 157 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'] 158 command_passed: success_message='Able to build without patch' patch='197' 159 """ 160 self._run_through_task(commit_queue, expected_stderr, ScriptError) 161 162 def test_red_build_failure(self): 163 commit_queue = MockCommitQueue([ 164 None, 165 None, 166 None, 167 ScriptError("MOCK build failure"), 168 ScriptError("MOCK clean build failure"), 169 ]) 170 expected_stderr = """run_webkit_patch: ['clean'] 171 command_passed: success_message='Cleaned working directory' patch='197' 172 run_webkit_patch: ['update'] 173 command_passed: success_message='Updated working directory' patch='197' 174 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 175 command_passed: success_message='Applied patch' patch='197' 176 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 177 command_failed: failure_message='Patch does not build' script_error='MOCK build failure' patch='197' 178 run_webkit_patch: ['build', '--force-clean', '--no-update', '--build-style=both'] 179 command_failed: failure_message='Unable to build without patch' script_error='MOCK clean build failure' patch='197' 180 """ 181 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 182 183 def test_flaky_test_failure(self): 184 commit_queue = MockCommitQueue([ 185 None, 186 None, 187 None, 188 None, 189 ScriptError("MOCK tests failure"), 190 ]) 191 # CommitQueueTask will only report flaky tests if we successfully parsed 192 # results.html and returned a LayoutTestResults object, so we fake one. 193 commit_queue.layout_test_results = lambda: LayoutTestResults([]) 194 expected_stderr = """run_webkit_patch: ['clean'] 195 command_passed: success_message='Cleaned working directory' patch='197' 196 run_webkit_patch: ['update'] 197 command_passed: success_message='Updated working directory' patch='197' 198 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 199 command_passed: success_message='Applied patch' patch='197' 200 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 201 command_passed: success_message='Built patch' patch='197' 202 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 203 command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='197' 204 archive_last_layout_test_results: patch='197' 205 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 206 command_passed: success_message='Passed tests' patch='197' 207 report_flaky_tests: patch='197' flaky_tests='[]' archive='mock-archive-197.zip' 208 run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 209 command_passed: success_message='Landed patch' patch='197' 210 """ 211 self._run_through_task(commit_queue, expected_stderr) 212 213 def test_failed_archive(self): 214 commit_queue = MockCommitQueue([ 215 None, 216 None, 217 None, 218 None, 219 ScriptError("MOCK tests failure"), 220 ]) 221 commit_queue.layout_test_results = lambda: LayoutTestResults([]) 222 # It's possible delegate to fail to archive layout tests, don't try to report 223 # flaky tests when that happens. 224 commit_queue.archive_last_layout_test_results = lambda patch: None 225 expected_stderr = """run_webkit_patch: ['clean'] 226 command_passed: success_message='Cleaned working directory' patch='197' 227 run_webkit_patch: ['update'] 228 command_passed: success_message='Updated working directory' patch='197' 229 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 230 command_passed: success_message='Applied patch' patch='197' 231 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 232 command_passed: success_message='Built patch' patch='197' 233 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 234 command_failed: failure_message='Patch does not pass tests' script_error='MOCK tests failure' patch='197' 235 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 236 command_passed: success_message='Passed tests' patch='197' 237 run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 238 command_passed: success_message='Landed patch' patch='197' 239 """ 240 self._run_through_task(commit_queue, expected_stderr) 241 242 def test_double_flaky_test_failure(self): 243 class DoubleFlakyCommitQueue(MockCommitQueue): 244 def __init__(self, error_plan): 245 MockCommitQueue.__init__(self, error_plan) 246 self._double_flaky_test_counter = 0 247 248 def run_command(self, command): 249 self._double_flaky_test_counter += 1 250 MockCommitQueue.run_command(self, command) 251 252 def _mock_test_result(self, testname): 253 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) 254 255 def layout_test_results(self): 256 if self._double_flaky_test_counter % 2: 257 return LayoutTestResults([self._mock_test_result('foo.html')]) 258 return LayoutTestResults([self._mock_test_result('bar.html')]) 259 260 commit_queue = DoubleFlakyCommitQueue([ 261 None, 262 None, 263 None, 264 None, 265 ScriptError("MOCK test failure"), 266 ScriptError("MOCK test failure again"), 267 ]) 268 # The (subtle) point of this test is that report_flaky_tests does not appear 269 # in the expected_stderr for this run. 270 # Note also that there is no attempt to run the tests w/o the patch. 271 expected_stderr = """run_webkit_patch: ['clean'] 272 command_passed: success_message='Cleaned working directory' patch='197' 273 run_webkit_patch: ['update'] 274 command_passed: success_message='Updated working directory' patch='197' 275 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 276 command_passed: success_message='Applied patch' patch='197' 277 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 278 command_passed: success_message='Built patch' patch='197' 279 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 280 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197' 281 archive_last_layout_test_results: patch='197' 282 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 283 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' 284 """ 285 tool = MockTool(log_executive=True) 286 patch = tool.bugs.fetch_attachment(197) 287 task = CommitQueueTask(commit_queue, patch) 288 success = OutputCapture().assert_outputs(self, task.run, expected_stderr=expected_stderr) 289 self.assertEqual(success, False) 290 291 def test_test_failure(self): 292 commit_queue = MockCommitQueue([ 293 None, 294 None, 295 None, 296 None, 297 ScriptError("MOCK test failure"), 298 ScriptError("MOCK test failure again"), 299 ]) 300 expected_stderr = """run_webkit_patch: ['clean'] 301 command_passed: success_message='Cleaned working directory' patch='197' 302 run_webkit_patch: ['update'] 303 command_passed: success_message='Updated working directory' patch='197' 304 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 305 command_passed: success_message='Applied patch' patch='197' 306 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 307 command_passed: success_message='Built patch' patch='197' 308 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 309 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197' 310 archive_last_layout_test_results: patch='197' 311 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 312 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' 313 archive_last_layout_test_results: patch='197' 314 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive'] 315 command_passed: success_message='Able to pass tests without patch' patch='197' 316 """ 317 self._run_through_task(commit_queue, expected_stderr, ScriptError) 318 319 def test_red_test_failure(self): 320 commit_queue = MockCommitQueue([ 321 None, 322 None, 323 None, 324 None, 325 ScriptError("MOCK test failure"), 326 ScriptError("MOCK test failure again"), 327 ScriptError("MOCK clean test failure"), 328 ]) 329 expected_stderr = """run_webkit_patch: ['clean'] 330 command_passed: success_message='Cleaned working directory' patch='197' 331 run_webkit_patch: ['update'] 332 command_passed: success_message='Updated working directory' patch='197' 333 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 334 command_passed: success_message='Applied patch' patch='197' 335 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 336 command_passed: success_message='Built patch' patch='197' 337 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 338 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure' patch='197' 339 archive_last_layout_test_results: patch='197' 340 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 341 command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' 342 archive_last_layout_test_results: patch='197' 343 run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive'] 344 command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='197' 345 """ 346 self._run_through_task(commit_queue, expected_stderr, expect_retry=True) 347 348 def test_land_failure(self): 349 commit_queue = MockCommitQueue([ 350 None, 351 None, 352 None, 353 None, 354 None, 355 ScriptError("MOCK land failure"), 356 ]) 357 expected_stderr = """run_webkit_patch: ['clean'] 358 command_passed: success_message='Cleaned working directory' patch='197' 359 run_webkit_patch: ['update'] 360 command_passed: success_message='Updated working directory' patch='197' 361 run_webkit_patch: ['apply-attachment', '--no-update', '--non-interactive', 197] 362 command_passed: success_message='Applied patch' patch='197' 363 run_webkit_patch: ['build', '--no-clean', '--no-update', '--build-style=both'] 364 command_passed: success_message='Built patch' patch='197' 365 run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] 366 command_passed: success_message='Passed tests' patch='197' 367 run_webkit_patch: ['land-attachment', '--force-clean', '--ignore-builders', '--non-interactive', '--parent-command=commit-queue', 197] 368 command_failed: failure_message='Unable to land patch' script_error='MOCK land failure' patch='197' 369 """ 370 # FIXME: This should really be expect_retry=True for a better user experiance. 371 self._run_through_task(commit_queue, expected_stderr, ScriptError) 372 373 def _expect_validate(self, patch, is_valid): 374 class MockDelegate(object): 375 def refetch_patch(self, patch): 376 return patch 377 378 task = CommitQueueTask(MockDelegate(), patch) 379 self.assertEquals(task._validate(), is_valid) 380 381 def _mock_patch(self, attachment_dict={}, bug_dict={'bug_status': 'NEW'}, committer="fake"): 382 bug = bugzilla.Bug(bug_dict, None) 383 patch = bugzilla.Attachment(attachment_dict, bug) 384 patch._committer = committer 385 return patch 386 387 def test_validate(self): 388 self._expect_validate(self._mock_patch(), True) 389 self._expect_validate(self._mock_patch({'is_obsolete': True}), False) 390 self._expect_validate(self._mock_patch(bug_dict={'bug_status': 'CLOSED'}), False) 391 self._expect_validate(self._mock_patch(committer=None), False) 392 self._expect_validate(self._mock_patch({'review': '-'}), False) 393
