1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 from webkitpy.common.system.executive import ScriptError 30 from webkitpy.common.net.layouttestresults import LayoutTestResults 31 from webkitpy.tool.bot.expectedfailures import ExpectedFailures 32 33 34 class CommitQueueTaskDelegate(object): 35 def run_command(self, command): 36 raise NotImplementedError("subclasses must implement") 37 38 def command_passed(self, message, patch): 39 raise NotImplementedError("subclasses must implement") 40 41 def command_failed(self, message, script_error, patch): 42 raise NotImplementedError("subclasses must implement") 43 44 def refetch_patch(self, patch): 45 raise NotImplementedError("subclasses must implement") 46 47 def layout_test_results(self): 48 raise NotImplementedError("subclasses must implement") 49 50 def archive_last_layout_test_results(self, patch): 51 raise NotImplementedError("subclasses must implement") 52 53 # We could make results_archive optional, but for now it's required. 54 def report_flaky_tests(self, patch, flaky_tests, results_archive): 55 raise NotImplementedError("subclasses must implement") 56 57 58 class CommitQueueTask(object): 59 def __init__(self, delegate, patch): 60 self._delegate = delegate 61 self._patch = patch 62 self._script_error = None 63 self._results_archive_from_patch_test_run = None 64 self._expected_failures = ExpectedFailures() 65 66 def _validate(self): 67 # Bugs might get closed, or patches might be obsoleted or r-'d while the 68 # commit-queue is processing. 69 self._patch = self._delegate.refetch_patch(self._patch) 70 if self._patch.is_obsolete(): 71 return False 72 if self._patch.bug().is_closed(): 73 return False 74 if not self._patch.committer(): 75 return False 76 if not self._patch.review() != "-": 77 return False 78 # Reviewer is not required. Missing reviewers will be caught during 79 # the ChangeLog check during landing. 80 return True 81 82 def _run_command(self, command, success_message, failure_message): 83 try: 84 self._delegate.run_command(command) 85 self._delegate.command_passed(success_message, patch=self._patch) 86 return True 87 except ScriptError, e: 88 self._script_error = e 89 self.failure_status_id = self._delegate.command_failed(failure_message, script_error=self._script_error, patch=self._patch) 90 return False 91 92 def _clean(self): 93 return self._run_command([ 94 "clean", 95 ], 96 "Cleaned working directory", 97 "Unable to clean working directory") 98 99 def _update(self): 100 # FIXME: Ideally the status server log message should include which revision we updated to. 101 return self._run_command([ 102 "update", 103 ], 104 "Updated working directory", 105 "Unable to update working directory") 106 107 def _apply(self): 108 return self._run_command([ 109 "apply-attachment", 110 "--no-update", 111 "--non-interactive", 112 self._patch.id(), 113 ], 114 "Applied patch", 115 "Patch does not apply") 116 117 def _build(self): 118 return self._run_command([ 119 "build", 120 "--no-clean", 121 "--no-update", 122 "--build-style=both", 123 ], 124 "Built patch", 125 "Patch does not build") 126 127 def _build_without_patch(self): 128 return self._run_command([ 129 "build", 130 "--force-clean", 131 "--no-update", 132 "--build-style=both", 133 ], 134 "Able to build without patch", 135 "Unable to build without patch") 136 137 def _test(self): 138 success = self._run_command([ 139 "build-and-test", 140 "--no-clean", 141 "--no-update", 142 # Notice that we don't pass --build, which means we won't build! 143 "--test", 144 "--non-interactive", 145 ], 146 "Passed tests", 147 "Patch does not pass tests") 148 149 self._expected_failures.shrink_expected_failures(self._delegate.layout_test_results(), success) 150 return success 151 152 def _build_and_test_without_patch(self): 153 success = self._run_command([ 154 "build-and-test", 155 "--force-clean", 156 "--no-update", 157 "--build", 158 "--test", 159 "--non-interactive", 160 ], 161 "Able to pass tests without patch", 162 "Unable to pass tests without patch (tree is red?)") 163 164 self._expected_failures.shrink_expected_failures(self._delegate.layout_test_results(), success) 165 return success 166 167 def _land(self): 168 # Unclear if this should pass --quiet or not. If --parent-command always does the reporting, then it should. 169 return self._run_command([ 170 "land-attachment", 171 "--force-clean", 172 "--ignore-builders", 173 "--non-interactive", 174 "--parent-command=commit-queue", 175 self._patch.id(), 176 ], 177 "Landed patch", 178 "Unable to land patch") 179 180 def _report_flaky_tests(self, flaky_test_results, results_archive): 181 self._delegate.report_flaky_tests(self._patch, flaky_test_results, results_archive) 182 183 def _results_failed_different_tests(self, first, second): 184 first_failing_tests = [] if not first else first.failing_tests() 185 second_failing_tests = [] if not second else second.failing_tests() 186 return first_failing_tests != second_failing_tests 187 188 def _test_patch(self): 189 if self._test(): 190 return True 191 192 # Note: archive_last_layout_test_results deletes the results directory, making these calls order-sensitve. 193 # We could remove this dependency by building the layout_test_results from the archive. 194 first_results = self._delegate.layout_test_results() 195 first_results_archive = self._delegate.archive_last_layout_test_results(self._patch) 196 197 if self._expected_failures.failures_were_expected(first_results): 198 return True 199 200 if self._test(): 201 # Only report flaky tests if we were successful at parsing results.html and archiving results. 202 if first_results and first_results_archive: 203 self._report_flaky_tests(first_results.failing_test_results(), first_results_archive) 204 return True 205 206 second_results = self._delegate.layout_test_results() 207 if self._results_failed_different_tests(first_results, second_results): 208 # We could report flaky tests here, but we would need to be careful 209 # to use similar checks to ExpectedFailures._can_trust_results 210 # to make sure we don't report constant failures as flakes when 211 # we happen to hit the --exit-after-N-failures limit. 212 # See https://bugs.webkit.org/show_bug.cgi?id=51272 213 return False 214 215 # Archive (and remove) second results so layout_test_results() after 216 # build_and_test_without_patch won't use second results instead of the clean-tree results. 217 second_results_archive = self._delegate.archive_last_layout_test_results(self._patch) 218 219 if self._build_and_test_without_patch(): 220 # The error from the previous ._test() run is real, report it. 221 return self.report_failure(first_results_archive) 222 223 clean_tree_results = self._delegate.layout_test_results() 224 self._expected_failures.grow_expected_failures(clean_tree_results) 225 226 return False # Tree must be redder than we expected, just retry later. 227 228 def results_archive_from_patch_test_run(self, patch): 229 assert(self._patch.id() == patch.id()) # CommitQueueTask is not currently re-useable. 230 return self._results_archive_from_patch_test_run 231 232 def report_failure(self, results_archive=None): 233 if not self._validate(): 234 return False 235 self._results_archive_from_patch_test_run = results_archive 236 raise self._script_error 237 238 def run(self): 239 if not self._validate(): 240 return False 241 if not self._clean(): 242 return False 243 if not self._update(): 244 return False 245 if not self._apply(): 246 return self.report_failure() 247 if not self._patch.is_rollout(): 248 if not self._build(): 249 if not self._build_without_patch(): 250 return False 251 return self.report_failure() 252 if not self._test_patch(): 253 return False 254 # Make sure the patch is still valid before landing (e.g., make sure 255 # no one has set commit-queue- since we started working on the patch.) 256 if not self._validate(): 257 return False 258 # FIXME: We should understand why the land failure occured and retry if possible. 259 if not self._land(): 260 return self.report_failure() 261 return True 262
