1 #!/usr/bin/env python 2 # Copyright (c) 2009, Google Inc. All rights reserved. 3 # Copyright (c) 2009 Apple Inc. All rights reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # * Redistributions of source code must retain the above copyright 10 # notice, this list of conditions and the following disclaimer. 11 # * Redistributions in binary form must reproduce the above 12 # copyright notice, this list of conditions and the following disclaimer 13 # in the documentation and/or other materials provided with the 14 # distribution. 15 # * Neither the name of Google Inc. nor the names of its 16 # contributors may be used to endorse or promote products derived from 17 # this software without specific prior written permission. 18 # 19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 import traceback 32 import os 33 34 from datetime import datetime 35 from optparse import make_option 36 from StringIO import StringIO 37 38 from webkitpy.bugzilla import CommitterValidator 39 from webkitpy.executive import ScriptError 40 from webkitpy.grammar import pluralize 41 from webkitpy.webkit_logging import error, log 42 from webkitpy.multicommandtool import Command 43 from webkitpy.patchcollection import PersistentPatchCollection, PersistentPatchCollectionDelegate 44 from webkitpy.statusserver import StatusServer 45 from webkitpy.stepsequence import StepSequenceErrorHandler 46 from webkitpy.queueengine import QueueEngine, QueueEngineDelegate 47 48 class AbstractQueue(Command, QueueEngineDelegate): 49 watchers = [ 50 "webkit-bot-watchers (at] googlegroups.com", 51 ] 52 53 _pass_status = "Pass" 54 _fail_status = "Fail" 55 _error_status = "Error" 56 57 def __init__(self, options=None): # Default values should never be collections (like []) as default values are shared between invocations 58 options_list = (options or []) + [ 59 make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue. Dangerous!"), 60 ] 61 Command.__init__(self, "Run the %s" % self.name, options=options_list) 62 63 def _cc_watchers(self, bug_id): 64 try: 65 self.tool.bugs.add_cc_to_bug(bug_id, self.watchers) 66 except Exception, e: 67 traceback.print_exc() 68 log("Failed to CC watchers.") 69 70 def _update_status(self, message, patch=None, results_file=None): 71 self.tool.status_server.update_status(self.name, message, patch, results_file) 72 73 def _did_pass(self, patch): 74 self._update_status(self._pass_status, patch) 75 76 def _did_fail(self, patch): 77 self._update_status(self._fail_status, patch) 78 79 def _did_error(self, patch, reason): 80 message = "%s: %s" % (self._error_status, reason) 81 self._update_status(message, patch) 82 83 def queue_log_path(self): 84 return "%s.log" % self.name 85 86 def work_item_log_path(self, patch): 87 return os.path.join("%s-logs" % self.name, "%s.log" % patch.bug_id()) 88 89 def begin_work_queue(self): 90 log("CAUTION: %s will discard all local changes in \"%s\"" % (self.name, self.tool.scm().checkout_root)) 91 if self.options.confirm: 92 response = self.tool.user.prompt("Are you sure? Type \"yes\" to continue: ") 93 if (response != "yes"): 94 error("User declined.") 95 log("Running WebKit %s." % self.name) 96 97 def should_continue_work_queue(self): 98 return True 99 100 def next_work_item(self): 101 raise NotImplementedError, "subclasses must implement" 102 103 def should_proceed_with_work_item(self, work_item): 104 raise NotImplementedError, "subclasses must implement" 105 106 def process_work_item(self, work_item): 107 raise NotImplementedError, "subclasses must implement" 108 109 def handle_unexpected_error(self, work_item, message): 110 raise NotImplementedError, "subclasses must implement" 111 112 def run_webkit_patch(self, args): 113 webkit_patch_args = [self.tool.path()] 114 # FIXME: This is a hack, we should have a more general way to pass global options. 115 webkit_patch_args += ["--status-host=%s" % self.tool.status_server.host] 116 webkit_patch_args += map(str, args) 117 self.tool.executive.run_and_throw_if_fail(webkit_patch_args) 118 119 def log_progress(self, patch_ids): 120 log("%s in %s [%s]" % (pluralize("patch", len(patch_ids)), self.name, ", ".join(map(str, patch_ids)))) 121 122 def execute(self, options, args, tool, engine=QueueEngine): 123 self.options = options 124 self.tool = tool 125 return engine(self.name, self).run() 126 127 @classmethod 128 def _update_status_for_script_error(cls, tool, state, script_error, is_error=False): 129 message = script_error.message 130 if is_error: 131 message = "Error: %s" % message 132 output = script_error.message_with_output(output_limit=5*1024*1024) # 5MB 133 return tool.status_server.update_status(cls.name, message, state["patch"], StringIO(output)) 134 135 136 class CommitQueue(AbstractQueue, StepSequenceErrorHandler): 137 name = "commit-queue" 138 def __init__(self): 139 AbstractQueue.__init__(self) 140 141 # AbstractQueue methods 142 143 def begin_work_queue(self): 144 AbstractQueue.begin_work_queue(self) 145 self.committer_validator = CommitterValidator(self.tool.bugs) 146 147 def _validate_patches_in_commit_queue(self): 148 # Not using BugzillaQueries.fetch_patches_from_commit_queue() so we can reject patches with invalid committers/reviewers. 149 bug_ids = self.tool.bugs.queries.fetch_bug_ids_from_commit_queue() 150 all_patches = sum([self.tool.bugs.fetch_bug(bug_id).commit_queued_patches(include_invalid=True) for bug_id in bug_ids], []) 151 return self.committer_validator.patches_after_rejecting_invalid_commiters_and_reviewers(all_patches) 152 153 def next_work_item(self): 154 patches = self._validate_patches_in_commit_queue() 155 # FIXME: We could sort the patches in a specific order here, was suggested by https://bugs.webkit.org/show_bug.cgi?id=33395 156 if not patches: 157 self._update_status("Empty queue") 158 return None 159 # Only bother logging if we have patches in the queue. 160 self.log_progress([patch.id() for patch in patches]) 161 return patches[0] 162 163 def _can_build_and_test(self): 164 try: 165 self.run_webkit_patch(["build-and-test", "--force-clean", "--non-interactive", "--build-style=both", "--quiet"]) 166 except ScriptError, e: 167 self._update_status("Unabled to successfully build and test", None) 168 return False 169 return True 170 171 def _builders_are_green(self): 172 red_builders_names = self.tool.buildbot.red_core_builders_names() 173 if red_builders_names: 174 red_builders_names = map(lambda name: "\"%s\"" % name, red_builders_names) # Add quotes around the names. 175 self._update_status("Builders [%s] are red. See http://build.webkit.org" % ", ".join(red_builders_names), None) 176 return False 177 return True 178 179 def should_proceed_with_work_item(self, patch): 180 if not self._builders_are_green(): 181 return False 182 if not self._can_build_and_test(): 183 return False 184 if not self._builders_are_green(): 185 return False 186 self._update_status("Landing patch", patch) 187 return True 188 189 def process_work_item(self, patch): 190 try: 191 self._cc_watchers(patch.bug_id()) 192 # We pass --no-update here because we've already validated 193 # that the current revision actually builds and passes the tests. 194 # If we update, we risk moving to a revision that doesn't! 195 self.run_webkit_patch(["land-attachment", "--force-clean", "--non-interactive", "--no-update", "--parent-command=commit-queue", "--build-style=both", "--quiet", patch.id()]) 196 self._did_pass(patch) 197 except ScriptError, e: 198 self._did_fail(patch) 199 raise e 200 201 def handle_unexpected_error(self, patch, message): 202 self.committer_validator.reject_patch_from_commit_queue(patch.id(), message) 203 204 # StepSequenceErrorHandler methods 205 206 @staticmethod 207 def _error_message_for_bug(tool, status_id, script_error): 208 if not script_error.output: 209 return script_error.message_with_output() 210 results_link = tool.status_server.results_url_for_status(status_id) 211 return "%s\nFull output: %s" % (script_error.message_with_output(), results_link) 212 213 @classmethod 214 def handle_script_error(cls, tool, state, script_error): 215 status_id = cls._update_status_for_script_error(tool, state, script_error) 216 validator = CommitterValidator(tool.bugs) 217 validator.reject_patch_from_commit_queue(state["patch"].id(), cls._error_message_for_bug(tool, status_id, script_error)) 218 219 220 class AbstractReviewQueue(AbstractQueue, PersistentPatchCollectionDelegate, StepSequenceErrorHandler): 221 def __init__(self, options=None): 222 AbstractQueue.__init__(self, options) 223 224 def _review_patch(self, patch): 225 raise NotImplementedError, "subclasses must implement" 226 227 # PersistentPatchCollectionDelegate methods 228 229 def collection_name(self): 230 return self.name 231 232 def fetch_potential_patch_ids(self): 233 return self.tool.bugs.queries.fetch_attachment_ids_from_review_queue() 234 235 def status_server(self): 236 return self.tool.status_server 237 238 def is_terminal_status(self, status): 239 return status == "Pass" or status == "Fail" or status.startswith("Error:") 240 241 # AbstractQueue methods 242 243 def begin_work_queue(self): 244 AbstractQueue.begin_work_queue(self) 245 self._patches = PersistentPatchCollection(self) 246 247 def next_work_item(self): 248 patch_id = self._patches.next() 249 if patch_id: 250 return self.tool.bugs.fetch_attachment(patch_id) 251 self._update_status("Empty queue") 252 253 def should_proceed_with_work_item(self, patch): 254 raise NotImplementedError, "subclasses must implement" 255 256 def process_work_item(self, patch): 257 try: 258 self._review_patch(patch) 259 self._did_pass(patch) 260 except ScriptError, e: 261 if e.exit_code != QueueEngine.handled_error_code: 262 self._did_fail(patch) 263 raise e 264 265 def handle_unexpected_error(self, patch, message): 266 log(message) 267 268 # StepSequenceErrorHandler methods 269 270 @classmethod 271 def handle_script_error(cls, tool, state, script_error): 272 log(script_error.message_with_output()) 273 274 275 class StyleQueue(AbstractReviewQueue): 276 name = "style-queue" 277 def __init__(self): 278 AbstractReviewQueue.__init__(self) 279 280 def should_proceed_with_work_item(self, patch): 281 self._update_status("Checking style", patch) 282 return True 283 284 def _review_patch(self, patch): 285 self.run_webkit_patch(["check-style", "--force-clean", "--non-interactive", "--parent-command=style-queue", patch.id()]) 286 287 @classmethod 288 def handle_script_error(cls, tool, state, script_error): 289 is_svn_apply = script_error.command_name() == "svn-apply" 290 status_id = cls._update_status_for_script_error(tool, state, script_error, is_error=is_svn_apply) 291 if is_svn_apply: 292 QueueEngine.exit_after_handled_error(script_error) 293 message = "Attachment %s did not pass %s:\n\n%s\n\nIf any of these errors are false positives, please file a bug against check-webkit-style." % (state["patch"].id(), cls.name, script_error.message_with_output(output_limit=3*1024)) 294 tool.bugs.post_comment_to_bug(state["patch"].bug_id(), message, cc=cls.watchers) 295 exit(1) 296
