1 # Copyright (c) 2009 Google Inc. All rights reserved. 2 # Copyright (c) 2009 Apple Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 31 from optparse import make_option 32 33 from webkitpy.tool import steps 34 35 from webkitpy.common.checkout.commitinfo import CommitInfo 36 from webkitpy.common.config.committers import CommitterList 37 from webkitpy.common.net.buildbot import BuildBot 38 from webkitpy.common.net.regressionwindow import RegressionWindow 39 from webkitpy.common.system.user import User 40 from webkitpy.tool.grammar import pluralize 41 from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand 42 from webkitpy.common.system.deprecated_logging import log 43 from webkitpy.layout_tests import port 44 45 46 class SuggestReviewers(AbstractDeclarativeCommand): 47 name = "suggest-reviewers" 48 help_text = "Suggest reviewers for a patch based on recent changes to the modified files." 49 50 def __init__(self): 51 options = [ 52 steps.Options.git_commit, 53 ] 54 AbstractDeclarativeCommand.__init__(self, options=options) 55 56 def execute(self, options, args, tool): 57 reviewers = tool.checkout().suggested_reviewers(options.git_commit) 58 print "\n".join([reviewer.full_name for reviewer in reviewers]) 59 60 61 class BugsToCommit(AbstractDeclarativeCommand): 62 name = "bugs-to-commit" 63 help_text = "List bugs in the commit-queue" 64 65 def execute(self, options, args, tool): 66 # FIXME: This command is poorly named. It's fetching the commit-queue list here. The name implies it's fetching pending-commit (all r+'d patches). 67 bug_ids = tool.bugs.queries.fetch_bug_ids_from_commit_queue() 68 for bug_id in bug_ids: 69 print "%s" % bug_id 70 71 72 class PatchesInCommitQueue(AbstractDeclarativeCommand): 73 name = "patches-in-commit-queue" 74 help_text = "List patches in the commit-queue" 75 76 def execute(self, options, args, tool): 77 patches = tool.bugs.queries.fetch_patches_from_commit_queue() 78 log("Patches in commit queue:") 79 for patch in patches: 80 print patch.url() 81 82 83 class PatchesToCommitQueue(AbstractDeclarativeCommand): 84 name = "patches-to-commit-queue" 85 help_text = "Patches which should be added to the commit queue" 86 def __init__(self): 87 options = [ 88 make_option("--bugs", action="store_true", dest="bugs", help="Output bug links instead of patch links"), 89 ] 90 AbstractDeclarativeCommand.__init__(self, options=options) 91 92 @staticmethod 93 def _needs_commit_queue(patch): 94 if patch.commit_queue() == "+": # If it's already cq+, ignore the patch. 95 log("%s already has cq=%s" % (patch.id(), patch.commit_queue())) 96 return False 97 98 # We only need to worry about patches from contributers who are not yet committers. 99 committer_record = CommitterList().committer_by_email(patch.attacher_email()) 100 if committer_record: 101 log("%s committer = %s" % (patch.id(), committer_record)) 102 return not committer_record 103 104 def execute(self, options, args, tool): 105 patches = tool.bugs.queries.fetch_patches_from_pending_commit_list() 106 patches_needing_cq = filter(self._needs_commit_queue, patches) 107 if options.bugs: 108 bugs_needing_cq = map(lambda patch: patch.bug_id(), patches_needing_cq) 109 bugs_needing_cq = sorted(set(bugs_needing_cq)) 110 for bug_id in bugs_needing_cq: 111 print "%s" % tool.bugs.bug_url_for_bug_id(bug_id) 112 else: 113 for patch in patches_needing_cq: 114 print "%s" % tool.bugs.attachment_url_for_id(patch.id(), action="edit") 115 116 117 class PatchesToReview(AbstractDeclarativeCommand): 118 name = "patches-to-review" 119 help_text = "List patches that are pending review" 120 121 def execute(self, options, args, tool): 122 patch_ids = tool.bugs.queries.fetch_attachment_ids_from_review_queue() 123 log("Patches pending review:") 124 for patch_id in patch_ids: 125 print patch_id 126 127 128 class LastGreenRevision(AbstractDeclarativeCommand): 129 name = "last-green-revision" 130 help_text = "Prints the last known good revision" 131 132 def execute(self, options, args, tool): 133 print self._tool.buildbot.last_green_revision() 134 135 136 class WhatBroke(AbstractDeclarativeCommand): 137 name = "what-broke" 138 help_text = "Print failing buildbots (%s) and what revisions broke them" % BuildBot.default_host 139 140 def _print_builder_line(self, builder_name, max_name_width, status_message): 141 print "%s : %s" % (builder_name.ljust(max_name_width), status_message) 142 143 def _print_blame_information_for_builder(self, builder_status, name_width, avoid_flakey_tests=True): 144 builder = self._tool.buildbot.builder_with_name(builder_status["name"]) 145 red_build = builder.build(builder_status["build_number"]) 146 regression_window = builder.find_regression_window(red_build) 147 if not regression_window.failing_build(): 148 self._print_builder_line(builder.name(), name_width, "FAIL (error loading build information)") 149 return 150 if not regression_window.build_before_failure(): 151 self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: sometime before %s?)" % regression_window.failing_build().revision()) 152 return 153 154 revisions = regression_window.revisions() 155 first_failure_message = "" 156 if (regression_window.failing_build() == builder.build(builder_status["build_number"])): 157 first_failure_message = " FIRST FAILURE, possibly a flaky test" 158 self._print_builder_line(builder.name(), name_width, "FAIL (blame-list: %s%s)" % (revisions, first_failure_message)) 159 for revision in revisions: 160 commit_info = self._tool.checkout().commit_info_for_revision(revision) 161 if commit_info: 162 print commit_info.blame_string(self._tool.bugs) 163 else: 164 print "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision 165 166 def execute(self, options, args, tool): 167 builder_statuses = tool.buildbot.builder_statuses() 168 longest_builder_name = max(map(len, map(lambda builder: builder["name"], builder_statuses))) 169 failing_builders = 0 170 for builder_status in builder_statuses: 171 # If the builder is green, print OK, exit. 172 if builder_status["is_green"]: 173 continue 174 self._print_blame_information_for_builder(builder_status, name_width=longest_builder_name) 175 failing_builders += 1 176 if failing_builders: 177 print "%s of %s are failing" % (failing_builders, pluralize("builder", len(builder_statuses))) 178 else: 179 print "All builders are passing!" 180 181 182 class ResultsFor(AbstractDeclarativeCommand): 183 name = "results-for" 184 help_text = "Print a list of failures for the passed revision from bots on %s" % BuildBot.default_host 185 argument_names = "REVISION" 186 187 def _print_layout_test_results(self, results): 188 if not results: 189 print " No results." 190 return 191 for title, files in results.parsed_results().items(): 192 print " %s" % title 193 for filename in files: 194 print " %s" % filename 195 196 def execute(self, options, args, tool): 197 builders = self._tool.buildbot.builders() 198 for builder in builders: 199 print "%s:" % builder.name() 200 build = builder.build_for_revision(args[0], allow_failed_lookups=True) 201 self._print_layout_test_results(build.layout_test_results()) 202 203 204 class FailureReason(AbstractDeclarativeCommand): 205 name = "failure-reason" 206 help_text = "Lists revisions where individual test failures started at %s" % BuildBot.default_host 207 208 def _blame_line_for_revision(self, revision): 209 try: 210 commit_info = self._tool.checkout().commit_info_for_revision(revision) 211 except Exception, e: 212 return "FAILED to fetch CommitInfo for r%s, exception: %s" % (revision, e) 213 if not commit_info: 214 return "FAILED to fetch CommitInfo for r%s, likely missing ChangeLog" % revision 215 return commit_info.blame_string(self._tool.bugs) 216 217 def _print_blame_information_for_transition(self, regression_window, failing_tests): 218 red_build = regression_window.failing_build() 219 print "SUCCESS: Build %s (r%s) was the first to show failures: %s" % (red_build._number, red_build.revision(), failing_tests) 220 print "Suspect revisions:" 221 for revision in regression_window.revisions(): 222 print self._blame_line_for_revision(revision) 223 224 def _explain_failures_for_builder(self, builder, start_revision): 225 print "Examining failures for \"%s\", starting at r%s" % (builder.name(), start_revision) 226 revision_to_test = start_revision 227 build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) 228 layout_test_results = build.layout_test_results() 229 if not layout_test_results: 230 # FIXME: This could be made more user friendly. 231 print "Failed to load layout test results; can't continue. (start revision = r%s)" % start_revision 232 return 1 233 234 results_to_explain = set(layout_test_results.failing_tests()) 235 last_build_with_results = build 236 print "Starting at %s" % revision_to_test 237 while results_to_explain: 238 revision_to_test -= 1 239 new_build = builder.build_for_revision(revision_to_test, allow_failed_lookups=True) 240 if not new_build: 241 print "No build for %s" % revision_to_test 242 continue 243 build = new_build 244 latest_results = build.layout_test_results() 245 if not latest_results: 246 print "No results build %s (r%s)" % (build._number, build.revision()) 247 continue 248 failures = set(latest_results.failing_tests()) 249 if len(failures) >= 20: 250 # FIXME: We may need to move this logic into the LayoutTestResults class. 251 # The buildbot stops runs after 20 failures so we don't have full results to work with here. 252 print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision()) 253 continue 254 fixed_results = results_to_explain - failures 255 if not fixed_results: 256 print "No change in build %s (r%s), %s unexplained failures (%s in this build)" % (build._number, build.revision(), len(results_to_explain), len(failures)) 257 last_build_with_results = build 258 continue 259 regression_window = RegressionWindow(build, last_build_with_results) 260 self._print_blame_information_for_transition(regression_window, fixed_results) 261 last_build_with_results = build 262 results_to_explain -= fixed_results 263 if results_to_explain: 264 print "Failed to explain failures: %s" % results_to_explain 265 return 1 266 print "Explained all results for %s" % builder.name() 267 return 0 268 269 def _builder_to_explain(self): 270 builder_statuses = self._tool.buildbot.builder_statuses() 271 red_statuses = [status for status in builder_statuses if not status["is_green"]] 272 print "%s failing" % (pluralize("builder", len(red_statuses))) 273 builder_choices = [status["name"] for status in red_statuses] 274 # We could offer an "All" choice here. 275 chosen_name = self._tool.user.prompt_with_list("Which builder to diagnose:", builder_choices) 276 # FIXME: prompt_with_list should really take a set of objects and a set of names and then return the object. 277 for status in red_statuses: 278 if status["name"] == chosen_name: 279 return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"]) 280 281 def execute(self, options, args, tool): 282 (builder, latest_revision) = self._builder_to_explain() 283 start_revision = self._tool.user.prompt("Revision to walk backwards from? [%s] " % latest_revision) or latest_revision 284 if not start_revision: 285 print "Revision required." 286 return 1 287 return self._explain_failures_for_builder(builder, start_revision=int(start_revision)) 288 289 290 class FindFlakyTests(AbstractDeclarativeCommand): 291 name = "find-flaky-tests" 292 help_text = "Lists tests that often fail for a single build at %s" % BuildBot.default_host 293 294 def _find_failures(self, builder, revision): 295 build = builder.build_for_revision(revision, allow_failed_lookups=True) 296 if not build: 297 print "No build for %s" % revision 298 return (None, None) 299 results = build.layout_test_results() 300 if not results: 301 print "No results build %s (r%s)" % (build._number, build.revision()) 302 return (None, None) 303 failures = set(results.failing_tests()) 304 if len(failures) >= 20: 305 # FIXME: We may need to move this logic into the LayoutTestResults class. 306 # The buildbot stops runs after 20 failures so we don't have full results to work with here. 307 print "Too many failures in build %s (r%s), ignoring." % (build._number, build.revision()) 308 return (None, None) 309 return (build, failures) 310 311 def _increment_statistics(self, flaky_tests, flaky_test_statistics): 312 for test in flaky_tests: 313 count = flaky_test_statistics.get(test, 0) 314 flaky_test_statistics[test] = count + 1 315 316 def _print_statistics(self, statistics): 317 print "=== Results ===" 318 print "Occurances Test name" 319 for value, key in sorted([(value, key) for key, value in statistics.items()]): 320 print "%10d %s" % (value, key) 321 322 def _walk_backwards_from(self, builder, start_revision, limit): 323 flaky_test_statistics = {} 324 all_previous_failures = set([]) 325 one_time_previous_failures = set([]) 326 previous_build = None 327 for i in range(limit): 328 revision = start_revision - i 329 print "Analyzing %s ... " % revision, 330 (build, failures) = self._find_failures(builder, revision) 331 if failures == None: 332 # Notice that we don't loop on the empty set! 333 continue 334 print "has %s failures" % len(failures) 335 flaky_tests = one_time_previous_failures - failures 336 if flaky_tests: 337 print "Flaky tests: %s %s" % (sorted(flaky_tests), 338 previous_build.results_url()) 339 self._increment_statistics(flaky_tests, flaky_test_statistics) 340 one_time_previous_failures = failures - all_previous_failures 341 all_previous_failures = failures 342 previous_build = build 343 self._print_statistics(flaky_test_statistics) 344 345 def _builder_to_analyze(self): 346 statuses = self._tool.buildbot.builder_statuses() 347 choices = [status["name"] for status in statuses] 348 chosen_name = self._tool.user.prompt_with_list("Which builder to analyze:", choices) 349 for status in statuses: 350 if status["name"] == chosen_name: 351 return (self._tool.buildbot.builder_with_name(chosen_name), status["built_revision"]) 352 353 def execute(self, options, args, tool): 354 (builder, latest_revision) = self._builder_to_analyze() 355 limit = self._tool.user.prompt("How many revisions to look through? [10000] ") or 10000 356 return self._walk_backwards_from(builder, latest_revision, limit=int(limit)) 357 358 359 class TreeStatus(AbstractDeclarativeCommand): 360 name = "tree-status" 361 help_text = "Print the status of the %s buildbots" % BuildBot.default_host 362 long_help = """Fetches build status from http://build.webkit.org/one_box_per_builder 363 and displayes the status of each builder.""" 364 365 def execute(self, options, args, tool): 366 for builder in tool.buildbot.builder_statuses(): 367 status_string = "ok" if builder["is_green"] else "FAIL" 368 print "%s : %s" % (status_string.ljust(4), builder["name"]) 369 370 371 class SkippedPorts(AbstractDeclarativeCommand): 372 name = "skipped-ports" 373 help_text = "Print the list of ports skipping the given layout test(s)" 374 long_help = """Scans the the Skipped file of each port and figure 375 out what ports are skipping the test(s). Categories are taken in account too.""" 376 argument_names = "TEST_NAME" 377 378 def execute(self, options, args, tool): 379 results = dict([(test_name, []) for test_name in args]) 380 for port_name, port_object in tool.port_factory.get_all().iteritems(): 381 for test_name in args: 382 if port_object.skips_layout_test(test_name): 383 results[test_name].append(port_name) 384 385 for test_name, ports in results.iteritems(): 386 if ports: 387 print "Ports skipping test %r: %s" % (test_name, ', '.join(ports)) 388 else: 389 print "Test %r is not skipped by any port." % test_name 390
