1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import codecs 30 import logging 31 import platform 32 import os.path 33 34 from webkitpy.common.net.layouttestresults import path_for_layout_test, LayoutTestResults 35 from webkitpy.common.config import urls 36 from webkitpy.tool.bot.botinfo import BotInfo 37 from webkitpy.tool.grammar import plural, pluralize, join_with_separators 38 39 _log = logging.getLogger(__name__) 40 41 42 class FlakyTestReporter(object): 43 def __init__(self, tool, bot_name): 44 self._tool = tool 45 self._bot_name = bot_name 46 self._bot_info = BotInfo(tool) 47 48 def _author_emails_for_test(self, flaky_test): 49 test_path = path_for_layout_test(flaky_test) 50 commit_infos = self._tool.checkout().recent_commit_infos_for_files([test_path]) 51 # This ignores authors which are not committers because we don't have their bugzilla_email. 52 return set([commit_info.author().bugzilla_email() for commit_info in commit_infos if commit_info.author()]) 53 54 def _bugzilla_email(self): 55 # FIXME: This is kinda a funny way to get the bugzilla email, 56 # we could also just create a Credentials object directly 57 # but some of the Credentials logic is in bugzilla.py too... 58 self._tool.bugs.authenticate() 59 return self._tool.bugs.username 60 61 # FIXME: This should move into common.config 62 _bot_emails = set([ 63 "commit-queue (at] webkit.org", # commit-queue 64 "eseidel (at] chromium.org", # old commit-queue 65 "webkit.review.bot (at] gmail.com", # style-queue, sheriff-bot, CrLx/Gtk EWS 66 "buildbot (at] hotmail.com", # Win EWS 67 # Mac EWS currently uses eric (at] webkit.org, but that's not normally a bot 68 ]) 69 70 def _lookup_bug_for_flaky_test(self, flaky_test): 71 bugs = self._tool.bugs.queries.fetch_bugs_matching_search(search_string=flaky_test) 72 if not bugs: 73 return None 74 # Match any bugs which are from known bots or the email this bot is using. 75 allowed_emails = self._bot_emails | set([self._bugzilla_email]) 76 bugs = filter(lambda bug: bug.reporter_email() in allowed_emails, bugs) 77 if not bugs: 78 return None 79 if len(bugs) > 1: 80 # FIXME: There are probably heuristics we could use for finding 81 # the right bug instead of the first, like open vs. closed. 82 _log.warn("Found %s %s matching '%s' filed by a bot, using the first." % (pluralize('bug', len(bugs)), [bug.id() for bug in bugs], flaky_test)) 83 return bugs[0] 84 85 def _view_source_url_for_test(self, test_path): 86 return urls.view_source_url("LayoutTests/%s" % test_path) 87 88 def _create_bug_for_flaky_test(self, flaky_test, author_emails, latest_flake_message): 89 format_values = { 90 'test': flaky_test, 91 'authors': join_with_separators(sorted(author_emails)), 92 'flake_message': latest_flake_message, 93 'test_url': self._view_source_url_for_test(flaky_test), 94 'bot_name': self._bot_name, 95 } 96 title = "Flaky Test: %(test)s" % format_values 97 description = """This is an automatically generated bug from the %(bot_name)s. 98 %(test)s has been flaky on the %(bot_name)s. 99 100 %(test)s was authored by %(authors)s. 101 %(test_url)s 102 103 %(flake_message)s 104 105 The bots will update this with information from each new failure. 106 107 If you believe this bug to be fixed or invalid, feel free to close. The bots will re-open if the flake re-occurs. 108 109 If you would like to track this test fix with another bug, please close this bug as a duplicate. The bots will follow the duplicate chain when making future comments. 110 """ % format_values 111 112 master_flake_bug = 50856 # MASTER: Flaky tests found by the commit-queue 113 return self._tool.bugs.create_bug(title, description, 114 component="Tools / Tests", 115 cc=",".join(author_emails), 116 blocked="50856") 117 118 # This is over-engineered, but it makes for pretty bug messages. 119 def _optional_author_string(self, author_emails): 120 if not author_emails: 121 return "" 122 heading_string = plural('author') if len(author_emails) > 1 else 'author' 123 authors_string = join_with_separators(sorted(author_emails)) 124 return " (%s: %s)" % (heading_string, authors_string) 125 126 def _latest_flake_message(self, flaky_result, patch): 127 failure_messages = [failure.message() for failure in flaky_result.failures] 128 flake_message = "The %s just saw %s flake (%s) while processing attachment %s on bug %s." % (self._bot_name, flaky_result.filename, ", ".join(failure_messages), patch.id(), patch.bug_id()) 129 return "%s\n%s" % (flake_message, self._bot_info.summary_text()) 130 131 def _results_diff_path_for_test(self, test_path): 132 # FIXME: This is a big hack. We should get this path from results.json 133 # except that old-run-webkit-tests doesn't produce a results.json 134 # so we just guess at the file path. 135 (test_path_root, _) = os.path.splitext(test_path) 136 return "%s-diffs.txt" % test_path_root 137 138 def _follow_duplicate_chain(self, bug): 139 while bug.is_closed() and bug.duplicate_of(): 140 bug = self._tool.bugs.fetch_bug(bug.duplicate_of()) 141 return bug 142 143 # Maybe this logic should move into Bugzilla? a reopen=True arg to post_comment? 144 def _update_bug_for_flaky_test(self, bug, latest_flake_message): 145 if bug.is_closed(): 146 self._tool.bugs.reopen_bug(bug.id(), latest_flake_message) 147 else: 148 self._tool.bugs.post_comment_to_bug(bug.id(), latest_flake_message) 149 150 # This method is needed because our archive paths include a leading tmp/layout-test-results 151 def _find_in_archive(self, path, archive): 152 for archived_path in archive.namelist(): 153 # Archives are currently created with full paths. 154 if archived_path.endswith(path): 155 return archived_path 156 return None 157 158 def _attach_failure_diff(self, flake_bug_id, flaky_test, results_archive_zip): 159 results_diff_path = self._results_diff_path_for_test(flaky_test) 160 # Check to make sure that the path makes sense. 161 # Since we're not actually getting this path from the results.html 162 # there is a chance it's wrong. 163 bot_id = self._tool.status_server.bot_id or "bot" 164 archive_path = self._find_in_archive(results_diff_path, results_archive_zip) 165 if archive_path: 166 results_diff = results_archive_zip.read(archive_path) 167 description = "Failure diff from %s" % bot_id 168 self._tool.bugs.add_attachment_to_bug(flake_bug_id, results_diff, description, filename="failure.diff") 169 else: 170 _log.warn("%s does not exist in results archive, uploading entire archive." % results_diff_path) 171 description = "Archive of layout-test-results from %s" % bot_id 172 # results_archive is a ZipFile object, grab the File object (.fp) to pass to Mechanize for uploading. 173 results_archive_file = results_archive_zip.fp 174 # Rewind the file object to start (since Mechanize won't do that automatically) 175 # See https://bugs.webkit.org/show_bug.cgi?id=54593 176 results_archive_file.seek(0) 177 self._tool.bugs.add_attachment_to_bug(flake_bug_id, results_archive_file, description, filename="layout-test-results.zip") 178 179 def report_flaky_tests(self, patch, flaky_test_results, results_archive): 180 message = "The %s encountered the following flaky tests while processing attachment %s:\n\n" % (self._bot_name, patch.id()) 181 for flaky_result in flaky_test_results: 182 flaky_test = flaky_result.filename 183 bug = self._lookup_bug_for_flaky_test(flaky_test) 184 latest_flake_message = self._latest_flake_message(flaky_result, patch) 185 author_emails = self._author_emails_for_test(flaky_test) 186 if not bug: 187 _log.info("Bug does not already exist for %s, creating." % flaky_test) 188 flake_bug_id = self._create_bug_for_flaky_test(flaky_test, author_emails, latest_flake_message) 189 else: 190 bug = self._follow_duplicate_chain(bug) 191 # FIXME: Ideally we'd only make one comment per flake, not two. But that's not possible 192 # in all cases (e.g. when reopening), so for now file attachment and comment are separate. 193 self._update_bug_for_flaky_test(bug, latest_flake_message) 194 flake_bug_id = bug.id() 195 196 self._attach_failure_diff(flake_bug_id, flaky_test, results_archive) 197 message += "%s bug %s%s\n" % (flaky_test, flake_bug_id, self._optional_author_string(author_emails)) 198 199 message += "The %s is continuing to process your patch." % self._bot_name 200 self._tool.bugs.post_comment_to_bug(patch.bug_id(), message) 201
