1 # Copyright (c) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import unittest 30 31 from webkitpy.common.config.committers import Committer 32 from webkitpy.common.system.filesystem_mock import MockFileSystem 33 from webkitpy.common.system.outputcapture import OutputCapture 34 from webkitpy.layout_tests.layout_package import test_results 35 from webkitpy.layout_tests.layout_package import test_failures 36 from webkitpy.thirdparty.mock import Mock 37 from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter 38 from webkitpy.tool.mocktool import MockTool, MockStatusServer 39 40 41 # Creating fake CommitInfos is a pain, so we use a mock one here. 42 class MockCommitInfo(object): 43 def __init__(self, author_email): 44 self._author_email = author_email 45 46 def author(self): 47 # It's definitely possible to have commits with authors who 48 # are not in our committers.py list. 49 if not self._author_email: 50 return None 51 return Committer("Mock Committer", self._author_email) 52 53 54 class FlakyTestReporterTest(unittest.TestCase): 55 def _mock_test_result(self, testname): 56 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) 57 58 def _assert_emails_for_test(self, emails): 59 tool = MockTool() 60 reporter = FlakyTestReporter(tool, 'dummy-queue') 61 commit_infos = [MockCommitInfo(email) for email in emails] 62 tool.checkout().recent_commit_infos_for_files = lambda paths: set(commit_infos) 63 self.assertEqual(reporter._author_emails_for_test([]), set(emails)) 64 65 def test_author_emails_for_test(self): 66 self._assert_emails_for_test([]) 67 self._assert_emails_for_test(["test1 (at] test.com", "test1 (at] test.com"]) 68 self._assert_emails_for_test(["test1 (at] test.com", "test2 (at] test.com"]) 69 70 def test_create_bug_for_flaky_test(self): 71 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') 72 expected_stderr = """MOCK create_bug 73 bug_title: Flaky Test: foo/bar.html 74 bug_description: This is an automatically generated bug from the dummy-queue. 75 foo/bar.html has been flaky on the dummy-queue. 76 77 foo/bar.html was authored by test (at] test.com. 78 http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html 79 80 FLAKE_MESSAGE 81 82 The bots will update this with information from each new failure. 83 84 If you believe this bug to be fixed or invalid, feel free to close. The bots will re-open if the flake re-occurs. 85 86 If you would like to track this test fix with another bug, please close this bug as a duplicate. The bots will follow the duplicate chain when making future comments. 87 88 component: Tools / Tests 89 cc: test (at] test.com 90 blocked: 50856 91 """ 92 OutputCapture().assert_outputs(self, reporter._create_bug_for_flaky_test, ['foo/bar.html', ['test (at] test.com'], 'FLAKE_MESSAGE'], expected_stderr=expected_stderr) 93 94 def test_follow_duplicate_chain(self): 95 tool = MockTool() 96 reporter = FlakyTestReporter(tool, 'dummy-queue') 97 bug = tool.bugs.fetch_bug(78) 98 self.assertEqual(reporter._follow_duplicate_chain(bug).id(), 76) 99 100 def test_report_flaky_tests_creating_bug(self): 101 tool = MockTool() 102 tool.filesystem = MockFileSystem({"/mock/foo/bar-diffs.txt": "mock"}) 103 tool.status_server = MockStatusServer(bot_id="mock-bot-id") 104 reporter = FlakyTestReporter(tool, 'dummy-queue') 105 reporter._lookup_bug_for_flaky_test = lambda bug_id: None 106 patch = tool.bugs.fetch_attachment(197) 107 expected_stderr = """MOCK create_bug 108 bug_title: Flaky Test: foo/bar.html 109 bug_description: This is an automatically generated bug from the dummy-queue. 110 foo/bar.html has been flaky on the dummy-queue. 111 112 foo/bar.html was authored by abarth (at] webkit.org. 113 http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html 114 115 The dummy-queue just saw foo/bar.html flake (Text diff mismatch) while processing attachment 197 on bug 42. 116 Bot: mock-bot-id Port: MockPort Platform: MockPlatform 1.0 117 118 The bots will update this with information from each new failure. 119 120 If you believe this bug to be fixed or invalid, feel free to close. The bots will re-open if the flake re-occurs. 121 122 If you would like to track this test fix with another bug, please close this bug as a duplicate. The bots will follow the duplicate chain when making future comments. 123 124 component: Tools / Tests 125 cc: abarth (at] webkit.org 126 blocked: 50856 127 MOCK add_attachment_to_bug: bug_id=78, description=Failure diff from mock-bot-id filename=failure.diff 128 MOCK bug comment: bug_id=42, cc=None 129 --- Begin comment --- 130 The dummy-queue encountered the following flaky tests while processing attachment 197: 131 132 foo/bar.html bug 78 (author: abarth (at] webkit.org) 133 The dummy-queue is continuing to process your patch. 134 --- End comment --- 135 136 """ 137 test_results = [self._mock_test_result('foo/bar.html')] 138 139 class MockZipFile(object): 140 def read(self, path): 141 return "" 142 143 def namelist(self): 144 return ['foo/bar-diffs.txt'] 145 146 OutputCapture().assert_outputs(self, reporter.report_flaky_tests, [patch, test_results, MockZipFile()], expected_stderr=expected_stderr) 147 148 def test_optional_author_string(self): 149 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') 150 self.assertEqual(reporter._optional_author_string([]), "") 151 self.assertEqual(reporter._optional_author_string(["foo (at] bar.com"]), " (author: foo (at] bar.com)") 152 self.assertEqual(reporter._optional_author_string(["a (at] b.com", "b (at] b.com"]), " (authors: a (at] b.com and b (at] b.com)") 153 154 def test_results_diff_path_for_test(self): 155 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') 156 self.assertEqual(reporter._results_diff_path_for_test("test.html"), "test-diffs.txt") 157 158 def test_find_in_archive(self): 159 reporter = FlakyTestReporter(MockTool(), 'dummy-queue') 160 161 class MockZipFile(object): 162 def namelist(self): 163 return ["tmp/layout-test-results/foo/bar-diffs.txt"] 164 165 reporter._find_in_archive("foo/bar-diffs.txt", MockZipFile()) 166 # This is not ideal, but its 167 reporter._find_in_archive("txt", MockZipFile()) 168
