1 # Copyright (C) 2009 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import unittest 30 31 from webkitpy.common.net.layouttestresults import LayoutTestResults 32 from webkitpy.common.net.buildbot import BuildBot, Builder, Build 33 from webkitpy.layout_tests.layout_package import test_results 34 from webkitpy.layout_tests.layout_package import test_failures 35 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup 36 37 38 class BuilderTest(unittest.TestCase): 39 def _mock_test_result(self, testname): 40 return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) 41 42 def _install_fetch_build(self, failure): 43 def _mock_fetch_build(build_number): 44 build = Build( 45 builder=self.builder, 46 build_number=build_number, 47 revision=build_number + 1000, 48 is_green=build_number < 4 49 ) 50 results = [self._mock_test_result(testname) for testname in failure(build_number)] 51 build._layout_test_results = LayoutTestResults(results) 52 return build 53 self.builder._fetch_build = _mock_fetch_build 54 55 def setUp(self): 56 self.buildbot = BuildBot() 57 self.builder = Builder(u"Test Builder \u2661", self.buildbot) 58 self._install_fetch_build(lambda build_number: ["test1", "test2"]) 59 60 def test_find_regression_window(self): 61 regression_window = self.builder.find_regression_window(self.builder.build(10)) 62 self.assertEqual(regression_window.build_before_failure().revision(), 1003) 63 self.assertEqual(regression_window.failing_build().revision(), 1004) 64 65 regression_window = self.builder.find_regression_window(self.builder.build(10), look_back_limit=2) 66 self.assertEqual(regression_window.build_before_failure(), None) 67 self.assertEqual(regression_window.failing_build().revision(), 1008) 68 69 def test_none_build(self): 70 self.builder._fetch_build = lambda build_number: None 71 regression_window = self.builder.find_regression_window(self.builder.build(10)) 72 self.assertEqual(regression_window.build_before_failure(), None) 73 self.assertEqual(regression_window.failing_build(), None) 74 75 def test_flaky_tests(self): 76 self._install_fetch_build(lambda build_number: ["test1"] if build_number % 2 else ["test2"]) 77 regression_window = self.builder.find_regression_window(self.builder.build(10)) 78 self.assertEqual(regression_window.build_before_failure().revision(), 1009) 79 self.assertEqual(regression_window.failing_build().revision(), 1010) 80 81 def test_failure_and_flaky(self): 82 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) 83 regression_window = self.builder.find_regression_window(self.builder.build(10)) 84 self.assertEqual(regression_window.build_before_failure().revision(), 1003) 85 self.assertEqual(regression_window.failing_build().revision(), 1004) 86 87 def test_no_results(self): 88 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number % 2 else ["test2"]) 89 regression_window = self.builder.find_regression_window(self.builder.build(10)) 90 self.assertEqual(regression_window.build_before_failure().revision(), 1003) 91 self.assertEqual(regression_window.failing_build().revision(), 1004) 92 93 def test_failure_after_flaky(self): 94 self._install_fetch_build(lambda build_number: ["test1", "test2"] if build_number > 6 else ["test3"]) 95 regression_window = self.builder.find_regression_window(self.builder.build(10)) 96 self.assertEqual(regression_window.build_before_failure().revision(), 1006) 97 self.assertEqual(regression_window.failing_build().revision(), 1007) 98 99 def test_find_blameworthy_regression_window(self): 100 self.assertEqual(self.builder.find_blameworthy_regression_window(10).revisions(), [1004]) 101 self.assertEqual(self.builder.find_blameworthy_regression_window(10, look_back_limit=2), None) 102 # Flakey test avoidance requires at least 2 red builds: 103 self.assertEqual(self.builder.find_blameworthy_regression_window(4), None) 104 self.assertEqual(self.builder.find_blameworthy_regression_window(4, avoid_flakey_tests=False).revisions(), [1004]) 105 # Green builder: 106 self.assertEqual(self.builder.find_blameworthy_regression_window(3), None) 107 108 def test_build_caching(self): 109 self.assertEqual(self.builder.build(10), self.builder.build(10)) 110 111 def test_build_and_revision_for_filename(self): 112 expectations = { 113 "r47483 (1)/" : (47483, 1), 114 "r47483 (1).zip" : (47483, 1), 115 } 116 for filename, revision_and_build in expectations.items(): 117 self.assertEqual(self.builder._revision_and_build_for_filename(filename), revision_and_build) 118 119 120 class BuildTest(unittest.TestCase): 121 def test_layout_test_results(self): 122 build = Build(None, None, None, None) 123 build._fetch_results_html = lambda: None 124 # Test that layout_test_results() returns None if the fetch fails. 125 self.assertEqual(build.layout_test_results(), None) 126 127 128 class BuildBotTest(unittest.TestCase): 129 130 _example_one_box_status = ''' 131 <table> 132 <tr> 133 <td class="box"><a href="builders/Windows%20Debug%20%28Tests%29">Windows Debug (Tests)</a></td> 134 <td align="center" class="LastBuild box success"><a href="builders/Windows%20Debug%20%28Tests%29/builds/3693">47380</a><br />build<br />successful</td> 135 <td align="center" class="Activity building">building<br />ETA in<br />~ 14 mins<br />at 13:40</td> 136 <tr> 137 <td class="box"><a href="builders/SnowLeopard%20Intel%20Release">SnowLeopard Intel Release</a></td> 138 <td class="LastBuild box" >no build</td> 139 <td align="center" class="Activity building">building<br />< 1 min</td> 140 <tr> 141 <td class="box"><a href="builders/Qt%20Linux%20Release">Qt Linux Release</a></td> 142 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Linux%20Release/builds/654">47383</a><br />failed<br />compile-webkit</td> 143 <td align="center" class="Activity idle">idle<br />3 pending</td> 144 <tr> 145 <td class="box"><a href="builders/Qt%20Windows%2032-bit%20Debug">Qt Windows 32-bit Debug</a></td> 146 <td align="center" class="LastBuild box failure"><a href="builders/Qt%20Windows%2032-bit%20Debug/builds/2090">60563</a><br />failed<br />failed<br />slave<br />lost</td> 147 <td align="center" class="Activity building">building<br />ETA in<br />~ 5 mins<br />at 08:25</td> 148 </table> 149 ''' 150 _expected_example_one_box_parsings = [ 151 { 152 'is_green': True, 153 'build_number' : 3693, 154 'name': u'Windows Debug (Tests)', 155 'built_revision': 47380, 156 'activity': 'building', 157 'pending_builds': 0, 158 }, 159 { 160 'is_green': False, 161 'build_number' : None, 162 'name': u'SnowLeopard Intel Release', 163 'built_revision': None, 164 'activity': 'building', 165 'pending_builds': 0, 166 }, 167 { 168 'is_green': False, 169 'build_number' : 654, 170 'name': u'Qt Linux Release', 171 'built_revision': 47383, 172 'activity': 'idle', 173 'pending_builds': 3, 174 }, 175 { 176 'is_green': True, 177 'build_number' : 2090, 178 'name': u'Qt Windows 32-bit Debug', 179 'built_revision': 60563, 180 'activity': 'building', 181 'pending_builds': 0, 182 }, 183 ] 184 185 def test_status_parsing(self): 186 buildbot = BuildBot() 187 188 soup = BeautifulSoup(self._example_one_box_status) 189 status_table = soup.find("table") 190 input_rows = status_table.findAll('tr') 191 192 for x in range(len(input_rows)): 193 status_row = input_rows[x] 194 expected_parsing = self._expected_example_one_box_parsings[x] 195 196 builder = buildbot._parse_builder_status_from_row(status_row) 197 198 # Make sure we aren't parsing more or less than we expect 199 self.assertEquals(builder.keys(), expected_parsing.keys()) 200 201 for key, expected_value in expected_parsing.items(): 202 self.assertEquals(builder[key], expected_value, ("Builder %d parse failure for key: %s: Actual='%s' Expected='%s'" % (x, key, builder[key], expected_value))) 203 204 def test_core_builder_methods(self): 205 buildbot = BuildBot() 206 207 # Override builder_statuses function to not touch the network. 208 def example_builder_statuses(): # We could use instancemethod() to bind 'self' but we don't need to. 209 return BuildBotTest._expected_example_one_box_parsings 210 buildbot.builder_statuses = example_builder_statuses 211 212 buildbot.core_builder_names_regexps = [ 'Leopard', "Windows.*Build" ] 213 self.assertEquals(buildbot.red_core_builders_names(), []) 214 self.assertTrue(buildbot.core_builders_are_green()) 215 216 buildbot.core_builder_names_regexps = [ 'SnowLeopard', 'Qt' ] 217 self.assertEquals(buildbot.red_core_builders_names(), [ u'SnowLeopard Intel Release', u'Qt Linux Release' ]) 218 self.assertFalse(buildbot.core_builders_are_green()) 219 220 def test_builder_name_regexps(self): 221 buildbot = BuildBot() 222 223 # For complete testing, this list should match the list of builders at build.webkit.org: 224 example_builders = [ 225 {'name': u'Leopard Intel Release (Build)', }, 226 {'name': u'Leopard Intel Release (Tests)', }, 227 {'name': u'Leopard Intel Debug (Build)', }, 228 {'name': u'Leopard Intel Debug (Tests)', }, 229 {'name': u'SnowLeopard Intel Release (Build)', }, 230 {'name': u'SnowLeopard Intel Release (Tests)', }, 231 {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', }, 232 {'name': u'SnowLeopard Intel Leaks', }, 233 {'name': u'Windows Release (Build)', }, 234 {'name': u'Windows 7 Release (Tests)', }, 235 {'name': u'Windows Debug (Build)', }, 236 {'name': u'Windows XP Debug (Tests)', }, 237 {'name': u'Windows 7 Release (WebKit2 Tests)', }, 238 {'name': u'GTK Linux 32-bit Release', }, 239 {'name': u'GTK Linux 32-bit Debug', }, 240 {'name': u'GTK Linux 64-bit Debug', }, 241 {'name': u'Qt Linux Release', }, 242 {'name': u'Qt Linux Release minimal', }, 243 {'name': u'Qt Linux ARMv7 Release', }, 244 {'name': u'Qt Windows 32-bit Release', }, 245 {'name': u'Qt Windows 32-bit Debug', }, 246 {'name': u'Chromium Win Release', }, 247 {'name': u'Chromium Mac Release', }, 248 {'name': u'Chromium Linux Release', }, 249 {'name': u'Chromium Win Release (Tests)', }, 250 {'name': u'Chromium Mac Release (Tests)', }, 251 {'name': u'Chromium Linux Release (Tests)', }, 252 {'name': u'New run-webkit-tests', }, 253 {'name': u'WinCairo Debug (Build)', }, 254 {'name': u'WinCE Release (Build)', }, 255 {'name': u'EFL Linux Release (Build)', }, 256 ] 257 name_regexps = [ 258 "SnowLeopard.*Build", 259 "SnowLeopard.*\(Test", 260 "SnowLeopard.*\(WebKit2 Test", 261 "Leopard.*", 262 "Windows.*Build", 263 "Windows.*\(Test", 264 "WinCairo", 265 "WinCE", 266 "EFL", 267 "GTK.*32", 268 "GTK.*64.*Debug", # Disallow the 64-bit Release bot which is broken. 269 "Qt", 270 "Chromium.*Release$", 271 ] 272 expected_builders = [ 273 {'name': u'Leopard Intel Release (Build)', }, 274 {'name': u'Leopard Intel Release (Tests)', }, 275 {'name': u'Leopard Intel Debug (Build)', }, 276 {'name': u'Leopard Intel Debug (Tests)', }, 277 {'name': u'SnowLeopard Intel Release (Build)', }, 278 {'name': u'SnowLeopard Intel Release (Tests)', }, 279 {'name': u'SnowLeopard Intel Release (WebKit2 Tests)', }, 280 {'name': u'Windows Release (Build)', }, 281 {'name': u'Windows 7 Release (Tests)', }, 282 {'name': u'Windows Debug (Build)', }, 283 {'name': u'Windows XP Debug (Tests)', }, 284 {'name': u'GTK Linux 32-bit Release', }, 285 {'name': u'GTK Linux 32-bit Debug', }, 286 {'name': u'GTK Linux 64-bit Debug', }, 287 {'name': u'Qt Linux Release', }, 288 {'name': u'Qt Linux Release minimal', }, 289 {'name': u'Qt Linux ARMv7 Release', }, 290 {'name': u'Qt Windows 32-bit Release', }, 291 {'name': u'Qt Windows 32-bit Debug', }, 292 {'name': u'Chromium Win Release', }, 293 {'name': u'Chromium Mac Release', }, 294 {'name': u'Chromium Linux Release', }, 295 {'name': u'WinCairo Debug (Build)', }, 296 {'name': u'WinCE Release (Build)', }, 297 {'name': u'EFL Linux Release (Build)', }, 298 ] 299 300 # This test should probably be updated if the default regexp list changes 301 self.assertEquals(buildbot.core_builder_names_regexps, name_regexps) 302 303 builders = buildbot._builder_statuses_with_names_matching_regexps(example_builders, name_regexps) 304 self.assertEquals(builders, expected_builders) 305 306 def test_builder_with_name(self): 307 buildbot = BuildBot() 308 309 builder = buildbot.builder_with_name("Test Builder") 310 self.assertEqual(builder.name(), "Test Builder") 311 self.assertEqual(builder.url(), "http://build.webkit.org/builders/Test%20Builder") 312 self.assertEqual(builder.url_encoded_name(), "Test%20Builder") 313 self.assertEqual(builder.results_url(), "http://build.webkit.org/results/Test%20Builder") 314 315 # Override _fetch_build_dictionary function to not touch the network. 316 def mock_fetch_build_dictionary(self, build_number): 317 build_dictionary = { 318 "sourceStamp": { 319 "revision" : 2 * build_number, 320 }, 321 "number" : int(build_number), 322 "results" : build_number % 2, # 0 means pass 323 } 324 return build_dictionary 325 buildbot._fetch_build_dictionary = mock_fetch_build_dictionary 326 327 build = builder.build(10) 328 self.assertEqual(build.builder(), builder) 329 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/10") 330 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r20%20%2810%29") 331 self.assertEqual(build.revision(), 20) 332 self.assertEqual(build.is_green(), True) 333 334 build = build.previous_build() 335 self.assertEqual(build.builder(), builder) 336 self.assertEqual(build.url(), "http://build.webkit.org/builders/Test%20Builder/builds/9") 337 self.assertEqual(build.results_url(), "http://build.webkit.org/results/Test%20Builder/r18%20%289%29") 338 self.assertEqual(build.revision(), 18) 339 self.assertEqual(build.is_green(), False) 340 341 self.assertEqual(builder.build(None), None) 342 343 _example_directory_listing = ''' 344 <h1>Directory listing for /results/SnowLeopard Intel Leaks/</h1> 345 346 <table> 347 <tr class="alt"> 348 <th>Filename</th> 349 <th>Size</th> 350 <th>Content type</th> 351 <th>Content encoding</th> 352 </tr> 353 <tr class="directory "> 354 <td><a href="r47483%20%281%29/"><b>r47483 (1)/</b></a></td> 355 <td><b></b></td> 356 <td><b>[Directory]</b></td> 357 <td><b></b></td> 358 </tr> 359 <tr class="file alt"> 360 <td><a href="r47484%20%282%29.zip">r47484 (2).zip</a></td> 361 <td>89K</td> 362 <td>[application/zip]</td> 363 <td></td> 364 </tr> 365 ''' 366 _expected_files = [ 367 { 368 "filename" : "r47483 (1)/", 369 "size" : "", 370 "type" : "[Directory]", 371 "encoding" : "", 372 }, 373 { 374 "filename" : "r47484 (2).zip", 375 "size" : "89K", 376 "type" : "[application/zip]", 377 "encoding" : "", 378 }, 379 ] 380 381 def test_parse_build_to_revision_map(self): 382 buildbot = BuildBot() 383 files = buildbot._parse_twisted_directory_listing(self._example_directory_listing) 384 self.assertEqual(self._expected_files, files) 385 386 # Revision, is_green 387 # Ordered from newest (highest number) to oldest. 388 fake_builder1 = [ 389 [2, False], 390 [1, True], 391 ] 392 fake_builder2 = [ 393 [2, False], 394 [1, True], 395 ] 396 fake_builders = [ 397 fake_builder1, 398 fake_builder2, 399 ] 400 def _build_from_fake(self, fake_builder, index): 401 if index >= len(fake_builder): 402 return None 403 fake_build = fake_builder[index] 404 build = Build( 405 builder=fake_builder, 406 build_number=index, 407 revision=fake_build[0], 408 is_green=fake_build[1], 409 ) 410 def mock_previous_build(): 411 return self._build_from_fake(fake_builder, index + 1) 412 build.previous_build = mock_previous_build 413 return build 414 415 def _fake_builds_at_index(self, index): 416 return [self._build_from_fake(builder, index) for builder in self.fake_builders] 417 418 def test_last_green_revision(self): 419 buildbot = BuildBot() 420 def mock_builds_from_builders(only_core_builders): 421 return self._fake_builds_at_index(0) 422 buildbot._latest_builds_from_builders = mock_builds_from_builders 423 self.assertEqual(buildbot.last_green_revision(), 1) 424 425 def _fetch_build(self, build_number): 426 if build_number == 5: 427 return "correct build" 428 return "wrong build" 429 430 def _fetch_revision_to_build_map(self): 431 return {'r5': 5, 'r2': 2, 'r3': 3} 432 433 def test_latest_cached_build(self): 434 b = Builder('builder', BuildBot()) 435 b._fetch_build = self._fetch_build 436 b._fetch_revision_to_build_map = self._fetch_revision_to_build_map 437 self.assertEquals("correct build", b.latest_cached_build()) 438 439 def results_url(self): 440 return "some-url" 441 442 def test_results_zip_url(self): 443 b = Build(None, 123, 123, False) 444 b.results_url = self.results_url 445 self.assertEquals("some-url.zip", b.results_zip_url()) 446 447 def test_results(self): 448 builder = Builder('builder', BuildBot()) 449 b = Build(builder, 123, 123, True) 450 self.assertTrue(b.results()) 451 452 453 if __name__ == '__main__': 454 unittest.main() 455
