1 #!/usr/bin/python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Unit tests for printing.py.""" 31 32 import optparse 33 import unittest 34 import logging 35 36 from webkitpy.common import array_stream 37 from webkitpy.common.system import logtesting 38 from webkitpy.layout_tests import port 39 40 from webkitpy.layout_tests.layout_package import printing 41 from webkitpy.layout_tests.layout_package import result_summary 42 from webkitpy.layout_tests.layout_package import test_expectations 43 from webkitpy.layout_tests.layout_package import test_failures 44 from webkitpy.layout_tests.layout_package import test_results 45 from webkitpy.layout_tests.layout_package import test_runner 46 47 48 def get_options(args): 49 print_options = printing.print_options() 50 option_parser = optparse.OptionParser(option_list=print_options) 51 return option_parser.parse_args(args) 52 53 54 class TestUtilityFunctions(unittest.TestCase): 55 def test_configure_logging(self): 56 options, args = get_options([]) 57 stream = array_stream.ArrayStream() 58 handler = printing._configure_logging(stream, options.verbose) 59 logging.info("this should be logged") 60 self.assertFalse(stream.empty()) 61 62 stream.reset() 63 logging.debug("this should not be logged") 64 self.assertTrue(stream.empty()) 65 66 printing._restore_logging(handler) 67 68 stream.reset() 69 options, args = get_options(['--verbose']) 70 handler = printing._configure_logging(stream, options.verbose) 71 logging.debug("this should be logged") 72 self.assertFalse(stream.empty()) 73 printing._restore_logging(handler) 74 75 def test_print_options(self): 76 options, args = get_options([]) 77 self.assertTrue(options is not None) 78 79 def test_parse_print_options(self): 80 def test_switches(args, expected_switches_str, 81 verbose=False, child_processes=1, 82 is_fully_parallel=False): 83 options, args = get_options(args) 84 if expected_switches_str: 85 expected_switches = set(expected_switches_str.split(',')) 86 else: 87 expected_switches = set() 88 switches = printing.parse_print_options(options.print_options, 89 verbose, 90 child_processes, 91 is_fully_parallel) 92 self.assertEqual(expected_switches, switches) 93 94 # test that we default to the default set of switches 95 test_switches([], printing.PRINT_DEFAULT) 96 97 # test that verbose defaults to everything 98 test_switches([], printing.PRINT_EVERYTHING, verbose=True) 99 100 # test that --print default does what it's supposed to 101 test_switches(['--print', 'default'], printing.PRINT_DEFAULT) 102 103 # test that --print nothing does what it's supposed to 104 test_switches(['--print', 'nothing'], None) 105 106 # test that --print everything does what it's supposed to 107 test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING) 108 109 # this tests that '--print X' overrides '--verbose' 110 test_switches(['--print', 'actual'], 'actual', verbose=True) 111 112 113 114 class Testprinter(unittest.TestCase): 115 def get_printer(self, args=None, single_threaded=False, 116 is_fully_parallel=False): 117 args = args or [] 118 printing_options = printing.print_options() 119 option_parser = optparse.OptionParser(option_list=printing_options) 120 options, args = option_parser.parse_args(args) 121 self._port = port.get('test', options) 122 nproc = 2 123 if single_threaded: 124 nproc = 1 125 126 regular_output = array_stream.ArrayStream() 127 buildbot_output = array_stream.ArrayStream() 128 printer = printing.Printer(self._port, options, regular_output, 129 buildbot_output, single_threaded, 130 is_fully_parallel) 131 return printer, regular_output, buildbot_output 132 133 def get_result(self, test, result_type=test_expectations.PASS, run_time=0): 134 failures = [] 135 if result_type == test_expectations.TIMEOUT: 136 failures = [test_failures.FailureTimeout()] 137 elif result_type == test_expectations.CRASH: 138 failures = [test_failures.FailureCrash()] 139 path = self._port._filesystem.join(self._port.layout_tests_dir(), test) 140 return test_results.TestResult(path, failures=failures, test_run_time=run_time) 141 142 def get_result_summary(self, tests, expectations_str): 143 test_paths = [self._port._filesystem.join(self._port.layout_tests_dir(), test) for 144 test in tests] 145 expectations = test_expectations.TestExpectations( 146 self._port, test_paths, expectations_str, 147 self._port.test_configuration(), 148 is_lint_mode=False) 149 150 rs = result_summary.ResultSummary(expectations, test_paths) 151 return test_paths, rs, expectations 152 153 def test_help_printer(self): 154 # Here and below we'll call the "regular" printer err and the 155 # buildbot printer out; this corresponds to how things run on the 156 # bots with stderr and stdout. 157 printer, err, out = self.get_printer() 158 159 # This routine should print something to stdout. testing what it is 160 # is kind of pointless. 161 printer.help_printing() 162 self.assertFalse(err.empty()) 163 self.assertTrue(out.empty()) 164 165 def do_switch_tests(self, method_name, switch, to_buildbot, 166 message='hello', exp_err=None, exp_bot=None): 167 def do_helper(method_name, switch, message, exp_err, exp_bot): 168 printer, err, bot = self.get_printer(['--print', switch]) 169 getattr(printer, method_name)(message) 170 self.assertEqual(err.get(), exp_err) 171 self.assertEqual(bot.get(), exp_bot) 172 173 if to_buildbot: 174 if exp_err is None: 175 exp_err = [] 176 if exp_bot is None: 177 exp_bot = [message + "\n"] 178 else: 179 if exp_err is None: 180 exp_err = [message + "\n"] 181 if exp_bot is None: 182 exp_bot = [] 183 do_helper(method_name, 'nothing', 'hello', [], []) 184 do_helper(method_name, switch, 'hello', exp_err, exp_bot) 185 do_helper(method_name, 'everything', 'hello', exp_err, exp_bot) 186 187 def test_configure_and_cleanup(self): 188 # This test verifies that calling cleanup repeatedly and deleting 189 # the object is safe. 190 printer, err, out = self.get_printer(['--print', 'everything']) 191 printer.cleanup() 192 printer.cleanup() 193 printer = None 194 195 def test_print_actual(self): 196 # Actual results need to be logged to the buildbot's stream. 197 self.do_switch_tests('print_actual', 'actual', to_buildbot=True) 198 199 def test_print_actual_buildbot(self): 200 # FIXME: Test that the format of the actual results matches what the 201 # buildbot is expecting. 202 pass 203 204 def test_print_config(self): 205 self.do_switch_tests('print_config', 'config', to_buildbot=False) 206 207 def test_print_expected(self): 208 self.do_switch_tests('print_expected', 'expected', to_buildbot=False) 209 210 def test_print_timing(self): 211 self.do_switch_tests('print_timing', 'timing', to_buildbot=False) 212 213 def test_print_update(self): 214 # Note that there shouldn't be a carriage return here; updates() 215 # are meant to be overwritten. 216 self.do_switch_tests('print_update', 'updates', to_buildbot=False, 217 message='hello', exp_err=['hello']) 218 219 def test_print_one_line_summary(self): 220 printer, err, out = self.get_printer(['--print', 'nothing']) 221 printer.print_one_line_summary(1, 1, 0) 222 self.assertTrue(err.empty()) 223 224 printer, err, out = self.get_printer(['--print', 'one-line-summary']) 225 printer.print_one_line_summary(1, 1, 0) 226 self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) 227 228 printer, err, out = self.get_printer(['--print', 'everything']) 229 printer.print_one_line_summary(1, 1, 0) 230 self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) 231 232 err.reset() 233 printer.print_one_line_summary(2, 1, 1) 234 self.assertEquals(err.get(), 235 ["1 test ran as expected, 1 didn't:\n", "\n"]) 236 237 err.reset() 238 printer.print_one_line_summary(3, 2, 1) 239 self.assertEquals(err.get(), 240 ["2 tests ran as expected, 1 didn't:\n", "\n"]) 241 242 err.reset() 243 printer.print_one_line_summary(3, 2, 0) 244 self.assertEquals(err.get(), 245 ['\n', "2 tests ran as expected (1 didn't run).\n", 246 '\n']) 247 248 249 def test_print_test_result(self): 250 # Note here that we don't use meaningful exp_str and got_str values; 251 # the actual contents of the string are treated opaquely by 252 # print_test_result() when tracing, and usually we don't want 253 # to test what exactly is printed, just that something 254 # was printed (or that nothing was printed). 255 # 256 # FIXME: this is actually some goofy layering; it would be nice 257 # we could refactor it so that the args weren't redundant. Maybe 258 # the TestResult should contain what was expected, and the 259 # strings could be derived from the TestResult? 260 printer, err, out = self.get_printer(['--print', 'nothing']) 261 result = self.get_result('passes/image.html') 262 printer.print_test_result(result, expected=False, exp_str='', 263 got_str='') 264 self.assertTrue(err.empty()) 265 266 printer, err, out = self.get_printer(['--print', 'unexpected']) 267 printer.print_test_result(result, expected=True, exp_str='', 268 got_str='') 269 self.assertTrue(err.empty()) 270 printer.print_test_result(result, expected=False, exp_str='', 271 got_str='') 272 self.assertEquals(err.get(), 273 [' passes/image.html -> unexpected pass\n']) 274 275 printer, err, out = self.get_printer(['--print', 'everything']) 276 printer.print_test_result(result, expected=True, exp_str='', 277 got_str='') 278 self.assertTrue(err.empty()) 279 280 printer.print_test_result(result, expected=False, exp_str='', 281 got_str='') 282 self.assertEquals(err.get(), 283 [' passes/image.html -> unexpected pass\n']) 284 285 printer, err, out = self.get_printer(['--print', 'nothing']) 286 printer.print_test_result(result, expected=False, exp_str='', 287 got_str='') 288 self.assertTrue(err.empty()) 289 290 printer, err, out = self.get_printer(['--print', 291 'trace-unexpected']) 292 printer.print_test_result(result, expected=True, exp_str='', 293 got_str='') 294 self.assertTrue(err.empty()) 295 296 printer, err, out = self.get_printer(['--print', 297 'trace-unexpected']) 298 printer.print_test_result(result, expected=False, exp_str='', 299 got_str='') 300 self.assertFalse(err.empty()) 301 302 printer, err, out = self.get_printer(['--print', 303 'trace-unexpected']) 304 result = self.get_result("passes/text.html") 305 printer.print_test_result(result, expected=False, exp_str='', 306 got_str='') 307 self.assertFalse(err.empty()) 308 309 err.reset() 310 printer.print_test_result(result, expected=False, exp_str='', 311 got_str='') 312 self.assertFalse(err.empty()) 313 314 printer, err, out = self.get_printer(['--print', 'trace-everything']) 315 result = self.get_result('passes/image.html') 316 printer.print_test_result(result, expected=True, exp_str='', 317 got_str='') 318 result = self.get_result('failures/expected/missing_text.html') 319 printer.print_test_result(result, expected=True, exp_str='', 320 got_str='') 321 result = self.get_result('failures/expected/missing_check.html') 322 printer.print_test_result(result, expected=True, exp_str='', 323 got_str='') 324 result = self.get_result('failures/expected/missing_image.html') 325 printer.print_test_result(result, expected=True, exp_str='', 326 got_str='') 327 self.assertFalse(err.empty()) 328 329 err.reset() 330 printer.print_test_result(result, expected=False, exp_str='', 331 got_str='') 332 333 def test_print_progress(self): 334 expectations = '' 335 336 # test that we print nothing 337 printer, err, out = self.get_printer(['--print', 'nothing']) 338 tests = ['passes/text.html', 'failures/expected/timeout.html', 339 'failures/expected/crash.html'] 340 paths, rs, exp = self.get_result_summary(tests, expectations) 341 342 printer.print_progress(rs, False, paths) 343 self.assertTrue(out.empty()) 344 self.assertTrue(err.empty()) 345 346 printer.print_progress(rs, True, paths) 347 self.assertTrue(out.empty()) 348 self.assertTrue(err.empty()) 349 350 # test regular functionality 351 printer, err, out = self.get_printer(['--print', 352 'one-line-progress']) 353 printer.print_progress(rs, False, paths) 354 self.assertTrue(out.empty()) 355 self.assertFalse(err.empty()) 356 357 err.reset() 358 out.reset() 359 printer.print_progress(rs, True, paths) 360 self.assertFalse(err.empty()) 361 self.assertTrue(out.empty()) 362 363 def test_print_progress__detailed(self): 364 tests = ['passes/text.html', 'failures/expected/timeout.html', 365 'failures/expected/crash.html'] 366 expectations = 'BUGX : failures/expected/timeout.html = TIMEOUT' 367 368 # first, test that it is disabled properly 369 # should still print one-line-progress 370 printer, err, out = self.get_printer( 371 ['--print', 'detailed-progress'], single_threaded=False) 372 paths, rs, exp = self.get_result_summary(tests, expectations) 373 printer.print_progress(rs, False, paths) 374 self.assertFalse(err.empty()) 375 self.assertTrue(out.empty()) 376 377 # now test the enabled paths 378 printer, err, out = self.get_printer( 379 ['--print', 'detailed-progress'], single_threaded=True) 380 paths, rs, exp = self.get_result_summary(tests, expectations) 381 printer.print_progress(rs, False, paths) 382 self.assertFalse(err.empty()) 383 self.assertTrue(out.empty()) 384 385 err.reset() 386 out.reset() 387 printer.print_progress(rs, True, paths) 388 self.assertFalse(err.empty()) 389 self.assertTrue(out.empty()) 390 391 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) 392 rs.add(self.get_result('failures/expected/timeout.html'), True) 393 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) 394 err.reset() 395 out.reset() 396 printer.print_progress(rs, False, paths) 397 self.assertFalse(err.empty()) 398 self.assertTrue(out.empty()) 399 400 # We only clear the meter when retrying w/ detailed-progress. 401 err.reset() 402 out.reset() 403 printer.print_progress(rs, True, paths) 404 self.assertFalse(err.empty()) 405 self.assertTrue(out.empty()) 406 407 printer, err, out = self.get_printer( 408 ['--print', 'detailed-progress,unexpected'], single_threaded=True) 409 paths, rs, exp = self.get_result_summary(tests, expectations) 410 printer.print_progress(rs, False, paths) 411 self.assertFalse(err.empty()) 412 self.assertTrue(out.empty()) 413 414 err.reset() 415 out.reset() 416 printer.print_progress(rs, True, paths) 417 self.assertFalse(err.empty()) 418 self.assertTrue(out.empty()) 419 420 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) 421 rs.add(self.get_result('failures/expected/timeout.html'), True) 422 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) 423 err.reset() 424 out.reset() 425 printer.print_progress(rs, False, paths) 426 self.assertFalse(err.empty()) 427 self.assertTrue(out.empty()) 428 429 # We only clear the meter when retrying w/ detailed-progress. 430 err.reset() 431 out.reset() 432 printer.print_progress(rs, True, paths) 433 self.assertFalse(err.empty()) 434 self.assertTrue(out.empty()) 435 436 def test_write_nothing(self): 437 printer, err, out = self.get_printer(['--print', 'nothing']) 438 printer.write("foo") 439 self.assertTrue(err.empty()) 440 441 def test_write_misc(self): 442 printer, err, out = self.get_printer(['--print', 'misc']) 443 printer.write("foo") 444 self.assertFalse(err.empty()) 445 err.reset() 446 printer.write("foo", "config") 447 self.assertTrue(err.empty()) 448 449 def test_write_everything(self): 450 printer, err, out = self.get_printer(['--print', 'everything']) 451 printer.write("foo") 452 self.assertFalse(err.empty()) 453 err.reset() 454 printer.write("foo", "config") 455 self.assertFalse(err.empty()) 456 457 def test_write_verbose(self): 458 printer, err, out = self.get_printer(['--verbose']) 459 printer.write("foo") 460 self.assertTrue(not err.empty() and "foo" in err.get()[0]) 461 self.assertTrue(out.empty()) 462 463 def test_print_unexpected_results(self): 464 # This routine is the only one that prints stuff that the bots 465 # care about. 466 # 467 # FIXME: there's some weird layering going on here. It seems 468 # like we shouldn't be both using an expectations string and 469 # having to specify whether or not the result was expected. 470 # This whole set of tests should probably be rewritten. 471 # 472 # FIXME: Plus, the fact that we're having to call into 473 # run_webkit_tests is clearly a layering inversion. 474 def get_unexpected_results(expected, passing, flaky): 475 """Return an unexpected results summary matching the input description. 476 477 There are a lot of different combinations of test results that 478 can be tested; this routine produces various combinations based 479 on the values of the input flags. 480 481 Args 482 expected: whether the tests ran as expected 483 passing: whether the tests should all pass 484 flaky: whether the tests should be flaky (if False, they 485 produce the same results on both runs; if True, they 486 all pass on the second run). 487 488 """ 489 paths, rs, exp = self.get_result_summary(tests, expectations) 490 if expected: 491 rs.add(self.get_result('passes/text.html', test_expectations.PASS), 492 expected) 493 rs.add(self.get_result('failures/expected/timeout.html', 494 test_expectations.TIMEOUT), expected) 495 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), 496 expected) 497 elif passing: 498 rs.add(self.get_result('passes/text.html'), expected) 499 rs.add(self.get_result('failures/expected/timeout.html'), expected) 500 rs.add(self.get_result('failures/expected/crash.html'), expected) 501 else: 502 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), 503 expected) 504 rs.add(self.get_result('failures/expected/timeout.html', 505 test_expectations.CRASH), expected) 506 rs.add(self.get_result('failures/expected/crash.html', 507 test_expectations.TIMEOUT), 508 expected) 509 retry = rs 510 if flaky: 511 paths, retry, exp = self.get_result_summary(tests, 512 expectations) 513 retry.add(self.get_result('passes/text.html'), True) 514 retry.add(self.get_result('failures/expected/timeout.html'), True) 515 retry.add(self.get_result('failures/expected/crash.html'), True) 516 unexpected_results = test_runner.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True) 517 return unexpected_results 518 519 tests = ['passes/text.html', 'failures/expected/timeout.html', 520 'failures/expected/crash.html'] 521 expectations = '' 522 523 printer, err, out = self.get_printer(['--print', 'nothing']) 524 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 525 printer.print_unexpected_results(ur) 526 self.assertTrue(err.empty()) 527 self.assertTrue(out.empty()) 528 529 printer, err, out = self.get_printer(['--print', 530 'unexpected-results']) 531 532 # test everything running as expected 533 ur = get_unexpected_results(expected=True, passing=False, flaky=False) 534 printer.print_unexpected_results(ur) 535 self.assertTrue(err.empty()) 536 self.assertTrue(out.empty()) 537 538 # test failures 539 err.reset() 540 out.reset() 541 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 542 printer.print_unexpected_results(ur) 543 self.assertTrue(err.empty()) 544 self.assertFalse(out.empty()) 545 546 # test unexpected flaky results 547 err.reset() 548 out.reset() 549 ur = get_unexpected_results(expected=False, passing=True, flaky=False) 550 printer.print_unexpected_results(ur) 551 self.assertTrue(err.empty()) 552 self.assertFalse(out.empty()) 553 554 # test unexpected passes 555 err.reset() 556 out.reset() 557 ur = get_unexpected_results(expected=False, passing=False, flaky=True) 558 printer.print_unexpected_results(ur) 559 self.assertTrue(err.empty()) 560 self.assertFalse(out.empty()) 561 562 err.reset() 563 out.reset() 564 printer, err, out = self.get_printer(['--print', 'everything']) 565 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 566 printer.print_unexpected_results(ur) 567 self.assertTrue(err.empty()) 568 self.assertFalse(out.empty()) 569 570 expectations = """ 571 BUGX : failures/expected/crash.html = CRASH 572 BUGX : failures/expected/timeout.html = TIMEOUT 573 """ 574 err.reset() 575 out.reset() 576 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 577 printer.print_unexpected_results(ur) 578 self.assertTrue(err.empty()) 579 self.assertFalse(out.empty()) 580 581 err.reset() 582 out.reset() 583 ur = get_unexpected_results(expected=False, passing=True, flaky=False) 584 printer.print_unexpected_results(ur) 585 self.assertTrue(err.empty()) 586 self.assertFalse(out.empty()) 587 588 # Test handling of --verbose as well. 589 err.reset() 590 out.reset() 591 printer, err, out = self.get_printer(['--verbose']) 592 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 593 printer.print_unexpected_results(ur) 594 self.assertTrue(err.empty()) 595 self.assertFalse(out.empty()) 596 597 def test_print_unexpected_results_buildbot(self): 598 # FIXME: Test that print_unexpected_results() produces the printer the 599 # buildbot is expecting. 600 pass 601 602 if __name__ == '__main__': 603 unittest.main() 604
