1 # Copyright (C) 2010 Google Inc. All rights reserved. 2 # 3 # Redistribution and use in source and binary forms, with or without 4 # modification, are permitted provided that the following conditions are 5 # met: 6 # 7 # * Redistributions of source code must retain the above copyright 8 # notice, this list of conditions and the following disclaimer. 9 # * Redistributions in binary form must reproduce the above 10 # copyright notice, this list of conditions and the following disclaimer 11 # in the documentation and/or other materials provided with the 12 # distribution. 13 # * Neither the name of Google Inc. nor the names of its 14 # contributors may be used to endorse or promote products derived from 15 # this software without specific prior written permission. 16 # 17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29 import webkitpy.thirdparty.unittest2 as unittest 30 31 from webkitpy.common.host_mock import MockHost 32 from webkitpy.common.system.outputcapture import OutputCapture 33 34 from webkitpy.layout_tests.models.test_configuration import * 35 from webkitpy.layout_tests.models.test_expectations import * 36 37 try: 38 from collections import OrderedDict 39 except ImportError: 40 # Needed for Python < 2.7 41 from webkitpy.thirdparty.ordered_dict import OrderedDict 42 43 44 class Base(unittest.TestCase): 45 # Note that all of these tests are written assuming the configuration 46 # being tested is Windows XP, Release build. 47 48 def __init__(self, testFunc): 49 host = MockHost() 50 self._port = host.port_factory.get('test-win-xp', None) 51 self._exp = None 52 unittest.TestCase.__init__(self, testFunc) 53 54 def get_test(self, test_name): 55 # FIXME: Remove this routine and just reference test names directly. 56 return test_name 57 58 def get_basic_tests(self): 59 return [self.get_test('failures/expected/text.html'), 60 self.get_test('failures/expected/image_checksum.html'), 61 self.get_test('failures/expected/crash.html'), 62 self.get_test('failures/expected/needsrebaseline.html'), 63 self.get_test('failures/expected/needsmanualrebaseline.html'), 64 self.get_test('failures/expected/missing_text.html'), 65 self.get_test('failures/expected/image.html'), 66 self.get_test('passes/text.html')] 67 68 def get_basic_expectations(self): 69 return """ 70 Bug(test) failures/expected/text.html [ Failure ] 71 Bug(test) failures/expected/crash.html [ WontFix ] 72 Bug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] 73 Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] 74 Bug(test) failures/expected/missing_image.html [ Rebaseline Missing ] 75 Bug(test) failures/expected/image_checksum.html [ WontFix ] 76 Bug(test) failures/expected/image.html [ WontFix Mac ] 77 """ 78 79 def parse_exp(self, expectations, overrides=None, is_lint_mode=False): 80 expectations_dict = OrderedDict() 81 expectations_dict['expectations'] = expectations 82 if overrides: 83 expectations_dict['overrides'] = overrides 84 self._port.expectations_dict = lambda: expectations_dict 85 expectations_to_lint = expectations_dict if is_lint_mode else None 86 self._exp = TestExpectations(self._port, self.get_basic_tests(), expectations_dict=expectations_to_lint, is_lint_mode=is_lint_mode) 87 88 def assert_exp_list(self, test, results): 89 self.assertEqual(self._exp.get_expectations(self.get_test(test)), set(results)) 90 91 def assert_exp(self, test, result): 92 self.assert_exp_list(test, [result]) 93 94 def assert_bad_expectations(self, expectations, overrides=None): 95 self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides) 96 97 98 class BasicTests(Base): 99 def test_basic(self): 100 self.parse_exp(self.get_basic_expectations()) 101 self.assert_exp('failures/expected/text.html', FAIL) 102 self.assert_exp_list('failures/expected/image_checksum.html', [WONTFIX, SKIP]) 103 self.assert_exp('passes/text.html', PASS) 104 self.assert_exp('failures/expected/image.html', PASS) 105 106 107 class MiscTests(Base): 108 def test_multiple_results(self): 109 self.parse_exp('Bug(x) failures/expected/text.html [ Crash Failure ]') 110 self.assertEqual(self._exp.get_expectations( 111 self.get_test('failures/expected/text.html')), 112 set([FAIL, CRASH])) 113 114 def test_result_was_expected(self): 115 # test basics 116 self.assertEqual(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False), True) 117 self.assertEqual(TestExpectations.result_was_expected(FAIL, set([PASS]), test_needs_rebaselining=False), False) 118 119 # test handling of SKIPped tests and results 120 self.assertEqual(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False), True) 121 122 # test handling of MISSING results and the REBASELINE specifier 123 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True), True) 124 self.assertEqual(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False), False) 125 126 self.assertTrue(TestExpectations.result_was_expected(PASS, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 127 self.assertTrue(TestExpectations.result_was_expected(MISSING, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 128 self.assertTrue(TestExpectations.result_was_expected(TEXT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 129 self.assertTrue(TestExpectations.result_was_expected(IMAGE, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 130 self.assertTrue(TestExpectations.result_was_expected(IMAGE_PLUS_TEXT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 131 self.assertTrue(TestExpectations.result_was_expected(AUDIO, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 132 self.assertFalse(TestExpectations.result_was_expected(TIMEOUT, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 133 self.assertFalse(TestExpectations.result_was_expected(CRASH, set([NEEDS_REBASELINE]), test_needs_rebaselining=False)) 134 135 def test_remove_pixel_failures(self): 136 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL])) 137 self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS])) 138 self.assertEqual(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS])) 139 self.assertEqual(TestExpectations.remove_pixel_failures(set([FAIL])), set([FAIL])) 140 self.assertEqual(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH])) 141 142 def test_suffixes_for_expectations(self): 143 self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL])), set(['txt', 'png', 'wav'])) 144 self.assertEqual(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png'])) 145 self.assertEqual(TestExpectations.suffixes_for_expectations(set([FAIL, IMAGE, CRASH])), set(['txt', 'png', 'wav'])) 146 self.assertEqual(TestExpectations.suffixes_for_expectations(set()), set()) 147 148 def test_category_expectations(self): 149 # This test checks unknown tests are not present in the 150 # expectations and that known test part of a test category is 151 # present in the expectations. 152 exp_str = 'Bug(x) failures/expected [ WontFix ]' 153 self.parse_exp(exp_str) 154 test_name = 'failures/expected/unknown-test.html' 155 unknown_test = self.get_test(test_name) 156 self.assertRaises(KeyError, self._exp.get_expectations, 157 unknown_test) 158 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) 159 160 def test_get_expectations_string(self): 161 self.parse_exp(self.get_basic_expectations()) 162 self.assertEqual(self._exp.get_expectations_string( 163 self.get_test('failures/expected/text.html')), 164 'FAIL') 165 166 def test_expectation_to_string(self): 167 # Normal cases are handled by other tests. 168 self.parse_exp(self.get_basic_expectations()) 169 self.assertRaises(ValueError, self._exp.expectation_to_string, 170 -1) 171 172 def test_get_test_set(self): 173 # Handle some corner cases for this routine not covered by other tests. 174 self.parse_exp(self.get_basic_expectations()) 175 s = self._exp.get_test_set(WONTFIX) 176 self.assertEqual(s, 177 set([self.get_test('failures/expected/crash.html'), 178 self.get_test('failures/expected/image_checksum.html')])) 179 180 def test_parse_warning(self): 181 try: 182 filesystem = self._port.host.filesystem 183 filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content') 184 self.get_test('disabled-test.html-disabled'), 185 self.parse_exp("Bug(user) [ FOO ] failures/expected/text.html [ Failure ]\n" 186 "Bug(user) non-existent-test.html [ Failure ]\n" 187 "Bug(user) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True) 188 self.assertFalse(True, "ParseError wasn't raised") 189 except ParseError, e: 190 warnings = ("expectations:1 Unrecognized specifier 'foo' failures/expected/text.html\n" 191 "expectations:2 Path does not exist. non-existent-test.html") 192 self.assertEqual(str(e), warnings) 193 194 def test_parse_warnings_are_logged_if_not_in_lint_mode(self): 195 oc = OutputCapture() 196 try: 197 oc.capture_output() 198 self.parse_exp('-- this should be a syntax error', is_lint_mode=False) 199 finally: 200 _, _, logs = oc.restore_output() 201 self.assertNotEquals(logs, '') 202 203 def test_error_on_different_platform(self): 204 # parse_exp uses a Windows port. Assert errors on Mac show up in lint mode. 205 self.assertRaises(ParseError, self.parse_exp, 206 'Bug(test) [ Mac ] failures/expected/text.html [ Failure ]\nBug(test) [ Mac ] failures/expected/text.html [ Failure ]', 207 is_lint_mode=True) 208 209 def test_error_on_different_build_type(self): 210 # parse_exp uses a Release port. Assert errors on DEBUG show up in lint mode. 211 self.assertRaises(ParseError, self.parse_exp, 212 'Bug(test) [ Debug ] failures/expected/text.html [ Failure ]\nBug(test) [ Debug ] failures/expected/text.html [ Failure ]', 213 is_lint_mode=True) 214 215 def test_overrides(self): 216 self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]", 217 "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]") 218 self.assert_exp_list('failures/expected/text.html', [FAIL, IMAGE]) 219 220 def test_overrides__directory(self): 221 self.parse_exp("Bug(exp) failures/expected/text.html [ Failure ]", 222 "Bug(override) failures/expected [ Crash ]") 223 self.assert_exp_list('failures/expected/text.html', [FAIL, CRASH]) 224 self.assert_exp_list('failures/expected/image.html', [CRASH]) 225 226 def test_overrides__duplicate(self): 227 self.assert_bad_expectations("Bug(exp) failures/expected/text.html [ Failure ]", 228 "Bug(override) failures/expected/text.html [ ImageOnlyFailure ]\n" 229 "Bug(override) failures/expected/text.html [ Crash ]\n") 230 231 def test_pixel_tests_flag(self): 232 def match(test, result, pixel_tests_enabled): 233 return self._exp.matches_an_expected_result( 234 self.get_test(test), result, pixel_tests_enabled) 235 236 self.parse_exp(self.get_basic_expectations()) 237 self.assertTrue(match('failures/expected/text.html', FAIL, True)) 238 self.assertTrue(match('failures/expected/text.html', FAIL, False)) 239 self.assertFalse(match('failures/expected/text.html', CRASH, True)) 240 self.assertFalse(match('failures/expected/text.html', CRASH, False)) 241 self.assertTrue(match('failures/expected/image_checksum.html', PASS, True)) 242 self.assertTrue(match('failures/expected/image_checksum.html', PASS, False)) 243 self.assertTrue(match('failures/expected/crash.html', PASS, False)) 244 self.assertTrue(match('failures/expected/needsrebaseline.html', TEXT, True)) 245 self.assertFalse(match('failures/expected/needsrebaseline.html', CRASH, True)) 246 self.assertTrue(match('failures/expected/needsmanualrebaseline.html', TEXT, True)) 247 self.assertFalse(match('failures/expected/needsmanualrebaseline.html', CRASH, True)) 248 self.assertTrue(match('passes/text.html', PASS, False)) 249 250 def test_more_specific_override_resets_skip(self): 251 self.parse_exp("Bug(x) failures/expected [ Skip ]\n" 252 "Bug(x) failures/expected/text.html [ ImageOnlyFailure ]\n") 253 self.assert_exp('failures/expected/text.html', IMAGE) 254 self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(), 255 'failures/expected/text.html') in 256 self._exp.get_tests_with_result_type(SKIP)) 257 258 def test_bot_test_expectations(self): 259 """Test that expectations are merged rather than overridden when using flaky option 'unexpected'.""" 260 test_name1 = 'failures/expected/text.html' 261 test_name2 = 'passes/text.html' 262 263 expectations_dict = OrderedDict() 264 expectations_dict['expectations'] = "Bug(x) %s [ ImageOnlyFailure ]\nBug(x) %s [ Slow ]\n" % (test_name1, test_name2) 265 self._port.expectations_dict = lambda: expectations_dict 266 267 expectations = TestExpectations(self._port, self.get_basic_tests()) 268 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)), set([IMAGE])) 269 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)), set([SLOW])) 270 271 def bot_expectations(): 272 return {test_name1: ['PASS', 'TIMEOUT'], test_name2: ['CRASH']} 273 self._port.bot_expectations = bot_expectations 274 self._port._options.ignore_flaky_tests = 'unexpected' 275 276 expectations = TestExpectations(self._port, self.get_basic_tests()) 277 self.assertEqual(expectations.get_expectations(self.get_test(test_name1)), set([PASS, IMAGE, TIMEOUT])) 278 self.assertEqual(expectations.get_expectations(self.get_test(test_name2)), set([CRASH, SLOW])) 279 280 class SkippedTests(Base): 281 def check(self, expectations, overrides, skips, lint=False, expected_results=[WONTFIX, SKIP, FAIL]): 282 port = MockHost().port_factory.get('test-win-xp') 283 port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html'), 'foo') 284 expectations_dict = OrderedDict() 285 expectations_dict['expectations'] = expectations 286 if overrides: 287 expectations_dict['overrides'] = overrides 288 port.expectations_dict = lambda: expectations_dict 289 port.skipped_layout_tests = lambda tests: set(skips) 290 expectations_to_lint = expectations_dict if lint else None 291 exp = TestExpectations(port, ['failures/expected/text.html'], expectations_dict=expectations_to_lint, is_lint_mode=lint) 292 self.assertEqual(exp.get_expectations('failures/expected/text.html'), set(expected_results)) 293 294 def test_skipped_tests_work(self): 295 self.check(expectations='', overrides=None, skips=['failures/expected/text.html'], expected_results=[WONTFIX, SKIP]) 296 297 def test_duplicate_skipped_test_fails_lint(self): 298 self.assertRaises(ParseError, self.check, expectations='Bug(x) failures/expected/text.html [ Failure ]\n', 299 overrides=None, skips=['failures/expected/text.html'], lint=True) 300 301 def test_skipped_file_overrides_expectations(self): 302 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, 303 skips=['failures/expected/text.html']) 304 305 def test_skipped_dir_overrides_expectations(self): 306 self.check(expectations='Bug(x) failures/expected/text.html [ Failure ]\n', overrides=None, 307 skips=['failures/expected']) 308 309 def test_skipped_file_overrides_overrides(self): 310 self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n', 311 skips=['failures/expected/text.html']) 312 313 def test_skipped_dir_overrides_overrides(self): 314 self.check(expectations='', overrides='Bug(x) failures/expected/text.html [ Failure ]\n', 315 skips=['failures/expected']) 316 317 def test_skipped_entry_dont_exist(self): 318 port = MockHost().port_factory.get('test-win-xp') 319 expectations_dict = OrderedDict() 320 expectations_dict['expectations'] = '' 321 port.expectations_dict = lambda: expectations_dict 322 port.skipped_layout_tests = lambda tests: set(['foo/bar/baz.html']) 323 capture = OutputCapture() 324 capture.capture_output() 325 exp = TestExpectations(port) 326 _, _, logs = capture.restore_output() 327 self.assertEqual('The following test foo/bar/baz.html from the Skipped list doesn\'t exist\n', logs) 328 329 def test_expectations_string(self): 330 self.parse_exp(self.get_basic_expectations()) 331 notrun = 'failures/expected/text.html' 332 self._exp.add_extra_skipped_tests([notrun]) 333 self.assertEqual('NOTRUN', self._exp.get_expectations_string(notrun)) 334 335 336 class ExpectationSyntaxTests(Base): 337 def test_unrecognized_expectation(self): 338 self.assert_bad_expectations('Bug(test) failures/expected/text.html [ Unknown ]') 339 340 def test_macro(self): 341 exp_str = 'Bug(test) [ Win ] failures/expected/text.html [ Failure ]' 342 self.parse_exp(exp_str) 343 self.assert_exp('failures/expected/text.html', FAIL) 344 345 def assert_tokenize_exp(self, line, bugs=None, specifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'): 346 bugs = bugs or [] 347 specifiers = specifiers or [] 348 expectations = expectations or [] 349 warnings = warnings or [] 350 filename = 'TestExpectations' 351 line_number = '1' 352 expectation_line = TestExpectationParser._tokenize_line(filename, line, line_number) 353 self.assertEqual(expectation_line.warnings, warnings) 354 self.assertEqual(expectation_line.name, name) 355 self.assertEqual(expectation_line.filename, filename) 356 self.assertEqual(expectation_line.line_numbers, line_number) 357 if not warnings: 358 self.assertEqual(expectation_line.specifiers, specifiers) 359 self.assertEqual(expectation_line.expectations, expectations) 360 361 def test_comments(self): 362 self.assert_tokenize_exp("# comment", name=None, comment="# comment") 363 self.assert_tokenize_exp("foo.html [ Pass ] # comment", comment="# comment", expectations=['PASS'], specifiers=[]) 364 365 def test_config_specifiers(self): 366 self.assert_tokenize_exp('[ Mac ] foo.html [ Failure ] ', specifiers=['MAC'], expectations=['FAIL']) 367 368 def test_unknown_config(self): 369 self.assert_tokenize_exp('[ Foo ] foo.html [ Pass ]', specifiers=['Foo'], expectations=['PASS']) 370 371 def test_unknown_expectation(self): 372 self.assert_tokenize_exp('foo.html [ Audio ]', warnings=['Unrecognized expectation "Audio"']) 373 374 def test_skip(self): 375 self.assert_tokenize_exp('foo.html [ Skip ]', specifiers=[], expectations=['SKIP']) 376 377 def test_slow(self): 378 self.assert_tokenize_exp('foo.html [ Slow ]', specifiers=[], expectations=['SLOW']) 379 380 def test_wontfix(self): 381 self.assert_tokenize_exp('foo.html [ WontFix ]', specifiers=[], expectations=['WONTFIX', 'SKIP']) 382 self.assert_tokenize_exp('foo.html [ WontFix ImageOnlyFailure ]', specifiers=[], expectations=['WONTFIX', 'SKIP'], 383 warnings=['A test marked Skip or WontFix must not have other expectations.']) 384 385 def test_blank_line(self): 386 self.assert_tokenize_exp('', name=None) 387 388 def test_warnings(self): 389 self.assert_tokenize_exp('[ Mac ]', warnings=['Did not find a test name.', 'Missing expectations.'], name=None) 390 self.assert_tokenize_exp('[ [', warnings=['unexpected "["', 'Missing expectations.'], name=None) 391 self.assert_tokenize_exp('crbug.com/12345 ]', warnings=['unexpected "]"', 'Missing expectations.'], name=None) 392 393 self.assert_tokenize_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.', 'Missing expectations.']) 394 self.assert_tokenize_exp('foo.html', warnings=['Missing expectations.']) 395 396 397 class SemanticTests(Base): 398 def test_bug_format(self): 399 self.assertRaises(ParseError, self.parse_exp, 'BUG1234 failures/expected/text.html [ Failure ]', is_lint_mode=True) 400 401 def test_bad_bugid(self): 402 try: 403 self.parse_exp('crbug/1234 failures/expected/text.html [ Failure ]', is_lint_mode=True) 404 self.fail('should have raised an error about a bad bug identifier') 405 except ParseError, exp: 406 self.assertEqual(len(exp.warnings), 3) 407 408 def test_missing_bugid(self): 409 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=False) 410 self.assertFalse(self._exp.has_warnings()) 411 412 try: 413 self.parse_exp('failures/expected/text.html [ Failure ]', is_lint_mode=True) 414 except ParseError, exp: 415 self.assertEqual(exp.warnings, ['expectations:1 Test lacks BUG specifier. failures/expected/text.html']) 416 417 def test_skip_and_wontfix(self): 418 # Skip is not allowed to have other expectations as well, because those 419 # expectations won't be exercised and may become stale . 420 self.parse_exp('failures/expected/text.html [ Failure Skip ]') 421 self.assertTrue(self._exp.has_warnings()) 422 423 self.parse_exp('failures/expected/text.html [ Crash WontFix ]') 424 self.assertTrue(self._exp.has_warnings()) 425 426 self.parse_exp('failures/expected/text.html [ Pass WontFix ]') 427 self.assertTrue(self._exp.has_warnings()) 428 429 def test_rebaseline(self): 430 # Can't lint a file w/ 'REBASELINE' in it. 431 self.assertRaises(ParseError, self.parse_exp, 432 'Bug(test) failures/expected/text.html [ Failure Rebaseline ]', 433 is_lint_mode=True) 434 435 def test_duplicates(self): 436 self.assertRaises(ParseError, self.parse_exp, """ 437 Bug(exp) failures/expected/text.html [ Failure ] 438 Bug(exp) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True) 439 440 self.assertRaises(ParseError, self.parse_exp, 441 self.get_basic_expectations(), overrides=""" 442 Bug(override) failures/expected/text.html [ Failure ] 443 Bug(override) failures/expected/text.html [ ImageOnlyFailure ]""", is_lint_mode=True) 444 445 def test_duplicate_with_line_before_preceding_line(self): 446 self.assert_bad_expectations("""Bug(exp) [ Debug ] failures/expected/text.html [ Failure ] 447 Bug(exp) [ Release ] failures/expected/text.html [ Failure ] 448 Bug(exp) [ Debug ] failures/expected/text.html [ Failure ] 449 """) 450 451 def test_missing_file(self): 452 self.parse_exp('Bug(test) missing_file.html [ Failure ]') 453 self.assertTrue(self._exp.has_warnings(), 1) 454 455 456 class PrecedenceTests(Base): 457 def test_file_over_directory(self): 458 # This tests handling precedence of specific lines over directories 459 # and tests expectations covering entire directories. 460 exp_str = """ 461 Bug(x) failures/expected/text.html [ Failure ] 462 Bug(y) failures/expected [ WontFix ] 463 """ 464 self.parse_exp(exp_str) 465 self.assert_exp('failures/expected/text.html', FAIL) 466 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) 467 468 exp_str = """ 469 Bug(x) failures/expected [ WontFix ] 470 Bug(y) failures/expected/text.html [ Failure ] 471 """ 472 self.parse_exp(exp_str) 473 self.assert_exp('failures/expected/text.html', FAIL) 474 self.assert_exp_list('failures/expected/crash.html', [WONTFIX, SKIP]) 475 476 def test_ambiguous(self): 477 self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n" 478 "Bug(test) [ Win ] passes/text.html [ Failure ]\n") 479 480 def test_more_specifiers(self): 481 self.assert_bad_expectations("Bug(test) [ Release ] passes/text.html [ Pass ]\n" 482 "Bug(test) [ Win Release ] passes/text.html [ Failure ]\n") 483 484 def test_order_in_file(self): 485 self.assert_bad_expectations("Bug(test) [ Win Release ] : passes/text.html [ Failure ]\n" 486 "Bug(test) [ Release ] : passes/text.html [ Pass ]\n") 487 488 def test_macro_overrides(self): 489 self.assert_bad_expectations("Bug(test) [ Win ] passes/text.html [ Pass ]\n" 490 "Bug(test) [ XP ] passes/text.html [ Failure ]\n") 491 492 493 class RemoveConfigurationsTest(Base): 494 def test_remove(self): 495 host = MockHost() 496 test_port = host.port_factory.get('test-win-xp', None) 497 test_port.test_exists = lambda test: True 498 test_port.test_isfile = lambda test: True 499 500 test_config = test_port.test_configuration() 501 test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Linux Win Release ] failures/expected/foo.html [ Failure ] 502 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] 503 """} 504 expectations = TestExpectations(test_port, self.get_basic_tests()) 505 506 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 507 508 self.assertEqual("""Bug(x) [ Linux Win7 Release ] failures/expected/foo.html [ Failure ] 509 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ] 510 """, actual_expectations) 511 512 def test_remove_needs_rebaseline(self): 513 host = MockHost() 514 test_port = host.port_factory.get('test-win-xp', None) 515 test_port.test_exists = lambda test: True 516 test_port.test_isfile = lambda test: True 517 518 test_config = test_port.test_configuration() 519 test_port.expectations_dict = lambda: {"expectations": """Bug(x) [ Win ] failures/expected/foo.html [ NeedsRebaseline ] 520 """} 521 expectations = TestExpectations(test_port, self.get_basic_tests()) 522 523 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 524 525 self.assertEqual("""Bug(x) [ XP Debug ] failures/expected/foo.html [ NeedsRebaseline ] 526 Bug(x) [ Win7 ] failures/expected/foo.html [ NeedsRebaseline ] 527 """, actual_expectations) 528 529 def test_remove_line_with_comments(self): 530 host = MockHost() 531 test_port = host.port_factory.get('test-win-xp', None) 532 test_port.test_exists = lambda test: True 533 test_port.test_isfile = lambda test: True 534 535 test_config = test_port.test_configuration() 536 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 537 538 # This comment line should get stripped. As should the preceding line. 539 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] 540 """} 541 expectations = TestExpectations(test_port) 542 543 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 544 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) 545 546 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 547 """, actual_expectations) 548 549 def test_remove_line_with_comments_at_start(self): 550 host = MockHost() 551 test_port = host.port_factory.get('test-win-xp', None) 552 test_port.test_exists = lambda test: True 553 test_port.test_isfile = lambda test: True 554 555 test_config = test_port.test_configuration() 556 test_port.expectations_dict = lambda: {'expectations': """ 557 # This comment line should get stripped. As should the preceding line. 558 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] 559 560 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 561 """} 562 expectations = TestExpectations(test_port) 563 564 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 565 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) 566 567 self.assertEqual(""" 568 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 569 """, actual_expectations) 570 571 def test_remove_line_with_comments_at_end_with_no_trailing_newline(self): 572 host = MockHost() 573 test_port = host.port_factory.get('test-win-xp', None) 574 test_port.test_exists = lambda test: True 575 test_port.test_isfile = lambda test: True 576 577 test_config = test_port.test_configuration() 578 test_port.expectations_dict = lambda: {'expectations': """Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 579 580 # This comment line should get stripped. As should the preceding line. 581 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ]"""} 582 expectations = TestExpectations(test_port) 583 584 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 585 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) 586 587 self.assertEqual("""Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ]""", actual_expectations) 588 589 def test_remove_line_leaves_comments_for_next_line(self): 590 host = MockHost() 591 test_port = host.port_factory.get('test-win-xp', None) 592 test_port.test_exists = lambda test: True 593 test_port.test_isfile = lambda test: True 594 595 test_config = test_port.test_configuration() 596 test_port.expectations_dict = lambda: {'expectations': """ 597 # This comment line should not get stripped. 598 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] 599 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 600 """} 601 expectations = TestExpectations(test_port) 602 603 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 604 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) 605 606 self.assertEqual(""" 607 # This comment line should not get stripped. 608 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 609 """, actual_expectations) 610 611 def test_remove_line_no_whitespace_lines(self): 612 host = MockHost() 613 test_port = host.port_factory.get('test-win-xp', None) 614 test_port.test_exists = lambda test: True 615 test_port.test_isfile = lambda test: True 616 617 test_config = test_port.test_configuration() 618 test_port.expectations_dict = lambda: {'expectations': """ 619 # This comment line should get stripped. 620 Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] 621 # This comment line should not get stripped. 622 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 623 """} 624 expectations = TestExpectations(test_port) 625 626 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 627 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) 628 629 self.assertEqual(""" # This comment line should not get stripped. 630 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 631 """, actual_expectations) 632 633 def test_remove_first_line(self): 634 host = MockHost() 635 test_port = host.port_factory.get('test-win-xp', None) 636 test_port.test_exists = lambda test: True 637 test_port.test_isfile = lambda test: True 638 639 test_config = test_port.test_configuration() 640 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win Release ] failures/expected/foo.html [ Failure ] 641 # This comment line should not get stripped. 642 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 643 """} 644 expectations = TestExpectations(test_port) 645 646 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 647 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) 648 649 self.assertEqual(""" # This comment line should not get stripped. 650 Bug(y) [ Win Debug ] failures/expected/foo.html [ Crash ] 651 """, actual_expectations) 652 653 def test_remove_flaky_line(self): 654 host = MockHost() 655 test_port = host.port_factory.get('test-win-xp', None) 656 test_port.test_exists = lambda test: True 657 test_port.test_isfile = lambda test: True 658 659 test_config = test_port.test_configuration() 660 test_port.expectations_dict = lambda: {'expectations': """Bug(x) [ Win ] failures/expected/foo.html [ Failure Timeout ] 661 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] 662 """} 663 expectations = TestExpectations(test_port) 664 665 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', test_config) 666 actual_expectations = expectations.remove_configuration_from_test('failures/expected/foo.html', host.port_factory.get('test-win-win7', None).test_configuration()) 667 668 self.assertEqual("""Bug(x) [ Win Debug ] failures/expected/foo.html [ Failure Timeout ] 669 Bug(y) [ Mac ] failures/expected/foo.html [ Crash ] 670 """, actual_expectations) 671 672 673 class RebaseliningTest(Base): 674 def test_get_rebaselining_failures(self): 675 # Make sure we find a test as needing a rebaseline even if it is not marked as a failure. 676 self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n') 677 self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) 678 679 self.parse_exp(self.get_basic_expectations()) 680 self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) 681 682 683 class TestExpectationsParserTests(unittest.TestCase): 684 def __init__(self, testFunc): 685 host = MockHost() 686 test_port = host.port_factory.get('test-win-xp', None) 687 self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros()) 688 unittest.TestCase.__init__(self, testFunc) 689 self._parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], is_lint_mode=False) 690 691 def test_expectation_line_for_test(self): 692 # This is kind of a silly test, but it at least ensures that we don't throw an error. 693 test_name = 'foo/test.html' 694 expectations = set(["PASS", "IMAGE"]) 695 696 expectation_line = TestExpectationLine() 697 expectation_line.original_string = test_name 698 expectation_line.name = test_name 699 expectation_line.filename = '<Bot TestExpectations>' 700 expectation_line.line_numbers = '0' 701 expectation_line.expectations = expectations 702 self._parser._parse_line(expectation_line) 703 704 self.assertEqual(self._parser.expectation_line_for_test(test_name, expectations), expectation_line) 705 706 707 class TestExpectationSerializationTests(unittest.TestCase): 708 def __init__(self, testFunc): 709 host = MockHost() 710 test_port = host.port_factory.get('test-win-xp', None) 711 self._converter = TestConfigurationConverter(test_port.all_test_configurations(), test_port.configuration_specifier_macros()) 712 unittest.TestCase.__init__(self, testFunc) 713 714 def _tokenize(self, line): 715 return TestExpectationParser._tokenize_line('path', line, 0) 716 717 def assert_round_trip(self, in_string, expected_string=None): 718 expectation = self._tokenize(in_string) 719 if expected_string is None: 720 expected_string = in_string 721 self.assertEqual(expected_string, expectation.to_string(self._converter)) 722 723 def assert_list_round_trip(self, in_string, expected_string=None): 724 host = MockHost() 725 parser = TestExpectationParser(host.port_factory.get('test-win-xp', None), [], is_lint_mode=False) 726 expectations = parser.parse('path', in_string) 727 if expected_string is None: 728 expected_string = in_string 729 self.assertEqual(expected_string, TestExpectations.list_to_string(expectations, self._converter)) 730 731 def test_unparsed_to_string(self): 732 expectation = TestExpectationLine() 733 734 self.assertEqual(expectation.to_string(self._converter), '') 735 expectation.comment = ' Qux.' 736 self.assertEqual(expectation.to_string(self._converter), '# Qux.') 737 expectation.name = 'bar' 738 self.assertEqual(expectation.to_string(self._converter), 'bar # Qux.') 739 expectation.specifiers = ['foo'] 740 # FIXME: case should be preserved here but we can't until we drop the old syntax. 741 self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar # Qux.') 742 expectation.expectations = ['bAz'] 743 self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ ] # Qux.') 744 expectation.expectations = ['bAz1', 'baZ2'] 745 self.assertEqual(expectation.to_string(self._converter), '[ FOO ] bar [ BAZ1 BAZ2 ] # Qux.') 746 expectation.specifiers = ['foo1', 'foO2'] 747 self.assertEqual(expectation.to_string(self._converter), '[ FOO1 FOO2 ] bar [ BAZ1 BAZ2 ] # Qux.') 748 expectation.warnings.append('Oh the horror.') 749 self.assertEqual(expectation.to_string(self._converter), '') 750 expectation.original_string = 'Yes it is!' 751 self.assertEqual(expectation.to_string(self._converter), 'Yes it is!') 752 753 def test_unparsed_list_to_string(self): 754 expectation = TestExpectationLine() 755 expectation.comment = 'Qux.' 756 expectation.name = 'bar' 757 expectation.specifiers = ['foo'] 758 expectation.expectations = ['bAz1', 'baZ2'] 759 # FIXME: case should be preserved here but we can't until we drop the old syntax. 760 self.assertEqual(TestExpectations.list_to_string([expectation]), '[ FOO ] bar [ BAZ1 BAZ2 ] #Qux.') 761 762 def test_parsed_to_string(self): 763 expectation_line = TestExpectationLine() 764 expectation_line.bugs = ['Bug(x)'] 765 expectation_line.name = 'test/name/for/realz.html' 766 expectation_line.parsed_expectations = set([IMAGE]) 767 self.assertEqual(expectation_line.to_string(self._converter), None) 768 expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release')]) 769 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP Release ] test/name/for/realz.html [ ImageOnlyFailure ]') 770 expectation_line.matching_configurations = set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]) 771 self.assertEqual(expectation_line.to_string(self._converter), 'Bug(x) [ XP ] test/name/for/realz.html [ ImageOnlyFailure ]') 772 773 def test_serialize_parsed_expectations(self): 774 expectation_line = TestExpectationLine() 775 expectation_line.parsed_expectations = set([]) 776 parsed_expectation_to_string = dict([[parsed_expectation, expectation_string] for expectation_string, parsed_expectation in TestExpectations.EXPECTATIONS.items()]) 777 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), '') 778 expectation_line.parsed_expectations = set([FAIL]) 779 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'fail') 780 expectation_line.parsed_expectations = set([PASS, IMAGE]) 781 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'image pass') 782 expectation_line.parsed_expectations = set([FAIL, PASS]) 783 self.assertEqual(expectation_line._serialize_parsed_expectations(parsed_expectation_to_string), 'pass fail') 784 785 def test_serialize_parsed_specifier_string(self): 786 expectation_line = TestExpectationLine() 787 expectation_line.bugs = ['garden-o-matic'] 788 expectation_line.parsed_specifiers = ['the', 'for'] 789 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, []), 'for the') 790 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, ['win']), 'for the win') 791 expectation_line.bugs = [] 792 expectation_line.parsed_specifiers = [] 793 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, []), '') 794 self.assertEqual(expectation_line._serialize_parsed_specifiers(self._converter, ['win']), 'win') 795 796 def test_format_line(self): 797 self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'name', ['EXPECTATIONS'], 'comment'), '[ MODIFIERS ] name [ EXPECTATIONS ] #comment') 798 self.assertEqual(TestExpectationLine._format_line([], ['MODIFIERS'], 'name', ['EXPECTATIONS'], None), '[ MODIFIERS ] name [ EXPECTATIONS ]') 799 800 def test_string_roundtrip(self): 801 self.assert_round_trip('') 802 self.assert_round_trip('[') 803 self.assert_round_trip('FOO [') 804 self.assert_round_trip('FOO ] bar') 805 self.assert_round_trip(' FOO [') 806 self.assert_round_trip(' [ FOO ] ') 807 self.assert_round_trip('[ FOO ] bar [ BAZ ]') 808 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.') 809 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.') 810 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux. ') 811 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux. ') 812 self.assert_round_trip('[ FOO ] ] ] bar BAZ') 813 self.assert_round_trip('[ FOO ] ] ] bar [ BAZ ]') 814 self.assert_round_trip('FOO ] ] bar ==== BAZ') 815 self.assert_round_trip('=') 816 self.assert_round_trip('#') 817 self.assert_round_trip('# ') 818 self.assert_round_trip('# Foo') 819 self.assert_round_trip('# Foo') 820 self.assert_round_trip('# Foo :') 821 self.assert_round_trip('# Foo : =') 822 823 def test_list_roundtrip(self): 824 self.assert_list_round_trip('') 825 self.assert_list_round_trip('\n') 826 self.assert_list_round_trip('\n\n') 827 self.assert_list_round_trip('bar') 828 self.assert_list_round_trip('bar\n# Qux.') 829 self.assert_list_round_trip('bar\n# Qux.\n') 830 831 def test_reconstitute_only_these(self): 832 lines = [] 833 reconstitute_only_these = [] 834 835 def add_line(matching_configurations, reconstitute): 836 expectation_line = TestExpectationLine() 837 expectation_line.original_string = "Nay" 838 expectation_line.bugs = ['Bug(x)'] 839 expectation_line.name = 'Yay' 840 expectation_line.parsed_expectations = set([IMAGE]) 841 expectation_line.matching_configurations = matching_configurations 842 lines.append(expectation_line) 843 if reconstitute: 844 reconstitute_only_these.append(expectation_line) 845 846 add_line(set([TestConfiguration('xp', 'x86', 'release')]), True) 847 add_line(set([TestConfiguration('xp', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug')]), False) 848 serialized = TestExpectations.list_to_string(lines, self._converter) 849 self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nBug(x) [ XP ] Yay [ ImageOnlyFailure ]") 850 serialized = TestExpectations.list_to_string(lines, self._converter, reconstitute_only_these=reconstitute_only_these) 851 self.assertEqual(serialized, "Bug(x) [ XP Release ] Yay [ ImageOnlyFailure ]\nNay") 852 853 def disabled_test_string_whitespace_stripping(self): 854 # FIXME: Re-enable this test once we rework the code to no longer support the old syntax. 855 self.assert_round_trip('\n', '') 856 self.assert_round_trip(' [ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') 857 self.assert_round_trip('[ FOO ] bar [ BAZ ]', '[ FOO ] bar [ BAZ ]') 858 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar [ BAZ ] # Qux.') 859 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar [ BAZ ] # Qux.') 860 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar [ BAZ ] # Qux.') 861 self.assert_round_trip('[ FOO ] bar [ BAZ ] # Qux.', '[ FOO ] bar [ BAZ ] # Qux.') 862
