1 #!/usr/bin/python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """Unit tests for test_expectations.py.""" 31 32 import unittest 33 34 from webkitpy.layout_tests import port 35 from webkitpy.layout_tests.port import base 36 from webkitpy.layout_tests.layout_package.test_expectations import * 37 38 class FunctionsTest(unittest.TestCase): 39 def test_result_was_expected(self): 40 # test basics 41 self.assertEquals(result_was_expected(PASS, set([PASS]), 42 False, False), True) 43 self.assertEquals(result_was_expected(TEXT, set([PASS]), 44 False, False), False) 45 46 # test handling of FAIL expectations 47 self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]), 48 False, False), True) 49 self.assertEquals(result_was_expected(IMAGE, set([FAIL]), 50 False, False), True) 51 self.assertEquals(result_was_expected(TEXT, set([FAIL]), 52 False, False), True) 53 self.assertEquals(result_was_expected(CRASH, set([FAIL]), 54 False, False), False) 55 56 # test handling of SKIPped tests and results 57 self.assertEquals(result_was_expected(SKIP, set([CRASH]), 58 False, True), True) 59 self.assertEquals(result_was_expected(SKIP, set([CRASH]), 60 False, False), False) 61 62 # test handling of MISSING results and the REBASELINE modifier 63 self.assertEquals(result_was_expected(MISSING, set([PASS]), 64 True, False), True) 65 self.assertEquals(result_was_expected(MISSING, set([PASS]), 66 False, False), False) 67 68 def test_remove_pixel_failures(self): 69 self.assertEquals(remove_pixel_failures(set([TEXT])), 70 set([TEXT])) 71 self.assertEquals(remove_pixel_failures(set([PASS])), 72 set([PASS])) 73 self.assertEquals(remove_pixel_failures(set([IMAGE])), 74 set([PASS])) 75 self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])), 76 set([TEXT])) 77 self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])), 78 set([PASS, CRASH])) 79 80 81 class Base(unittest.TestCase): 82 # Note that all of these tests are written assuming the configuration 83 # being tested is Windows XP, Release build. 84 85 def __init__(self, testFunc, setUp=None, tearDown=None, description=None): 86 self._port = port.get('test-win-xp', None) 87 self._fs = self._port._filesystem 88 self._exp = None 89 unittest.TestCase.__init__(self, testFunc) 90 91 def get_test(self, test_name): 92 return self._fs.join(self._port.layout_tests_dir(), test_name) 93 94 def get_basic_tests(self): 95 return [self.get_test('failures/expected/text.html'), 96 self.get_test('failures/expected/image_checksum.html'), 97 self.get_test('failures/expected/crash.html'), 98 self.get_test('failures/expected/missing_text.html'), 99 self.get_test('failures/expected/image.html'), 100 self.get_test('passes/text.html')] 101 102 def get_basic_expectations(self): 103 return """ 104 BUG_TEST : failures/expected/text.html = TEXT 105 BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH 106 BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING 107 BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE 108 BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE 109 """ 110 111 def parse_exp(self, expectations, overrides=None, is_lint_mode=False): 112 test_config = self._port.test_configuration() 113 self._exp = TestExpectations(self._port, 114 tests=self.get_basic_tests(), 115 expectations=expectations, 116 test_config=test_config, 117 is_lint_mode=is_lint_mode, 118 overrides=overrides) 119 120 def assert_exp(self, test, result): 121 self.assertEquals(self._exp.get_expectations(self.get_test(test)), 122 set([result])) 123 124 125 class BasicTests(Base): 126 def test_basic(self): 127 self.parse_exp(self.get_basic_expectations()) 128 self.assert_exp('failures/expected/text.html', TEXT) 129 self.assert_exp('failures/expected/image_checksum.html', IMAGE) 130 self.assert_exp('passes/text.html', PASS) 131 self.assert_exp('failures/expected/image.html', PASS) 132 133 134 class MiscTests(Base): 135 def test_multiple_results(self): 136 self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH') 137 self.assertEqual(self._exp.get_expectations( 138 self.get_test('failures/expected/text.html')), 139 set([TEXT, CRASH])) 140 141 def test_category_expectations(self): 142 # This test checks unknown tests are not present in the 143 # expectations and that known test part of a test category is 144 # present in the expectations. 145 exp_str = """ 146 BUGX WONTFIX : failures/expected = IMAGE 147 """ 148 self.parse_exp(exp_str) 149 test_name = 'failures/expected/unknown-test.html' 150 unknown_test = self.get_test(test_name) 151 self.assertRaises(KeyError, self._exp.get_expectations, 152 unknown_test) 153 self.assert_exp('failures/expected/crash.html', IMAGE) 154 155 def test_get_options(self): 156 self.parse_exp(self.get_basic_expectations()) 157 self.assertEqual(self._exp.get_options( 158 self.get_test('passes/text.html')), []) 159 160 def test_expectations_json_for_all_platforms(self): 161 self.parse_exp(self.get_basic_expectations()) 162 json_str = self._exp.get_expectations_json_for_all_platforms() 163 # FIXME: test actual content? 164 self.assertTrue(json_str) 165 166 def test_get_expectations_string(self): 167 self.parse_exp(self.get_basic_expectations()) 168 self.assertEquals(self._exp.get_expectations_string( 169 self.get_test('failures/expected/text.html')), 170 'TEXT') 171 172 def test_expectation_to_string(self): 173 # Normal cases are handled by other tests. 174 self.parse_exp(self.get_basic_expectations()) 175 self.assertRaises(ValueError, self._exp.expectation_to_string, 176 -1) 177 178 def test_get_test_set(self): 179 # Handle some corner cases for this routine not covered by other tests. 180 self.parse_exp(self.get_basic_expectations()) 181 s = self._exp._expected_failures.get_test_set(WONTFIX) 182 self.assertEqual(s, 183 set([self.get_test('failures/expected/crash.html'), 184 self.get_test('failures/expected/image_checksum.html')])) 185 s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH) 186 self.assertEqual(s, 187 set([self.get_test('failures/expected/crash.html')])) 188 s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH, 189 include_skips=False) 190 self.assertEqual(s, set([])) 191 192 def test_parse_error_fatal(self): 193 try: 194 self.parse_exp("""FOO : failures/expected/text.html = TEXT 195 SKIP : failures/expected/image.html""") 196 self.assertFalse(True, "ParseError wasn't raised") 197 except ParseError, e: 198 self.assertTrue(e.fatal) 199 exp_errors = [u"Line:1 Unrecognized option 'foo' failures/expected/text.html", 200 u"Line:2 Missing expectations. [' failures/expected/image.html']"] 201 self.assertEqual(str(e), '\n'.join(map(str, exp_errors))) 202 self.assertEqual(e.errors, exp_errors) 203 204 def test_parse_error_nonfatal(self): 205 try: 206 self.parse_exp('SKIP : failures/expected/text.html = TEXT', 207 is_lint_mode=True) 208 self.assertFalse(True, "ParseError wasn't raised") 209 except ParseError, e: 210 self.assertFalse(e.fatal) 211 exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html'] 212 self.assertEqual(str(e), '\n'.join(map(str, exp_errors))) 213 self.assertEqual(e.errors, exp_errors) 214 215 def test_overrides(self): 216 self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT", 217 "BUG_OVERRIDE : failures/expected/text.html = IMAGE") 218 self.assert_exp('failures/expected/text.html', IMAGE) 219 220 def test_overrides__duplicate(self): 221 self.assertRaises(ParseError, self.parse_exp, 222 "BUG_EXP: failures/expected/text.html = TEXT", 223 """ 224 BUG_OVERRIDE : failures/expected/text.html = IMAGE 225 BUG_OVERRIDE : failures/expected/text.html = CRASH 226 """) 227 228 def test_pixel_tests_flag(self): 229 def match(test, result, pixel_tests_enabled): 230 return self._exp.matches_an_expected_result( 231 self.get_test(test), result, pixel_tests_enabled) 232 233 self.parse_exp(self.get_basic_expectations()) 234 self.assertTrue(match('failures/expected/text.html', TEXT, True)) 235 self.assertTrue(match('failures/expected/text.html', TEXT, False)) 236 self.assertFalse(match('failures/expected/text.html', CRASH, True)) 237 self.assertFalse(match('failures/expected/text.html', CRASH, False)) 238 self.assertTrue(match('failures/expected/image_checksum.html', IMAGE, 239 True)) 240 self.assertTrue(match('failures/expected/image_checksum.html', PASS, 241 False)) 242 self.assertTrue(match('failures/expected/crash.html', SKIP, False)) 243 self.assertTrue(match('passes/text.html', PASS, False)) 244 245 def test_more_specific_override_resets_skip(self): 246 self.parse_exp("BUGX SKIP : failures/expected = TEXT\n" 247 "BUGX : failures/expected/text.html = IMAGE\n") 248 self.assert_exp('failures/expected/text.html', IMAGE) 249 self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(), 250 'failures/expected/text.html') in 251 self._exp.get_tests_with_result_type(SKIP)) 252 253 class ExpectationSyntaxTests(Base): 254 def test_missing_expectation(self): 255 # This is missing the expectation. 256 self.assertRaises(ParseError, self.parse_exp, 257 'BUG_TEST: failures/expected/text.html') 258 259 def test_missing_colon(self): 260 # This is missing the modifiers and the ':' 261 self.assertRaises(ParseError, self.parse_exp, 262 'failures/expected/text.html = TEXT') 263 264 def disabled_test_too_many_colons(self): 265 # FIXME: Enable this test and fix the underlying bug. 266 self.assertRaises(ParseError, self.parse_exp, 267 'BUG_TEST: failures/expected/text.html = PASS :') 268 269 def test_too_many_equals_signs(self): 270 self.assertRaises(ParseError, self.parse_exp, 271 'BUG_TEST: failures/expected/text.html = TEXT = IMAGE') 272 273 def test_unrecognized_expectation(self): 274 self.assertRaises(ParseError, self.parse_exp, 275 'BUG_TEST: failures/expected/text.html = UNKNOWN') 276 277 def test_macro(self): 278 exp_str = """ 279 BUG_TEST WIN-XP : failures/expected/text.html = TEXT 280 """ 281 self.parse_exp(exp_str) 282 self.assert_exp('failures/expected/text.html', TEXT) 283 284 285 class SemanticTests(Base): 286 def test_bug_format(self): 287 self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT') 288 289 def test_missing_bugid(self): 290 # This should log a non-fatal error. 291 self.parse_exp('SLOW : failures/expected/text.html = TEXT') 292 self.assertEqual( 293 len(self._exp._expected_failures.get_non_fatal_errors()), 1) 294 295 def test_slow_and_timeout(self): 296 # A test cannot be SLOW and expected to TIMEOUT. 297 self.assertRaises(ParseError, self.parse_exp, 298 'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT') 299 300 def test_rebaseline(self): 301 # Can't lint a file w/ 'REBASELINE' in it. 302 self.assertRaises(ParseError, self.parse_exp, 303 'BUG_TEST REBASELINE : failures/expected/text.html = TEXT', 304 is_lint_mode=True) 305 306 def test_duplicates(self): 307 self.assertRaises(ParseError, self.parse_exp, """ 308 BUG_EXP : failures/expected/text.html = TEXT 309 BUG_EXP : failures/expected/text.html = IMAGE""") 310 311 self.assertRaises(ParseError, self.parse_exp, 312 self.get_basic_expectations(), overrides=""" 313 BUG_OVERRIDE : failures/expected/text.html = TEXT 314 BUG_OVERRIDE : failures/expected/text.html = IMAGE""", ) 315 316 def test_missing_file(self): 317 # This should log a non-fatal error. 318 self.parse_exp('BUG_TEST : missing_file.html = TEXT') 319 self.assertEqual( 320 len(self._exp._expected_failures.get_non_fatal_errors()), 1) 321 322 323 class PrecedenceTests(Base): 324 def test_file_over_directory(self): 325 # This tests handling precedence of specific lines over directories 326 # and tests expectations covering entire directories. 327 exp_str = """ 328 BUGX : failures/expected/text.html = TEXT 329 BUGX WONTFIX : failures/expected = IMAGE 330 """ 331 self.parse_exp(exp_str) 332 self.assert_exp('failures/expected/text.html', TEXT) 333 self.assert_exp('failures/expected/crash.html', IMAGE) 334 335 exp_str = """ 336 BUGX WONTFIX : failures/expected = IMAGE 337 BUGX : failures/expected/text.html = TEXT 338 """ 339 self.parse_exp(exp_str) 340 self.assert_exp('failures/expected/text.html', TEXT) 341 self.assert_exp('failures/expected/crash.html', IMAGE) 342 343 def test_ambiguous(self): 344 self.assertRaises(ParseError, self.parse_exp, """ 345 BUG_TEST RELEASE : passes/text.html = PASS 346 BUG_TEST WIN : passes/text.html = FAIL 347 """) 348 349 def test_more_modifiers(self): 350 exp_str = """ 351 BUG_TEST RELEASE : passes/text.html = PASS 352 BUG_TEST WIN RELEASE : passes/text.html = TEXT 353 """ 354 self.assertRaises(ParseError, self.parse_exp, exp_str) 355 356 def test_order_in_file(self): 357 exp_str = """ 358 BUG_TEST WIN RELEASE : passes/text.html = TEXT 359 BUG_TEST RELEASE : passes/text.html = PASS 360 """ 361 self.assertRaises(ParseError, self.parse_exp, exp_str) 362 363 def test_version_overrides(self): 364 exp_str = """ 365 BUG_TEST WIN : passes/text.html = PASS 366 BUG_TEST WIN XP : passes/text.html = TEXT 367 """ 368 self.assertRaises(ParseError, self.parse_exp, exp_str) 369 370 def test_macro_overrides(self): 371 exp_str = """ 372 BUG_TEST WIN : passes/text.html = PASS 373 BUG_TEST WIN-XP : passes/text.html = TEXT 374 """ 375 self.assertRaises(ParseError, self.parse_exp, exp_str) 376 377 378 class RebaseliningTest(Base): 379 """Test rebaselining-specific functionality.""" 380 def assertRemove(self, input_expectations, tests, expected_expectations): 381 self.parse_exp(input_expectations) 382 actual_expectations = self._exp.remove_rebaselined_tests(tests) 383 self.assertEqual(expected_expectations, actual_expectations) 384 385 def test_remove(self): 386 self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n' 387 'BUGY : failures/expected/image.html = IMAGE\n' 388 'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n', 389 ['failures/expected/text.html'], 390 'BUGY : failures/expected/image.html = IMAGE\n' 391 'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n') 392 393 def test_no_get_rebaselining_failures(self): 394 self.parse_exp(self.get_basic_expectations()) 395 self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) 396 397 398 class ModifierTests(unittest.TestCase): 399 def setUp(self): 400 port_obj = port.get('test-win-xp', None) 401 self.config = port_obj.test_configuration() 402 self.matcher = ModifierMatcher(self.config) 403 404 def match(self, modifiers, expected_num_matches=-1, values=None, num_errors=0): 405 matcher = self.matcher 406 if values: 407 matcher = ModifierMatcher(self.FakeTestConfiguration(values)) 408 match_result = matcher.match(modifiers) 409 self.assertEqual(len(match_result.warnings), 0) 410 self.assertEqual(len(match_result.errors), num_errors) 411 self.assertEqual(match_result.num_matches, expected_num_matches, 412 'match(%s, %s) returned -> %d, expected %d' % 413 (modifiers, str(self.config.values()), 414 match_result.num_matches, expected_num_matches)) 415 416 def test_bad_match_modifier(self): 417 self.match(['foo'], num_errors=1) 418 419 def test_none(self): 420 self.match([], 0) 421 422 def test_one(self): 423 self.match(['xp'], 1) 424 self.match(['win'], 1) 425 self.match(['release'], 1) 426 self.match(['cpu'], 1) 427 self.match(['x86'], 1) 428 self.match(['leopard'], -1) 429 self.match(['gpu'], -1) 430 self.match(['debug'], -1) 431 432 def test_two(self): 433 self.match(['xp', 'release'], 2) 434 self.match(['win7', 'release'], -1) 435 self.match(['win7', 'xp'], 1) 436 437 def test_three(self): 438 self.match(['win7', 'xp', 'release'], 2) 439 self.match(['xp', 'debug', 'x86'], -1) 440 self.match(['xp', 'release', 'x86'], 3) 441 self.match(['xp', 'cpu', 'release'], 3) 442 443 def test_four(self): 444 self.match(['xp', 'release', 'cpu', 'x86'], 4) 445 self.match(['win7', 'xp', 'release', 'cpu'], 3) 446 self.match(['win7', 'xp', 'debug', 'cpu'], -1) 447 448 def test_case_insensitivity(self): 449 self.match(['Win'], num_errors=1) 450 self.match(['WIN'], num_errors=1) 451 self.match(['win'], 1) 452 453 def test_duplicates(self): 454 self.match(['release', 'release'], num_errors=1) 455 self.match(['win-xp', 'xp'], num_errors=1) 456 self.match(['win-xp', 'win-xp'], num_errors=1) 457 self.match(['xp', 'release', 'xp', 'release'], num_errors=2) 458 self.match(['rebaseline', 'rebaseline'], num_errors=1) 459 460 def test_unknown_option(self): 461 self.match(['vms'], num_errors=1) 462 463 def test_duplicate_bugs(self): 464 # BUG* regexes can appear multiple times. 465 self.match(['bugfoo', 'bugbar'], 0) 466 467 def test_invalid_combinations(self): 468 # FIXME: This should probably raise an error instead of NO_MATCH. 469 self.match(['mac', 'xp'], num_errors=0) 470 471 def test_regexes_are_ignored(self): 472 self.match(['bug123xy', 'rebaseline', 'wontfix', 'slow', 'skip'], 0) 473 474 def test_none_is_invalid(self): 475 self.match(['none'], num_errors=1) 476 477 478 if __name__ == '__main__': 479 unittest.main() 480
