1 #!/usr/bin/env python 2 # Copyright (C) 2010 Google Inc. All rights reserved. 3 # 4 # Redistribution and use in source and binary forms, with or without 5 # modification, are permitted provided that the following conditions are 6 # met: 7 # 8 # * Redistributions of source code must retain the above copyright 9 # notice, this list of conditions and the following disclaimer. 10 # * Redistributions in binary form must reproduce the above 11 # copyright notice, this list of conditions and the following disclaimer 12 # in the documentation and/or other materials provided with the 13 # distribution. 14 # * Neither the name of Google Inc. nor the names of its 15 # contributors may be used to endorse or promote products derived from 16 # this software without specific prior written permission. 17 # 18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 """A helper class for reading in and dealing with tests expectations 31 for layout tests. 32 """ 33 34 import itertools 35 import logging 36 import re 37 38 import webkitpy.thirdparty.simplejson as simplejson 39 40 _log = logging.getLogger("webkitpy.layout_tests.layout_package." 41 "test_expectations") 42 43 # Test expectation and modifier constants. 44 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX, 45 SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(16) 46 47 # Test expectation file update action constants 48 (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4) 49 50 51 def result_was_expected(result, expected_results, test_needs_rebaselining, 52 test_is_skipped): 53 """Returns whether we got a result we were expecting. 54 Args: 55 result: actual result of a test execution 56 expected_results: set of results listed in test_expectations 57 test_needs_rebaselining: whether test was marked as REBASELINE 58 test_is_skipped: whether test was marked as SKIP""" 59 if result in expected_results: 60 return True 61 if result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and FAIL in expected_results: 62 return True 63 if result == MISSING and test_needs_rebaselining: 64 return True 65 if result == SKIP and test_is_skipped: 66 return True 67 return False 68 69 70 def remove_pixel_failures(expected_results): 71 """Returns a copy of the expected results for a test, except that we 72 drop any pixel failures and return the remaining expectations. For example, 73 if we're not running pixel tests, then tests expected to fail as IMAGE 74 will PASS.""" 75 expected_results = expected_results.copy() 76 if IMAGE in expected_results: 77 expected_results.remove(IMAGE) 78 expected_results.add(PASS) 79 if IMAGE_PLUS_TEXT in expected_results: 80 expected_results.remove(IMAGE_PLUS_TEXT) 81 expected_results.add(TEXT) 82 return expected_results 83 84 85 class TestExpectations: 86 TEST_LIST = "test_expectations.txt" 87 88 def __init__(self, port, tests, expectations, test_config, 89 is_lint_mode, overrides=None): 90 """Loads and parses the test expectations given in the string. 91 Args: 92 port: handle to object containing platform-specific functionality 93 tests: list of all of the test files 94 expectations: test expectations as a string 95 test_config: specific values to check against when 96 parsing the file (usually port.test_config(), 97 but may be different when linting or doing other things). 98 is_lint_mode: If True, just parse the expectations string 99 looking for errors. 100 overrides: test expectations that are allowed to override any 101 entries in |expectations|. This is used by callers 102 that need to manage two sets of expectations (e.g., upstream 103 and downstream expectations). 104 """ 105 self._expected_failures = TestExpectationsFile(port, expectations, 106 tests, test_config, is_lint_mode, 107 overrides=overrides) 108 109 # TODO(ojan): Allow for removing skipped tests when getting the list of 110 # tests to run, but not when getting metrics. 111 # TODO(ojan): Replace the Get* calls here with the more sane API exposed 112 # by TestExpectationsFile below. Maybe merge the two classes entirely? 113 114 def get_expectations_json_for_all_platforms(self): 115 return ( 116 self._expected_failures.get_expectations_json_for_all_platforms()) 117 118 def get_rebaselining_failures(self): 119 return (self._expected_failures.get_test_set(REBASELINE, FAIL) | 120 self._expected_failures.get_test_set(REBASELINE, IMAGE) | 121 self._expected_failures.get_test_set(REBASELINE, TEXT) | 122 self._expected_failures.get_test_set(REBASELINE, 123 IMAGE_PLUS_TEXT) | 124 self._expected_failures.get_test_set(REBASELINE, AUDIO)) 125 126 def get_options(self, test): 127 return self._expected_failures.get_options(test) 128 129 def get_expectations(self, test): 130 return self._expected_failures.get_expectations(test) 131 132 def get_expectations_string(self, test): 133 """Returns the expectatons for the given test as an uppercase string. 134 If there are no expectations for the test, then "PASS" is returned.""" 135 expectations = self.get_expectations(test) 136 retval = [] 137 138 for expectation in expectations: 139 retval.append(self.expectation_to_string(expectation)) 140 141 return " ".join(retval) 142 143 def expectation_to_string(self, expectation): 144 """Return the uppercased string equivalent of a given expectation.""" 145 for item in TestExpectationsFile.EXPECTATIONS.items(): 146 if item[1] == expectation: 147 return item[0].upper() 148 raise ValueError(expectation) 149 150 def get_tests_with_result_type(self, result_type): 151 return self._expected_failures.get_tests_with_result_type(result_type) 152 153 def get_tests_with_timeline(self, timeline): 154 return self._expected_failures.get_tests_with_timeline(timeline) 155 156 def matches_an_expected_result(self, test, result, 157 pixel_tests_are_enabled): 158 expected_results = self._expected_failures.get_expectations(test) 159 if not pixel_tests_are_enabled: 160 expected_results = remove_pixel_failures(expected_results) 161 return result_was_expected(result, expected_results, 162 self.is_rebaselining(test), self.has_modifier(test, SKIP)) 163 164 def is_rebaselining(self, test): 165 return self._expected_failures.has_modifier(test, REBASELINE) 166 167 def has_modifier(self, test, modifier): 168 return self._expected_failures.has_modifier(test, modifier) 169 170 def remove_rebaselined_tests(self, tests): 171 return self._expected_failures.remove_rebaselined_tests(tests) 172 173 174 def strip_comments(line): 175 """Strips comments from a line and return None if the line is empty 176 or else the contents of line with leading and trailing spaces removed 177 and all other whitespace collapsed""" 178 179 commentIndex = line.find('//') 180 if commentIndex is -1: 181 commentIndex = len(line) 182 183 line = re.sub(r'\s+', ' ', line[:commentIndex].strip()) 184 if line == '': 185 return None 186 else: 187 return line 188 189 190 class ParseError(Exception): 191 def __init__(self, fatal, errors): 192 self.fatal = fatal 193 self.errors = errors 194 195 def __str__(self): 196 return '\n'.join(map(str, self.errors)) 197 198 def __repr__(self): 199 return 'ParseError(fatal=%s, errors=%s)' % (self.fatal, self.errors) 200 201 202 class ModifiersAndExpectations: 203 """A holder for modifiers and expectations on a test that serializes to 204 JSON.""" 205 206 def __init__(self, modifiers, expectations): 207 self.modifiers = modifiers 208 self.expectations = expectations 209 210 211 class ExpectationsJsonEncoder(simplejson.JSONEncoder): 212 """JSON encoder that can handle ModifiersAndExpectations objects.""" 213 def default(self, obj): 214 # A ModifiersAndExpectations object has two fields, each of which 215 # is a dict. Since JSONEncoders handle all the builtin types directly, 216 # the only time this routine should be called is on the top level 217 # object (i.e., the encoder shouldn't recurse). 218 assert isinstance(obj, ModifiersAndExpectations) 219 return {"modifiers": obj.modifiers, 220 "expectations": obj.expectations} 221 222 223 class TestExpectationsFile: 224 """Test expectation files consist of lines with specifications of what 225 to expect from layout test cases. The test cases can be directories 226 in which case the expectations apply to all test cases in that 227 directory and any subdirectory. The format of the file is along the 228 lines of: 229 230 LayoutTests/fast/js/fixme.js = FAIL 231 LayoutTests/fast/js/flaky.js = FAIL PASS 232 LayoutTests/fast/js/crash.js = CRASH TIMEOUT FAIL PASS 233 ... 234 235 To add other options: 236 SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 237 DEBUG : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 238 DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 239 LINUX DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 240 LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 241 242 SKIP: Doesn't run the test. 243 SLOW: The test takes a long time to run, but does not timeout indefinitely. 244 WONTFIX: For tests that we never intend to pass on a given platform. 245 246 Notes: 247 -A test cannot be both SLOW and TIMEOUT 248 -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, AUDIO, or FAIL. 249 FAIL is a legacy value that currently means either IMAGE, 250 TEXT, or IMAGE+TEXT. Once we have finished migrating the expectations, 251 we should change FAIL to have the meaning of IMAGE+TEXT and remove the 252 IMAGE+TEXT identifier. 253 -A test can be included twice, but not via the same path. 254 -If a test is included twice, then the more precise path wins. 255 -CRASH tests cannot be WONTFIX 256 """ 257 258 EXPECTATIONS = {'pass': PASS, 259 'fail': FAIL, 260 'text': TEXT, 261 'image': IMAGE, 262 'image+text': IMAGE_PLUS_TEXT, 263 'audio': AUDIO, 264 'timeout': TIMEOUT, 265 'crash': CRASH, 266 'missing': MISSING} 267 268 EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped'), 269 PASS: ('pass', 'passes'), 270 FAIL: ('failure', 'failures'), 271 TEXT: ('text diff mismatch', 272 'text diff mismatch'), 273 IMAGE: ('image mismatch', 'image mismatch'), 274 IMAGE_PLUS_TEXT: ('image and text mismatch', 275 'image and text mismatch'), 276 AUDIO: ('audio mismatch', 'audio mismatch'), 277 CRASH: ('DumpRenderTree crash', 278 'DumpRenderTree crashes'), 279 TIMEOUT: ('test timed out', 'tests timed out'), 280 MISSING: ('no expected result found', 281 'no expected results found')} 282 283 EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT, 284 TEXT, IMAGE, AUDIO, FAIL, SKIP) 285 286 BUILD_TYPES = ('debug', 'release') 287 288 MODIFIERS = {'skip': SKIP, 289 'wontfix': WONTFIX, 290 'slow': SLOW, 291 'rebaseline': REBASELINE, 292 'none': NONE} 293 294 TIMELINES = {'wontfix': WONTFIX, 295 'now': NOW} 296 297 RESULT_TYPES = {'skip': SKIP, 298 'pass': PASS, 299 'fail': FAIL, 300 'flaky': FLAKY} 301 302 def __init__(self, port, expectations, full_test_list, 303 test_config, is_lint_mode, overrides=None): 304 # See argument documentation in TestExpectation(), above. 305 306 self._port = port 307 self._fs = port._filesystem 308 self._expectations = expectations 309 self._full_test_list = full_test_list 310 self._test_config = test_config 311 self._is_lint_mode = is_lint_mode 312 self._overrides = overrides 313 self._errors = [] 314 self._non_fatal_errors = [] 315 316 # Maps relative test paths as listed in the expectations file to a 317 # list of maps containing modifiers and expectations for each time 318 # the test is listed in the expectations file. We use this to 319 # keep a representation of the entire list of expectations, even 320 # invalid ones. 321 self._all_expectations = {} 322 323 # Maps a test to its list of expectations. 324 self._test_to_expectations = {} 325 326 # Maps a test to its list of options (string values) 327 self._test_to_options = {} 328 329 # Maps a test to its list of modifiers: the constants associated with 330 # the options minus any bug or platform strings 331 self._test_to_modifiers = {} 332 333 # Maps a test to the base path that it was listed with in the list and 334 # the number of matches that base path had. 335 self._test_list_paths = {} 336 337 self._modifier_to_tests = self._dict_of_sets(self.MODIFIERS) 338 self._expectation_to_tests = self._dict_of_sets(self.EXPECTATIONS) 339 self._timeline_to_tests = self._dict_of_sets(self.TIMELINES) 340 self._result_type_to_tests = self._dict_of_sets(self.RESULT_TYPES) 341 342 self._read(self._get_iterable_expectations(self._expectations), 343 overrides_allowed=False) 344 345 # List of tests that are in the overrides file (used for checking for 346 # duplicates inside the overrides file itself). Note that just because 347 # a test is in this set doesn't mean it's necessarily overridding a 348 # expectation in the regular expectations; the test might not be 349 # mentioned in the regular expectations file at all. 350 self._overridding_tests = set() 351 352 if overrides: 353 self._read(self._get_iterable_expectations(self._overrides), 354 overrides_allowed=True) 355 356 self._handle_any_read_errors() 357 self._process_tests_without_expectations() 358 359 def _handle_any_read_errors(self): 360 if len(self._errors) or len(self._non_fatal_errors): 361 _log.error("FAILURES FOR %s" % str(self._test_config)) 362 363 for error in self._errors: 364 _log.error(error) 365 for error in self._non_fatal_errors: 366 _log.error(error) 367 368 if len(self._errors): 369 raise ParseError(fatal=True, errors=self._errors) 370 if len(self._non_fatal_errors) and self._is_lint_mode: 371 raise ParseError(fatal=False, errors=self._non_fatal_errors) 372 373 def _process_tests_without_expectations(self): 374 expectations = set([PASS]) 375 options = [] 376 modifiers = [] 377 num_matches = 0 378 if self._full_test_list: 379 for test in self._full_test_list: 380 if not test in self._test_list_paths: 381 self._add_test(test, modifiers, num_matches, expectations, 382 options, overrides_allowed=False) 383 384 def _dict_of_sets(self, strings_to_constants): 385 """Takes a dict of strings->constants and returns a dict mapping 386 each constant to an empty set.""" 387 d = {} 388 for c in strings_to_constants.values(): 389 d[c] = set() 390 return d 391 392 def _get_iterable_expectations(self, expectations_str): 393 """Returns an object that can be iterated over. Allows for not caring 394 about whether we're iterating over a file or a new-line separated 395 string.""" 396 iterable = [x + "\n" for x in expectations_str.split("\n")] 397 # Strip final entry if it's empty to avoid added in an extra 398 # newline. 399 if iterable[-1] == "\n": 400 return iterable[:-1] 401 return iterable 402 403 def get_test_set(self, modifier, expectation=None, include_skips=True): 404 if expectation is None: 405 tests = self._modifier_to_tests[modifier] 406 else: 407 tests = (self._expectation_to_tests[expectation] & 408 self._modifier_to_tests[modifier]) 409 410 if not include_skips: 411 tests = tests - self.get_test_set(SKIP, expectation) 412 413 return tests 414 415 def get_tests_with_result_type(self, result_type): 416 return self._result_type_to_tests[result_type] 417 418 def get_tests_with_timeline(self, timeline): 419 return self._timeline_to_tests[timeline] 420 421 def get_options(self, test): 422 """This returns the entire set of options for the given test 423 (the modifiers plus the BUGXXXX identifier). This is used by the 424 LTTF dashboard.""" 425 return self._test_to_options[test] 426 427 def has_modifier(self, test, modifier): 428 return test in self._modifier_to_tests[modifier] 429 430 def get_expectations(self, test): 431 return self._test_to_expectations[test] 432 433 def get_expectations_json_for_all_platforms(self): 434 # Specify separators in order to get compact encoding. 435 return ExpectationsJsonEncoder(separators=(',', ':')).encode( 436 self._all_expectations) 437 438 def get_non_fatal_errors(self): 439 return self._non_fatal_errors 440 441 def remove_rebaselined_tests(self, tests): 442 """Returns a copy of the expectations with the tests removed.""" 443 lines = [] 444 for (lineno, line) in enumerate(self._get_iterable_expectations(self._expectations)): 445 test, options, _ = self.parse_expectations_line(line, lineno) 446 if not (test and test in tests and 'rebaseline' in options): 447 lines.append(line) 448 return ''.join(lines) 449 450 def parse_expectations_line(self, line, lineno): 451 """Parses a line from test_expectations.txt and returns a tuple 452 with the test path, options as a list, expectations as a list.""" 453 line = strip_comments(line) 454 if not line: 455 return (None, None, None) 456 457 options = [] 458 if line.find(":") is -1: 459 self._add_error(lineno, "Missing a ':'", line) 460 return (None, None, None) 461 462 parts = line.split(':') 463 464 # FIXME: verify that there is exactly one colon in the line. 465 466 options = self._get_options_list(parts[0]) 467 test_and_expectation = parts[1].split('=') 468 test = test_and_expectation[0].strip() 469 if (len(test_and_expectation) is not 2): 470 self._add_error(lineno, "Missing expectations.", 471 test_and_expectation) 472 expectations = None 473 else: 474 expectations = self._get_options_list(test_and_expectation[1]) 475 476 return (test, options, expectations) 477 478 def _add_to_all_expectations(self, test, options, expectations): 479 # Make all paths unix-style so the dashboard doesn't need to. 480 test = test.replace('\\', '/') 481 if not test in self._all_expectations: 482 self._all_expectations[test] = [] 483 self._all_expectations[test].append( 484 ModifiersAndExpectations(options, expectations)) 485 486 def _read(self, expectations, overrides_allowed): 487 """For each test in an expectations iterable, generate the 488 expectations for it.""" 489 lineno = 0 490 matcher = ModifierMatcher(self._test_config) 491 for line in expectations: 492 lineno += 1 493 self._process_line(line, lineno, matcher, overrides_allowed) 494 495 def _process_line(self, line, lineno, matcher, overrides_allowed): 496 test_list_path, options, expectations = \ 497 self.parse_expectations_line(line, lineno) 498 if not expectations: 499 return 500 501 self._add_to_all_expectations(test_list_path, 502 " ".join(options).upper(), 503 " ".join(expectations).upper()) 504 505 num_matches = self._check_options(matcher, options, lineno, 506 test_list_path) 507 if num_matches == ModifierMatcher.NO_MATCH: 508 return 509 510 expectations = self._parse_expectations(expectations, lineno, 511 test_list_path) 512 513 self._check_options_against_expectations(options, expectations, 514 lineno, test_list_path) 515 516 if self._check_path_does_not_exist(lineno, test_list_path): 517 return 518 519 if not self._full_test_list: 520 tests = [test_list_path] 521 else: 522 tests = self._expand_tests(test_list_path) 523 524 modifiers = [o for o in options if o in self.MODIFIERS] 525 self._add_tests(tests, expectations, test_list_path, lineno, 526 modifiers, num_matches, options, overrides_allowed) 527 528 def _get_options_list(self, listString): 529 return [part.strip().lower() for part in listString.strip().split(' ')] 530 531 def _parse_expectations(self, expectations, lineno, test_list_path): 532 result = set() 533 for part in expectations: 534 if not part in self.EXPECTATIONS: 535 self._add_error(lineno, 'Unsupported expectation: %s' % part, 536 test_list_path) 537 continue 538 expectation = self.EXPECTATIONS[part] 539 result.add(expectation) 540 return result 541 542 def _check_options(self, matcher, options, lineno, test_list_path): 543 match_result = self._check_syntax(matcher, options, lineno, 544 test_list_path) 545 self._check_semantics(options, lineno, test_list_path) 546 return match_result.num_matches 547 548 def _check_syntax(self, matcher, options, lineno, test_list_path): 549 match_result = matcher.match(options) 550 for error in match_result.errors: 551 self._add_error(lineno, error, test_list_path) 552 for warning in match_result.warnings: 553 self._log_non_fatal_error(lineno, warning, test_list_path) 554 return match_result 555 556 def _check_semantics(self, options, lineno, test_list_path): 557 has_wontfix = 'wontfix' in options 558 has_bug = False 559 for opt in options: 560 if opt.startswith('bug'): 561 has_bug = True 562 if re.match('bug\d+', opt): 563 self._add_error(lineno, 564 'BUG\d+ is not allowed, must be one of ' 565 'BUGCR\d+, BUGWK\d+, BUGV8_\d+, ' 566 'or a non-numeric bug identifier.', test_list_path) 567 568 if not has_bug and not has_wontfix: 569 self._log_non_fatal_error(lineno, 'Test lacks BUG modifier.', 570 test_list_path) 571 572 if self._is_lint_mode and 'rebaseline' in options: 573 self._add_error(lineno, 574 'REBASELINE should only be used for running rebaseline.py. ' 575 'Cannot be checked in.', test_list_path) 576 577 def _check_options_against_expectations(self, options, expectations, 578 lineno, test_list_path): 579 if 'slow' in options and TIMEOUT in expectations: 580 self._add_error(lineno, 581 'A test can not be both SLOW and TIMEOUT. If it times out ' 582 'indefinitely, then it should be just TIMEOUT.', test_list_path) 583 584 def _check_path_does_not_exist(self, lineno, test_list_path): 585 full_path = self._fs.join(self._port.layout_tests_dir(), 586 test_list_path) 587 full_path = self._fs.normpath(full_path) 588 # WebKit's way of skipping tests is to add a -disabled suffix. 589 # So we should consider the path existing if the path or the 590 # -disabled version exists. 591 if (not self._port.path_exists(full_path) 592 and not self._port.path_exists(full_path + '-disabled')): 593 # Log a non fatal error here since you hit this case any 594 # time you update test_expectations.txt without syncing 595 # the LayoutTests directory 596 self._log_non_fatal_error(lineno, 'Path does not exist.', 597 test_list_path) 598 return True 599 return False 600 601 def _expand_tests(self, test_list_path): 602 """Convert the test specification to an absolute, normalized 603 path and make sure directories end with the OS path separator.""" 604 # FIXME: full_test_list can quickly contain a big amount of 605 # elements. We should consider at some point to use a more 606 # efficient structure instead of a list. Maybe a dictionary of 607 # lists to represent the tree of tests, leaves being test 608 # files and nodes being categories. 609 610 path = self._fs.join(self._port.layout_tests_dir(), test_list_path) 611 path = self._fs.normpath(path) 612 if self._fs.isdir(path): 613 # this is a test category, return all the tests of the category. 614 path = self._fs.join(path, '') 615 616 return [test for test in self._full_test_list if test.startswith(path)] 617 618 # this is a test file, do a quick check if it's in the 619 # full test suite. 620 result = [] 621 if path in self._full_test_list: 622 result = [path, ] 623 return result 624 625 def _add_tests(self, tests, expectations, test_list_path, lineno, 626 modifiers, num_matches, options, overrides_allowed): 627 for test in tests: 628 if self._already_seen_better_match(test, test_list_path, 629 num_matches, lineno, overrides_allowed): 630 continue 631 632 self._clear_expectations_for_test(test, test_list_path) 633 self._test_list_paths[test] = (self._fs.normpath(test_list_path), 634 num_matches, lineno) 635 self._add_test(test, modifiers, num_matches, expectations, options, 636 overrides_allowed) 637 638 def _add_test(self, test, modifiers, num_matches, expectations, options, 639 overrides_allowed): 640 """Sets the expected state for a given test. 641 642 This routine assumes the test has not been added before. If it has, 643 use _clear_expectations_for_test() to reset the state prior to 644 calling this. 645 646 Args: 647 test: test to add 648 modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.) 649 num_matches: number of modifiers that matched the configuration 650 expectations: sequence of expectations (PASS, IMAGE, etc.) 651 options: sequence of keywords and bug identifiers. 652 overrides_allowed: whether we're parsing the regular expectations 653 or the overridding expectations""" 654 self._test_to_expectations[test] = expectations 655 for expectation in expectations: 656 self._expectation_to_tests[expectation].add(test) 657 658 self._test_to_options[test] = options 659 self._test_to_modifiers[test] = set() 660 for modifier in modifiers: 661 mod_value = self.MODIFIERS[modifier] 662 self._modifier_to_tests[mod_value].add(test) 663 self._test_to_modifiers[test].add(mod_value) 664 665 if 'wontfix' in modifiers: 666 self._timeline_to_tests[WONTFIX].add(test) 667 else: 668 self._timeline_to_tests[NOW].add(test) 669 670 if 'skip' in modifiers: 671 self._result_type_to_tests[SKIP].add(test) 672 elif expectations == set([PASS]): 673 self._result_type_to_tests[PASS].add(test) 674 elif len(expectations) > 1: 675 self._result_type_to_tests[FLAKY].add(test) 676 else: 677 self._result_type_to_tests[FAIL].add(test) 678 679 if overrides_allowed: 680 self._overridding_tests.add(test) 681 682 def _clear_expectations_for_test(self, test, test_list_path): 683 """Remove prexisting expectations for this test. 684 This happens if we are seeing a more precise path 685 than a previous listing. 686 """ 687 if test in self._test_list_paths: 688 self._test_to_expectations.pop(test, '') 689 self._remove_from_sets(test, self._expectation_to_tests) 690 self._remove_from_sets(test, self._modifier_to_tests) 691 self._remove_from_sets(test, self._timeline_to_tests) 692 self._remove_from_sets(test, self._result_type_to_tests) 693 694 self._test_list_paths[test] = self._fs.normpath(test_list_path) 695 696 def _remove_from_sets(self, test, dict): 697 """Removes the given test from the sets in the dictionary. 698 699 Args: 700 test: test to look for 701 dict: dict of sets of files""" 702 for set_of_tests in dict.itervalues(): 703 if test in set_of_tests: 704 set_of_tests.remove(test) 705 706 def _already_seen_better_match(self, test, test_list_path, num_matches, 707 lineno, overrides_allowed): 708 """Returns whether we've seen a better match already in the file. 709 710 Returns True if we've already seen a test_list_path that matches more of the test 711 than this path does 712 """ 713 # FIXME: See comment below about matching test configs and num_matches. 714 715 if not test in self._test_list_paths: 716 # We've never seen this test before. 717 return False 718 719 prev_base_path, prev_num_matches, prev_lineno = self._test_list_paths[test] 720 base_path = self._fs.normpath(test_list_path) 721 722 if len(prev_base_path) > len(base_path): 723 # The previous path matched more of the test. 724 return True 725 726 if len(prev_base_path) < len(base_path): 727 # This path matches more of the test. 728 return False 729 730 if overrides_allowed and test not in self._overridding_tests: 731 # We have seen this path, but that's okay because it is 732 # in the overrides and the earlier path was in the 733 # expectations (not the overrides). 734 return False 735 736 # At this point we know we have seen a previous exact match on this 737 # base path, so we need to check the two sets of modifiers. 738 739 if overrides_allowed: 740 expectation_source = "override" 741 else: 742 expectation_source = "expectation" 743 744 # FIXME: This code was originally designed to allow lines that matched 745 # more modifiers to override lines that matched fewer modifiers. 746 # However, we currently view these as errors. If we decide to make 747 # this policy permanent, we can probably simplify this code 748 # and the ModifierMatcher code a fair amount. 749 # 750 # To use the "more modifiers wins" policy, change the "_add_error" lines for overrides 751 # to _log_non_fatal_error() and change the commented-out "return False". 752 753 if prev_num_matches == num_matches: 754 self._add_error(lineno, 755 'Duplicate or ambiguous %s.' % expectation_source, 756 test) 757 return True 758 759 if prev_num_matches < num_matches: 760 self._add_error(lineno, 761 'More specific entry on line %d overrides line %d' % 762 (lineno, prev_lineno), test_list_path) 763 # FIXME: return False if we want more specific to win. 764 return True 765 766 self._add_error(lineno, 767 'More specific entry on line %d overrides line %d' % 768 (prev_lineno, lineno), test_list_path) 769 return True 770 771 def _add_error(self, lineno, msg, path): 772 """Reports an error that will prevent running the tests. Does not 773 immediately raise an exception because we'd like to aggregate all the 774 errors so they can all be printed out.""" 775 self._errors.append('Line:%s %s %s' % (lineno, msg, path)) 776 777 def _log_non_fatal_error(self, lineno, msg, path): 778 """Reports an error that will not prevent running the tests. These are 779 still errors, but not bad enough to warrant breaking test running.""" 780 self._non_fatal_errors.append('Line:%s %s %s' % (lineno, msg, path)) 781 782 783 class ModifierMatchResult(object): 784 def __init__(self, options): 785 self.num_matches = ModifierMatcher.NO_MATCH 786 self.options = options 787 self.errors = [] 788 self.warnings = [] 789 self.modifiers = [] 790 self._matched_regexes = set() 791 self._matched_macros = set() 792 793 794 class ModifierMatcher(object): 795 796 """ 797 This class manages the interpretation of the "modifiers" for a given 798 line in the expectations file. Modifiers are the tokens that appear to the 799 left of the colon on a line. For example, "BUG1234", "DEBUG", and "WIN" are 800 all modifiers. This class gets what the valid modifiers are, and which 801 modifiers are allowed to exist together on a line, from the 802 TestConfiguration object that is passed in to the call. 803 804 This class detects *intra*-line errors like unknown modifiers, but 805 does not detect *inter*-line modifiers like duplicate expectations. 806 807 More importantly, this class is also used to determine if a given line 808 matches the port in question. Matches are ranked according to the number 809 of modifiers that match on a line. A line with no modifiers matches 810 everything and has a score of zero. A line with one modifier matches only 811 ports that have that modifier and gets a score of 1, and so one. Ports 812 that don't match at all get a score of -1. 813 814 Given two lines in a file that apply to the same test, if both expectations 815 match the current config, then the expectation is considered ambiguous, 816 even if one expectation matches more of the config than the other. For 817 example, in: 818 819 BUG1 RELEASE : foo.html = FAIL 820 BUG1 WIN RELEASE : foo.html = PASS 821 BUG2 WIN : bar.html = FAIL 822 BUG2 DEBUG : bar.html = PASS 823 824 lines 1 and 2 would produce an error on a Win XP Release bot (the scores 825 would be 1 and 2, respectively), and lines three and four would produce 826 a duplicate expectation on a Win Debug bot since both the 'win' and the 827 'debug' expectations would apply (both had scores of 1). 828 829 In addition to the definitions of all of the modifiers, the class 830 supports "macros" that are expanded prior to interpretation, and "ignore 831 regexes" that can be used to skip over modifiers like the BUG* modifiers. 832 """ 833 MACROS = { 834 'mac-snowleopard': ['mac', 'snowleopard'], 835 'mac-leopard': ['mac', 'leopard'], 836 'win-xp': ['win', 'xp'], 837 'win-vista': ['win', 'vista'], 838 'win-win7': ['win', 'win7'], 839 } 840 841 # We don't include the "none" modifier because it isn't actually legal. 842 REGEXES_TO_IGNORE = (['bug\w+'] + 843 TestExpectationsFile.MODIFIERS.keys()[:-1]) 844 DUPLICATE_REGEXES_ALLOWED = ['bug\w+'] 845 846 # Magic value returned when the options don't match. 847 NO_MATCH = -1 848 849 # FIXME: The code currently doesn't detect combinations of modifiers 850 # that are syntactically valid but semantically invalid, like 851 # 'MAC XP'. See ModifierMatchTest.test_invalid_combinations() in the 852 # _unittest.py file. 853 854 def __init__(self, test_config): 855 """Initialize a ModifierMatcher argument with the TestConfiguration it 856 should be matched against.""" 857 self.test_config = test_config 858 self.allowed_configurations = test_config.all_test_configurations() 859 self.macros = self.MACROS 860 861 self.regexes_to_ignore = {} 862 for regex_str in self.REGEXES_TO_IGNORE: 863 self.regexes_to_ignore[regex_str] = re.compile(regex_str) 864 865 # Keep a set of all of the legal modifiers for quick checking. 866 self._all_modifiers = set() 867 868 # Keep a dict mapping values back to their categories. 869 self._categories_for_modifiers = {} 870 for config in self.allowed_configurations: 871 for category, modifier in config.items(): 872 self._categories_for_modifiers[modifier] = category 873 self._all_modifiers.add(modifier) 874 875 def match(self, options): 876 """Checks a list of options against the config set in the constructor. 877 Options may be either actual modifier strings, "macro" strings 878 that get expanded to a list of modifiers, or strings that are allowed 879 to be ignored. All of the options must be passed in in lower case. 880 881 Returns the number of matching categories, or NO_MATCH (-1) if it 882 doesn't match or there were errors found. Matches are prioritized 883 by the number of matching categories, because the more specific 884 the options list, the more categories will match. 885 886 The results of the most recent match are available in the 'options', 887 'modifiers', 'num_matches', 'errors', and 'warnings' properties. 888 """ 889 result = ModifierMatchResult(options) 890 self._parse(result) 891 if result.errors: 892 return result 893 self._count_matches(result) 894 return result 895 896 def _parse(self, result): 897 # FIXME: Should we warn about lines having every value in a category? 898 for option in result.options: 899 self._parse_one(option, result) 900 901 def _parse_one(self, option, result): 902 if option in self._all_modifiers: 903 self._add_modifier(option, result) 904 elif option in self.macros: 905 self._expand_macro(option, result) 906 elif not self._matches_any_regex(option, result): 907 result.errors.append("Unrecognized option '%s'" % option) 908 909 def _add_modifier(self, option, result): 910 if option in result.modifiers: 911 result.errors.append("More than one '%s'" % option) 912 else: 913 result.modifiers.append(option) 914 915 def _expand_macro(self, macro, result): 916 if macro in result._matched_macros: 917 result.errors.append("More than one '%s'" % macro) 918 return 919 920 mods = [] 921 for modifier in self.macros[macro]: 922 if modifier in result.options: 923 result.errors.append("Can't specify both modifier '%s' and " 924 "macro '%s'" % (modifier, macro)) 925 else: 926 mods.append(modifier) 927 result._matched_macros.add(macro) 928 result.modifiers.extend(mods) 929 930 def _matches_any_regex(self, option, result): 931 for regex_str, pattern in self.regexes_to_ignore.iteritems(): 932 if pattern.match(option): 933 self._handle_regex_match(regex_str, result) 934 return True 935 return False 936 937 def _handle_regex_match(self, regex_str, result): 938 if (regex_str in result._matched_regexes and 939 regex_str not in self.DUPLICATE_REGEXES_ALLOWED): 940 result.errors.append("More than one option matching '%s'" % 941 regex_str) 942 else: 943 result._matched_regexes.add(regex_str) 944 945 def _count_matches(self, result): 946 """Returns the number of modifiers that match the test config.""" 947 categorized_modifiers = self._group_by_category(result.modifiers) 948 result.num_matches = 0 949 for category, modifier in self.test_config.items(): 950 if category in categorized_modifiers: 951 if modifier in categorized_modifiers[category]: 952 result.num_matches += 1 953 else: 954 result.num_matches = self.NO_MATCH 955 return 956 957 def _group_by_category(self, modifiers): 958 # Returns a dict of category name -> list of modifiers. 959 modifiers_by_category = {} 960 for m in modifiers: 961 modifiers_by_category.setdefault(self._category(m), []).append(m) 962 return modifiers_by_category 963 964 def _category(self, modifier): 965 return self._categories_for_modifiers[modifier] 966