Home | History | Annotate | Download | only in checkers 
      1 #!/usr/bin/python
      2 # Copyright (C) 2010 Google Inc. All rights reserved.
      3 #
      4 # Redistribution and use in source and binary forms, with or without
      5 # modification, are permitted provided that the following conditions are
      6 # met:
      7 #
      8 #     * Redistributions of source code must retain the above copyright
      9 # notice, this list of conditions and the following disclaimer.
     10 #     * Redistributions in binary form must reproduce the above
     11 # copyright notice, this list of conditions and the following disclaimer
     12 # in the documentation and/or other materials provided with the
     13 # distribution.
     14 #     * Neither the name of Google Inc. nor the names of its
     15 # contributors may be used to endorse or promote products derived from
     16 # this software without specific prior written permission.
     17 #
     18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     29 
     30 """Unit tests for test_expectations.py."""
     31 
     32 import os
     33 import sys
     34 import unittest
     35 
     36 # We need following workaround hack to run this unit tests in stand-alone.
     37 try:
     38     d = os.path.dirname(__file__)
     39 except NameError:
     40     d = os.path.dirname(sys.argv[0])
     41 sys.path.append(os.path.abspath(os.path.join(d, '../../../')))
     42 
     43 from test_expectations import TestExpectationsChecker
     44 from webkitpy.style_references import port
     45 from webkitpy.style_references import test_expectations as test_expectations_style
     46 
     47 
     48 class ErrorCollector(object):
     49     """An error handler class for unit tests."""
     50 
     51     def __init__(self):
     52         self._errors = []
     53 
     54     def __call__(self, lineno, category, confidence, message):
     55         self._errors.append('%s  [%s] [%d]' % (message, category, confidence))
     56 
     57     def get_errors(self):
     58         return ''.join(self._errors)
     59 
     60     def reset_errors(self):
     61         self._errors = []
     62 
     63 
     64 class TestExpectationsTestCase(unittest.TestCase):
     65     """TestCase for test_expectations.py"""
     66 
     67     def setUp(self):
     68         self._error_collector = ErrorCollector()
     69         port_obj = port.get('test')
     70         self._test_file = port_obj._filesystem.join(port_obj.layout_tests_dir(), 'passes/text.html')
     71 
     72     def process_expectations(self, expectations, overrides=None):
     73         self._checker = TestExpectationsChecker()
     74 
     75     def assert_lines_lint(self, lines, expected):
     76         self._error_collector.reset_errors()
     77         checker = TestExpectationsChecker('test/test_expectations.txt',
     78                                           self._error_collector)
     79         checker.check_test_expectations(expectations_str='\n'.join(lines),
     80                                         tests=[self._test_file],
     81                                         overrides=None)
     82         checker.check_tabs(lines)
     83         self.assertEqual(expected, self._error_collector.get_errors())
     84 
     85     def test_valid_expectations(self):
     86         self.assert_lines_lint(
     87             ["BUGCR1234 MAC : passes/text.html = PASS FAIL"],
     88             "")
     89         self.assert_lines_lint(
     90             ["SKIP BUGCR1234 : passes/text.html = TIMEOUT PASS"],
     91             "")
     92         self.assert_lines_lint(
     93             ["BUGCR1234 DEBUG : passes/text.html = TIMEOUT PASS"],
     94             "")
     95         self.assert_lines_lint(
     96             ["BUGCR1234 DEBUG SKIP : passes/text.html = TIMEOUT PASS"],
     97             "")
     98         self.assert_lines_lint(
     99             ["BUGCR1234 MAC DEBUG SKIP : passes/text.html = TIMEOUT PASS"],
    100             "")
    101         self.assert_lines_lint(
    102             ["BUGCR1234 DEBUG MAC : passes/text.html = TIMEOUT PASS"],
    103             "")
    104         self.assert_lines_lint(
    105             ["SLOW BUGCR1234 : passes/text.html = PASS"],
    106             "")
    107         self.assert_lines_lint(
    108             ["WONTFIX SKIP : passes/text.html = TIMEOUT"],
    109             "")
    110 
    111     def test_modifier_errors(self):
    112         self.assert_lines_lint(
    113             ["BUG1234 : passes/text.html = FAIL"],
    114             "BUG\\d+ is not allowed, must be one of BUGCR\\d+, BUGWK\\d+, BUGV8_\\d+, or a non-numeric bug identifier. passes/text.html  [test/expectations] [5]")
    115 
    116     def test_valid_modifiers(self):
    117         self.assert_lines_lint(
    118             ["INVALID-MODIFIER : passes/text.html = PASS"],
    119             "Unrecognized option 'invalid-modifier' "
    120             "passes/text.html  [test/expectations] [5]")
    121         self.assert_lines_lint(
    122             ["SKIP : passes/text.html = PASS"],
    123             "Test lacks BUG modifier. "
    124             "passes/text.html  [test/expectations] [2]")
    125 
    126     def test_expectation_errors(self):
    127         self.assert_lines_lint(
    128             ["missing expectations"],
    129             "Missing a ':' missing expectations  [test/expectations] [5]")
    130         self.assert_lines_lint(
    131             ["SLOW : passes/text.html = TIMEOUT"],
    132             "A test can not be both SLOW and TIMEOUT. "
    133             "If it times out indefinitely, then it should be just TIMEOUT. "
    134             "passes/text.html  [test/expectations] [5]")
    135         self.assert_lines_lint(
    136             ["BUGWK1 : does/not/exist.html = FAIL"],
    137             "Path does not exist. does/not/exist.html  [test/expectations] [2]")
    138 
    139     def test_parse_expectations(self):
    140         self.assert_lines_lint(
    141             ["BUGWK1 : passes/text.html = PASS"],
    142             "")
    143         self.assert_lines_lint(
    144             ["BUGWK1 : passes/text.html = UNSUPPORTED"],
    145             "Unsupported expectation: unsupported "
    146             "passes/text.html  [test/expectations] [5]")
    147         self.assert_lines_lint(
    148             ["BUGWK1 : passes/text.html = PASS UNSUPPORTED"],
    149             "Unsupported expectation: unsupported "
    150             "passes/text.html  [test/expectations] [5]")
    151 
    152     def test_already_seen_test(self):
    153         self.assert_lines_lint(
    154             ["BUGWK1 : passes/text.html = PASS",
    155              "BUGWK1 : passes/text.html = TIMEOUT"],
    156             "Duplicate or ambiguous expectation. %s  [test/expectations] [5]" % self._test_file)
    157 
    158     def test_tab(self):
    159         self.assert_lines_lint(
    160             ["\tBUGWK1 : passes/text.html = PASS"],
    161             "Line contains tab character.  [whitespace/tab] [5]")
    162 
    163 if __name__ == '__main__':
    164     unittest.main()
    165