Home | History | Annotate | Download | only in layout_tests 
      1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """A module to analyze test expectations for Webkit layout tests."""
      6 
      7 import urllib2
      8 
      9 from webkitpy.layout_tests.models.test_expectations import *
     10 
     11 # Default location for chromium test expectation file.
     12 # TODO(imasaki): support multiple test expectations files.
     13 DEFAULT_TEST_EXPECTATIONS_LOCATION = (
     14     'http://src.chromium.org/blink/trunk/LayoutTests/TestExpectations')
     15 
     16 # The following is from test expectation syntax. The detail can be found in
     17 # http://www.chromium.org/developers/testing/webkit-layout-tests#TOC-Test-Expectations
     18 # <decision> ::== [SKIP] [WONTFIX] [SLOW]
     19 DECISION_NAMES = ['SKIP', 'WONTFIX', 'SLOW']
     20 # <config> ::== RELEASE | DEBUG
     21 CONFIG_NAMES = ['RELEASE', 'DEBUG']
     22 # Only hard code keywords we don't expect to change.  Determine the rest from
     23 # the format of the status line.
     24 KNOWN_TE_KEYWORDS = DECISION_NAMES + CONFIG_NAMES
     25 
     26 
     27 class TestExpectations(object):
     28   """A class to model the content of test expectation file for analysis.
     29 
     30   This class retrieves the TestExpectations file via HTTP from WebKit and uses
     31   the WebKit layout test processor to process each line.
     32 
     33   The resulting dictionary is stored in |all_test_expectation_info| and looks
     34   like:
     35 
     36     {'<test name>': [{'<modifier0>': True, '<modifier1>': True, ...,
     37                      'Platforms: ['<platform0>', ... ], 'Bugs': ['....']}]}
     38 
     39   Duplicate keys are merged (though technically they shouldn't exist).
     40 
     41   Example:
     42     crbug.com/145590 [ Android ] \
     43         platform/chromium/media/video-frame-size-change.html [ Timeout ]
     44     webkit.org/b/84724 [ SnowLeopard ] \
     45         platform/chromium/media/video-frame-size-change.html \
     46         [ ImageOnlyFailure Pass ]
     47 
     48   {'platform/chromium/media/video-frame-size-change.html': [{'IMAGE': True,
     49    'Bugs': ['BUGWK84724', 'BUGCR145590'], 'Comments': '',
     50    'Platforms': ['SNOWLEOPARD', 'ANDROID'], 'TIMEOUT': True, 'PASS': True}]}
     51   """
     52 
     53   def __init__(self, url=DEFAULT_TEST_EXPECTATIONS_LOCATION):
     54     """Read the test expectation file from the specified URL and parse it.
     55 
     56     Args:
     57       url: A URL string for the test expectation file.
     58 
     59     Raises:
     60       NameError when the test expectation file cannot be retrieved from |url|.
     61     """
     62     self.all_test_expectation_info = {}
     63     resp = urllib2.urlopen(url)
     64     if resp.code != 200:
     65       raise NameError('Test expectation file does not exist in %s' % url)
     66     # Start parsing each line.
     67     for line in resp.read().split('\n'):
     68       line = line.strip()
     69       # Skip comments.
     70       if line.startswith('#'):
     71         continue
     72       testname, te_info = self.ParseLine(line)
     73       if not testname or not te_info:
     74         continue
     75       if testname in self.all_test_expectation_info:
     76         # Merge keys if entry already exists.
     77         for k in te_info.keys():
     78           if (isinstance(te_info[k], list) and
     79               k in self.all_test_expectation_info[testname]):
     80             self.all_test_expectation_info[testname][0][k] += te_info[k]
     81           else:
     82             self.all_test_expectation_info[testname][0][k] = te_info[k]
     83       else:
     84         self.all_test_expectation_info[testname] = [te_info]
     85 
     86   @staticmethod
     87   def ParseLine(line):
     88     """Parses the provided line using WebKit's TextExpecations parser.
     89 
     90     Returns:
     91       Tuple of test name, test expectations dictionary.  See class documentation
     92       for the format of the dictionary
     93     """
     94     test_expectation_info = {}
     95     parsed = TestExpectationParser._tokenize_line('TestExpectations', line, 0)
     96     if parsed.is_invalid():
     97       return None, None
     98 
     99     test_expectation_info['Comments'] = parsed.comment or ''
    100 
    101     # Split the modifiers dictionary into the format we want.
    102     remaining_modifiers = list(parsed.modifiers)
    103     test_expectation_info['Bugs'] = []
    104     for m in parsed.modifiers:
    105       if (m.startswith(WEBKIT_BUG_PREFIX) or
    106           m.startswith(CHROMIUM_BUG_PREFIX) or
    107           m.startswith(V8_BUG_PREFIX) or
    108           m.startswith(NAMED_BUG_PREFIX)):
    109         test_expectation_info['Bugs'].append(m)
    110         remaining_modifiers.remove(m)
    111       elif m in KNOWN_TE_KEYWORDS:
    112         test_expectation_info[m] = True
    113         remaining_modifiers.remove(m)
    114 
    115     # The modifiers left over should all be platform names.
    116     test_expectation_info['Platforms'] = list(remaining_modifiers)
    117 
    118     # Shovel the expectations and modifiers in as "<key>: True" entries.  Ugly,
    119     # but required by the rest of the pipeline for parsing.
    120     for m in parsed.expectations + remaining_modifiers:
    121       test_expectation_info[m] = True
    122 
    123     return parsed.name, test_expectation_info
    124