Home | History | Annotate | Download | only in webkit 
      1 # Copyright 2013 the V8 project authors. All rights reserved.
      2 # Redistribution and use in source and binary forms, with or without
      3 # modification, are permitted provided that the following conditions are
      4 # met:
      5 #
      6 #     * Redistributions of source code must retain the above copyright
      7 #       notice, this list of conditions and the following disclaimer.
      8 #     * Redistributions in binary form must reproduce the above
      9 #       copyright notice, this list of conditions and the following
     10 #       disclaimer in the documentation and/or other materials provided
     11 #       with the distribution.
     12 #     * Neither the name of Google Inc. nor the names of its
     13 #       contributors may be used to endorse or promote products derived
     14 #       from this software without specific prior written permission.
     15 #
     16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27 
     28 import itertools
     29 import os
     30 import re
     31 
     32 from testrunner.local import testsuite
     33 from testrunner.objects import testcase
     34 
     35 FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
     36 FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
     37 SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
     38 
     39 
     40 # TODO (machenbach): Share commonalities with mjstest.
     41 class WebkitTestSuite(testsuite.TestSuite):
     42 
     43   def __init__(self, name, root):
     44     super(WebkitTestSuite, self).__init__(name, root)
     45 
     46   def ListTests(self, context):
     47     tests = []
     48     for dirname, dirs, files in os.walk(self.root):
     49       for dotted in [x for x in dirs if x.startswith('.')]:
     50         dirs.remove(dotted)
     51       if 'resources' in dirs:
     52         dirs.remove('resources')
     53 
     54       dirs.sort()
     55       files.sort()
     56       for filename in files:
     57         if filename.endswith(".js"):
     58           fullpath = os.path.join(dirname, filename)
     59           relpath = fullpath[len(self.root) + 1 : -3]
     60           testname = relpath.replace(os.path.sep, "/")
     61           test = testcase.TestCase(self, testname)
     62           tests.append(test)
     63     return tests
     64 
     65   def GetFlagsForTestCase(self, testcase, context):
     66     source = self.GetSourceForTest(testcase)
     67     flags = [] + context.mode_flags
     68     flags_match = re.findall(FLAGS_PATTERN, source)
     69     for match in flags_match:
     70       flags += match.strip().split()
     71 
     72     files_list = []  # List of file names to append to command arguments.
     73     files_match = FILES_PATTERN.search(source);
     74     # Accept several lines of 'Files:'.
     75     while True:
     76       if files_match:
     77         files_list += files_match.group(1).strip().split()
     78         files_match = FILES_PATTERN.search(source, files_match.end())
     79       else:
     80         break
     81     files = [ os.path.normpath(os.path.join(self.root, '..', '..', f))
     82               for f in files_list ]
     83     testfilename = os.path.join(self.root, testcase.path + self.suffix())
     84     if SELF_SCRIPT_PATTERN.search(source):
     85       env = ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")]
     86       files = env + files
     87     files.append(os.path.join(self.root, "resources/standalone-pre.js"))
     88     files.append(testfilename)
     89     files.append(os.path.join(self.root, "resources/standalone-post.js"))
     90 
     91     flags += files
     92     if context.isolates:
     93       flags.append("--isolate")
     94       flags += files
     95 
     96     return testcase.flags + flags
     97 
     98   def GetSourceForTest(self, testcase):
     99     filename = os.path.join(self.root, testcase.path + self.suffix())
    100     with open(filename) as f:
    101       return f.read()
    102 
    103   # TODO(machenbach): Share with test/message/testcfg.py
    104   def _IgnoreLine(self, string):
    105     """Ignore empty lines, valgrind output and Android output."""
    106     if not string: return True
    107     return (string.startswith("==") or string.startswith("**") or
    108             string.startswith("ANDROID") or
    109             # These five patterns appear in normal Native Client output.
    110             string.startswith("DEBUG MODE ENABLED") or
    111             string.startswith("tools/nacl-run.py") or
    112             string.find("BYPASSING ALL ACL CHECKS") > 0 or
    113             string.find("Native Client module will be loaded") > 0 or
    114             string.find("NaClHostDescOpen:") > 0 or
    115             # FIXME(machenbach): The test driver shouldn't try to use slow
    116             # asserts if they weren't compiled. This fails in optdebug=2.
    117             string == "Warning: unknown flag --enable-slow-asserts." or
    118             string == "Try --help for options")
    119 
    120   def IsFailureOutput(self, output, testpath):
    121     if super(WebkitTestSuite, self).IsFailureOutput(output, testpath):
    122       return True
    123     file_name = os.path.join(self.root, testpath) + "-expected.txt"
    124     with file(file_name, "r") as expected:
    125       expected_lines = expected.readlines()
    126 
    127     def ExpIterator():
    128       for line in expected_lines:
    129         if line.startswith("#") or not line.strip(): continue
    130         yield line.strip()
    131 
    132     def ActIterator(lines):
    133       for line in lines:
    134         if self._IgnoreLine(line.strip()): continue
    135         yield line.strip()
    136 
    137     def ActBlockIterator():
    138       """Iterates over blocks of actual output lines."""
    139       lines = output.stdout.splitlines()
    140       start_index = 0
    141       found_eqeq = False
    142       for index, line in enumerate(lines):
    143         # If a stress test separator is found:
    144         if line.startswith("=="):
    145           # Iterate over all lines before a separator except the first.
    146           if not found_eqeq:
    147             found_eqeq = True
    148           else:
    149             yield ActIterator(lines[start_index:index])
    150           # The next block of ouput lines starts after the separator.
    151           start_index = index + 1
    152       # Iterate over complete output if no separator was found.
    153       if not found_eqeq:
    154         yield ActIterator(lines)
    155 
    156     for act_iterator in ActBlockIterator():
    157       for (expected, actual) in itertools.izip_longest(
    158           ExpIterator(), act_iterator, fillvalue=''):
    159         if expected != actual:
    160           return True
    161       return False
    162 
    163 
    164 def GetSuite(name, root):
    165   return WebkitTestSuite(name, root)
    166