Home | History | Annotate | Download | only in local 
      1 # Copyright 2012 the V8 project authors. All rights reserved.
      2 # Redistribution and use in source and binary forms, with or without
      3 # modification, are permitted provided that the following conditions are
      4 # met:
      5 #
      6 #     * Redistributions of source code must retain the above copyright
      7 #       notice, this list of conditions and the following disclaimer.
      8 #     * Redistributions in binary form must reproduce the above
      9 #       copyright notice, this list of conditions and the following
     10 #       disclaimer in the documentation and/or other materials provided
     11 #       with the distribution.
     12 #     * Neither the name of Google Inc. nor the names of its
     13 #       contributors may be used to endorse or promote products derived
     14 #       from this software without specific prior written permission.
     15 #
     16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27 
     28 
     29 import imp
     30 import os
     31 
     32 from . import statusfile
     33 from . import utils
     34 
     35 class TestSuite(object):
     36 
     37   @staticmethod
     38   def LoadTestSuite(root):
     39     name = root.split(os.path.sep)[-1]
     40     f = None
     41     try:
     42       (f, pathname, description) = imp.find_module("testcfg", [root])
     43       module = imp.load_module("testcfg", f, pathname, description)
     44       suite = module.GetSuite(name, root)
     45     finally:
     46       if f:
     47         f.close()
     48     return suite
     49 
     50   def __init__(self, name, root):
     51     self.name = name  # string
     52     self.root = root  # string containing path
     53     self.tests = None  # list of TestCase objects
     54     self.rules = None  # dictionary mapping test path to list of outcomes
     55     self.wildcards = None  # dictionary mapping test paths to list of outcomes
     56     self.total_duration = None  # float, assigned on demand
     57 
     58   def shell(self):
     59     return "d8"
     60 
     61   def suffix(self):
     62     return ".js"
     63 
     64   def status_file(self):
     65     return "%s/%s.status" % (self.root, self.name)
     66 
     67   # Used in the status file and for stdout printing.
     68   def CommonTestName(self, testcase):
     69     if utils.IsWindows():
     70       return testcase.path.replace("\\", "/")
     71     else:
     72       return testcase.path
     73 
     74   def ListTests(self, context):
     75     raise NotImplementedError
     76 
     77   def VariantFlags(self):
     78     return None
     79 
     80   def DownloadData(self):
     81     pass
     82 
     83   def ReadStatusFile(self, variables):
     84     (self.rules, self.wildcards) = \
     85         statusfile.ReadStatusFile(self.status_file(), variables)
     86 
     87   def ReadTestCases(self, context):
     88     self.tests = self.ListTests(context)
     89 
     90   @staticmethod
     91   def _FilterFlaky(flaky, mode):
     92     return (mode == "run" and not flaky) or (mode == "skip" and flaky)
     93 
     94   def FilterTestCasesByStatus(self, warn_unused_rules, flaky_tests="dontcare"):
     95     filtered = []
     96     used_rules = set()
     97     for t in self.tests:
     98       flaky = False
     99       testname = self.CommonTestName(t)
    100       if testname in self.rules:
    101         used_rules.add(testname)
    102         # Even for skipped tests, as the TestCase object stays around and
    103         # PrintReport() uses it.
    104         t.outcomes = self.rules[testname]
    105         if statusfile.DoSkip(t.outcomes):
    106           continue  # Don't add skipped tests to |filtered|.
    107         flaky = statusfile.IsFlaky(t.outcomes)
    108       skip = False
    109       for rule in self.wildcards:
    110         assert rule[-1] == '*'
    111         if testname.startswith(rule[:-1]):
    112           used_rules.add(rule)
    113           t.outcomes = self.wildcards[rule]
    114           if statusfile.DoSkip(t.outcomes):
    115             skip = True
    116             break  # "for rule in self.wildcards"
    117           flaky = flaky or statusfile.IsFlaky(t.outcomes)
    118       if skip or self._FilterFlaky(flaky, flaky_tests):
    119         continue  # "for t in self.tests"
    120       filtered.append(t)
    121     self.tests = filtered
    122 
    123     if not warn_unused_rules:
    124       return
    125 
    126     for rule in self.rules:
    127       if rule not in used_rules:
    128         print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
    129     for rule in self.wildcards:
    130       if rule not in used_rules:
    131         print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
    132 
    133   def FilterTestCasesByArgs(self, args):
    134     filtered = []
    135     filtered_args = []
    136     for a in args:
    137       argpath = a.split(os.path.sep)
    138       if argpath[0] != self.name:
    139         continue
    140       if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
    141         return  # Don't filter, run all tests in this suite.
    142       path = os.path.sep.join(argpath[1:])
    143       if path[-1] == '*':
    144         path = path[:-1]
    145       filtered_args.append(path)
    146     for t in self.tests:
    147       for a in filtered_args:
    148         if t.path.startswith(a):
    149           filtered.append(t)
    150           break
    151     self.tests = filtered
    152 
    153   def GetFlagsForTestCase(self, testcase, context):
    154     raise NotImplementedError
    155 
    156   def GetSourceForTest(self, testcase):
    157     return "(no source available)"
    158 
    159   def IsFailureOutput(self, output, testpath):
    160     return output.exit_code != 0
    161 
    162   def IsNegativeTest(self, testcase):
    163     return False
    164 
    165   def HasFailed(self, testcase):
    166     execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
    167     if self.IsNegativeTest(testcase):
    168       return not execution_failed
    169     else:
    170       return execution_failed
    171 
    172   def HasUnexpectedOutput(self, testcase):
    173     if testcase.output.HasCrashed():
    174       outcome = statusfile.CRASH
    175     elif testcase.output.HasTimedOut():
    176       outcome = statusfile.TIMEOUT
    177     elif self.HasFailed(testcase):
    178       outcome = statusfile.FAIL
    179     else:
    180       outcome = statusfile.PASS
    181     if not testcase.outcomes:
    182       return outcome != statusfile.PASS
    183     return not outcome in testcase.outcomes
    184 
    185   def StripOutputForTransmit(self, testcase):
    186     if not self.HasUnexpectedOutput(testcase):
    187       testcase.output.stdout = ""
    188       testcase.output.stderr = ""
    189 
    190   def CalculateTotalDuration(self):
    191     self.total_duration = 0.0
    192     for t in self.tests:
    193       self.total_duration += t.duration
    194     return self.total_duration
    195