Home | History | Annotate | Download | only in local 
      1 # Copyright 2012 the V8 project authors. All rights reserved.
      2 # Redistribution and use in source and binary forms, with or without
      3 # modification, are permitted provided that the following conditions are
      4 # met:
      5 #
      6 #     * Redistributions of source code must retain the above copyright
      7 #       notice, this list of conditions and the following disclaimer.
      8 #     * Redistributions in binary form must reproduce the above
      9 #       copyright notice, this list of conditions and the following
     10 #       disclaimer in the documentation and/or other materials provided
     11 #       with the distribution.
     12 #     * Neither the name of Google Inc. nor the names of its
     13 #       contributors may be used to endorse or promote products derived
     14 #       from this software without specific prior written permission.
     15 #
     16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27 
     28 
     29 import imp
     30 import os
     31 
     32 from . import commands
     33 from . import statusfile
     34 from . import utils
     35 from ..objects import testcase
     36 
     37 # Use this to run several variants of the tests.
     38 ALL_VARIANT_FLAGS = {
     39   "default": [[]],
     40   "stress": [["--stress-opt", "--always-opt"]],
     41   "turbofan": [["--turbo"]],
     42   "turbofan_opt": [["--turbo", "--always-opt"]],
     43   "nocrankshaft": [["--nocrankshaft"]],
     44   "ignition": [["--ignition"]],
     45   "ignition_turbofan": [["--ignition", "--turbo", "--turbo-from-bytecode"]],
     46   "preparser": [["--min-preparse-length=0"]],
     47 }
     48 
     49 # FAST_VARIANTS implies no --always-opt.
     50 FAST_VARIANT_FLAGS = {
     51   "default": [[]],
     52   "stress": [["--stress-opt"]],
     53   "turbofan": [["--turbo"]],
     54   "nocrankshaft": [["--nocrankshaft"]],
     55   "ignition": [["--ignition"]],
     56   "ignition_turbofan": [["--ignition", "--turbo", "--turbo-from-bytecode"]],
     57   "preparser": [["--min-preparse-length=0"]],
     58 }
     59 
     60 ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt",
     61                     "nocrankshaft", "ignition", "ignition_turbofan",
     62                     "preparser"])
     63 FAST_VARIANTS = set(["default", "turbofan"])
     64 STANDARD_VARIANT = set(["default"])
     65 IGNITION_VARIANT = set(["ignition"])
     66 
     67 
     68 class VariantGenerator(object):
     69   def __init__(self, suite, variants):
     70     self.suite = suite
     71     self.all_variants = ALL_VARIANTS & variants
     72     self.fast_variants = FAST_VARIANTS & variants
     73     self.standard_variant = STANDARD_VARIANT & variants
     74 
     75   def FilterVariantsByTest(self, testcase):
     76     result = self.all_variants
     77     if testcase.outcomes:
     78       if statusfile.OnlyStandardVariant(testcase.outcomes):
     79         return self.standard_variant
     80       if statusfile.OnlyFastVariants(testcase.outcomes):
     81         result = self.fast_variants
     82       if statusfile.NoIgnitionVariant(testcase.outcomes):
     83         result = result - IGNITION_VARIANT
     84     return result
     85 
     86   def GetFlagSets(self, testcase, variant):
     87     if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
     88       return FAST_VARIANT_FLAGS[variant]
     89     else:
     90       return ALL_VARIANT_FLAGS[variant]
     91 
     92 
     93 class TestSuite(object):
     94 
     95   @staticmethod
     96   def LoadTestSuite(root, global_init=True):
     97     name = root.split(os.path.sep)[-1]
     98     f = None
     99     try:
    100       (f, pathname, description) = imp.find_module("testcfg", [root])
    101       module = imp.load_module("testcfg", f, pathname, description)
    102       return module.GetSuite(name, root)
    103     except ImportError:
    104       # Use default if no testcfg is present.
    105       return GoogleTestSuite(name, root)
    106     finally:
    107       if f:
    108         f.close()
    109 
    110   def __init__(self, name, root):
    111     # Note: This might be called concurrently from different processes.
    112     self.name = name  # string
    113     self.root = root  # string containing path
    114     self.tests = None  # list of TestCase objects
    115     self.rules = None  # dictionary mapping test path to list of outcomes
    116     self.wildcards = None  # dictionary mapping test paths to list of outcomes
    117     self.total_duration = None  # float, assigned on demand
    118 
    119   def shell(self):
    120     return "d8"
    121 
    122   def suffix(self):
    123     return ".js"
    124 
    125   def status_file(self):
    126     return "%s/%s.status" % (self.root, self.name)
    127 
    128   # Used in the status file and for stdout printing.
    129   def CommonTestName(self, testcase):
    130     if utils.IsWindows():
    131       return testcase.path.replace("\\", "/")
    132     else:
    133       return testcase.path
    134 
    135   def ListTests(self, context):
    136     raise NotImplementedError
    137 
    138   def _VariantGeneratorFactory(self):
    139     """The variant generator class to be used."""
    140     return VariantGenerator
    141 
    142   def CreateVariantGenerator(self, variants):
    143     """Return a generator for the testing variants of this suite.
    144 
    145     Args:
    146       variants: List of variant names to be run as specified by the test
    147                 runner.
    148     Returns: An object of type VariantGenerator.
    149     """
    150     return self._VariantGeneratorFactory()(self, set(variants))
    151 
    152   def DownloadData(self):
    153     pass
    154 
    155   def ReadStatusFile(self, variables):
    156     (self.rules, self.wildcards) = \
    157         statusfile.ReadStatusFile(self.status_file(), variables)
    158 
    159   def ReadTestCases(self, context):
    160     self.tests = self.ListTests(context)
    161 
    162   @staticmethod
    163   def _FilterSlow(slow, mode):
    164     return (mode == "run" and not slow) or (mode == "skip" and slow)
    165 
    166   @staticmethod
    167   def _FilterPassFail(pass_fail, mode):
    168     return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)
    169 
    170   def FilterTestCasesByStatus(self, warn_unused_rules,
    171                               slow_tests="dontcare",
    172                               pass_fail_tests="dontcare"):
    173     filtered = []
    174     used_rules = set()
    175     for t in self.tests:
    176       slow = False
    177       pass_fail = False
    178       testname = self.CommonTestName(t)
    179       if testname in self.rules:
    180         used_rules.add(testname)
    181         # Even for skipped tests, as the TestCase object stays around and
    182         # PrintReport() uses it.
    183         t.outcomes = self.rules[testname]
    184         if statusfile.DoSkip(t.outcomes):
    185           continue  # Don't add skipped tests to |filtered|.
    186         for outcome in t.outcomes:
    187           if outcome.startswith('Flags: '):
    188             t.flags += outcome[7:].split()
    189         slow = statusfile.IsSlow(t.outcomes)
    190         pass_fail = statusfile.IsPassOrFail(t.outcomes)
    191       skip = False
    192       for rule in self.wildcards:
    193         assert rule[-1] == '*'
    194         if testname.startswith(rule[:-1]):
    195           used_rules.add(rule)
    196           t.outcomes |= self.wildcards[rule]
    197           if statusfile.DoSkip(t.outcomes):
    198             skip = True
    199             break  # "for rule in self.wildcards"
    200           slow = slow or statusfile.IsSlow(t.outcomes)
    201           pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
    202       if (skip
    203           or self._FilterSlow(slow, slow_tests)
    204           or self._FilterPassFail(pass_fail, pass_fail_tests)):
    205         continue  # "for t in self.tests"
    206       filtered.append(t)
    207     self.tests = filtered
    208 
    209     if not warn_unused_rules:
    210       return
    211 
    212     for rule in self.rules:
    213       if rule not in used_rules:
    214         print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
    215     for rule in self.wildcards:
    216       if rule not in used_rules:
    217         print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
    218 
    219   def FilterTestCasesByArgs(self, args):
    220     """Filter test cases based on command-line arguments.
    221 
    222     An argument with an asterisk in the end will match all test cases
    223     that have the argument as a prefix. Without asterisk, only exact matches
    224     will be used with the exeption of the test-suite name as argument.
    225     """
    226     filtered = []
    227     globs = []
    228     exact_matches = []
    229     for a in args:
    230       argpath = a.split('/')
    231       if argpath[0] != self.name:
    232         continue
    233       if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
    234         return  # Don't filter, run all tests in this suite.
    235       path = '/'.join(argpath[1:])
    236       if path[-1] == '*':
    237         path = path[:-1]
    238         globs.append(path)
    239       else:
    240         exact_matches.append(path)
    241     for t in self.tests:
    242       for a in globs:
    243         if t.path.startswith(a):
    244           filtered.append(t)
    245           break
    246       for a in exact_matches:
    247         if t.path == a:
    248           filtered.append(t)
    249           break
    250     self.tests = filtered
    251 
    252   def GetFlagsForTestCase(self, testcase, context):
    253     raise NotImplementedError
    254 
    255   def GetSourceForTest(self, testcase):
    256     return "(no source available)"
    257 
    258   def IsFailureOutput(self, testcase):
    259     return testcase.output.exit_code != 0
    260 
    261   def IsNegativeTest(self, testcase):
    262     return False
    263 
    264   def HasFailed(self, testcase):
    265     execution_failed = self.IsFailureOutput(testcase)
    266     if self.IsNegativeTest(testcase):
    267       return not execution_failed
    268     else:
    269       return execution_failed
    270 
    271   def GetOutcome(self, testcase):
    272     if testcase.output.HasCrashed():
    273       return statusfile.CRASH
    274     elif testcase.output.HasTimedOut():
    275       return statusfile.TIMEOUT
    276     elif self.HasFailed(testcase):
    277       return statusfile.FAIL
    278     else:
    279       return statusfile.PASS
    280 
    281   def HasUnexpectedOutput(self, testcase):
    282     outcome = self.GetOutcome(testcase)
    283     return not outcome in (testcase.outcomes or [statusfile.PASS])
    284 
    285   def StripOutputForTransmit(self, testcase):
    286     if not self.HasUnexpectedOutput(testcase):
    287       testcase.output.stdout = ""
    288       testcase.output.stderr = ""
    289 
    290   def CalculateTotalDuration(self):
    291     self.total_duration = 0.0
    292     for t in self.tests:
    293       self.total_duration += t.duration
    294     return self.total_duration
    295 
    296 
    297 class StandardVariantGenerator(VariantGenerator):
    298   def FilterVariantsByTest(self, testcase):
    299     return self.standard_variant
    300 
    301 
    302 class GoogleTestSuite(TestSuite):
    303   def __init__(self, name, root):
    304     super(GoogleTestSuite, self).__init__(name, root)
    305 
    306   def ListTests(self, context):
    307     shell = os.path.abspath(os.path.join(context.shell_dir, self.shell()))
    308     if utils.IsWindows():
    309       shell += ".exe"
    310     output = commands.Execute(context.command_prefix +
    311                               [shell, "--gtest_list_tests"] +
    312                               context.extra_flags)
    313     if output.exit_code != 0:
    314       print output.stdout
    315       print output.stderr
    316       raise Exception("Test executable failed to list the tests.")
    317     tests = []
    318     test_case = ''
    319     for line in output.stdout.splitlines():
    320       test_desc = line.strip().split()[0]
    321       if test_desc.endswith('.'):
    322         test_case = test_desc
    323       elif test_case and test_desc:
    324         test = testcase.TestCase(self, test_case + test_desc)
    325         tests.append(test)
    326     tests.sort(key=lambda t: t.path)
    327     return tests
    328 
    329   def GetFlagsForTestCase(self, testcase, context):
    330     return (testcase.flags + ["--gtest_filter=" + testcase.path] +
    331             ["--gtest_random_seed=%s" % context.random_seed] +
    332             ["--gtest_print_time=0"] +
    333             context.mode_flags)
    334 
    335   def _VariantGeneratorFactory(self):
    336     return StandardVariantGenerator
    337 
    338   def shell(self):
    339     return self.name
    340