Home | History | Annotate | Download | only in test262 
      1 # Copyright 2012 the V8 project authors. All rights reserved.
      2 # Redistribution and use in source and binary forms, with or without
      3 # modification, are permitted provided that the following conditions are
      4 # met:
      5 #
      6 #     * Redistributions of source code must retain the above copyright
      7 #       notice, this list of conditions and the following disclaimer.
      8 #     * Redistributions in binary form must reproduce the above
      9 #       copyright notice, this list of conditions and the following
     10 #       disclaimer in the documentation and/or other materials provided
     11 #       with the distribution.
     12 #     * Neither the name of Google Inc. nor the names of its
     13 #       contributors may be used to endorse or promote products derived
     14 #       from this software without specific prior written permission.
     15 #
     16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27 
     28 
     29 import hashlib
     30 import imp
     31 import os
     32 import shutil
     33 import sys
     34 import tarfile
     35 
     36 
     37 from testrunner.local import statusfile
     38 from testrunner.local import testsuite
     39 from testrunner.local import utils
     40 from testrunner.objects import testcase
     41 
     42 DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
     43 ARCHIVE = DATA + ".tar"
     44 
     45 TEST_262_HARNESS_FILES = ["sta.js", "assert.js"]
     46 TEST_262_NATIVE_FILES = ["detachArrayBuffer.js"]
     47 
     48 TEST_262_SUITE_PATH = ["data", "test"]
     49 TEST_262_HARNESS_PATH = ["data", "harness"]
     50 TEST_262_TOOLS_PATH = ["data", "tools", "packaging"]
     51 
     52 ALL_VARIANT_FLAGS_STRICT = dict(
     53     (v, [flags + ["--use-strict"] for flags in flag_sets])
     54     for v, flag_sets in testsuite.ALL_VARIANT_FLAGS.iteritems()
     55 )
     56 
     57 FAST_VARIANT_FLAGS_STRICT = dict(
     58     (v, [flags + ["--use-strict"] for flags in flag_sets])
     59     for v, flag_sets in testsuite.FAST_VARIANT_FLAGS.iteritems()
     60 )
     61 
     62 ALL_VARIANT_FLAGS_BOTH = dict(
     63     (v, [flags for flags in testsuite.ALL_VARIANT_FLAGS[v] +
     64                             ALL_VARIANT_FLAGS_STRICT[v]])
     65     for v in testsuite.ALL_VARIANT_FLAGS
     66 )
     67 
     68 FAST_VARIANT_FLAGS_BOTH = dict(
     69     (v, [flags for flags in testsuite.FAST_VARIANT_FLAGS[v] +
     70                             FAST_VARIANT_FLAGS_STRICT[v]])
     71     for v in testsuite.FAST_VARIANT_FLAGS
     72 )
     73 
     74 ALL_VARIANTS = {
     75   'nostrict': testsuite.ALL_VARIANT_FLAGS,
     76   'strict': ALL_VARIANT_FLAGS_STRICT,
     77   'both': ALL_VARIANT_FLAGS_BOTH,
     78 }
     79 
     80 FAST_VARIANTS = {
     81   'nostrict': testsuite.FAST_VARIANT_FLAGS,
     82   'strict': FAST_VARIANT_FLAGS_STRICT,
     83   'both': FAST_VARIANT_FLAGS_BOTH,
     84 }
     85 
     86 class Test262VariantGenerator(testsuite.VariantGenerator):
     87   def GetFlagSets(self, testcase, variant):
     88     if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
     89       variant_flags = FAST_VARIANTS
     90     else:
     91       variant_flags = ALL_VARIANTS
     92 
     93     test_record = self.suite.GetTestRecord(testcase)
     94     if "noStrict" in test_record:
     95       return variant_flags["nostrict"][variant]
     96     if "onlyStrict" in test_record:
     97       return variant_flags["strict"][variant]
     98     return variant_flags["both"][variant]
     99 
    100 
    101 class Test262TestSuite(testsuite.TestSuite):
    102 
    103   def __init__(self, name, root):
    104     super(Test262TestSuite, self).__init__(name, root)
    105     self.testroot = os.path.join(self.root, *TEST_262_SUITE_PATH)
    106     self.harnesspath = os.path.join(self.root, *TEST_262_HARNESS_PATH)
    107     self.harness = [os.path.join(self.harnesspath, f)
    108                     for f in TEST_262_HARNESS_FILES]
    109     self.harness += [os.path.join(self.root, "harness-adapt.js")]
    110     self.ParseTestRecord = None
    111 
    112   def ListTests(self, context):
    113     tests = []
    114     for dirname, dirs, files in os.walk(self.testroot):
    115       for dotted in [x for x in dirs if x.startswith(".")]:
    116         dirs.remove(dotted)
    117       if context.noi18n and "intl402" in dirs:
    118         dirs.remove("intl402")
    119       dirs.sort()
    120       files.sort()
    121       for filename in files:
    122         if filename.endswith(".js"):
    123           fullpath = os.path.join(dirname, filename)
    124           relpath = fullpath[len(self.testroot) + 1 : -3]
    125           testname = relpath.replace(os.path.sep, "/")
    126           case = testcase.TestCase(self, testname)
    127           tests.append(case)
    128     return tests
    129 
    130   def GetFlagsForTestCase(self, testcase, context):
    131     return (testcase.flags + context.mode_flags + self.harness +
    132             self.GetIncludesForTest(testcase) + ["--harmony"] +
    133             (["--module"] if "module" in self.GetTestRecord(testcase) else []) +
    134             [os.path.join(self.testroot, testcase.path + ".js")] +
    135             (["--throws"] if "negative" in self.GetTestRecord(testcase)
    136                           else []) +
    137             (["--allow-natives-syntax"]
    138              if "detachArrayBuffer.js" in
    139                 self.GetTestRecord(testcase).get("includes", [])
    140              else []))
    141 
    142   def _VariantGeneratorFactory(self):
    143     return Test262VariantGenerator
    144 
    145   def LoadParseTestRecord(self):
    146     if not self.ParseTestRecord:
    147       root = os.path.join(self.root, *TEST_262_TOOLS_PATH)
    148       f = None
    149       try:
    150         (f, pathname, description) = imp.find_module("parseTestRecord", [root])
    151         module = imp.load_module("parseTestRecord", f, pathname, description)
    152         self.ParseTestRecord = module.parseTestRecord
    153       except:
    154         raise ImportError("Cannot load parseTestRecord; you may need to "
    155                           "gclient sync for test262")
    156       finally:
    157         if f:
    158           f.close()
    159     return self.ParseTestRecord
    160 
    161   def GetTestRecord(self, testcase):
    162     if not hasattr(testcase, "test_record"):
    163       ParseTestRecord = self.LoadParseTestRecord()
    164       testcase.test_record = ParseTestRecord(self.GetSourceForTest(testcase),
    165                                              testcase.path)
    166     return testcase.test_record
    167 
    168   def BasePath(self, filename):
    169     return self.root if filename in TEST_262_NATIVE_FILES else self.harnesspath
    170 
    171   def GetIncludesForTest(self, testcase):
    172     test_record = self.GetTestRecord(testcase)
    173     if "includes" in test_record:
    174       return [os.path.join(self.BasePath(filename), filename)
    175               for filename in test_record.get("includes", [])]
    176     else:
    177       includes = []
    178     return includes
    179 
    180   def GetSourceForTest(self, testcase):
    181     filename = os.path.join(self.testroot, testcase.path + ".js")
    182     with open(filename) as f:
    183       return f.read()
    184 
    185   def _ParseException(self, str):
    186     for line in str.split("\n")[::-1]:
    187       if line and not line[0].isspace() and ":" in line:
    188         return line.split(":")[0]
    189 
    190 
    191   def IsFailureOutput(self, testcase):
    192     output = testcase.output
    193     test_record = self.GetTestRecord(testcase)
    194     if output.exit_code != 0:
    195       return True
    196     if "negative" in test_record:
    197       if self._ParseException(output.stdout) != test_record["negative"]:
    198         return True
    199     return "FAILED!" in output.stdout
    200 
    201   def HasUnexpectedOutput(self, testcase):
    202     outcome = self.GetOutcome(testcase)
    203     if (statusfile.FAIL_SLOPPY in testcase.outcomes and
    204         "--use-strict" not in testcase.flags):
    205       return outcome != statusfile.FAIL
    206     return not outcome in (testcase.outcomes or [statusfile.PASS])
    207 
    208   def DownloadData(self):
    209     print "Test262 download is deprecated. It's part of DEPS."
    210 
    211     # Clean up old directories and archive files.
    212     directory_old_name = os.path.join(self.root, "data.old")
    213     if os.path.exists(directory_old_name):
    214       shutil.rmtree(directory_old_name)
    215 
    216     archive_files = [f for f in os.listdir(self.root)
    217                      if f.startswith("tc39-test262-")]
    218     if len(archive_files) > 0:
    219       print "Clobber outdated test archives ..."
    220       for f in archive_files:
    221         os.remove(os.path.join(self.root, f))
    222 
    223     # The archive is created only on swarming. Local checkouts have the
    224     # data folder.
    225     if os.path.exists(ARCHIVE) and not os.path.exists(DATA):
    226       print "Extracting archive..."
    227       tar = tarfile.open(ARCHIVE)
    228       tar.extractall(path=os.path.dirname(ARCHIVE))
    229       tar.close()
    230 
    231 
    232 def GetSuite(name, root):
    233   return Test262TestSuite(name, root)
    234