Home | History | Annotate | Download | only in benchmarks
      1 # Copyright 2013 the V8 project authors. All rights reserved.
      2 # Redistribution and use in source and binary forms, with or without
      3 # modification, are permitted provided that the following conditions are
      4 # met:
      5 #
      6 #     * Redistributions of source code must retain the above copyright
      7 #       notice, this list of conditions and the following disclaimer.
      8 #     * Redistributions in binary form must reproduce the above
      9 #       copyright notice, this list of conditions and the following
     10 #       disclaimer in the documentation and/or other materials provided
     11 #       with the distribution.
     12 #     * Neither the name of Google Inc. nor the names of its
     13 #       contributors may be used to endorse or promote products derived
     14 #       from this software without specific prior written permission.
     15 #
     16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
     17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
     18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
     19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
     20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
     21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
     22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27 
     28 
     29 import os
     30 import shutil
     31 import subprocess
     32 import tarfile
     33 
     34 from testrunner.local import testsuite
     35 from testrunner.objects import testcase
     36 
     37 
     38 class BenchmarksTestSuite(testsuite.TestSuite):
     39 
     40   def __init__(self, name, root):
     41     super(BenchmarksTestSuite, self).__init__(name, root)
     42     self.testroot = root
     43 
     44   def ListTests(self, context):
     45     tests = []
     46     for test in [
     47         "kraken/ai-astar",
     48         "kraken/audio-beat-detection",
     49         "kraken/audio-dft",
     50         "kraken/audio-fft",
     51         "kraken/audio-oscillator",
     52         "kraken/imaging-darkroom",
     53         "kraken/imaging-desaturate",
     54         "kraken/imaging-gaussian-blur",
     55         "kraken/json-parse-financial",
     56         "kraken/json-stringify-tinderbox",
     57         "kraken/stanford-crypto-aes",
     58         "kraken/stanford-crypto-ccm",
     59         "kraken/stanford-crypto-pbkdf2",
     60         "kraken/stanford-crypto-sha256-iterative",
     61 
     62         "octane/box2d",
     63         "octane/code-load",
     64         "octane/crypto",
     65         "octane/deltablue",
     66         "octane/earley-boyer",
     67         "octane/gbemu-part1",
     68         "octane/mandreel",
     69         "octane/navier-stokes",
     70         "octane/pdfjs",
     71         "octane/raytrace",
     72         "octane/regexp",
     73         "octane/richards",
     74         "octane/splay",
     75         "octane/typescript",
     76         "octane/zlib",
     77 
     78         "sunspider/3d-cube",
     79         "sunspider/3d-morph",
     80         "sunspider/3d-raytrace",
     81         "sunspider/access-binary-trees",
     82         "sunspider/access-fannkuch",
     83         "sunspider/access-nbody",
     84         "sunspider/access-nsieve",
     85         "sunspider/bitops-3bit-bits-in-byte",
     86         "sunspider/bitops-bits-in-byte",
     87         "sunspider/bitops-bitwise-and",
     88         "sunspider/bitops-nsieve-bits",
     89         "sunspider/controlflow-recursive",
     90         "sunspider/crypto-aes",
     91         "sunspider/crypto-md5",
     92         "sunspider/crypto-sha1",
     93         "sunspider/date-format-tofte",
     94         "sunspider/date-format-xparb",
     95         "sunspider/math-cordic",
     96         "sunspider/math-partial-sums",
     97         "sunspider/math-spectral-norm",
     98         "sunspider/regexp-dna",
     99         "sunspider/string-base64",
    100         "sunspider/string-fasta",
    101         "sunspider/string-tagcloud",
    102         "sunspider/string-unpack-code",
    103         "sunspider/string-validate-input"]:
    104       tests.append(testcase.TestCase(self, test))
    105     return tests
    106 
    107   def GetFlagsForTestCase(self, testcase, context):
    108     result = []
    109     result += context.mode_flags
    110     if testcase.path.startswith("kraken"):
    111       result.append(os.path.join(self.testroot, "%s-data.js" % testcase.path))
    112       result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
    113     elif testcase.path.startswith("octane"):
    114       result.append(os.path.join(self.testroot, "octane/base.js"))
    115       result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
    116       if testcase.path.startswith("octane/gbemu"):
    117         result.append(os.path.join(self.testroot, "octane/gbemu-part2.js"))
    118       elif testcase.path.startswith("octane/typescript"):
    119         result.append(os.path.join(self.testroot,
    120                                    "octane/typescript-compiler.js"))
    121         result.append(os.path.join(self.testroot, "octane/typescript-input.js"))
    122       elif testcase.path.startswith("octane/zlib"):
    123         result.append(os.path.join(self.testroot, "octane/zlib-data.js"))
    124       result += ["-e", "BenchmarkSuite.RunSuites({});"]
    125     elif testcase.path.startswith("sunspider"):
    126       result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
    127     return testcase.flags + result
    128 
    129   def GetSourceForTest(self, testcase):
    130     filename = os.path.join(self.testroot, testcase.path + ".js")
    131     with open(filename) as f:
    132       return f.read()
    133 
    134   def _DownloadIfNecessary(self, url, revision, target_dir):
    135     # Maybe we're still up to date?
    136     revision_file = "CHECKED_OUT_%s" % target_dir
    137     checked_out_revision = None
    138     if os.path.exists(revision_file):
    139       with open(revision_file) as f:
    140         checked_out_revision = f.read()
    141     if checked_out_revision == revision:
    142       return
    143 
    144     # If we have a local archive file with the test data, extract it.
    145     if os.path.exists(target_dir):
    146       shutil.rmtree(target_dir)
    147     archive_file = "downloaded_%s_%s.tar.gz" % (target_dir, revision)
    148     if os.path.exists(archive_file):
    149       with tarfile.open(archive_file, "r:gz") as tar:
    150         tar.extractall()
    151       with open(revision_file, "w") as f:
    152         f.write(revision)
    153       return
    154 
    155     # No cached copy. Check out via SVN, and pack as .tar.gz for later use.
    156     command = "svn co %s -r %s %s" % (url, revision, target_dir)
    157     code = subprocess.call(command, shell=True)
    158     if code != 0:
    159       raise Exception("Error checking out %s benchmark" % target_dir)
    160     with tarfile.open(archive_file, "w:gz") as tar:
    161       tar.add("%s" % target_dir)
    162     with open(revision_file, "w") as f:
    163       f.write(revision)
    164 
    165   def DownloadData(self):
    166     old_cwd = os.getcwd()
    167     os.chdir(os.path.abspath(self.root))
    168 
    169     self._DownloadIfNecessary(
    170         ("http://svn.webkit.org/repository/webkit/trunk/PerformanceTests/"
    171          "SunSpider/tests/sunspider-1.0.2/"),
    172         "159499", "sunspider")
    173 
    174     self._DownloadIfNecessary(
    175         ("http://kraken-mirror.googlecode.com/svn/trunk/kraken/tests/"
    176          "kraken-1.1/"),
    177         "8", "kraken")
    178 
    179     self._DownloadIfNecessary(
    180         "http://octane-benchmark.googlecode.com/svn/trunk/",
    181         "26", "octane")
    182 
    183     os.chdir(old_cwd)
    184 
    185   def VariantFlags(self, testcase, default_flags):
    186     # Both --nocrankshaft and --stressopt are very slow. Add TF but without
    187     # always opt to match the way the benchmarks are run for performance
    188     # testing.
    189     return [[], ["--turbo-filter=*"]]
    190 
    191 
    192 def GetSuite(name, root):
    193   return BenchmarksTestSuite(name, root)
    194