1 # Copyright 2013 the V8 project authors. All rights reserved. 2 # Redistribution and use in source and binary forms, with or without 3 # modification, are permitted provided that the following conditions are 4 # met: 5 # 6 # * Redistributions of source code must retain the above copyright 7 # notice, this list of conditions and the following disclaimer. 8 # * Redistributions in binary form must reproduce the above 9 # copyright notice, this list of conditions and the following 10 # disclaimer in the documentation and/or other materials provided 11 # with the distribution. 12 # * Neither the name of Google Inc. nor the names of its 13 # contributors may be used to endorse or promote products derived 14 # from this software without specific prior written permission. 15 # 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 29 import os 30 import shutil 31 import subprocess 32 import tarfile 33 34 from testrunner.local import testsuite 35 from testrunner.objects import testcase 36 37 38 class BenchmarksTestSuite(testsuite.TestSuite): 39 40 def __init__(self, name, root): 41 super(BenchmarksTestSuite, self).__init__(name, root) 42 self.testroot = root 43 44 def ListTests(self, context): 45 tests = [] 46 for test in [ 47 "kraken/ai-astar", 48 "kraken/audio-beat-detection", 49 "kraken/audio-dft", 50 "kraken/audio-fft", 51 "kraken/audio-oscillator", 52 "kraken/imaging-darkroom", 53 "kraken/imaging-desaturate", 54 "kraken/imaging-gaussian-blur", 55 "kraken/json-parse-financial", 56 "kraken/json-stringify-tinderbox", 57 "kraken/stanford-crypto-aes", 58 "kraken/stanford-crypto-ccm", 59 "kraken/stanford-crypto-pbkdf2", 60 "kraken/stanford-crypto-sha256-iterative", 61 62 "octane/box2d", 63 "octane/code-load", 64 "octane/crypto", 65 "octane/deltablue", 66 "octane/earley-boyer", 67 "octane/gbemu", 68 "octane/mandreel", 69 "octane/navier-stokes", 70 "octane/pdfjs", 71 "octane/raytrace", 72 "octane/regexp", 73 "octane/richards", 74 "octane/splay", 75 76 "sunspider/3d-cube", 77 "sunspider/3d-morph", 78 "sunspider/3d-raytrace", 79 "sunspider/access-binary-trees", 80 "sunspider/access-fannkuch", 81 "sunspider/access-nbody", 82 "sunspider/access-nsieve", 83 "sunspider/bitops-3bit-bits-in-byte", 84 "sunspider/bitops-bits-in-byte", 85 "sunspider/bitops-bitwise-and", 86 "sunspider/bitops-nsieve-bits", 87 "sunspider/controlflow-recursive", 88 "sunspider/crypto-aes", 89 "sunspider/crypto-md5", 90 "sunspider/crypto-sha1", 91 "sunspider/date-format-tofte", 92 "sunspider/date-format-xparb", 93 "sunspider/math-cordic", 94 "sunspider/math-partial-sums", 95 "sunspider/math-spectral-norm", 96 "sunspider/regexp-dna", 97 "sunspider/string-base64", 98 "sunspider/string-fasta", 99 "sunspider/string-tagcloud", 100 "sunspider/string-unpack-code", 101 "sunspider/string-validate-input"]: 102 tests.append(testcase.TestCase(self, test)) 103 return tests 104 105 def GetFlagsForTestCase(self, testcase, context): 106 result = [] 107 result += context.mode_flags 108 if testcase.path.startswith("kraken"): 109 result.append(os.path.join(self.testroot, "%s-data.js" % testcase.path)) 110 result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 111 elif testcase.path.startswith("octane"): 112 result.append(os.path.join(self.testroot, "octane/base.js")) 113 result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 114 result += ["-e", "BenchmarkSuite.RunSuites({});"] 115 elif testcase.path.startswith("sunspider"): 116 result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 117 return testcase.flags + result 118 119 def GetSourceForTest(self, testcase): 120 filename = os.path.join(self.testroot, testcase.path + ".js") 121 with open(filename) as f: 122 return f.read() 123 124 def _DownloadIfNecessary(self, url, revision, target_dir): 125 # Maybe we're still up to date? 126 revision_file = "CHECKED_OUT_%s" % target_dir 127 checked_out_revision = None 128 if os.path.exists(revision_file): 129 with open(revision_file) as f: 130 checked_out_revision = f.read() 131 if checked_out_revision == revision: 132 return 133 134 # If we have a local archive file with the test data, extract it. 135 if os.path.exists(target_dir): 136 shutil.rmtree(target_dir) 137 archive_file = "downloaded_%s_%s.tar.gz" % (target_dir, revision) 138 if os.path.exists(archive_file): 139 with tarfile.open(archive_file, "r:gz") as tar: 140 tar.extractall() 141 with open(revision_file, "w") as f: 142 f.write(revision) 143 return 144 145 # No cached copy. Check out via SVN, and pack as .tar.gz for later use. 146 command = "svn co %s -r %s %s" % (url, revision, target_dir) 147 code = subprocess.call(command, shell=True) 148 if code != 0: 149 raise Exception("Error checking out %s benchmark" % target_dir) 150 with tarfile.open(archive_file, "w:gz") as tar: 151 tar.add("%s" % target_dir) 152 with open(revision_file, "w") as f: 153 f.write(revision) 154 155 def DownloadData(self): 156 old_cwd = os.getcwd() 157 os.chdir(os.path.abspath(self.root)) 158 159 self._DownloadIfNecessary( 160 ("http://svn.webkit.org/repository/webkit/trunk/PerformanceTests/" 161 "SunSpider/tests/sunspider-1.0/"), 162 "153700", "sunspider") 163 164 self._DownloadIfNecessary( 165 ("http://kraken-mirror.googlecode.com/svn/trunk/kraken/tests/" 166 "kraken-1.1/"), 167 "8", "kraken") 168 169 self._DownloadIfNecessary( 170 "http://octane-benchmark.googlecode.com/svn/trunk/", 171 "22", "octane") 172 173 os.chdir(old_cwd) 174 175 def VariantFlags(self): 176 # Both --nocrankshaft and --stressopt are very slow. 177 return [[]] 178 179 180 def GetSuite(name, root): 181 return BenchmarksTestSuite(name, root) 182
