1 # Copyright 2013 the V8 project authors. All rights reserved. 2 # Redistribution and use in source and binary forms, with or without 3 # modification, are permitted provided that the following conditions are 4 # met: 5 # 6 # * Redistributions of source code must retain the above copyright 7 # notice, this list of conditions and the following disclaimer. 8 # * Redistributions in binary form must reproduce the above 9 # copyright notice, this list of conditions and the following 10 # disclaimer in the documentation and/or other materials provided 11 # with the distribution. 12 # * Neither the name of Google Inc. nor the names of its 13 # contributors may be used to endorse or promote products derived 14 # from this software without specific prior written permission. 15 # 16 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 29 import os 30 import shutil 31 import subprocess 32 import tarfile 33 34 from testrunner.local import testsuite 35 from testrunner.objects import testcase 36 37 38 class BenchmarksTestSuite(testsuite.TestSuite): 39 40 def __init__(self, name, root): 41 super(BenchmarksTestSuite, self).__init__(name, root) 42 self.testroot = root 43 44 def ListTests(self, context): 45 tests = [] 46 for test in [ 47 "kraken/ai-astar", 48 "kraken/audio-beat-detection", 49 "kraken/audio-dft", 50 "kraken/audio-fft", 51 "kraken/audio-oscillator", 52 "kraken/imaging-darkroom", 53 "kraken/imaging-desaturate", 54 "kraken/imaging-gaussian-blur", 55 "kraken/json-parse-financial", 56 "kraken/json-stringify-tinderbox", 57 "kraken/stanford-crypto-aes", 58 "kraken/stanford-crypto-ccm", 59 "kraken/stanford-crypto-pbkdf2", 60 "kraken/stanford-crypto-sha256-iterative", 61 62 "octane/box2d", 63 "octane/code-load", 64 "octane/crypto", 65 "octane/deltablue", 66 "octane/earley-boyer", 67 "octane/gbemu-part1", 68 "octane/mandreel", 69 "octane/navier-stokes", 70 "octane/pdfjs", 71 "octane/raytrace", 72 "octane/regexp", 73 "octane/richards", 74 "octane/splay", 75 "octane/typescript", 76 "octane/zlib", 77 78 "sunspider/3d-cube", 79 "sunspider/3d-morph", 80 "sunspider/3d-raytrace", 81 "sunspider/access-binary-trees", 82 "sunspider/access-fannkuch", 83 "sunspider/access-nbody", 84 "sunspider/access-nsieve", 85 "sunspider/bitops-3bit-bits-in-byte", 86 "sunspider/bitops-bits-in-byte", 87 "sunspider/bitops-bitwise-and", 88 "sunspider/bitops-nsieve-bits", 89 "sunspider/controlflow-recursive", 90 "sunspider/crypto-aes", 91 "sunspider/crypto-md5", 92 "sunspider/crypto-sha1", 93 "sunspider/date-format-tofte", 94 "sunspider/date-format-xparb", 95 "sunspider/math-cordic", 96 "sunspider/math-partial-sums", 97 "sunspider/math-spectral-norm", 98 "sunspider/regexp-dna", 99 "sunspider/string-base64", 100 "sunspider/string-fasta", 101 "sunspider/string-tagcloud", 102 "sunspider/string-unpack-code", 103 "sunspider/string-validate-input"]: 104 tests.append(testcase.TestCase(self, test)) 105 return tests 106 107 def GetFlagsForTestCase(self, testcase, context): 108 result = [] 109 result += context.mode_flags 110 if testcase.path.startswith("kraken"): 111 result.append(os.path.join(self.testroot, "%s-data.js" % testcase.path)) 112 result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 113 elif testcase.path.startswith("octane"): 114 result.append(os.path.join(self.testroot, "octane/base.js")) 115 result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 116 if testcase.path.startswith("octane/gbemu"): 117 result.append(os.path.join(self.testroot, "octane/gbemu-part2.js")) 118 elif testcase.path.startswith("octane/typescript"): 119 result.append(os.path.join(self.testroot, 120 "octane/typescript-compiler.js")) 121 result.append(os.path.join(self.testroot, "octane/typescript-input.js")) 122 elif testcase.path.startswith("octane/zlib"): 123 result.append(os.path.join(self.testroot, "octane/zlib-data.js")) 124 result += ["-e", "BenchmarkSuite.RunSuites({});"] 125 elif testcase.path.startswith("sunspider"): 126 result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 127 return testcase.flags + result 128 129 def GetSourceForTest(self, testcase): 130 filename = os.path.join(self.testroot, testcase.path + ".js") 131 with open(filename) as f: 132 return f.read() 133 134 def _DownloadIfNecessary(self, url, revision, target_dir): 135 # Maybe we're still up to date? 136 revision_file = "CHECKED_OUT_%s" % target_dir 137 checked_out_revision = None 138 if os.path.exists(revision_file): 139 with open(revision_file) as f: 140 checked_out_revision = f.read() 141 if checked_out_revision == revision: 142 return 143 144 # If we have a local archive file with the test data, extract it. 145 if os.path.exists(target_dir): 146 shutil.rmtree(target_dir) 147 archive_file = "downloaded_%s_%s.tar.gz" % (target_dir, revision) 148 if os.path.exists(archive_file): 149 with tarfile.open(archive_file, "r:gz") as tar: 150 tar.extractall() 151 with open(revision_file, "w") as f: 152 f.write(revision) 153 return 154 155 # No cached copy. Check out via SVN, and pack as .tar.gz for later use. 156 command = "svn co %s -r %s %s" % (url, revision, target_dir) 157 code = subprocess.call(command, shell=True) 158 if code != 0: 159 raise Exception("Error checking out %s benchmark" % target_dir) 160 with tarfile.open(archive_file, "w:gz") as tar: 161 tar.add("%s" % target_dir) 162 with open(revision_file, "w") as f: 163 f.write(revision) 164 165 def DownloadData(self): 166 old_cwd = os.getcwd() 167 os.chdir(os.path.abspath(self.root)) 168 169 self._DownloadIfNecessary( 170 ("http://svn.webkit.org/repository/webkit/trunk/PerformanceTests/" 171 "SunSpider/tests/sunspider-1.0.2/"), 172 "159499", "sunspider") 173 174 self._DownloadIfNecessary( 175 ("http://kraken-mirror.googlecode.com/svn/trunk/kraken/tests/" 176 "kraken-1.1/"), 177 "8", "kraken") 178 179 self._DownloadIfNecessary( 180 "http://octane-benchmark.googlecode.com/svn/trunk/", 181 "26", "octane") 182 183 os.chdir(old_cwd) 184 185 def VariantFlags(self, testcase, default_flags): 186 # Both --nocrankshaft and --stressopt are very slow. Add TF but without 187 # always opt to match the way the benchmarks are run for performance 188 # testing. 189 return [[], ["--turbo-filter=*"]] 190 191 192 def GetSuite(name, root): 193 return BenchmarksTestSuite(name, root) 194
