1 #!/usr/bin/env python 2 # Copyright 2014 the V8 project authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 from collections import namedtuple 7 import coverage 8 import json 9 from mock import DEFAULT 10 from mock import MagicMock 11 import os 12 from os import path, sys 13 import shutil 14 import tempfile 15 import unittest 16 17 # Requires python-coverage and python-mock. Native python coverage 18 # version >= 3.7.1 should be installed to get the best speed. 19 20 TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf") 21 22 V8_JSON = { 23 "path": ["."], 24 "binary": "d7", 25 "flags": ["--flag"], 26 "main": "run.js", 27 "run_count": 1, 28 "results_regexp": "^%s: (.+)$", 29 "tests": [ 30 {"name": "Richards"}, 31 {"name": "DeltaBlue"}, 32 ] 33 } 34 35 V8_NESTED_SUITES_JSON = { 36 "path": ["."], 37 "flags": ["--flag"], 38 "run_count": 1, 39 "units": "score", 40 "tests": [ 41 {"name": "Richards", 42 "path": ["richards"], 43 "binary": "d7", 44 "main": "run.js", 45 "resources": ["file1.js", "file2.js"], 46 "run_count": 2, 47 "results_regexp": "^Richards: (.+)$"}, 48 {"name": "Sub", 49 "path": ["sub"], 50 "tests": [ 51 {"name": "Leaf", 52 "path": ["leaf"], 53 "run_count_x64": 3, 54 "units": "ms", 55 "main": "run.js", 56 "results_regexp": "^Simple: (.+) ms.$"}, 57 ] 58 }, 59 {"name": "DeltaBlue", 60 "path": ["delta_blue"], 61 "main": "run.js", 62 "flags": ["--flag2"], 63 "results_regexp": "^DeltaBlue: (.+)$"}, 64 {"name": "ShouldntRun", 65 "path": ["."], 66 "archs": ["arm"], 67 "main": "run.js"}, 68 ] 69 } 70 71 V8_GENERIC_JSON = { 72 "path": ["."], 73 "binary": "cc", 74 "flags": ["--flag"], 75 "generic": True, 76 "run_count": 1, 77 "units": "ms", 78 } 79 80 Output = namedtuple("Output", "stdout, stderr, timed_out") 81 82 class PerfTest(unittest.TestCase): 83 @classmethod 84 def setUpClass(cls): 85 cls.base = path.dirname(path.dirname(path.abspath(__file__))) 86 sys.path.append(cls.base) 87 cls._cov = coverage.coverage( 88 include=([os.path.join(cls.base, "run_perf.py")])) 89 cls._cov.start() 90 import run_perf 91 from testrunner.local import commands 92 global commands 93 global run_perf 94 95 @classmethod 96 def tearDownClass(cls): 97 cls._cov.stop() 98 print "" 99 print cls._cov.report() 100 101 def setUp(self): 102 self.maxDiff = None 103 if path.exists(TEST_WORKSPACE): 104 shutil.rmtree(TEST_WORKSPACE) 105 os.makedirs(TEST_WORKSPACE) 106 107 def tearDown(self): 108 if path.exists(TEST_WORKSPACE): 109 shutil.rmtree(TEST_WORKSPACE) 110 111 def _WriteTestInput(self, json_content): 112 self._test_input = path.join(TEST_WORKSPACE, "test.json") 113 with open(self._test_input, "w") as f: 114 f.write(json.dumps(json_content)) 115 116 def _MockCommand(self, *args, **kwargs): 117 # Fake output for each test run. 118 test_outputs = [Output(stdout=arg, 119 stderr=None, 120 timed_out=kwargs.get("timed_out", False)) 121 for arg in args[1]] 122 def execute(*args, **kwargs): 123 return test_outputs.pop() 124 commands.Execute = MagicMock(side_effect=execute) 125 126 # Check that d8 is called from the correct cwd for each test run. 127 dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]] 128 def chdir(*args, **kwargs): 129 self.assertEquals(dirs.pop(), args[0]) 130 os.chdir = MagicMock(side_effect=chdir) 131 132 def _CallMain(self, *args): 133 self._test_output = path.join(TEST_WORKSPACE, "results.json") 134 all_args=[ 135 "--json-test-results", 136 self._test_output, 137 self._test_input, 138 ] 139 all_args += args 140 return run_perf.Main(all_args) 141 142 def _LoadResults(self): 143 with open(self._test_output) as f: 144 return json.load(f) 145 146 def _VerifyResults(self, suite, units, traces): 147 self.assertEquals([ 148 {"units": units, 149 "graphs": [suite, trace["name"]], 150 "results": trace["results"], 151 "stddev": trace["stddev"]} for trace in traces], 152 self._LoadResults()["traces"]) 153 154 def _VerifyErrors(self, errors): 155 self.assertEquals(errors, self._LoadResults()["errors"]) 156 157 def _VerifyMock(self, binary, *args, **kwargs): 158 arg = [path.join(path.dirname(self.base), binary)] 159 arg += args 160 commands.Execute.assert_called_with( 161 arg, timeout=kwargs.get("timeout", 60)) 162 163 def _VerifyMockMultiple(self, *args, **kwargs): 164 expected = [] 165 for arg in args: 166 a = [path.join(path.dirname(self.base), arg[0])] 167 a += arg[1:] 168 expected.append(((a,), {"timeout": kwargs.get("timeout", 60)})) 169 self.assertEquals(expected, commands.Execute.call_args_list) 170 171 def testOneRun(self): 172 self._WriteTestInput(V8_JSON) 173 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) 174 self.assertEquals(0, self._CallMain()) 175 self._VerifyResults("test", "score", [ 176 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 177 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 178 ]) 179 self._VerifyErrors([]) 180 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 181 182 def testTwoRuns_Units_SuiteName(self): 183 test_input = dict(V8_JSON) 184 test_input["run_count"] = 2 185 test_input["name"] = "v8" 186 test_input["units"] = "ms" 187 self._WriteTestInput(test_input) 188 self._MockCommand([".", "."], 189 ["Richards: 100\nDeltaBlue: 200\n", 190 "Richards: 50\nDeltaBlue: 300\n"]) 191 self.assertEquals(0, self._CallMain()) 192 self._VerifyResults("v8", "ms", [ 193 {"name": "Richards", "results": ["50", "100"], "stddev": ""}, 194 {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, 195 ]) 196 self._VerifyErrors([]) 197 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 198 199 def testTwoRuns_SubRegexp(self): 200 test_input = dict(V8_JSON) 201 test_input["run_count"] = 2 202 del test_input["results_regexp"] 203 test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$" 204 test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$" 205 self._WriteTestInput(test_input) 206 self._MockCommand([".", "."], 207 ["Richards: 100\nDeltaBlue: 200\n", 208 "Richards: 50\nDeltaBlue: 300\n"]) 209 self.assertEquals(0, self._CallMain()) 210 self._VerifyResults("test", "score", [ 211 {"name": "Richards", "results": ["50", "100"], "stddev": ""}, 212 {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, 213 ]) 214 self._VerifyErrors([]) 215 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 216 217 def testNestedSuite(self): 218 self._WriteTestInput(V8_NESTED_SUITES_JSON) 219 self._MockCommand(["delta_blue", "sub/leaf", "richards"], 220 ["DeltaBlue: 200\n", 221 "Simple: 1 ms.\n", 222 "Simple: 2 ms.\n", 223 "Simple: 3 ms.\n", 224 "Richards: 100\n", 225 "Richards: 50\n"]) 226 self.assertEquals(0, self._CallMain()) 227 self.assertEquals([ 228 {"units": "score", 229 "graphs": ["test", "Richards"], 230 "results": ["50", "100"], 231 "stddev": ""}, 232 {"units": "ms", 233 "graphs": ["test", "Sub", "Leaf"], 234 "results": ["3", "2", "1"], 235 "stddev": ""}, 236 {"units": "score", 237 "graphs": ["test", "DeltaBlue"], 238 "results": ["200"], 239 "stddev": ""}, 240 ], self._LoadResults()["traces"]) 241 self._VerifyErrors([]) 242 self._VerifyMockMultiple( 243 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", 244 "file2.js", "run.js"), 245 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", 246 "file2.js", "run.js"), 247 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 248 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 249 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 250 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) 251 252 def testOneRunStdDevRegExp(self): 253 test_input = dict(V8_JSON) 254 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 255 self._WriteTestInput(test_input) 256 self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n" 257 "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"]) 258 self.assertEquals(0, self._CallMain()) 259 self._VerifyResults("test", "score", [ 260 {"name": "Richards", "results": ["1.234"], "stddev": "0.23"}, 261 {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"}, 262 ]) 263 self._VerifyErrors([]) 264 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 265 266 def testTwoRunsStdDevRegExp(self): 267 test_input = dict(V8_JSON) 268 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 269 test_input["run_count"] = 2 270 self._WriteTestInput(test_input) 271 self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n" 272 "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n", 273 "Richards: 2\nRichards-stddev: 0.5\n" 274 "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"]) 275 self.assertEquals(1, self._CallMain()) 276 self._VerifyResults("test", "score", [ 277 {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"}, 278 {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"}, 279 ]) 280 self._VerifyErrors( 281 ["Test Richards should only run once since a stddev is provided " 282 "by the test.", 283 "Test DeltaBlue should only run once since a stddev is provided " 284 "by the test.", 285 "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test " 286 "DeltaBlue."]) 287 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 288 289 def testBuildbot(self): 290 self._WriteTestInput(V8_JSON) 291 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 292 self.assertEquals(0, self._CallMain("--buildbot")) 293 self._VerifyResults("test", "score", [ 294 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 295 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 296 ]) 297 self._VerifyErrors([]) 298 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 299 300 def testBuildbotWithTotal(self): 301 test_input = dict(V8_JSON) 302 test_input["total"] = True 303 self._WriteTestInput(test_input) 304 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 305 self.assertEquals(0, self._CallMain("--buildbot")) 306 self._VerifyResults("test", "score", [ 307 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 308 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 309 {"name": "Total", "results": ["3626.49109719"], "stddev": ""}, 310 ]) 311 self._VerifyErrors([]) 312 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 313 314 def testBuildbotWithTotalAndErrors(self): 315 test_input = dict(V8_JSON) 316 test_input["total"] = True 317 self._WriteTestInput(test_input) 318 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) 319 self.assertEquals(1, self._CallMain("--buildbot")) 320 self._VerifyResults("test", "score", [ 321 {"name": "Richards", "results": [], "stddev": ""}, 322 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 323 ]) 324 self._VerifyErrors( 325 ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.", 326 "Not all traces have the same number of results."]) 327 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 328 329 def testRegexpNoMatch(self): 330 self._WriteTestInput(V8_JSON) 331 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) 332 self.assertEquals(1, self._CallMain()) 333 self._VerifyResults("test", "score", [ 334 {"name": "Richards", "results": [], "stddev": ""}, 335 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 336 ]) 337 self._VerifyErrors( 338 ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."]) 339 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 340 341 def testOneRunGeneric(self): 342 test_input = dict(V8_GENERIC_JSON) 343 self._WriteTestInput(test_input) 344 self._MockCommand(["."], [ 345 "Trace(Test1), Result(1.234), StdDev(0.23)\n" 346 "Trace(Test2), Result(10657567), StdDev(106)\n"]) 347 self.assertEquals(0, self._CallMain()) 348 self._VerifyResults("test", "ms", [ 349 {"name": "Test1", "results": ["1.234"], "stddev": "0.23"}, 350 {"name": "Test2", "results": ["10657567"], "stddev": "106"}, 351 ]) 352 self._VerifyErrors([]) 353 self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "") 354 355 def testOneRunTimingOut(self): 356 test_input = dict(V8_JSON) 357 test_input["timeout"] = 70 358 self._WriteTestInput(test_input) 359 self._MockCommand(["."], [""], timed_out=True) 360 self.assertEquals(1, self._CallMain()) 361 self._VerifyResults("test", "score", [ 362 {"name": "Richards", "results": [], "stddev": ""}, 363 {"name": "DeltaBlue", "results": [], "stddev": ""}, 364 ]) 365 self._VerifyErrors([ 366 "Regexp \"^Richards: (.+)$\" didn't match for test Richards.", 367 "Regexp \"^DeltaBlue: (.+)$\" didn't match for test DeltaBlue.", 368 ]) 369 self._VerifyMock( 370 path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70) 371
