Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 # Copyright 2014 the V8 project authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """
      7 Performance runner for d8.
      8 
      9 Call e.g. with tools/run-perf.py --arch ia32 some_suite.json
     10 
     11 The suite json format is expected to be:
     12 {
     13   "path": <relative path chunks to perf resources and main file>,
     14   "name": <optional suite name, file name is default>,
     15   "archs": [<architecture name for which this suite is run>, ...],
     16   "binary": <name of binary to run, default "d8">,
     17   "flags": [<flag to d8>, ...],
     18   "run_count": <how often will this suite run (optional)>,
     19   "run_count_XXX": <how often will this suite run for arch XXX (optional)>,
     20   "resources": [<js file to be loaded before main>, ...]
     21   "main": <main js perf runner file>,
     22   "results_regexp": <optional regexp>,
     23   "results_processor": <optional python results processor script>,
     24   "units": <the unit specification for the performance dashboard>,
     25   "tests": [
     26     {
     27       "name": <name of the trace>,
     28       "results_regexp": <optional more specific regexp>,
     29       "results_processor": <optional python results processor script>,
     30       "units": <the unit specification for the performance dashboard>,
     31     }, ...
     32   ]
     33 }
     34 
     35 The tests field can also nest other suites in arbitrary depth. A suite
     36 with a "main" file is a leaf suite that can contain one more level of
     37 tests.
     38 
     39 A suite's results_regexp is expected to have one string place holder
     40 "%s" for the trace name. A trace's results_regexp overwrites suite
     41 defaults.
     42 
     43 A suite's results_processor may point to an optional python script. If
     44 specified, it is called after running the tests like this (with a path
     45 relatve to the suite level's path):
     46 <results_processor file> <same flags as for d8> <suite level name> <output>
     47 
     48 The <output> is a temporary file containing d8 output. The results_regexp will
     49 be applied to the output of this script.
     50 
     51 A suite without "tests" is considered a performance test itself.
     52 
     53 Full example (suite with one runner):
     54 {
     55   "path": ["."],
     56   "flags": ["--expose-gc"],
     57   "archs": ["ia32", "x64"],
     58   "run_count": 5,
     59   "run_count_ia32": 3,
     60   "main": "run.js",
     61   "results_regexp": "^%s: (.+)$",
     62   "units": "score",
     63   "tests": [
     64     {"name": "Richards"},
     65     {"name": "DeltaBlue"},
     66     {"name": "NavierStokes",
     67      "results_regexp": "^NavierStokes: (.+)$"}
     68   ]
     69 }
     70 
     71 Full example (suite with several runners):
     72 {
     73   "path": ["."],
     74   "flags": ["--expose-gc"],
     75   "archs": ["ia32", "x64"],
     76   "run_count": 5,
     77   "units": "score",
     78   "tests": [
     79     {"name": "Richards",
     80      "path": ["richards"],
     81      "main": "run.js",
     82      "run_count": 3,
     83      "results_regexp": "^Richards: (.+)$"},
     84     {"name": "NavierStokes",
     85      "path": ["navier_stokes"],
     86      "main": "run.js",
     87      "results_regexp": "^NavierStokes: (.+)$"}
     88   ]
     89 }
     90 
     91 Path pieces are concatenated. D8 is always run with the suite's path as cwd.
     92 """
     93 
     94 import json
     95 import math
     96 import optparse
     97 import os
     98 import re
     99 import sys
    100 
    101 from testrunner.local import commands
    102 from testrunner.local import utils
    103 
    104 ARCH_GUESS = utils.DefaultArch()
    105 SUPPORTED_ARCHS = ["android_arm",
    106                    "android_arm64",
    107                    "android_ia32",
    108                    "arm",
    109                    "ia32",
    110                    "mips",
    111                    "mipsel",
    112                    "nacl_ia32",
    113                    "nacl_x64",
    114                    "x64",
    115                    "arm64"]
    116 
    117 GENERIC_RESULTS_RE = re.compile(
    118     r"^Trace\(([^\)]+)\), Result\(([^\)]+)\), StdDev\(([^\)]+)\)$")
    119 
    120 
    121 def GeometricMean(values):
    122   """Returns the geometric mean of a list of values.
    123 
    124   The mean is calculated using log to avoid overflow.
    125   """
    126   values = map(float, values)
    127   return str(math.exp(sum(map(math.log, values)) / len(values)))
    128 
    129 
    130 class Results(object):
    131   """Place holder for result traces."""
    132   def __init__(self, traces=None, errors=None):
    133     self.traces = traces or []
    134     self.errors = errors or []
    135 
    136   def ToDict(self):
    137     return {"traces": self.traces, "errors": self.errors}
    138 
    139   def WriteToFile(self, file_name):
    140     with open(file_name, "w") as f:
    141       f.write(json.dumps(self.ToDict()))
    142 
    143   def __add__(self, other):
    144     self.traces += other.traces
    145     self.errors += other.errors
    146     return self
    147 
    148   def __str__(self):  # pragma: no cover
    149     return str(self.ToDict())
    150 
    151 
    152 class Node(object):
    153   """Represents a node in the suite tree structure."""
    154   def __init__(self, *args):
    155     self._children = []
    156 
    157   def AppendChild(self, child):
    158     self._children.append(child)
    159 
    160 
    161 class DefaultSentinel(Node):
    162   """Fake parent node with all default values."""
    163   def __init__(self):
    164     super(DefaultSentinel, self).__init__()
    165     self.binary = "d8"
    166     self.run_count = 10
    167     self.timeout = 60
    168     self.path = []
    169     self.graphs = []
    170     self.flags = []
    171     self.resources = []
    172     self.results_regexp = None
    173     self.stddev_regexp = None
    174     self.units = "score"
    175     self.total = False
    176 
    177 
    178 class Graph(Node):
    179   """Represents a suite definition.
    180 
    181   Can either be a leaf or an inner node that provides default values.
    182   """
    183   def __init__(self, suite, parent, arch):
    184     super(Graph, self).__init__()
    185     self._suite = suite
    186 
    187     assert isinstance(suite.get("path", []), list)
    188     assert isinstance(suite["name"], basestring)
    189     assert isinstance(suite.get("flags", []), list)
    190     assert isinstance(suite.get("resources", []), list)
    191 
    192     # Accumulated values.
    193     self.path = parent.path[:] + suite.get("path", [])
    194     self.graphs = parent.graphs[:] + [suite["name"]]
    195     self.flags = parent.flags[:] + suite.get("flags", [])
    196     self.resources = parent.resources[:] + suite.get("resources", [])
    197 
    198     # Descrete values (with parent defaults).
    199     self.binary = suite.get("binary", parent.binary)
    200     self.run_count = suite.get("run_count", parent.run_count)
    201     self.run_count = suite.get("run_count_%s" % arch, self.run_count)
    202     self.timeout = suite.get("timeout", parent.timeout)
    203     self.units = suite.get("units", parent.units)
    204     self.total = suite.get("total", parent.total)
    205 
    206     # A regular expression for results. If the parent graph provides a
    207     # regexp and the current suite has none, a string place holder for the
    208     # suite name is expected.
    209     # TODO(machenbach): Currently that makes only sense for the leaf level.
    210     # Multiple place holders for multiple levels are not supported.
    211     if parent.results_regexp:
    212       regexp_default = parent.results_regexp % re.escape(suite["name"])
    213     else:
    214       regexp_default = None
    215     self.results_regexp = suite.get("results_regexp", regexp_default)
    216 
    217     # A similar regular expression for the standard deviation (optional).
    218     if parent.stddev_regexp:
    219       stddev_default = parent.stddev_regexp % re.escape(suite["name"])
    220     else:
    221       stddev_default = None
    222     self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
    223 
    224 
    225 class Trace(Graph):
    226   """Represents a leaf in the suite tree structure.
    227 
    228   Handles collection of measurements.
    229   """
    230   def __init__(self, suite, parent, arch):
    231     super(Trace, self).__init__(suite, parent, arch)
    232     assert self.results_regexp
    233     self.results = []
    234     self.errors = []
    235     self.stddev = ""
    236 
    237   def ConsumeOutput(self, stdout):
    238     try:
    239       self.results.append(
    240           re.search(self.results_regexp, stdout, re.M).group(1))
    241     except:
    242       self.errors.append("Regexp \"%s\" didn't match for test %s."
    243                          % (self.results_regexp, self.graphs[-1]))
    244 
    245     try:
    246       if self.stddev_regexp and self.stddev:
    247         self.errors.append("Test %s should only run once since a stddev "
    248                            "is provided by the test." % self.graphs[-1])
    249       if self.stddev_regexp:
    250         self.stddev = re.search(self.stddev_regexp, stdout, re.M).group(1)
    251     except:
    252       self.errors.append("Regexp \"%s\" didn't match for test %s."
    253                          % (self.stddev_regexp, self.graphs[-1]))
    254 
    255   def GetResults(self):
    256     return Results([{
    257       "graphs": self.graphs,
    258       "units": self.units,
    259       "results": self.results,
    260       "stddev": self.stddev,
    261     }], self.errors)
    262 
    263 
    264 class Runnable(Graph):
    265   """Represents a runnable suite definition (i.e. has a main file).
    266   """
    267   @property
    268   def main(self):
    269     return self._suite.get("main", "")
    270 
    271   def ChangeCWD(self, suite_path):
    272     """Changes the cwd to to path defined in the current graph.
    273 
    274     The tests are supposed to be relative to the suite configuration.
    275     """
    276     suite_dir = os.path.abspath(os.path.dirname(suite_path))
    277     bench_dir = os.path.normpath(os.path.join(*self.path))
    278     os.chdir(os.path.join(suite_dir, bench_dir))
    279 
    280   def GetCommand(self, shell_dir):
    281     # TODO(machenbach): This requires +.exe if run on windows.
    282     return (
    283       [os.path.join(shell_dir, self.binary)] +
    284       self.flags +
    285       self.resources +
    286       [self.main]
    287     )
    288 
    289   def Run(self, runner):
    290     """Iterates over several runs and handles the output for all traces."""
    291     for stdout in runner():
    292       for trace in self._children:
    293         trace.ConsumeOutput(stdout)
    294     res = reduce(lambda r, t: r + t.GetResults(), self._children, Results())
    295 
    296     if not res.traces or not self.total:
    297       return res
    298 
    299     # Assume all traces have the same structure.
    300     if len(set(map(lambda t: len(t["results"]), res.traces))) != 1:
    301       res.errors.append("Not all traces have the same number of results.")
    302       return res
    303 
    304     # Calculate the geometric means for all traces. Above we made sure that
    305     # there is at least one trace and that the number of results is the same
    306     # for each trace.
    307     n_results = len(res.traces[0]["results"])
    308     total_results = [GeometricMean(t["results"][i] for t in res.traces)
    309                      for i in range(0, n_results)]
    310     res.traces.append({
    311       "graphs": self.graphs + ["Total"],
    312       "units": res.traces[0]["units"],
    313       "results": total_results,
    314       "stddev": "",
    315     })
    316     return res
    317 
    318 class RunnableTrace(Trace, Runnable):
    319   """Represents a runnable suite definition that is a leaf."""
    320   def __init__(self, suite, parent, arch):
    321     super(RunnableTrace, self).__init__(suite, parent, arch)
    322 
    323   def Run(self, runner):
    324     """Iterates over several runs and handles the output."""
    325     for stdout in runner():
    326       self.ConsumeOutput(stdout)
    327     return self.GetResults()
    328 
    329 
    330 class RunnableGeneric(Runnable):
    331   """Represents a runnable suite definition with generic traces."""
    332   def __init__(self, suite, parent, arch):
    333     super(RunnableGeneric, self).__init__(suite, parent, arch)
    334 
    335   def Run(self, runner):
    336     """Iterates over several runs and handles the output."""
    337     traces = {}
    338     for stdout in runner():
    339       for line in stdout.strip().splitlines():
    340         match = GENERIC_RESULTS_RE.match(line)
    341         if match:
    342           trace = match.group(1)
    343           result = match.group(2)
    344           stddev = match.group(3)
    345           trace_result = traces.setdefault(trace, Results([{
    346             "graphs": self.graphs + [trace],
    347             "units": self.units,
    348             "results": [],
    349             "stddev": "",
    350           }], []))
    351           trace_result.traces[0]["results"].append(result)
    352           trace_result.traces[0]["stddev"] = stddev
    353 
    354     return reduce(lambda r, t: r + t, traces.itervalues(), Results())
    355 
    356 
    357 def MakeGraph(suite, arch, parent):
    358   """Factory method for making graph objects."""
    359   if isinstance(parent, Runnable):
    360     # Below a runnable can only be traces.
    361     return Trace(suite, parent, arch)
    362   elif suite.get("main"):
    363     # A main file makes this graph runnable.
    364     if suite.get("tests"):
    365       # This graph has subgraphs (traces).
    366       return Runnable(suite, parent, arch)
    367     else:
    368       # This graph has no subgraphs, it's a leaf.
    369       return RunnableTrace(suite, parent, arch)
    370   elif suite.get("generic"):
    371     # This is a generic suite definition. It is either a runnable executable
    372     # or has a main js file.
    373     return RunnableGeneric(suite, parent, arch)
    374   elif suite.get("tests"):
    375     # This is neither a leaf nor a runnable.
    376     return Graph(suite, parent, arch)
    377   else:  # pragma: no cover
    378     raise Exception("Invalid suite configuration.")
    379 
    380 
    381 def BuildGraphs(suite, arch, parent=None):
    382   """Builds a tree structure of graph objects that corresponds to the suite
    383   configuration.
    384   """
    385   parent = parent or DefaultSentinel()
    386 
    387   # TODO(machenbach): Implement notion of cpu type?
    388   if arch not in suite.get("archs", ["ia32", "x64"]):
    389     return None
    390 
    391   graph = MakeGraph(suite, arch, parent)
    392   for subsuite in suite.get("tests", []):
    393     BuildGraphs(subsuite, arch, graph)
    394   parent.AppendChild(graph)
    395   return graph
    396 
    397 
    398 def FlattenRunnables(node):
    399   """Generator that traverses the tree structure and iterates over all
    400   runnables.
    401   """
    402   if isinstance(node, Runnable):
    403     yield node
    404   elif isinstance(node, Node):
    405     for child in node._children:
    406       for result in FlattenRunnables(child):
    407         yield result
    408   else:  # pragma: no cover
    409     raise Exception("Invalid suite configuration.")
    410 
    411 
    412 # TODO: Implement results_processor.
    413 def Main(args):
    414   parser = optparse.OptionParser()
    415   parser.add_option("--arch",
    416                     help=("The architecture to run tests for, "
    417                           "'auto' or 'native' for auto-detect"),
    418                     default="x64")
    419   parser.add_option("--buildbot",
    420                     help="Adapt to path structure used on buildbots",
    421                     default=False, action="store_true")
    422   parser.add_option("--json-test-results",
    423                     help="Path to a file for storing json results.")
    424   parser.add_option("--outdir", help="Base directory with compile output",
    425                     default="out")
    426   (options, args) = parser.parse_args(args)
    427 
    428   if len(args) == 0:  # pragma: no cover
    429     parser.print_help()
    430     return 1
    431 
    432   if options.arch in ["auto", "native"]:  # pragma: no cover
    433     options.arch = ARCH_GUESS
    434 
    435   if not options.arch in SUPPORTED_ARCHS:  # pragma: no cover
    436     print "Unknown architecture %s" % options.arch
    437     return 1
    438 
    439   workspace = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
    440 
    441   if options.buildbot:
    442     shell_dir = os.path.join(workspace, options.outdir, "Release")
    443   else:
    444     shell_dir = os.path.join(workspace, options.outdir,
    445                              "%s.release" % options.arch)
    446 
    447   results = Results()
    448   for path in args:
    449     path = os.path.abspath(path)
    450 
    451     if not os.path.exists(path):  # pragma: no cover
    452       results.errors.append("Configuration file %s does not exist." % path)
    453       continue
    454 
    455     with open(path) as f:
    456       suite = json.loads(f.read())
    457 
    458     # If no name is given, default to the file name without .json.
    459     suite.setdefault("name", os.path.splitext(os.path.basename(path))[0])
    460 
    461     for runnable in FlattenRunnables(BuildGraphs(suite, options.arch)):
    462       print ">>> Running suite: %s" % "/".join(runnable.graphs)
    463       runnable.ChangeCWD(path)
    464 
    465       def Runner():
    466         """Output generator that reruns several times."""
    467         for i in xrange(0, max(1, runnable.run_count)):
    468           # TODO(machenbach): Allow timeout per arch like with run_count per
    469           # arch.
    470           output = commands.Execute(runnable.GetCommand(shell_dir),
    471                                     timeout=runnable.timeout)
    472           print ">>> Stdout (#%d):" % (i + 1)
    473           print output.stdout
    474           if output.stderr:  # pragma: no cover
    475             # Print stderr for debugging.
    476             print ">>> Stderr (#%d):" % (i + 1)
    477             print output.stderr
    478           if output.timed_out:
    479             print ">>> Test timed out after %ss." % runnable.timeout
    480           yield output.stdout
    481 
    482       # Let runnable iterate over all runs and handle output.
    483       results += runnable.Run(Runner)
    484 
    485   if options.json_test_results:
    486     results.WriteToFile(options.json_test_results)
    487   else:  # pragma: no cover
    488     print results
    489 
    490   return min(1, len(results.errors))
    491 
    492 if __name__ == "__main__":  # pragma: no cover
    493   sys.exit(Main(sys.argv[1:]))
    494