Lines Matching full:suite
11 The suite json format is expected to be:
14 "name": <optional suite name, file name is default>,
15 "archs": [<architecture name for which this suite is run>, ...],
18 "run_count": <how often will this suite run (optional)>,
19 "run_count_XXX": <how often will this suite run for arch XXX (optional)>,
35 The tests field can also nest other suites in arbitrary depth. A suite
36 with a "main" file is a leaf suite that can contain one more level of
39 A suite's results_regexp is expected to have one string place holder
40 "%s" for the trace name. A trace's results_regexp overwrites suite
43 A suite's results_processor may point to an optional python script. If
45 relatve to the suite level's path):
46 <results_processor file> <same flags as for d8> <suite level name> <output>
51 A suite without "tests" is considered a performance test itself.
53 Full example (suite with one runner):
71 Full example (suite with several runners):
91 Path pieces are concatenated. D8 is always run with the suite's path as cwd.
153 """Represents a node in the suite tree structure."""
179 """Represents a suite definition.
183 def __init__(self, suite, parent, arch):
185 self._suite = suite
187 assert isinstance(suite.get("path", []), list)
188 assert isinstance(suite["name"], basestring)
189 assert isinstance(suite.get("flags", []), list)
190 assert isinstance(suite.get("resources", []), list)
193 self.path = parent.path[:] + suite.get("path", [])
194 self.graphs = parent.graphs[:] + [suite["name"]]
195 self.flags = parent.flags[:] + suite.get("flags", [])
196 self.resources = parent.resources[:] + suite.get("resources", [])
199 self.binary = suite.get("binary", parent.binary)
200 self.run_count = suite.get("run_count", parent.run_count)
201 self.run_count = suite.get("run_count_%s" % arch, self.run_count)
202 self.timeout = suite.get("timeout", parent.timeout)
203 self.units = suite.get("units", parent.units)
204 self.total = suite.get("total", parent.total)
207 # regexp and the current suite has none, a string place holder for the
208 # suite name is expected.
212 regexp_default = parent.results_regexp % re.escape(suite["name"])
215 self.results_regexp = suite.get("results_regexp", regexp_default)
219 stddev_default = parent.stddev_regexp % re.escape(suite["name"])
222 self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
226 """Represents a leaf in the suite tree structure.
230 def __init__(self, suite, parent, arch):
231 super(Trace, self).__init__(suite, parent, arch)
265 """Represents a runnable suite definition (i.e. has a main file).
274 The tests are supposed to be relative to the suite configuration.
319 """Represents a runnable suite definition that is a leaf."""
320 def __init__(self, suite, parent, arch):
321 super(RunnableTrace, self).__init__(suite, parent, arch)
331 """Represents a runnable suite definition with generic traces."""
332 def __init__(self, suite, parent, arch):
333 super(RunnableGeneric, self).__init__(suite, parent, arch)
357 def MakeGraph(suite, arch, parent):
361 return Trace(suite, parent, arch)
362 elif suite.get("main"):
364 if suite.get("tests"):
366 return Runnable(suite, parent, arch)
369 return RunnableTrace(suite, parent, arch)
370 elif suite.get("generic"):
371 # This is a generic suite definition. It is either a runnable executable
373 return RunnableGeneric(suite, parent, arch)
374 elif suite.get("tests"):
376 return Graph(suite, parent, arch)
378 raise Exception("Invalid suite configuration.")
381 def BuildGraphs(suite, arch, parent=None):
382 """Builds a tree structure of graph objects that corresponds to the suite
388 if arch not in suite.get("archs", ["ia32", "x64"]):
391 graph = MakeGraph(suite, arch, parent)
392 for subsuite in suite.get("tests", []):
409 raise Exception("Invalid suite configuration.")
456 suite = json.loads(f.read())
459 suite.setdefault("name", os.path.splitext(os.path.basename(path))[0])
461 for runnable in FlattenRunnables(BuildGraphs(suite, options.arch)):
462 print ">>> Running suite: %s" % "/".join(runnable.graphs)
