Lines Matching refs:suite
11 The suite json format is expected to be:
14 "name": <optional suite name, file name is default>,
15 "archs": [<architecture name for which this suite is run>, ...],
19 "run_count": <how often will this suite run (optional)>,
20 "run_count_XXX": <how often will this suite run for arch XXX (optional)>,
36 The tests field can also nest other suites in arbitrary depth. A suite
37 with a "main" file is a leaf suite that can contain one more level of
40 A suite's results_regexp is expected to have one string place holder
41 "%s" for the trace name. A trace's results_regexp overwrites suite
44 A suite's results_processor may point to an optional python script. If
46 suite level's path). It is expected to read the measurement's output text
51 A suite without "tests" is considered a performance test itself.
53 Full example (suite with one runner):
72 Full example (suite with several runners):
92 Path pieces are concatenated. D8 is always run with the suite's path as cwd.
245 # We assume the results processor is relative to the suite.
314 suite_units: Measurement default units as defined by the benchmark suite.
362 """Represents a node in the suite tree structure."""
390 """Represents a suite definition.
394 def __init__(self, suite, parent, arch):
396 self._suite = suite
398 assert isinstance(suite.get("path", []), list)
399 assert isinstance(suite["name"], basestring)
400 assert isinstance(suite.get("flags", []), list)
401 assert isinstance(suite.get("test_flags", []), list)
402 assert isinstance(suite.get("resources", []), list)
405 self.path = parent.path[:] + suite.get("path", [])
406 self.graphs = parent.graphs[:] + [suite["name"]]
407 self.flags = parent.flags[:] + suite.get("flags", [])
408 self.test_flags = parent.test_flags[:] + suite.get("test_flags", [])
411 self.resources = suite.get("resources", [])
414 self.binary = suite.get("binary", parent.binary)
415 self.run_count = suite.get("run_count", parent.run_count)
416 self.run_count = suite.get("run_count_%s" % arch, self.run_count)
417 self.timeout = suite.get("timeout", parent.timeout)
418 self.timeout = suite.get("timeout_%s" % arch, self.timeout)
419 self.units = suite.get("units", parent.units)
420 self.total = suite.get("total", parent.total)
421 self.results_processor = suite.get(
425 # regexp and the current suite has none, a string place holder for the
426 # suite name is expected.
430 regexp_default = parent.results_regexp % re.escape(suite["name"])
433 self.results_regexp = suite.get("results_regexp", regexp_default)
437 stddev_default = parent.stddev_regexp % re.escape(suite["name"])
440 self.stddev_regexp = suite.get("stddev_regexp", stddev_default)
444 """Represents a leaf in the suite tree structure."""
445 def __init__(self, suite, parent, arch):
446 super(TraceConfig, self).__init__(suite, parent, arch)
463 """Represents a runnable suite definition (i.e. has a main file).
481 The tests are supposed to be relative to the suite configuration.
525 """Represents a runnable suite definition that is a leaf."""
526 def __init__(self, suite, parent, arch):
527 super(RunnableTraceConfig, self).__init__(suite, parent, arch)
543 """Represents a runnable suite definition with generic traces."""
544 def __init__(self, suite, parent, arch):
545 super(RunnableGenericConfig, self).__init__(suite, parent, arch)
555 def MakeGraphConfig(suite, arch, parent):
559 return TraceConfig(suite, parent, arch)
560 elif suite.get("main") is not None:
562 if suite.get("tests"):
564 return RunnableConfig(suite, parent, arch)
567 return RunnableTraceConfig(suite, parent, arch)
568 elif suite.get("generic"):
569 # This is a generic suite definition. It is either a runnable executable
571 return RunnableGenericConfig(suite, parent, arch)
572 elif suite.get("tests"):
574 return GraphConfig(suite, parent, arch)
576 raise Exception("Invalid suite configuration.")
579 def BuildGraphConfigs(suite, arch, parent):
580 """Builds a tree structure of graph objects that corresponds to the suite
585 if arch not in suite.get("archs", SUPPORTED_ARCHS):
588 graph = MakeGraphConfig(suite, arch, parent)
589 for subsuite in suite.get("tests", []):
607 raise Exception("Invalid suite configuration.")
999 "for shorter completion time of suite, with potentially "
1067 suite = json.loads(f.read())
1070 suite.setdefault("name", os.path.splitext(os.path.basename(path))[0])
1072 # Setup things common to one test suite.
1077 root = BuildGraphConfigs(suite, options.arch, default_parent)
1085 print ">>> Running suite: %s" % "/".join(runnable.graphs)
