Home | History | Annotate | Download | only in telemetry 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import optparse
      6 
      7 from telemetry import decorators
      8 from telemetry.internal import story_runner
      9 from telemetry.internal.util import command_line
     10 from telemetry.page import page_test
     11 from telemetry.web_perf import timeline_based_measurement
     12 
     13 Disabled = decorators.Disabled
     14 Enabled = decorators.Enabled
     15 
     16 
     17 class InvalidOptionsError(Exception):
     18   """Raised for invalid benchmark options."""
     19   pass
     20 
     21 
     22 class BenchmarkMetadata(object):
     23   def __init__(self, name, description='', rerun_options=None):
     24     self._name = name
     25     self._description = description
     26     self._rerun_options = rerun_options
     27 
     28   @property
     29   def name(self):
     30     return self._name
     31 
     32   @property
     33   def description(self):
     34     return self._description
     35 
     36   @property
     37   def rerun_options(self):
     38     return self._rerun_options
     39 
     40   def AsDict(self):
     41     return {
     42       'type': 'telemetry_benchmark',
     43       'name': self._name,
     44       'description': self._description,
     45       'rerun_options': self._rerun_options,
     46     }
     47 
     48 
     49 class Benchmark(command_line.Command):
     50   """Base class for a Telemetry benchmark.
     51 
     52   A benchmark packages a measurement and a PageSet together.
     53   Benchmarks default to using TBM unless you override the value of
     54   Benchmark.test, or override the CreatePageTest method.
     55 
     56   New benchmarks should override CreateStorySet.
     57   """
     58   options = {}
     59   page_set = None
     60   test = timeline_based_measurement.TimelineBasedMeasurement
     61 
     62   def __init__(self, max_failures=None):
     63     """Creates a new Benchmark.
     64 
     65     Args:
     66       max_failures: The number of story run's failures before bailing
     67           from executing subsequent page runs. If None, we never bail.
     68     """
     69     self._max_failures = max_failures
     70     self._has_original_tbm_options = (
     71         self.CreateTimelineBasedMeasurementOptions.__func__ ==
     72         Benchmark.CreateTimelineBasedMeasurementOptions.__func__)
     73     has_original_create_page_test = (
     74         self.CreatePageTest.__func__ == Benchmark.CreatePageTest.__func__)
     75     assert self._has_original_tbm_options or has_original_create_page_test, (
     76         'Cannot override both CreatePageTest and '
     77         'CreateTimelineBasedMeasurementOptions.')
     78 
     79   # pylint: disable=unused-argument
     80   @classmethod
     81   def ShouldDisable(cls, possible_browser):
     82     """Override this method to disable a benchmark under specific conditions.
     83 
     84      Supports logic too complex for simple Enabled and Disabled decorators.
     85      Decorators are still respected in cases where this function returns False.
     86      """
     87     return False
     88 
     89   def Run(self, finder_options):
     90     """Do not override this method."""
     91     return story_runner.RunBenchmark(self, finder_options)
     92 
     93   @property
     94   def max_failures(self):
     95     return self._max_failures
     96 
     97   @classmethod
     98   def Name(cls):
     99     return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
    100 
    101   @classmethod
    102   def ShouldTearDownStateAfterEachStoryRun(cls):
    103     """Override this method to tear down state after each story run.
    104 
    105     Tearing down all states after each story run, e.g., clearing profiles,
    106     stopping the browser, stopping local server, etc. So the browser will not be
    107     reused among multiple stories. This is particularly useful to get the
    108     startup part of launching the browser in each story.
    109 
    110     This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
    111     not by PageTest based benchmarks.
    112     """
    113     return False
    114 
    115   @classmethod
    116   def AddCommandLineArgs(cls, parser):
    117     group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
    118     cls.AddBenchmarkCommandLineArgs(group)
    119 
    120     if cls.HasTraceRerunDebugOption():
    121       group.add_option(
    122           '--rerun-with-debug-trace',
    123           action='store_true',
    124           help='Rerun option that enables more extensive tracing.')
    125 
    126     if group.option_list:
    127       parser.add_option_group(group)
    128 
    129   @classmethod
    130   def AddBenchmarkCommandLineArgs(cls, group):
    131     del group  # unused
    132 
    133   @classmethod
    134   def HasTraceRerunDebugOption(cls):
    135     return False
    136 
    137   def GetTraceRerunCommands(self):
    138     if self.HasTraceRerunDebugOption():
    139       return [['Debug Trace', '--rerun-with-debug-trace']]
    140     return []
    141 
    142   def SetupTraceRerunOptions(self, browser_options, tbm_options):
    143     if self.HasTraceRerunDebugOption():
    144       if browser_options.rerun_with_debug_trace:
    145         self.SetupBenchmarkDebugTraceRerunOptions(tbm_options)
    146       else:
    147         self.SetupBenchmarkDefaultTraceRerunOptions(tbm_options)
    148 
    149   def SetupBenchmarkDefaultTraceRerunOptions(self, tbm_options):
    150     """Setup tracing categories associated with default trace option."""
    151 
    152   def SetupBenchmarkDebugTraceRerunOptions(self, tbm_options):
    153     """Setup tracing categories associated with debug trace option."""
    154 
    155   @classmethod
    156   def SetArgumentDefaults(cls, parser):
    157     default_values = parser.get_default_values()
    158     invalid_options = [
    159         o for o in cls.options if not hasattr(default_values, o)]
    160     if invalid_options:
    161       raise InvalidOptionsError('Invalid benchmark options: %s',
    162                                 ', '.join(invalid_options))
    163     parser.set_defaults(**cls.options)
    164 
    165   @classmethod
    166   def ProcessCommandLineArgs(cls, parser, args):
    167     pass
    168 
    169   # pylint: disable=unused-argument
    170   @classmethod
    171   def ValueCanBeAddedPredicate(cls, value, is_first_result):
    172     """Returns whether |value| can be added to the test results.
    173 
    174     Override this method to customize the logic of adding values to test
    175     results.
    176 
    177     Args:
    178       value: a value.Value instance (except failure.FailureValue,
    179         skip.SkipValue or trace.TraceValue which will always be added).
    180       is_first_result: True if |value| is the first result for its
    181           corresponding story.
    182 
    183     Returns:
    184       True if |value| should be added to the test results.
    185       Otherwise, it returns False.
    186     """
    187     return True
    188 
    189   def CustomizeBrowserOptions(self, options):
    190     """Add browser options that are required by this benchmark."""
    191 
    192   def GetMetadata(self):
    193     return BenchmarkMetadata(
    194         self.Name(), self.__doc__, self.GetTraceRerunCommands())
    195 
    196   def CreateTimelineBasedMeasurementOptions(self):
    197     """Return the TimelineBasedMeasurementOptions for this Benchmark.
    198 
    199     Override this method to configure a TimelineBasedMeasurement benchmark.
    200     Otherwise, override CreatePageTest for PageTest tests. Do not override
    201     both methods.
    202     """
    203     return timeline_based_measurement.Options()
    204 
    205   def CreatePageTest(self, options):  # pylint: disable=unused-argument
    206     """Return the PageTest for this Benchmark.
    207 
    208     Override this method for PageTest tests.
    209     Override, override CreateTimelineBasedMeasurementOptions to configure
    210     TimelineBasedMeasurement tests. Do not override both methods.
    211 
    212     Args:
    213       options: a browser_options.BrowserFinderOptions instance
    214     Returns:
    215       |test()| if |test| is a PageTest class.
    216       Otherwise, a TimelineBasedMeasurement instance.
    217     """
    218     is_page_test = issubclass(self.test, page_test.PageTest)
    219     is_tbm = self.test == timeline_based_measurement.TimelineBasedMeasurement
    220     if not is_page_test and not is_tbm:
    221       raise TypeError('"%s" is not a PageTest or a TimelineBasedMeasurement.' %
    222                       self.test.__name__)
    223     if is_page_test:
    224       assert self._has_original_tbm_options, (
    225           'Cannot override CreateTimelineBasedMeasurementOptions '
    226           'with a PageTest.')
    227       return self.test()  # pylint: disable=no-value-for-parameter
    228 
    229     opts = self.CreateTimelineBasedMeasurementOptions()
    230     self.SetupTraceRerunOptions(options, opts)
    231     return timeline_based_measurement.TimelineBasedMeasurement(opts)
    232 
    233   def CreateStorySet(self, options):
    234     """Creates the instance of StorySet used to run the benchmark.
    235 
    236     Can be overridden by subclasses.
    237     """
    238     del options  # unused
    239     # TODO(aiolos, nednguyen, eakufner): replace class attribute page_set with
    240     # story_set.
    241     if not self.page_set:
    242       raise NotImplementedError('This test has no "page_set" attribute.')
    243     return self.page_set()  # pylint: disable=not-callable
    244 
    245 
    246 def AddCommandLineArgs(parser):
    247   story_runner.AddCommandLineArgs(parser)
    248 
    249 
    250 def ProcessCommandLineArgs(parser, args):
    251   story_runner.ProcessCommandLineArgs(parser, args)
    252