Home | History | Annotate | Download | only in results 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import collections
      6 import copy
      7 import datetime
      8 import logging
      9 import random
     10 import sys
     11 import traceback
     12 
     13 from py_utils import cloud_storage  # pylint: disable=import-error
     14 
     15 from telemetry.internal.results import json_output_formatter
     16 from telemetry.internal.results import progress_reporter as reporter_module
     17 from telemetry.internal.results import story_run
     18 from telemetry import value as value_module
     19 from telemetry.value import failure
     20 from telemetry.value import skip
     21 from telemetry.value import trace
     22 
     23 
     24 class IterationInfo(object):
     25   def __init__(self):
     26     self._benchmark_name = None
     27     self._benchmark_start_ms = None
     28     self._label = None
     29     self._story_display_name = ''
     30     self._story_grouping_keys = {}
     31     self._story_repeat_counter = 0
     32     self._story_url = ''
     33     self._storyset_repeat_counter = 0
     34 
     35   @property
     36   def benchmark_name(self):
     37     return self._benchmark_name
     38 
     39   @benchmark_name.setter
     40   def benchmark_name(self, benchmark_name):
     41     assert self.benchmark_name is None, (
     42       'benchmark_name must be set exactly once')
     43     self._benchmark_name = benchmark_name
     44 
     45   @property
     46   def benchmark_start_ms(self):
     47     return self._benchmark_start_ms
     48 
     49   @benchmark_start_ms.setter
     50   def benchmark_start_ms(self, benchmark_start_ms):
     51     assert self.benchmark_start_ms is None, (
     52       'benchmark_start_ms must be set exactly once')
     53     self._benchmark_start_ms = benchmark_start_ms
     54 
     55   @property
     56   def label(self):
     57     return self._label
     58 
     59   @label.setter
     60   def label(self, label):
     61     assert self.label is None, 'label cannot be set more than once'
     62     self._label = label
     63 
     64   @property
     65   def story_display_name(self):
     66     return self._story_display_name
     67 
     68   @property
     69   def story_url(self):
     70     return self._story_url
     71 
     72   @property
     73   def story_grouping_keys(self):
     74     return self._story_grouping_keys
     75 
     76   @property
     77   def storyset_repeat_counter(self):
     78     return self._storyset_repeat_counter
     79 
     80   @property
     81   def story_repeat_counter(self):
     82     return self._story_repeat_counter
     83 
     84   def WillRunStory(self, story, storyset_repeat_counter, story_repeat_counter):
     85     self._story_display_name = story.display_name
     86     self._story_url = story.url
     87     if story.grouping_keys:
     88       self._story_grouping_keys = story.grouping_keys
     89     self._storyset_repeat_counter = storyset_repeat_counter
     90     self._story_repeat_counter = story_repeat_counter
     91 
     92   def AsDict(self):
     93     assert self.benchmark_name is not None, (
     94         'benchmark_name must be set exactly once')
     95     assert self.benchmark_start_ms is not None, (
     96         'benchmark_start_ms must be set exactly once')
     97     d = {}
     98     d['benchmarkName'] = self.benchmark_name
     99     d['benchmarkStartMs'] = self.benchmark_start_ms
    100     if self.label:
    101       d['label'] = self.label
    102     d['storyDisplayName'] = self.story_display_name
    103     d['storyGroupingKeys'] = self.story_grouping_keys
    104     d['storyRepeatCounter'] = self.story_repeat_counter
    105     d['storyUrl'] = self.story_url
    106     d['storysetRepeatCounter'] = self.storyset_repeat_counter
    107     return d
    108 
    109 
    110 class PageTestResults(object):
    111   def __init__(self, output_formatters=None,
    112                progress_reporter=None, trace_tag='', output_dir=None,
    113                value_can_be_added_predicate=lambda v, is_first: True):
    114     """
    115     Args:
    116       output_formatters: A list of output formatters. The output
    117           formatters are typically used to format the test results, such
    118           as CsvPivotTableOutputFormatter, which output the test results as CSV.
    119       progress_reporter: An instance of progress_reporter.ProgressReporter,
    120           to be used to output test status/results progressively.
    121       trace_tag: A string to append to the buildbot trace name. Currently only
    122           used for buildbot.
    123       output_dir: A string specified the directory where to store the test
    124           artifacts, e.g: trace, videos,...
    125       value_can_be_added_predicate: A function that takes two arguments:
    126           a value.Value instance (except failure.FailureValue, skip.SkipValue
    127           or trace.TraceValue) and a boolean (True when the value is part of
    128           the first result for the story). It returns True if the value
    129           can be added to the test results and False otherwise.
    130     """
    131     # TODO(chrishenry): Figure out if trace_tag is still necessary.
    132 
    133     super(PageTestResults, self).__init__()
    134     self._progress_reporter = (
    135         progress_reporter if progress_reporter is not None
    136         else reporter_module.ProgressReporter())
    137     self._output_formatters = (
    138         output_formatters if output_formatters is not None else [])
    139     self._trace_tag = trace_tag
    140     self._output_dir = output_dir
    141     self._value_can_be_added_predicate = value_can_be_added_predicate
    142 
    143     self._current_page_run = None
    144     self._all_page_runs = []
    145     self._all_stories = set()
    146     self._representative_value_for_each_value_name = {}
    147     self._all_summary_values = []
    148     self._serialized_trace_file_ids_to_paths = {}
    149     self._pages_to_profiling_files = collections.defaultdict(list)
    150     self._pages_to_profiling_files_cloud_url = collections.defaultdict(list)
    151 
    152     # You'd expect this to be a set(), but Values are dictionaries, which are
    153     # unhashable. We could wrap Values with custom __eq/hash__, but we don't
    154     # actually need set-ness in python.
    155     self._value_set = []
    156 
    157     self._iteration_info = IterationInfo()
    158 
    159   @property
    160   def iteration_info(self):
    161     return self._iteration_info
    162 
    163   @property
    164   def value_set(self):
    165     return self._value_set
    166 
    167   def __copy__(self):
    168     cls = self.__class__
    169     result = cls.__new__(cls)
    170     for k, v in self.__dict__.items():
    171       if isinstance(v, collections.Container):
    172         v = copy.copy(v)
    173       setattr(result, k, v)
    174     return result
    175 
    176   @property
    177   def pages_to_profiling_files(self):
    178     return self._pages_to_profiling_files
    179 
    180   @property
    181   def serialized_trace_file_ids_to_paths(self):
    182     return self._serialized_trace_file_ids_to_paths
    183 
    184   @property
    185   def pages_to_profiling_files_cloud_url(self):
    186     return self._pages_to_profiling_files_cloud_url
    187 
    188   @property
    189   def all_page_specific_values(self):
    190     values = []
    191     for run in self._all_page_runs:
    192       values += run.values
    193     if self._current_page_run:
    194       values += self._current_page_run.values
    195     return values
    196 
    197   @property
    198   def all_summary_values(self):
    199     return self._all_summary_values
    200 
    201   @property
    202   def current_page(self):
    203     assert self._current_page_run, 'Not currently running test.'
    204     return self._current_page_run.story
    205 
    206   @property
    207   def current_page_run(self):
    208     assert self._current_page_run, 'Not currently running test.'
    209     return self._current_page_run
    210 
    211   @property
    212   def all_page_runs(self):
    213     return self._all_page_runs
    214 
    215   @property
    216   def pages_that_succeeded(self):
    217     """Returns the set of pages that succeeded."""
    218     pages = set(run.story for run in self.all_page_runs)
    219     pages.difference_update(self.pages_that_failed)
    220     return pages
    221 
    222   @property
    223   def pages_that_failed(self):
    224     """Returns the set of failed pages."""
    225     failed_pages = set()
    226     for run in self.all_page_runs:
    227       if run.failed:
    228         failed_pages.add(run.story)
    229     return failed_pages
    230 
    231   @property
    232   def failures(self):
    233     values = self.all_page_specific_values
    234     return [v for v in values if isinstance(v, failure.FailureValue)]
    235 
    236   @property
    237   def skipped_values(self):
    238     values = self.all_page_specific_values
    239     return [v for v in values if isinstance(v, skip.SkipValue)]
    240 
    241   def _GetStringFromExcInfo(self, err):
    242     return ''.join(traceback.format_exception(*err))
    243 
    244   def CleanUp(self):
    245     """Clean up any TraceValues contained within this results object."""
    246     for run in self._all_page_runs:
    247       for v in run.values:
    248         if isinstance(v, trace.TraceValue):
    249           v.CleanUp()
    250           run.values.remove(v)
    251 
    252   def __enter__(self):
    253     return self
    254 
    255   def __exit__(self, _, __, ___):
    256     self.CleanUp()
    257 
    258   def WillRunPage(self, page, storyset_repeat_counter=0,
    259                   story_repeat_counter=0):
    260     assert not self._current_page_run, 'Did not call DidRunPage.'
    261     self._current_page_run = story_run.StoryRun(page)
    262     self._progress_reporter.WillRunPage(self)
    263     self.iteration_info.WillRunStory(
    264         page, storyset_repeat_counter, story_repeat_counter)
    265 
    266   def DidRunPage(self, page):  # pylint: disable=unused-argument
    267     """
    268     Args:
    269       page: The current page under test.
    270     """
    271     assert self._current_page_run, 'Did not call WillRunPage.'
    272     self._progress_reporter.DidRunPage(self)
    273     self._all_page_runs.append(self._current_page_run)
    274     self._all_stories.add(self._current_page_run.story)
    275     self._current_page_run = None
    276 
    277   def AddValue(self, value):
    278     assert self._current_page_run, 'Not currently running test.'
    279     self._ValidateValue(value)
    280     is_first_result = (
    281       self._current_page_run.story not in self._all_stories)
    282 
    283     story_keys = self._current_page_run.story.grouping_keys
    284 
    285     if story_keys:
    286       for k, v in story_keys.iteritems():
    287         assert k not in value.grouping_keys, (
    288             'Tried to add story grouping key ' + k + ' already defined by ' +
    289             'value')
    290         value.grouping_keys[k] = v
    291 
    292       # We sort by key name to make building the tir_label deterministic.
    293       story_keys_label = '_'.join(v for _, v in sorted(story_keys.iteritems()))
    294       if value.tir_label:
    295         assert value.tir_label == story_keys_label, (
    296             'Value has an explicit tir_label (%s) that does not match the '
    297             'one computed from story_keys (%s)' % (value.tir_label, story_keys))
    298       else:
    299         value.tir_label = story_keys_label
    300 
    301     if not (isinstance(value, skip.SkipValue) or
    302             isinstance(value, failure.FailureValue) or
    303             isinstance(value, trace.TraceValue) or
    304             self._value_can_be_added_predicate(value, is_first_result)):
    305       return
    306     # TODO(eakuefner/chrishenry): Add only one skip per pagerun assert here
    307     self._current_page_run.AddValue(value)
    308     self._progress_reporter.DidAddValue(value)
    309 
    310   def AddProfilingFile(self, page, file_handle):
    311     self._pages_to_profiling_files[page].append(file_handle)
    312 
    313   def AddSummaryValue(self, value):
    314     assert value.page is None
    315     self._ValidateValue(value)
    316     self._all_summary_values.append(value)
    317 
    318   def _ValidateValue(self, value):
    319     assert isinstance(value, value_module.Value)
    320     if value.name not in self._representative_value_for_each_value_name:
    321       self._representative_value_for_each_value_name[value.name] = value
    322     representative_value = self._representative_value_for_each_value_name[
    323         value.name]
    324     assert value.IsMergableWith(representative_value)
    325 
    326   def PrintSummary(self):
    327     self._progress_reporter.DidFinishAllTests(self)
    328 
    329     # Only serialize the trace if output_format is json.
    330     if (self._output_dir and
    331         any(isinstance(o, json_output_formatter.JsonOutputFormatter)
    332             for o in self._output_formatters)):
    333       self._SerializeTracesToDirPath(self._output_dir)
    334     for output_formatter in self._output_formatters:
    335       output_formatter.Format(self)
    336 
    337   def FindValues(self, predicate):
    338     """Finds all values matching the specified predicate.
    339 
    340     Args:
    341       predicate: A function that takes a Value and returns a bool.
    342     Returns:
    343       A list of values matching |predicate|.
    344     """
    345     values = []
    346     for value in self.all_page_specific_values:
    347       if predicate(value):
    348         values.append(value)
    349     return values
    350 
    351   def FindPageSpecificValuesForPage(self, page, value_name):
    352     return self.FindValues(lambda v: v.page == page and v.name == value_name)
    353 
    354   def FindAllPageSpecificValuesNamed(self, value_name):
    355     return self.FindValues(lambda v: v.name == value_name)
    356 
    357   def FindAllPageSpecificValuesFromIRNamed(self, tir_label, value_name):
    358     return self.FindValues(lambda v: v.name == value_name
    359                            and v.tir_label == tir_label)
    360 
    361   def FindAllTraceValues(self):
    362     return self.FindValues(lambda v: isinstance(v, trace.TraceValue))
    363 
    364   def _SerializeTracesToDirPath(self, dir_path):
    365     """ Serialize all trace values to files in dir_path and return a list of
    366     file handles to those files. """
    367     for value in self.FindAllTraceValues():
    368       fh = value.Serialize(dir_path)
    369       self._serialized_trace_file_ids_to_paths[fh.id] = fh.GetAbsPath()
    370 
    371   def UploadTraceFilesToCloud(self, bucket):
    372     for value in self.FindAllTraceValues():
    373       value.UploadToCloud(bucket)
    374 
    375   def UploadProfilingFilesToCloud(self, bucket):
    376     for page, file_handle_list in self._pages_to_profiling_files.iteritems():
    377       for file_handle in file_handle_list:
    378         remote_path = ('profiler-file-id_%s-%s%-d%s' % (
    379             file_handle.id,
    380             datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
    381             random.randint(1, 100000),
    382             file_handle.extension))
    383         try:
    384           cloud_url = cloud_storage.Insert(
    385               bucket, remote_path, file_handle.GetAbsPath())
    386           sys.stderr.write(
    387               'View generated profiler files online at %s for page %s\n' %
    388               (cloud_url, page.display_name))
    389           self._pages_to_profiling_files_cloud_url[page].append(cloud_url)
    390         except cloud_storage.PermissionError as e:
    391           logging.error('Cannot upload profiling files to cloud storage due to '
    392                         ' permission error: %s' % e.message)
    393