Home | History | Annotate | Download | only in results 
      1 # Copyright 2014 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import collections
      6 import copy
      7 import datetime
      8 import json
      9 import logging
     10 import os
     11 import random
     12 import sys
     13 import tempfile
     14 import traceback
     15 
     16 from py_utils import cloud_storage  # pylint: disable=import-error
     17 
     18 from telemetry import value as value_module
     19 from telemetry.internal.results import chart_json_output_formatter
     20 from telemetry.internal.results import json_output_formatter
     21 from telemetry.internal.results import progress_reporter as reporter_module
     22 from telemetry.internal.results import story_run
     23 from telemetry.value import failure
     24 from telemetry.value import skip
     25 from telemetry.value import trace
     26 
     27 from tracing.value import convert_chart_json
     28 
     29 class TelemetryInfo(object):
     30   def __init__(self):
     31     self._benchmark_name = None
     32     self._benchmark_start_ms = None
     33     self._label = None
     34     self._story_display_name = ''
     35     self._story_grouping_keys = {}
     36     self._storyset_repeat_counter = 0
     37 
     38   @property
     39   def benchmark_name(self):
     40     return self._benchmark_name
     41 
     42   @benchmark_name.setter
     43   def benchmark_name(self, benchmark_name):
     44     assert self.benchmark_name is None, (
     45       'benchmark_name must be set exactly once')
     46     self._benchmark_name = benchmark_name
     47 
     48   @property
     49   def benchmark_start_ms(self):
     50     return self._benchmark_start_ms
     51 
     52   @benchmark_start_ms.setter
     53   def benchmark_start_ms(self, benchmark_start_ms):
     54     assert self.benchmark_start_ms is None, (
     55       'benchmark_start_ms must be set exactly once')
     56     self._benchmark_start_ms = benchmark_start_ms
     57 
     58   @property
     59   def label(self):
     60     return self._label
     61 
     62   @label.setter
     63   def label(self, label):
     64     assert self.label is None, 'label cannot be set more than once'
     65     self._label = label
     66 
     67   @property
     68   def story_display_name(self):
     69     return self._story_display_name
     70 
     71   @property
     72   def story_grouping_keys(self):
     73     return self._story_grouping_keys
     74 
     75   @property
     76   def storyset_repeat_counter(self):
     77     return self._storyset_repeat_counter
     78 
     79   def WillRunStory(self, story, storyset_repeat_counter):
     80     self._story_display_name = story.display_name
     81     if story.grouping_keys:
     82       self._story_grouping_keys = story.grouping_keys
     83     self._storyset_repeat_counter = storyset_repeat_counter
     84 
     85   def AsDict(self):
     86     assert self.benchmark_name is not None, (
     87         'benchmark_name must be set exactly once')
     88     assert self.benchmark_start_ms is not None, (
     89         'benchmark_start_ms must be set exactly once')
     90     d = {}
     91     d['benchmarkName'] = self.benchmark_name
     92     d['benchmarkStartMs'] = self.benchmark_start_ms
     93     if self.label:
     94       d['label'] = self.label
     95     d['storyDisplayName'] = self.story_display_name
     96     d['storyGroupingKeys'] = self.story_grouping_keys
     97     d['storysetRepeatCounter'] = self.storyset_repeat_counter
     98     return d
     99 
    100 
    101 class PageTestResults(object):
    102   def __init__(self, output_formatters=None,
    103                progress_reporter=None, trace_tag='', output_dir=None,
    104                value_can_be_added_predicate=lambda v, is_first: True,
    105                benchmark_enabled=True):
    106     """
    107     Args:
    108       output_formatters: A list of output formatters. The output
    109           formatters are typically used to format the test results, such
    110           as CsvPivotTableOutputFormatter, which output the test results as CSV.
    111       progress_reporter: An instance of progress_reporter.ProgressReporter,
    112           to be used to output test status/results progressively.
    113       trace_tag: A string to append to the buildbot trace name. Currently only
    114           used for buildbot.
    115       output_dir: A string specified the directory where to store the test
    116           artifacts, e.g: trace, videos,...
    117       value_can_be_added_predicate: A function that takes two arguments:
    118           a value.Value instance (except failure.FailureValue, skip.SkipValue
    119           or trace.TraceValue) and a boolean (True when the value is part of
    120           the first result for the story). It returns True if the value
    121           can be added to the test results and False otherwise.
    122     """
    123     # TODO(chrishenry): Figure out if trace_tag is still necessary.
    124 
    125     super(PageTestResults, self).__init__()
    126     self._progress_reporter = (
    127         progress_reporter if progress_reporter is not None
    128         else reporter_module.ProgressReporter())
    129     self._output_formatters = (
    130         output_formatters if output_formatters is not None else [])
    131     self._trace_tag = trace_tag
    132     self._output_dir = output_dir
    133     self._value_can_be_added_predicate = value_can_be_added_predicate
    134 
    135     self._current_page_run = None
    136     self._all_page_runs = []
    137     self._all_stories = set()
    138     self._representative_value_for_each_value_name = {}
    139     self._all_summary_values = []
    140     self._serialized_trace_file_ids_to_paths = {}
    141     self._pages_to_profiling_files = collections.defaultdict(list)
    142     self._pages_to_profiling_files_cloud_url = collections.defaultdict(list)
    143 
    144     # You'd expect this to be a set(), but Values are dictionaries, which are
    145     # unhashable. We could wrap Values with custom __eq/hash__, but we don't
    146     # actually need set-ness in python.
    147     self._value_set = []
    148 
    149     self._telemetry_info = TelemetryInfo()
    150 
    151     # State of the benchmark this set of results represents.
    152     self._benchmark_enabled = benchmark_enabled
    153 
    154   @property
    155   def telemetry_info(self):
    156     return self._telemetry_info
    157 
    158   @property
    159   def value_set(self):
    160     return self._value_set
    161 
    162   def AsHistogramDicts(self, benchmark_metadata):
    163     if self.value_set:
    164       return self.value_set
    165     chart_json = chart_json_output_formatter.ResultsAsChartDict(
    166         benchmark_metadata, self.all_page_specific_values,
    167         self.all_summary_values)
    168     info = self.telemetry_info
    169     chart_json['label'] = info.label
    170     chart_json['benchmarkStartMs'] = info.benchmark_start_ms
    171 
    172     file_descriptor, chart_json_path = tempfile.mkstemp()
    173     os.close(file_descriptor)
    174     json.dump(chart_json, file(chart_json_path, 'w'))
    175 
    176     vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
    177 
    178     os.remove(chart_json_path)
    179 
    180     if vinn_result.returncode != 0:
    181       logging.error('Error converting chart json to Histograms:\n' +
    182           vinn_result.stdout)
    183       return []
    184     return json.loads(vinn_result.stdout)
    185 
    186   def __copy__(self):
    187     cls = self.__class__
    188     result = cls.__new__(cls)
    189     for k, v in self.__dict__.items():
    190       if isinstance(v, collections.Container):
    191         v = copy.copy(v)
    192       setattr(result, k, v)
    193     return result
    194 
    195   @property
    196   def pages_to_profiling_files(self):
    197     return self._pages_to_profiling_files
    198 
    199   @property
    200   def serialized_trace_file_ids_to_paths(self):
    201     return self._serialized_trace_file_ids_to_paths
    202 
    203   @property
    204   def pages_to_profiling_files_cloud_url(self):
    205     return self._pages_to_profiling_files_cloud_url
    206 
    207   @property
    208   def all_page_specific_values(self):
    209     values = []
    210     for run in self._all_page_runs:
    211       values += run.values
    212     if self._current_page_run:
    213       values += self._current_page_run.values
    214     return values
    215 
    216   @property
    217   def all_summary_values(self):
    218     return self._all_summary_values
    219 
    220   @property
    221   def current_page(self):
    222     assert self._current_page_run, 'Not currently running test.'
    223     return self._current_page_run.story
    224 
    225   @property
    226   def current_page_run(self):
    227     assert self._current_page_run, 'Not currently running test.'
    228     return self._current_page_run
    229 
    230   @property
    231   def all_page_runs(self):
    232     return self._all_page_runs
    233 
    234   @property
    235   def pages_that_succeeded(self):
    236     """Returns the set of pages that succeeded."""
    237     pages = set(run.story for run in self.all_page_runs)
    238     pages.difference_update(self.pages_that_failed)
    239     return pages
    240 
    241   @property
    242   def pages_that_failed(self):
    243     """Returns the set of failed pages."""
    244     failed_pages = set()
    245     for run in self.all_page_runs:
    246       if run.failed:
    247         failed_pages.add(run.story)
    248     return failed_pages
    249 
    250   @property
    251   def failures(self):
    252     values = self.all_page_specific_values
    253     return [v for v in values if isinstance(v, failure.FailureValue)]
    254 
    255   @property
    256   def skipped_values(self):
    257     values = self.all_page_specific_values
    258     return [v for v in values if isinstance(v, skip.SkipValue)]
    259 
    260   def _GetStringFromExcInfo(self, err):
    261     return ''.join(traceback.format_exception(*err))
    262 
    263   def CleanUp(self):
    264     """Clean up any TraceValues contained within this results object."""
    265     for run in self._all_page_runs:
    266       for v in run.values:
    267         if isinstance(v, trace.TraceValue):
    268           v.CleanUp()
    269           run.values.remove(v)
    270 
    271   def __enter__(self):
    272     return self
    273 
    274   def __exit__(self, _, __, ___):
    275     self.CleanUp()
    276 
    277   def WillRunPage(self, page, storyset_repeat_counter=0):
    278     assert not self._current_page_run, 'Did not call DidRunPage.'
    279     self._current_page_run = story_run.StoryRun(page)
    280     self._progress_reporter.WillRunPage(self)
    281     self.telemetry_info.WillRunStory(
    282         page, storyset_repeat_counter)
    283 
    284   def DidRunPage(self, page):  # pylint: disable=unused-argument
    285     """
    286     Args:
    287       page: The current page under test.
    288     """
    289     assert self._current_page_run, 'Did not call WillRunPage.'
    290     self._progress_reporter.DidRunPage(self)
    291     self._all_page_runs.append(self._current_page_run)
    292     self._all_stories.add(self._current_page_run.story)
    293     self._current_page_run = None
    294 
    295   def AddValue(self, value):
    296     assert self._current_page_run, 'Not currently running test.'
    297     assert self._benchmark_enabled, 'Cannot add value to disabled results'
    298     self._ValidateValue(value)
    299     is_first_result = (
    300       self._current_page_run.story not in self._all_stories)
    301 
    302     story_keys = self._current_page_run.story.grouping_keys
    303 
    304     if story_keys:
    305       for k, v in story_keys.iteritems():
    306         assert k not in value.grouping_keys, (
    307             'Tried to add story grouping key ' + k + ' already defined by ' +
    308             'value')
    309         value.grouping_keys[k] = v
    310 
    311       # We sort by key name to make building the tir_label deterministic.
    312       story_keys_label = '_'.join(v for _, v in sorted(story_keys.iteritems()))
    313       if value.tir_label:
    314         assert value.tir_label == story_keys_label, (
    315             'Value has an explicit tir_label (%s) that does not match the '
    316             'one computed from story_keys (%s)' % (value.tir_label, story_keys))
    317       else:
    318         value.tir_label = story_keys_label
    319 
    320     if not (isinstance(value, skip.SkipValue) or
    321             isinstance(value, failure.FailureValue) or
    322             isinstance(value, trace.TraceValue) or
    323             self._value_can_be_added_predicate(value, is_first_result)):
    324       return
    325     # TODO(eakuefner/chrishenry): Add only one skip per pagerun assert here
    326     self._current_page_run.AddValue(value)
    327     self._progress_reporter.DidAddValue(value)
    328 
    329   def AddProfilingFile(self, page, file_handle):
    330     self._pages_to_profiling_files[page].append(file_handle)
    331 
    332   def AddSummaryValue(self, value):
    333     assert value.page is None
    334     self._ValidateValue(value)
    335     self._all_summary_values.append(value)
    336 
    337   def _ValidateValue(self, value):
    338     assert isinstance(value, value_module.Value)
    339     if value.name not in self._representative_value_for_each_value_name:
    340       self._representative_value_for_each_value_name[value.name] = value
    341     representative_value = self._representative_value_for_each_value_name[
    342         value.name]
    343     assert value.IsMergableWith(representative_value)
    344 
    345   def PrintSummary(self):
    346     if self._benchmark_enabled:
    347       self._progress_reporter.DidFinishAllTests(self)
    348 
    349       # Only serialize the trace if output_format is json.
    350       if (self._output_dir and
    351           any(isinstance(o, json_output_formatter.JsonOutputFormatter)
    352               for o in self._output_formatters)):
    353         self._SerializeTracesToDirPath(self._output_dir)
    354       for output_formatter in self._output_formatters:
    355         output_formatter.Format(self)
    356         output_formatter.PrintViewResults()
    357     else:
    358       for output_formatter in self._output_formatters:
    359         output_formatter.FormatDisabled()
    360 
    361   def FindValues(self, predicate):
    362     """Finds all values matching the specified predicate.
    363 
    364     Args:
    365       predicate: A function that takes a Value and returns a bool.
    366     Returns:
    367       A list of values matching |predicate|.
    368     """
    369     values = []
    370     for value in self.all_page_specific_values:
    371       if predicate(value):
    372         values.append(value)
    373     return values
    374 
    375   def FindPageSpecificValuesForPage(self, page, value_name):
    376     return self.FindValues(lambda v: v.page == page and v.name == value_name)
    377 
    378   def FindAllPageSpecificValuesNamed(self, value_name):
    379     return self.FindValues(lambda v: v.name == value_name)
    380 
    381   def FindAllPageSpecificValuesFromIRNamed(self, tir_label, value_name):
    382     return self.FindValues(lambda v: v.name == value_name
    383                            and v.tir_label == tir_label)
    384 
    385   def FindAllTraceValues(self):
    386     return self.FindValues(lambda v: isinstance(v, trace.TraceValue))
    387 
    388   def _SerializeTracesToDirPath(self, dir_path):
    389     """ Serialize all trace values to files in dir_path and return a list of
    390     file handles to those files. """
    391     for value in self.FindAllTraceValues():
    392       fh = value.Serialize(dir_path)
    393       self._serialized_trace_file_ids_to_paths[fh.id] = fh.GetAbsPath()
    394 
    395   def UploadTraceFilesToCloud(self, bucket):
    396     for value in self.FindAllTraceValues():
    397       value.UploadToCloud(bucket)
    398 
    399   def UploadProfilingFilesToCloud(self, bucket):
    400     for page, file_handle_list in self._pages_to_profiling_files.iteritems():
    401       for file_handle in file_handle_list:
    402         remote_path = ('profiler-file-id_%s-%s%-d%s' % (
    403             file_handle.id,
    404             datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'),
    405             random.randint(1, 100000),
    406             file_handle.extension))
    407         try:
    408           cloud_url = cloud_storage.Insert(
    409               bucket, remote_path, file_handle.GetAbsPath())
    410           sys.stderr.write(
    411               'View generated profiler files online at %s for page %s\n' %
    412               (cloud_url, page.display_name))
    413           self._pages_to_profiling_files_cloud_url[page].append(cloud_url)
    414         except cloud_storage.PermissionError as e:
    415           logging.error('Cannot upload profiling files to cloud storage due to '
    416                         ' permission error: %s' % e.message)
    417