toolchain-utils/crosperf/results_report.py

# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A module to handle the report format."""
from __future__ import print_function

import datetime
import functools
import itertools
import json
import os
import re

from cros_utils.tabulator import AmeanResult
from cros_utils.tabulator import Cell
from cros_utils.tabulator import CoeffVarFormat
from cros_utils.tabulator import CoeffVarResult
from cros_utils.tabulator import Column
from cros_utils.tabulator import Format
from cros_utils.tabulator import GmeanRatioResult
from cros_utils.tabulator import LiteralResult
from cros_utils.tabulator import MaxResult
from cros_utils.tabulator import MinResult
from cros_utils.tabulator import PValueFormat
from cros_utils.tabulator import PValueResult
from cros_utils.tabulator import RatioFormat
from cros_utils.tabulator import RawResult
from cros_utils.tabulator import StdResult
from cros_utils.tabulator import TableFormatter
from cros_utils.tabulator import TableGenerator
from cros_utils.tabulator import TablePrinter
from update_telemetry_defaults import TelemetryDefaults

from column_chart import ColumnChart
from results_organizer import OrganizeResults

import results_report_templates as templates


def ParseChromeosImage(chromeos_image):
  """Parse the chromeos_image string for the image and version.

  The chromeos_image string will probably be in one of two formats:
  1: <path-to-chroot>/src/build/images/<board>/<ChromeOS-version>.<datetime>/ \
     chromiumos_test_image.bin
  2: <path-to-chroot>/chroot/tmp/<buildbot-build>/<ChromeOS-version>/ \
      chromiumos_test_image.bin

  We parse these strings to find the 'chromeos_version' to store in the
  json archive (without the .datatime bit in the first case); and also
  the 'chromeos_image', which would be all of the first case, but only the
  part after '/chroot/tmp' in the second case.

  Args:
      chromeos_image: string containing the path to the chromeos_image that
      crosperf used for the test.

  Returns:
      version, image: The results of parsing the input string, as explained
      above.
  """
  # Find the Chromeos Version, e.g. R45-2345.0.0.....
  # chromeos_image should have been something like:
  # <path>/<board-trybot-release>/<chromeos-version>/chromiumos_test_image.bin"
  if chromeos_image.endswith('/chromiumos_test_image.bin'):
    full_version = chromeos_image.split('/')[-2]
    # Strip the date and time off of local builds (which have the format
    # "R43-2345.0.0.date-and-time").
    version, _ = os.path.splitext(full_version)
  else:
    version = ''

  # Find the chromeos image.  If it's somewhere in .../chroot/tmp/..., then
  # it's an official image that got downloaded, so chop off the download path
  # to make the official image name more clear.
  official_image_path = '/chroot/tmp'
  if official_image_path in chromeos_image:
    image = chromeos_image.split(official_image_path, 1)[1]
  else:
    image = chromeos_image
  return version, image


def _AppendUntilLengthIs(gen, the_list, target_len):
  """Appends to `list` until `list` is `target_len` elements long.

  Uses `gen` to generate elements.
  """
  the_list.extend(gen() for _ in xrange(target_len - len(the_list)))
  return the_list


def _FilterPerfReport(event_threshold, report):
  """Filters out entries with `< event_threshold` percent in a perf report."""
  def filter_dict(m):
    return {fn_name: pct for fn_name, pct in m.iteritems()
            if pct >= event_threshold}
  return {event: filter_dict(m) for event, m in report.iteritems()}


class _PerfTable(object):
  """Generates dicts from a perf table.

  Dicts look like:
  {'benchmark_name': {'perf_event_name': [LabelData]}}
  where LabelData is a list of perf dicts, each perf dict coming from the same
  label.
  Each perf dict looks like {'function_name': 0.10, ...} (where 0.10 is the
  percentage of time spent in function_name).
  """

  def __init__(self, benchmark_names_and_iterations, label_names,
               read_perf_report, event_threshold=None):
    """Constructor.

    read_perf_report is a function that takes a label name, benchmark name, and
    benchmark iteration, and returns a dictionary describing the perf output for
    that given run.
    """
    self.event_threshold = event_threshold
    self._label_indices = {name: i for i, name in enumerate(label_names)}
    self.perf_data = {}
    for label in label_names:
      for bench_name, bench_iterations in benchmark_names_and_iterations:
        for i in xrange(bench_iterations):
          report = read_perf_report(label, bench_name, i)
          self._ProcessPerfReport(report, label, bench_name, i)

  def _ProcessPerfReport(self, perf_report, label, benchmark_name, iteration):
    """Add the data from one run to the dict."""
    perf_of_run = perf_report
    if self.event_threshold is not None:
      perf_of_run = _FilterPerfReport(self.event_threshold, perf_report)
    if benchmark_name not in self.perf_data:
      self.perf_data[benchmark_name] = {event: [] for event in perf_of_run}
    ben_data = self.perf_data[benchmark_name]
    label_index = self._label_indices[label]
    for event in ben_data:
      _AppendUntilLengthIs(list, ben_data[event], label_index + 1)
      data_for_label = ben_data[event][label_index]
      _AppendUntilLengthIs(dict, data_for_label, iteration + 1)
      data_for_label[iteration] = perf_of_run[event] if perf_of_run else {}


def _GetResultsTableHeader(ben_name, iterations):
  benchmark_info = ('Benchmark:  {0};  Iterations: {1}'
                    .format(ben_name, iterations))
  cell = Cell()
  cell.string_value = benchmark_info
  cell.header = True
  return [[cell]]


def _ParseColumn(columns, iteration):
  new_column = []
  for column in columns:
    if column.result.__class__.__name__ != 'RawResult':
      new_column.append(column)
    else:
      new_column.extend(Column(LiteralResult(i), Format(), str(i + 1))
                        for i in xrange(iteration))
  return new_column


def _GetTables(benchmark_results, columns, table_type):
  iter_counts = benchmark_results.iter_counts
  result = benchmark_results.run_keyvals
  tables = []
  for bench_name, runs in result.iteritems():
    iterations = iter_counts[bench_name]
    ben_table = _GetResultsTableHeader(bench_name, iterations)

    all_runs_empty = all(not dict for label in runs for dict in label)
    if all_runs_empty:
      cell = Cell()
      cell.string_value = ('This benchmark contains no result.'
                           ' Is the benchmark name valid?')
      cell_table = [[cell]]
    else:
      table = TableGenerator(runs, benchmark_results.label_names).GetTable()
      parsed_columns = _ParseColumn(columns, iterations)
      tf = TableFormatter(table, parsed_columns)
      cell_table = tf.GetCellTable(table_type)
    tables.append(ben_table)
    tables.append(cell_table)
  return tables


def _GetPerfTables(benchmark_results, columns, table_type):
  p_table = _PerfTable(benchmark_results.benchmark_names_and_iterations,
                       benchmark_results.label_names,
                       benchmark_results.read_perf_report)

  tables = []
  for benchmark in p_table.perf_data:
    iterations = benchmark_results.iter_counts[benchmark]
    ben_table = _GetResultsTableHeader(benchmark, iterations)
    tables.append(ben_table)
    benchmark_data = p_table.perf_data[benchmark]
    table = []
    for event in benchmark_data:
      tg = TableGenerator(benchmark_data[event],
                          benchmark_results.label_names,
                          sort=TableGenerator.SORT_BY_VALUES_DESC)
      table = tg.GetTable(ResultsReport.PERF_ROWS)
      parsed_columns = _ParseColumn(columns, iterations)
      tf = TableFormatter(table, parsed_columns)
      tf.GenerateCellTable(table_type)
      tf.AddColumnName()
      tf.AddLabelName()
      tf.AddHeader(str(event))
      table = tf.GetCellTable(table_type, headers=False)
      tables.append(table)
  return tables


class ResultsReport(object):
  """Class to handle the report format."""
  MAX_COLOR_CODE = 255
  PERF_ROWS = 5

  def __init__(self, results):
    self.benchmark_results = results

  def _GetTablesWithColumns(self, columns, table_type, perf):
    get_tables = _GetPerfTables if perf else _GetTables
    return get_tables(self.benchmark_results, columns, table_type)

  def GetFullTables(self, perf=False):
    columns = [Column(RawResult(), Format()),
               Column(MinResult(), Format()),
               Column(MaxResult(), Format()),
               Column(AmeanResult(), Format()),
               Column(StdResult(), Format(), 'StdDev'),
               Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'),
               Column(GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'),
               Column(PValueResult(), PValueFormat(), 'p-value')]
    return self._GetTablesWithColumns(columns, 'full', perf)

  def GetSummaryTables(self, perf=False):
    columns = [Column(AmeanResult(), Format()),
               Column(StdResult(), Format(), 'StdDev'),
               Column(CoeffVarResult(), CoeffVarFormat(), 'StdDev/Mean'),
               Column(GmeanRatioResult(), RatioFormat(), 'GmeanSpeedup'),
               Column(PValueResult(), PValueFormat(), 'p-value')]
    return self._GetTablesWithColumns(columns, 'summary', perf)


def _PrintTable(tables, out_to):
  # tables may be None.
  if not tables:
    return ''

  if out_to == 'HTML':
    out_type = TablePrinter.HTML
  elif out_to == 'PLAIN':
    out_type = TablePrinter.PLAIN
  elif out_to == 'CONSOLE':
    out_type = TablePrinter.CONSOLE
  elif out_to == 'TSV':
    out_type = TablePrinter.TSV
  elif out_to == 'EMAIL':
    out_type = TablePrinter.EMAIL
  else:
    raise ValueError('Invalid out_to value: %s' % (out_to,))

  printers = (TablePrinter(table, out_type) for table in tables)
  return ''.join(printer.Print() for printer in printers)


class TextResultsReport(ResultsReport):
  """Class to generate text result report."""

  H1_STR = '==========================================='
  H2_STR = '-------------------------------------------'

  def __init__(self, results, email=False, experiment=None):
    super(TextResultsReport, self).__init__(results)
    self.email = email
    self.experiment = experiment

  @staticmethod
  def _MakeTitle(title):
    header_line = TextResultsReport.H1_STR
    # '' at the end gives one newline.
    return '\n'.join([header_line, title, header_line, ''])

  @staticmethod
  def _MakeSection(title, body):
    header_line = TextResultsReport.H2_STR
    # '\n' at the end gives us two newlines.
    return '\n'.join([header_line, title, header_line, body, '\n'])

  @staticmethod
  def FromExperiment(experiment, email=False):
    results = BenchmarkResults.FromExperiment(experiment)
    return TextResultsReport(results, email, experiment)

  def GetStatusTable(self):
    """Generate the status table by the tabulator."""
    table = [['', '']]
    columns = [Column(LiteralResult(iteration=0), Format(), 'Status'),
               Column(LiteralResult(iteration=1), Format(), 'Failing Reason')]

    for benchmark_run in self.experiment.benchmark_runs:
      status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(),
                                     benchmark_run.failure_reason]]
      table.append(status)
    cell_table = TableFormatter(table, columns).GetCellTable('status')
    return [cell_table]

  def GetReport(self):
    """Generate the report for email and console."""
    output_type = 'EMAIL' if self.email else 'CONSOLE'
    experiment = self.experiment

    sections = []
    if experiment is not None:
      title_contents = "Results report for '%s'" % (experiment.name, )
    else:
      title_contents = 'Results report'
    sections.append(self._MakeTitle(title_contents))

    summary_table = _PrintTable(self.GetSummaryTables(perf=False), output_type)
    sections.append(self._MakeSection('Summary', summary_table))

    if experiment is not None:
      table = _PrintTable(self.GetStatusTable(), output_type)
      sections.append(self._MakeSection('Benchmark Run Status', table))

    perf_table = _PrintTable(self.GetSummaryTables(perf=True), output_type)
    if perf_table:
      sections.append(self._MakeSection('Perf Data', perf_table))

    if experiment is not None:
      experiment_file = experiment.experiment_file
      sections.append(self._MakeSection('Experiment File', experiment_file))

      cpu_info = experiment.machine_manager.GetAllCPUInfo(experiment.labels)
      sections.append(self._MakeSection('CPUInfo', cpu_info))

    return '\n'.join(sections)


def _GetHTMLCharts(label_names, test_results):
  charts = []
  for item, runs in test_results.iteritems():
    # Fun fact: label_names is actually *entirely* useless as a param, since we
    # never add headers. We still need to pass it anyway.
    table = TableGenerator(runs, label_names).GetTable()
    columns = [Column(AmeanResult(), Format()), Column(MinResult(), Format()),
               Column(MaxResult(), Format())]
    tf = TableFormatter(table, columns)
    data_table = tf.GetCellTable('full', headers=False)

    for cur_row_data in data_table:
      test_key = cur_row_data[0].string_value
      title = '{0}: {1}'.format(item, test_key.replace('/', ''))
      chart = ColumnChart(title, 300, 200)
      chart.AddColumn('Label', 'string')
      chart.AddColumn('Average', 'number')
      chart.AddColumn('Min', 'number')
      chart.AddColumn('Max', 'number')
      chart.AddSeries('Min', 'line', 'black')
      chart.AddSeries('Max', 'line', 'black')
      cur_index = 1
      for label in label_names:
        chart.AddRow([label,
                      cur_row_data[cur_index].value,
                      cur_row_data[cur_index + 1].value,
                      cur_row_data[cur_index + 2].value])
        if isinstance(cur_row_data[cur_index].value, str):
          chart = None
          break
        cur_index += 3
      if chart:
        charts.append(chart)
  return charts


class HTMLResultsReport(ResultsReport):
  """Class to generate html result report."""

  def __init__(self, benchmark_results, experiment=None):
    super(HTMLResultsReport, self).__init__(benchmark_results)
    self.experiment = experiment

  @staticmethod
  def FromExperiment(experiment):
    return HTMLResultsReport(BenchmarkResults.FromExperiment(experiment),
                             experiment=experiment)

  def GetReport(self):
    label_names = self.benchmark_results.label_names
    test_results = self.benchmark_results.run_keyvals
    charts = _GetHTMLCharts(label_names, test_results)
    chart_javascript = ''.join(chart.GetJavascript() for chart in charts)
    chart_divs = ''.join(chart.GetDiv() for chart in charts)

    summary_table = self.GetSummaryTables()
    full_table = self.GetFullTables()
    perf_table = self.GetSummaryTables(perf=True)
    experiment_file = ''
    if self.experiment is not None:
      experiment_file = self.experiment.experiment_file
    # Use kwargs for sanity, and so that testing is a bit easier.
    return templates.GenerateHTMLPage(perf_table=perf_table,
                                      chart_js=chart_javascript,
                                      summary_table=summary_table,
                                      print_table=_PrintTable,
                                      chart_divs=chart_divs,
                                      full_table=full_table,
                                      experiment_file=experiment_file)


def ParseStandardPerfReport(report_data):
  """Parses the output of `perf report`.

  It'll parse the following:
  {{garbage}}
  # Samples: 1234M of event 'foo'

  1.23% command shared_object location function::name

  1.22% command shared_object location function2::name

  # Samples: 999K of event 'bar'

  0.23% command shared_object location function3::name
  {{etc.}}

  Into:
    {'foo': {'function::name': 1.23, 'function2::name': 1.22},
     'bar': {'function3::name': 0.23, etc.}}
  """
  # This function fails silently on its if it's handed a string (as opposed to a
  # list of lines). So, auto-split if we do happen to get a string.
  if isinstance(report_data, basestring):
    report_data = report_data.splitlines()

  # Samples: N{K,M,G} of event 'event-name'
  samples_regex = re.compile(r"#\s+Samples: \d+\S? of event '([^']+)'")

  # We expect lines like:
  # N.NN%  command  samples  shared_object  [location] symbol
  #
  # Note that we're looking at stripped lines, so there is no space at the
  # start.
  perf_regex = re.compile(r'^(\d+(?:.\d*)?)%' # N.NN%
                          r'\s*\d+' # samples count (ignored)
                          r'\s*\S+' # command (ignored)
                          r'\s*\S+' # shared_object (ignored)
                          r'\s*\[.\]' # location (ignored)
                          r'\s*(\S.+)' # function
                         )

  stripped_lines = (l.strip() for l in report_data)
  nonempty_lines = (l for l in stripped_lines if l)
  # Ignore all lines before we see samples_regex
  interesting_lines = itertools.dropwhile(lambda x: not samples_regex.match(x),
                                          nonempty_lines)

  first_sample_line = next(interesting_lines, None)
  # Went through the entire file without finding a 'samples' header. Quit.
  if first_sample_line is None:
    return {}

  sample_name = samples_regex.match(first_sample_line).group(1)
  current_result = {}
  results = {sample_name: current_result}
  for line in interesting_lines:
    samples_match = samples_regex.match(line)
    if samples_match:
      sample_name = samples_match.group(1)
      current_result = {}
      results[sample_name] = current_result
      continue

    match = perf_regex.match(line)
    if not match:
      continue
    percentage_str, func_name = match.groups()
    try:
      percentage = float(percentage_str)
    except ValueError:
      # Couldn't parse it; try to be "resilient".
      continue
    current_result[func_name] = percentage
  return results


def _ReadExperimentPerfReport(results_directory, label_name, benchmark_name,
                              benchmark_iteration):
  """Reads a perf report for the given benchmark. Returns {} on failure.

  The result should be a map of maps; it should look like:
  {perf_event_name: {function_name: pct_time_spent}}, e.g.
  {'cpu_cycles': {'_malloc': 10.0, '_free': 0.3, ...}}
  """
  raw_dir_name = label_name + benchmark_name + str(benchmark_iteration + 1)
  dir_name = ''.join(c for c in raw_dir_name if c.isalnum())
  file_name = os.path.join(results_directory, dir_name, 'perf.data.report.0')
  try:
    with open(file_name) as in_file:
      return ParseStandardPerfReport(in_file)
  except IOError:
    # Yes, we swallow any IO-related errors.
    return {}


# Split out so that testing (specifically: mocking) is easier
def _ExperimentToKeyvals(experiment, for_json_report):
  """Converts an experiment to keyvals."""
  return OrganizeResults(experiment.benchmark_runs, experiment.labels,
                         json_report=for_json_report)


class BenchmarkResults(object):
  """The minimum set of fields that any ResultsReport will take."""
  def __init__(self, label_names, benchmark_names_and_iterations, run_keyvals,
               read_perf_report=None):
    if read_perf_report is None:
      def _NoPerfReport(*_args, **_kwargs):
        return {}
      read_perf_report = _NoPerfReport

    self.label_names = label_names
    self.benchmark_names_and_iterations = benchmark_names_and_iterations
    self.iter_counts = dict(benchmark_names_and_iterations)
    self.run_keyvals = run_keyvals
    self.read_perf_report = read_perf_report

  @staticmethod
  def FromExperiment(experiment, for_json_report=False):
    label_names = [label.name for label in experiment.labels]
    benchmark_names_and_iterations = [(benchmark.name, benchmark.iterations)
                                      for benchmark in experiment.benchmarks]
    run_keyvals = _ExperimentToKeyvals(experiment, for_json_report)
    read_perf_report = functools.partial(_ReadExperimentPerfReport,
                                         experiment.results_directory)
    return BenchmarkResults(label_names, benchmark_names_and_iterations,
                            run_keyvals, read_perf_report)


def _GetElemByName(name, from_list):
  """Gets an element from the given list by its name field.

  Raises an error if it doesn't find exactly one match.
  """
  elems = [e for e in from_list if e.name == name]
  if len(elems) != 1:
    raise ValueError('Expected 1 item named %s, found %d' % (name, len(elems)))
  return elems[0]


def _Unlist(l):
  """If l is a list, extracts the first element of l. Otherwise, returns l."""
  return l[0] if isinstance(l, list) else l

class JSONResultsReport(ResultsReport):
  """Class that generates JSON reports for experiments."""

  def __init__(self, benchmark_results, date=None, time=None, experiment=None,
               json_args=None):
    """Construct a JSONResultsReport.

    json_args is the dict of arguments we pass to json.dumps in GetReport().
    """
    super(JSONResultsReport, self).__init__(benchmark_results)

    defaults = TelemetryDefaults()
    defaults.ReadDefaultsFile()
    summary_field_defaults = defaults.GetDefault()
    if summary_field_defaults is None:
      summary_field_defaults = {}
    self.summary_field_defaults = summary_field_defaults

    if json_args is None:
      json_args = {}
    self.json_args = json_args

    self.experiment = experiment
    if not date:
      timestamp = datetime.datetime.strftime(datetime.datetime.now(),
                                             '%Y-%m-%d %H:%M:%S')
      date, time = timestamp.split(' ')
    self.date = date
    self.time = time

  @staticmethod
  def FromExperiment(experiment, date=None, time=None, json_args=None):
    benchmark_results = BenchmarkResults.FromExperiment(experiment,
                                                        for_json_report=True)
    return JSONResultsReport(benchmark_results, date, time, experiment,
                             json_args)

  def GetReportObjectIgnoringExperiment(self):
    """Gets the JSON report object specifically for the output data.

    Ignores any experiment-specific fields (e.g. board, machine checksum, ...).
    """
    benchmark_results = self.benchmark_results
    label_names = benchmark_results.label_names
    summary_field_defaults = self.summary_field_defaults
    final_results = []
    for test, test_results in benchmark_results.run_keyvals.iteritems():
      for label_name, label_results in zip(label_names, test_results):
        for iter_results in label_results:
          passed = iter_results.get('retval') == 0
          json_results = {
              'date': self.date,
              'time': self.time,
              'label': label_name,
              'test_name': test,
              'pass': passed,
          }
          final_results.append(json_results)

          if not passed:
            continue

          # Get overall results.
          summary_fields = summary_field_defaults.get(test)
          if summary_fields is not None:
            value = []
            json_results['overall_result'] = value
            for f in summary_fields:
              v = iter_results.get(f)
              if v is None:
                continue
              # New telemetry results format: sometimes we get a list of lists
              # now.
              v = _Unlist(_Unlist(v))
              value.append((f, float(v)))

          # Get detailed results.
          detail_results = {}
          json_results['detailed_results'] = detail_results
          for k, v in iter_results.iteritems():
            if k == 'retval' or k == 'PASS' or k == ['PASS'] or v == 'PASS':
              continue

            v = _Unlist(v)
            if 'machine' in k:
              json_results[k] = v
            elif v is not None:
              if isinstance(v, list):
                detail_results[k] = [float(d) for d in v]
              else:
                detail_results[k] = float(v)
    return final_results

  def GetReportObject(self):
    """Generate the JSON report, returning it as a python object."""
    report_list = self.GetReportObjectIgnoringExperiment()
    if self.experiment is not None:
      self._AddExperimentSpecificFields(report_list)
    return report_list

  def _AddExperimentSpecificFields(self, report_list):
    """Add experiment-specific data to the JSON report."""
    board = self.experiment.labels[0].board
    manager = self.experiment.machine_manager
    for report in report_list:
      label_name = report['label']
      label = _GetElemByName(label_name, self.experiment.labels)

      img_path = os.path.realpath(os.path.expanduser(label.chromeos_image))
      ver, img = ParseChromeosImage(img_path)

      report.update({
          'board': board,
          'chromeos_image': img,
          'chromeos_version': ver,
          'chrome_version': label.chrome_version,
          'compiler': label.compiler
      })

      if not report['pass']:
        continue
      if 'machine_checksum' not in report:
        report['machine_checksum'] = manager.machine_checksum[label_name]
      if 'machine_string' not in report:
        report['machine_string'] = manager.machine_checksum_string[label_name]

  def GetReport(self):
    """Dump the results of self.GetReportObject() to a string as JSON."""
    # This exists for consistency with the other GetReport methods.
    # Specifically, they all return strings, so it's a bit awkward if the JSON
    # results reporter returns an object.
    return json.dumps(self.GetReportObject(), **self.json_args)