Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/python
      2 
      3 # Copyright (c) 2014 The Chromium Authors. All rights reserved.
      4 # Use of this source code is governed by a BSD-style license that can be
      5 # found in the LICENSE file.
      6 
      7 
      8 """Generate new bench expectations from results of trybots on a code review."""
      9 
     10 
     11 import collections
     12 import compare_codereview
     13 import os
     14 import re
     15 import shutil
     16 import subprocess
     17 import sys
     18 
     19 
     20 BENCH_DATA_URL = 'gs://chromium-skia-gm/perfdata/%s/%s/*'
     21 CHECKOUT_PATH = os.path.realpath(os.path.join(
     22     os.path.dirname(os.path.abspath(__file__)), os.pardir))
     23 TMP_BENCH_DATA_DIR = os.path.join(CHECKOUT_PATH, '.bench_data')
     24 
     25 
     26 TryBuild = collections.namedtuple(
     27     'TryBuild', ['builder_name', 'build_number', 'is_finished'])
     28 
     29 
     30 def find_all_builds(codereview_url):
     31   """Finds and returns information about trybot runs for a code review.
     32 
     33   Args:
     34     codereview_url: URL of the codereview in question.
     35 
     36   Returns:
     37       List of NamedTuples: (builder_name, build_number, is_finished)
     38   """
     39   results = compare_codereview.CodeReviewHTMLParser().parse(codereview_url)
     40   try_builds = []
     41   for builder, data in results.iteritems():
     42     if builder.startswith('Perf'):
     43       build_num = data.url.split('/')[-1] if data.url else None
     44       is_finished = (data.status not in ('pending', 'try-pending') and
     45                      build_num is not None)
     46       try_builds.append(TryBuild(builder_name=builder,
     47                                  build_number=build_num,
     48                                  is_finished=is_finished))
     49   return try_builds
     50 
     51 
     52 def _all_trybots_finished(try_builds):
     53   """Return True iff all of the given try jobs have finished.
     54 
     55   Args:
     56       try_builds: list of TryBuild instances.
     57 
     58   Returns:
     59       True if all of the given try jobs have finished, otherwise False.
     60   """
     61   for try_build in try_builds:
     62     if not try_build.is_finished:
     63       return False
     64   return True
     65 
     66 
     67 def all_trybots_finished(codereview_url):
     68   """Return True iff all of the try jobs on the given codereview have finished.
     69 
     70   Args:
     71       codereview_url: string; URL of the codereview.
     72 
     73   Returns:
     74       True if all of the try jobs have finished, otherwise False.
     75   """
     76   return _all_trybots_finished(find_all_builds(codereview_url))
     77 
     78 
     79 def get_bench_data(builder, build_num, dest_dir):
     80   """Download the bench data for the given builder at the given build_num.
     81 
     82   Args:
     83       builder: string; name of the builder.
     84       build_num: string; build number.
     85       dest_dir: string; destination directory for the bench data.
     86   """
     87   url = BENCH_DATA_URL % (builder, build_num)
     88   subprocess.check_call(['gsutil', 'cp', '-R', url, dest_dir],
     89                          stdout=subprocess.PIPE,
     90                          stderr=subprocess.PIPE)
     91 
     92 
     93 def find_revision_from_downloaded_data(dest_dir):
     94   """Finds the revision at which the downloaded data was generated.
     95 
     96   Args:
     97       dest_dir: string; directory holding the downloaded data.
     98 
     99   Returns:
    100       The revision (git commit hash) at which the downloaded data was
    101       generated, or None if no revision can be found.
    102   """
    103   for data_file in os.listdir(dest_dir):
    104     match = re.match('bench_(?P<revision>[0-9a-fA-F]{2,40})_data.*', data_file)
    105     if match:
    106       return match.group('revision')
    107   return None
    108 
    109 
    110 class TrybotNotFinishedError(Exception):
    111   pass
    112 
    113 
    114 def gen_bench_expectations_from_codereview(codereview_url,
    115                                            error_on_unfinished=True):
    116   """Generate bench expectations from a code review.
    117 
    118   Scans the given code review for Perf trybot runs. Downloads the results of
    119   finished trybots and uses them to generate new expectations for their
    120   waterfall counterparts.
    121 
    122   Args:
    123       url: string; URL of the code review.
    124       error_on_unfinished: bool; throw an error if any trybot has not finished.
    125   """
    126   try_builds = find_all_builds(codereview_url)
    127 
    128   # Verify that all trybots have finished running.
    129   if error_on_unfinished and not _all_trybots_finished(try_builds):
    130     raise TrybotNotFinishedError('Not all trybots have finished.')
    131 
    132   failed_data_pull = []
    133   failed_gen_expectations = []
    134 
    135   if os.path.isdir(TMP_BENCH_DATA_DIR):
    136     shutil.rmtree(TMP_BENCH_DATA_DIR)
    137 
    138   for try_build in try_builds:
    139     try_builder = try_build.builder_name
    140     builder = try_builder.replace('-Trybot', '')
    141 
    142     # Download the data.
    143     dest_dir = os.path.join(TMP_BENCH_DATA_DIR, builder)
    144     os.makedirs(dest_dir)
    145     try:
    146       get_bench_data(try_builder, try_build.build_number, dest_dir)
    147     except subprocess.CalledProcessError:
    148       failed_data_pull.append(try_builder)
    149       continue
    150 
    151     # Find the revision at which the data was generated.
    152     revision = find_revision_from_downloaded_data(dest_dir)
    153     if not revision:
    154       # If we can't find a revision, then something is wrong with the data we
    155       # downloaded. Skip this builder.
    156       failed_data_pull.append(try_builder)
    157       continue
    158 
    159     # Generate new expectations.
    160     output_file = os.path.join(CHECKOUT_PATH, 'expectations', 'bench',
    161                                'bench_expectations_%s.txt' % builder)
    162     try:
    163       subprocess.check_call(['python',
    164                              os.path.join(CHECKOUT_PATH, 'bench',
    165                                           'gen_bench_expectations.py'),
    166                              '-b', builder, '-o', output_file,
    167                              '-d', dest_dir, '-r', revision])
    168     except subprocess.CalledProcessError:
    169       failed_gen_expectations.append(builder)
    170 
    171   failure = ''
    172   if failed_data_pull:
    173     failure += 'Failed to load data for: %s\n\n' % ','.join(failed_data_pull)
    174   if failed_gen_expectations:
    175     failure += 'Failed to generate expectations for: %s\n\n' % ','.join(
    176         failed_gen_expectations)
    177   if failure:
    178     raise Exception(failure)
    179 
    180 
    181 if __name__ == '__main__':
    182   gen_bench_expectations_from_codereview(sys.argv[1])
    183 
    184