Home | History | Annotate | Download | only in telemetry_Benchmarks 
      1 #!/usr/bin/env python
      2 
      3 """
      4 This file generates all telemetry_Benchmarks control files from a master list.
      5 """
      6 
      7 from datetime import datetime
      8 import os
      9 import re
     10 
     11 # This test list is a subset of telemetry benchmark tests. The full list can be
     12 # obtained by executing
     13 # /build/${BOARD}/usr/local/telemetry/src/tools/perf/list_benchmarks
     14 
     15 # PLEASE READ THIS:
     16 
     17 # PERF_TESTS: these tests run on each build: tot, tot-1, tot-2 and expensive to
     18 # run.
     19 
     20 # PERF_DAILY_RUN_TESTS: these tests run on a nightly build: tot. If you are
     21 # trying to gain confidence for a new test, adding your test in this list is a
     22 # good start.
     23 
     24 # For adding a new test to any of these lists, please add rohitbm, lafeenstra,
     25 # haddowk in the change.
     26 
     27 PERF_PER_BUILD_TESTS = (
     28     'cros_ui_smoothness',
     29     'jetstream',
     30     'kraken',
     31     'loading.desktop',
     32     'octane',
     33     'page_cycler_v2.typical_25',
     34     'rendering.desktop',
     35     'speedometer',
     36     'speedometer2',
     37 )
     38 
     39 PERF_DAILY_RUN_TESTS = (
     40     'blink_perf.image_decoder',
     41     'cros_tab_switching.typical_24',
     42     'dromaeo',
     43     'media.desktop',
     44     'memory.desktop',
     45     'smoothness.tough_pinch_zoom_cases',
     46     'system_health.memory_desktop',
     47     'webrtc',
     48 )
     49 
     50 PERF_WEEKLY_RUN_TESTS = (
     51 )
     52 
     53 ALL_TESTS = (PERF_PER_BUILD_TESTS +
     54              PERF_DAILY_RUN_TESTS +
     55              PERF_WEEKLY_RUN_TESTS)
     56 
     57 EXTRA_ARGS_MAP = {
     58     'loading.desktop': '--story-tag-filter=typical',
     59     'rendering.desktop': '--story-tag-filter=top_real_world_desktop',
     60     'system_health.memory_desktop': '--pageset-repeat=1',
     61 }
     62 
     63 DEFAULT_YEAR = str(datetime.now().year)
     64 
     65 DEFAULT_AUTHOR = 'Chrome OS Team'
     66 
     67 CONTROLFILE_TEMPLATE = (
     68 """# Copyright {year} The Chromium OS Authors. All rights reserved.
     69 # Use of this source code is governed by a BSD-style license that can be
     70 # found in the LICENSE file.
     71 
     72 # Do not edit this file! It was created by generate_controlfiles.py.
     73 
     74 from autotest_lib.client.common_lib import utils
     75 
     76 AUTHOR = '{author}'
     77 NAME = 'telemetry_Benchmarks.{test}'
     78 {attributes}
     79 TIME = 'LONG'
     80 TEST_CATEGORY = 'Benchmark'
     81 TEST_CLASS = 'performance'
     82 TEST_TYPE = 'server'
     83 
     84 DOC = '''
     85 This server side test suite executes the Telemetry Benchmark:
     86 {test}
     87 This is part of Chrome for Chrome OS performance testing.
     88 
     89 Pass local=True to run with local telemetry and no AFE server.
     90 '''
     91 
     92 def run_benchmark(machine):
     93     host = hosts.create_host(machine)
     94     dargs = utils.args_to_dict(args)
     95     dargs['extra_args'] = '{extra_args}'.split()
     96     job.run_test('telemetry_Benchmarks', host=host,
     97                  benchmark='{test}',
     98                  tag='{test}',
     99                  args=dargs)
    100 
    101 parallel_simple(run_benchmark, machines)""")
    102 
    103 
    104 def _get_suite(test):
    105     if test in PERF_PER_BUILD_TESTS:
    106         return 'ATTRIBUTES = \'suite:crosbolt_perf_perbuild\''
    107     elif test in PERF_DAILY_RUN_TESTS:
    108         return 'ATTRIBUTES = \'suite:crosbolt_perf_nightly\''
    109     elif test in PERF_WEEKLY_RUN_TESTS:
    110         return 'ATTRIBUTES = \'suite:crosbolt_perf_weekly\''
    111     return ''
    112 
    113 
    114 def get_existing_fields(filename):
    115     """Returns the existing copyright year and author of the control file."""
    116     if not os.path.isfile(filename):
    117         return (DEFAULT_YEAR, DEFAULT_AUTHOR)
    118 
    119     copyright_year = DEFAULT_YEAR
    120     author = DEFAULT_AUTHOR
    121     copyright_pattern = re.compile(
    122             '# Copyright (\d+) The Chromium OS Authors.')
    123     author_pattern = re.compile("AUTHOR = '(.+)'")
    124     with open(filename) as f:
    125         for line in f:
    126             match_year = copyright_pattern.match(line)
    127             if match_year:
    128                 copyright_year = match_year.group(1)
    129             match_author = author_pattern.match(line)
    130             if match_author:
    131                 author = match_author.group(1)
    132     return (copyright_year, author)
    133 
    134 
    135 def generate_control(test):
    136     """Generates control file from the template."""
    137     filename = 'control.%s' % test
    138     copyright_year, author = get_existing_fields(filename)
    139     extra_args = EXTRA_ARGS_MAP.get(test, '')
    140 
    141     with open(filename, 'w+') as f:
    142         content = CONTROLFILE_TEMPLATE.format(
    143                 attributes=_get_suite(test),
    144                 author=author,
    145                 extra_args=extra_args,
    146                 test=test,
    147                 year=copyright_year)
    148         f.write(content)
    149 
    150 
    151 def check_unmanaged_control_files():
    152     """Prints warning if there is unmanaged control file."""
    153     for filename in os.listdir('.'):
    154         if not filename.startswith('control.'):
    155             continue
    156         test = filename[len('control.'):]
    157         if test not in ALL_TESTS:
    158             print 'warning, unmanaged control file:', test
    159 
    160 
    161 def main():
    162     """The main function."""
    163     for test in ALL_TESTS:
    164         generate_control(test)
    165     check_unmanaged_control_files()
    166 
    167 
    168 if __name__ == "__main__":
    169     main()
    170