1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 import logging 5 import re 6 import os 7 import StringIO 8 9 import common 10 from autotest_lib.client.common_lib import error 11 from autotest_lib.server import test 12 from autotest_lib.server import utils 13 from autotest_lib.site_utils import test_runner_utils 14 from autotest_lib.server.cros import telemetry_runner 15 16 17 TELEMETRY_TIMEOUT_MINS = 60 18 DUT_COMMON_SSH_OPTIONS = ['-o StrictHostKeyChecking=no', 19 '-o UserKnownHostsFile=/dev/null', 20 '-o BatchMode=yes', 21 '-o ConnectTimeout=30', 22 '-o ServerAliveInterval=900', 23 '-o ServerAliveCountMax=3', 24 '-o ConnectionAttempts=4', 25 '-o Protocol=2'] 26 DUT_SCP_OPTIONS = ' '.join(DUT_COMMON_SSH_OPTIONS) 27 28 CHROME_SRC_ROOT = '/var/cache/chromeos-cache/distfiles/target/' 29 CLIENT_CHROME_ROOT = '/usr/local/telemetry/src' 30 RUN_BENCHMARK = 'tools/perf/run_benchmark' 31 32 RSA_KEY = '-i %s' % test_runner_utils.TEST_KEY_PATH 33 DUT_CHROME_RESULTS_DIR = '/usr/local/telemetry/src/tools/perf' 34 35 # Result Statuses 36 SUCCESS_STATUS = 'SUCCESS' 37 WARNING_STATUS = 'WARNING' 38 FAILED_STATUS = 'FAILED' 39 40 # Regex for the RESULT output lines understood by chrome buildbot. 41 # Keep in sync with 42 # chromium/tools/build/scripts/slave/performance_log_processor.py. 43 RESULTS_REGEX = re.compile(r'(?P<IMPORTANT>\*)?RESULT ' 44 r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= ' 45 r'(?P<VALUE>[\{\[]?[-\d\., ]+[\}\]]?)(' 46 r' ?(?P<UNITS>.+))?') 47 HISTOGRAM_REGEX = re.compile(r'(?P<IMPORTANT>\*)?HISTOGRAM ' 48 r'(?P<GRAPH>[^:]*): (?P<TRACE>[^=]*)= ' 49 r'(?P<VALUE_JSON>{.*})(?P<UNITS>.+)?') 50 51 52 def _find_chrome_root_dir(): 53 # Look for chrome source root, either externally mounted, or inside 54 # the chroot. Prefer chrome-src-internal source tree to chrome-src. 55 sources_list = ('chrome-src-internal', 'chrome-src') 56 57 dir_list = [os.path.join(CHROME_SRC_ROOT, x) for x in sources_list] 58 if 'CHROME_ROOT' in os.environ: 59 dir_list.insert(0, os.environ['CHROME_ROOT']) 60 61 for dir in dir_list: 62 if os.path.exists(dir): 63 chrome_root_dir = dir 64 break 65 else: 66 raise error.TestError('Chrome source directory not found.') 67 68 logging.info('Using Chrome source tree at %s', chrome_root_dir) 69 return os.path.join(chrome_root_dir, 'src') 70 71 72 def _ensure_deps(dut, test_name): 73 """ 74 Ensure the dependencies are locally available on DUT. 75 76 @param dut: The autotest host object representing DUT. 77 @param test_name: Name of the telemetry test. 78 """ 79 # Get DEPs using host's telemetry. 80 chrome_root_dir = _find_chrome_root_dir() 81 format_string = ('python %s/tools/perf/fetch_benchmark_deps.py %s') 82 command = format_string % (chrome_root_dir, test_name) 83 logging.info('Getting DEPs: %s', command) 84 stdout = StringIO.StringIO() 85 stderr = StringIO.StringIO() 86 try: 87 result = utils.run(command, stdout_tee=stdout, 88 stderr_tee=stderr) 89 except error.CmdError as e: 90 logging.debug('Error occurred getting DEPs: %s\n %s\n', 91 stdout.getvalue(), stderr.getvalue()) 92 raise error.TestFail('Error occurred while getting DEPs.') 93 94 # Download DEPs to DUT. 95 # send_file() relies on rsync over ssh. Couldn't be better. 96 stdout_str = stdout.getvalue() 97 stdout.close() 98 stderr.close() 99 for dep in stdout_str.split(): 100 src = os.path.join(chrome_root_dir, dep) 101 dst = os.path.join(CLIENT_CHROME_ROOT, dep) 102 if not os.path.isfile(src): 103 raise error.TestFail('Error occurred while saving DEPs.') 104 logging.info('Copying: %s -> %s', src, dst) 105 try: 106 dut.send_file(src, dst) 107 except: 108 raise error.TestFail('Error occurred while sending DEPs to dut.\n') 109 110 111 class telemetry_Crosperf(test.test): 112 """Run one or more telemetry benchmarks under the crosperf script.""" 113 version = 1 114 115 def scp_telemetry_results(self, client_ip, dut): 116 """Copy telemetry results from dut. 117 118 @param client_ip: The ip address of the DUT 119 @param dut: The autotest host object representing DUT. 120 121 @returns status code for scp command. 122 """ 123 cmd=[] 124 src = ('root@%s:%s/results-chart.json' % 125 (dut.hostname if dut else client_ip, DUT_CHROME_RESULTS_DIR)) 126 cmd.extend(['scp', DUT_SCP_OPTIONS, RSA_KEY, '-v', 127 src, self.resultsdir]) 128 command = ' '.join(cmd) 129 130 logging.debug('Retrieving Results: %s', command) 131 try: 132 result = utils.run(command, 133 timeout=TELEMETRY_TIMEOUT_MINS * 60) 134 exit_code = result.exit_status 135 except Exception as e: 136 logging.error('Failed to retrieve results: %s', e) 137 raise 138 139 logging.debug('command return value: %d', exit_code) 140 return exit_code 141 142 def run_once(self, args, client_ip='', dut=None): 143 """ 144 Run a single telemetry test. 145 146 @param args: A dictionary of the arguments that were passed 147 to this test. 148 @param client_ip: The ip address of the DUT 149 @param dut: The autotest host object representing DUT. 150 151 @returns A TelemetryResult instance with the results of this execution. 152 """ 153 test_name = args['test'] 154 test_args = '' 155 if 'test_args' in args: 156 test_args = args['test_args'] 157 158 # Decide whether the test will run locally or by a remote server. 159 if args.get('run_local', 'false').lower() == 'true': 160 # The telemetry scripts will run on DUT. 161 _ensure_deps(dut, test_name) 162 format_string = ('python %s --browser=system ' 163 '--output-format=chartjson %s %s') 164 command = format_string % (os.path.join(CLIENT_CHROME_ROOT, 165 RUN_BENCHMARK), 166 test_args, test_name) 167 runner = dut 168 else: 169 # The telemetry scripts will run on server. 170 format_string = ('python %s --browser=cros-chrome --remote=%s ' 171 '--output-dir="%s" ' 172 '--output-format=chartjson %s %s') 173 command = format_string % (os.path.join(_find_chrome_root_dir(), 174 RUN_BENCHMARK), 175 client_ip, self.resultsdir, test_args, 176 test_name) 177 runner = utils 178 179 # Run the test. 180 stdout = StringIO.StringIO() 181 stderr = StringIO.StringIO() 182 try: 183 logging.info('CMD: %s', command) 184 result = runner.run(command, stdout_tee=stdout, stderr_tee=stderr, 185 timeout=TELEMETRY_TIMEOUT_MINS*60) 186 exit_code = result.exit_status 187 except error.CmdError as e: 188 logging.debug('Error occurred executing telemetry.') 189 exit_code = e.result_obj.exit_status 190 raise error.TestFail('An error occurred while executing ' 191 'telemetry test.') 192 except: 193 logging.debug('Telemetry aborted with unknown error.') 194 exit_code = -1 195 raise 196 finally: 197 stdout_str = stdout.getvalue() 198 stderr_str = stderr.getvalue() 199 stdout.close() 200 stderr.close() 201 logging.info('Telemetry completed with exit code: %d.' 202 '\nstdout:%s\nstderr:%s', exit_code, 203 stdout_str, stderr_str) 204 205 # Copy the results-chart.json file into the test_that results 206 # directory, if necessary. 207 if args.get('run_local', 'false').lower() == 'true': 208 result = self.scp_telemetry_results(client_ip, dut) 209 else: 210 filepath = os.path.join(self.resultsdir, 'results-chart.json') 211 if not os.path.exists(filepath): 212 raise RuntimeError('Missing results file: %s' % filepath) 213 214 return result 215
