1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Run specific test on specific environment.""" 6 7 import json 8 import logging 9 import os 10 import re 11 import shutil 12 import string 13 import tempfile 14 import time 15 import zipfile 16 17 from devil.utils import zip_utils 18 from pylib.base import base_test_result 19 from pylib.base import test_run 20 from pylib.remote.device import appurify_constants 21 from pylib.remote.device import appurify_sanitized 22 from pylib.remote.device import remote_device_helper 23 24 _DEVICE_OFFLINE_RE = re.compile('error: device not found') 25 _LONG_MSG_RE = re.compile('longMsg=(.*)$') 26 _SHORT_MSG_RE = re.compile('shortMsg=(.*)$') 27 28 class RemoteDeviceTestRun(test_run.TestRun): 29 """Run tests on a remote device.""" 30 31 _TEST_RUN_KEY = 'test_run' 32 _TEST_RUN_ID_KEY = 'test_run_id' 33 34 WAIT_TIME = 5 35 COMPLETE = 'complete' 36 HEARTBEAT_INTERVAL = 300 37 38 def __init__(self, env, test_instance): 39 """Constructor. 40 41 Args: 42 env: Environment the tests will run in. 43 test_instance: The test that will be run. 44 """ 45 super(RemoteDeviceTestRun, self).__init__(env, test_instance) 46 self._env = env 47 self._test_instance = test_instance 48 self._app_id = '' 49 self._test_id = '' 50 self._results = '' 51 self._test_run_id = '' 52 self._results_temp_dir = None 53 54 #override 55 def SetUp(self): 56 """Set up a test run.""" 57 if self._env.trigger: 58 self._TriggerSetUp() 59 elif self._env.collect: 60 assert isinstance(self._env.collect, basestring), ( 61 'File for storing test_run_id must be a string.') 62 with open(self._env.collect, 'r') as persisted_data_file: 63 persisted_data = json.loads(persisted_data_file.read()) 64 self._env.LoadFrom(persisted_data) 65 self.LoadFrom(persisted_data) 66 67 def _TriggerSetUp(self): 68 """Set up the triggering of a test run.""" 69 raise NotImplementedError 70 71 #override 72 def RunTests(self): 73 """Run the test.""" 74 if self._env.trigger: 75 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 76 logging.WARNING): 77 test_start_res = appurify_sanitized.api.tests_run( 78 self._env.token, self._env.device_type_id, self._app_id, 79 self._test_id) 80 remote_device_helper.TestHttpResponse( 81 test_start_res, 'Unable to run test.') 82 self._test_run_id = test_start_res.json()['response']['test_run_id'] 83 logging.info('Test run id: %s', self._test_run_id) 84 85 if self._env.collect: 86 current_status = '' 87 timeout_counter = 0 88 heartbeat_counter = 0 89 while self._GetTestStatus(self._test_run_id) != self.COMPLETE: 90 if self._results['detailed_status'] != current_status: 91 logging.info('Test status: %s', self._results['detailed_status']) 92 current_status = self._results['detailed_status'] 93 timeout_counter = 0 94 heartbeat_counter = 0 95 if heartbeat_counter > self.HEARTBEAT_INTERVAL: 96 logging.info('Test status: %s', self._results['detailed_status']) 97 heartbeat_counter = 0 98 99 timeout = self._env.timeouts.get( 100 current_status, self._env.timeouts['unknown']) 101 if timeout_counter > timeout: 102 raise remote_device_helper.RemoteDeviceError( 103 'Timeout while in %s state for %s seconds' 104 % (current_status, timeout), 105 is_infra_error=True) 106 time.sleep(self.WAIT_TIME) 107 timeout_counter += self.WAIT_TIME 108 heartbeat_counter += self.WAIT_TIME 109 self._DownloadTestResults(self._env.results_path) 110 111 if self._results['results']['exception']: 112 raise remote_device_helper.RemoteDeviceError( 113 self._results['results']['exception'], is_infra_error=True) 114 115 return self._ParseTestResults() 116 117 #override 118 def TearDown(self): 119 """Tear down the test run.""" 120 if self._env.collect: 121 self._CollectTearDown() 122 elif self._env.trigger: 123 assert isinstance(self._env.trigger, basestring), ( 124 'File for storing test_run_id must be a string.') 125 with open(self._env.trigger, 'w') as persisted_data_file: 126 persisted_data = {} 127 self.DumpTo(persisted_data) 128 self._env.DumpTo(persisted_data) 129 persisted_data_file.write(json.dumps(persisted_data)) 130 131 def _CollectTearDown(self): 132 if self._GetTestStatus(self._test_run_id) != self.COMPLETE: 133 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 134 logging.WARNING): 135 test_abort_res = appurify_sanitized.api.tests_abort( 136 self._env.token, self._test_run_id, reason='Test runner exiting.') 137 remote_device_helper.TestHttpResponse(test_abort_res, 138 'Unable to abort test.') 139 if self._results_temp_dir: 140 shutil.rmtree(self._results_temp_dir) 141 142 def __enter__(self): 143 """Set up the test run when used as a context manager.""" 144 self.SetUp() 145 return self 146 147 def __exit__(self, exc_type, exc_val, exc_tb): 148 """Tear down the test run when used as a context manager.""" 149 self.TearDown() 150 151 def DumpTo(self, persisted_data): 152 test_run_data = { 153 self._TEST_RUN_ID_KEY: self._test_run_id, 154 } 155 persisted_data[self._TEST_RUN_KEY] = test_run_data 156 157 def LoadFrom(self, persisted_data): 158 test_run_data = persisted_data[self._TEST_RUN_KEY] 159 self._test_run_id = test_run_data[self._TEST_RUN_ID_KEY] 160 161 def _ParseTestResults(self): 162 raise NotImplementedError 163 164 def _GetTestByName(self, test_name): 165 """Gets test_id for specific test. 166 167 Args: 168 test_name: Test to find the ID of. 169 """ 170 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 171 logging.WARNING): 172 test_list_res = appurify_sanitized.api.tests_list(self._env.token) 173 remote_device_helper.TestHttpResponse(test_list_res, 174 'Unable to get tests list.') 175 for test in test_list_res.json()['response']: 176 if test['test_type'] == test_name: 177 return test['test_id'] 178 raise remote_device_helper.RemoteDeviceError( 179 'No test found with name %s' % (test_name)) 180 181 def _DownloadTestResults(self, results_path): 182 """Download the test results from remote device service. 183 184 Downloads results in temporary location, and then copys results 185 to results_path if results_path is not set to None. 186 187 Args: 188 results_path: Path to download appurify results zipfile. 189 190 Returns: 191 Path to downloaded file. 192 """ 193 194 if self._results_temp_dir is None: 195 self._results_temp_dir = tempfile.mkdtemp() 196 logging.info('Downloading results to %s.', self._results_temp_dir) 197 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 198 logging.WARNING): 199 appurify_sanitized.utils.wget(self._results['results']['url'], 200 self._results_temp_dir + '/results') 201 if results_path: 202 logging.info('Copying results to %s', results_path) 203 if not os.path.exists(os.path.dirname(results_path)): 204 os.makedirs(os.path.dirname(results_path)) 205 shutil.copy(self._results_temp_dir + '/results', results_path) 206 return self._results_temp_dir + '/results' 207 208 def _GetTestStatus(self, test_run_id): 209 """Checks the state of the test, and sets self._results 210 211 Args: 212 test_run_id: Id of test on on remote service. 213 """ 214 215 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 216 logging.WARNING): 217 test_check_res = appurify_sanitized.api.tests_check_result( 218 self._env.token, test_run_id) 219 remote_device_helper.TestHttpResponse(test_check_res, 220 'Unable to get test status.') 221 self._results = test_check_res.json()['response'] 222 return self._results['status'] 223 224 def _AmInstrumentTestSetup(self, app_path, test_path, runner_package, 225 environment_variables, extra_apks=None): 226 config = {'runner': runner_package} 227 if environment_variables: 228 config['environment_vars'] = ','.join( 229 '%s=%s' % (k, v) for k, v in environment_variables.iteritems()) 230 231 self._app_id = self._UploadAppToDevice(app_path) 232 233 data_deps = self._test_instance.GetDataDependencies() 234 if data_deps: 235 with tempfile.NamedTemporaryFile(suffix='.zip') as test_with_deps: 236 sdcard_files = [] 237 additional_apks = [] 238 host_test = os.path.basename(test_path) 239 with zipfile.ZipFile(test_with_deps.name, 'w') as zip_file: 240 zip_file.write(test_path, host_test, zipfile.ZIP_DEFLATED) 241 for h, _ in data_deps: 242 if os.path.isdir(h): 243 zip_utils.WriteToZipFile(zip_file, h, '.') 244 sdcard_files.extend(os.listdir(h)) 245 else: 246 zip_utils.WriteToZipFile(zip_file, h, os.path.basename(h)) 247 sdcard_files.append(os.path.basename(h)) 248 for a in extra_apks or (): 249 zip_utils.WriteToZipFile(zip_file, a, os.path.basename(a)) 250 additional_apks.append(os.path.basename(a)) 251 252 config['sdcard_files'] = ','.join(sdcard_files) 253 config['host_test'] = host_test 254 if additional_apks: 255 config['additional_apks'] = ','.join(additional_apks) 256 self._test_id = self._UploadTestToDevice( 257 'robotium', test_with_deps.name, app_id=self._app_id) 258 else: 259 self._test_id = self._UploadTestToDevice('robotium', test_path) 260 261 logging.info('Setting config: %s', config) 262 appurify_configs = {} 263 if self._env.network_config: 264 appurify_configs['network'] = self._env.network_config 265 self._SetTestConfig('robotium', config, **appurify_configs) 266 267 def _UploadAppToDevice(self, app_path): 268 """Upload app to device.""" 269 logging.info('Uploading %s to remote service as %s.', app_path, 270 self._test_instance.suite) 271 with open(app_path, 'rb') as apk_src: 272 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 273 logging.WARNING): 274 upload_results = appurify_sanitized.api.apps_upload( 275 self._env.token, apk_src, 'raw', name=self._test_instance.suite) 276 remote_device_helper.TestHttpResponse( 277 upload_results, 'Unable to upload %s.' % app_path) 278 return upload_results.json()['response']['app_id'] 279 280 def _UploadTestToDevice(self, test_type, test_path, app_id=None): 281 """Upload test to device 282 Args: 283 test_type: Type of test that is being uploaded. Ex. uirobot, gtest.. 284 """ 285 logging.info('Uploading %s to remote service.', test_path) 286 with open(test_path, 'rb') as test_src: 287 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 288 logging.WARNING): 289 upload_results = appurify_sanitized.api.tests_upload( 290 self._env.token, test_src, 'raw', test_type, app_id=app_id) 291 remote_device_helper.TestHttpResponse(upload_results, 292 'Unable to upload %s.' % test_path) 293 return upload_results.json()['response']['test_id'] 294 295 def _SetTestConfig(self, runner_type, runner_configs, 296 network=appurify_constants.NETWORK.WIFI_1_BAR, 297 pcap=0, profiler=0, videocapture=0): 298 """Generates and uploads config file for test. 299 Args: 300 runner_configs: Configs specific to the runner you are using. 301 network: Config to specify the network environment the devices running 302 the tests will be in. 303 pcap: Option to set the recording the of network traffic from the device. 304 profiler: Option to set the recording of CPU, memory, and network 305 transfer usage in the tests. 306 videocapture: Option to set video capture during the tests. 307 308 """ 309 logging.info('Generating config file for test.') 310 with tempfile.TemporaryFile() as config: 311 config_data = [ 312 '[appurify]', 313 'network=%s' % network, 314 'pcap=%s' % pcap, 315 'profiler=%s' % profiler, 316 'videocapture=%s' % videocapture, 317 '[%s]' % runner_type 318 ] 319 config_data.extend( 320 '%s=%s' % (k, v) for k, v in runner_configs.iteritems()) 321 config.write(''.join('%s\n' % l for l in config_data)) 322 config.flush() 323 config.seek(0) 324 with appurify_sanitized.SanitizeLogging(self._env.verbose_count, 325 logging.WARNING): 326 config_response = appurify_sanitized.api.config_upload( 327 self._env.token, config, self._test_id) 328 remote_device_helper.TestHttpResponse( 329 config_response, 'Unable to upload test config.') 330 331 def _LogLogcat(self, level=logging.CRITICAL): 332 """Prints out logcat downloaded from remote service. 333 Args: 334 level: logging level to print at. 335 336 Raises: 337 KeyError: If appurify_results/logcat.txt file cannot be found in 338 downloaded zip. 339 """ 340 zip_file = self._DownloadTestResults(None) 341 with zipfile.ZipFile(zip_file) as z: 342 try: 343 logcat = z.read('appurify_results/logcat.txt') 344 printable_logcat = ''.join(c for c in logcat if c in string.printable) 345 for line in printable_logcat.splitlines(): 346 logging.log(level, line) 347 except KeyError: 348 logging.error('No logcat found.') 349 350 def _LogAdbTraceLog(self): 351 zip_file = self._DownloadTestResults(None) 352 with zipfile.ZipFile(zip_file) as z: 353 adb_trace_log = z.read('adb_trace.log') 354 for line in adb_trace_log.splitlines(): 355 logging.critical(line) 356 357 def _DidDeviceGoOffline(self): 358 zip_file = self._DownloadTestResults(None) 359 with zipfile.ZipFile(zip_file) as z: 360 adb_trace_log = z.read('adb_trace.log') 361 if any(_DEVICE_OFFLINE_RE.search(l) for l in adb_trace_log.splitlines()): 362 return True 363 return False 364 365 def _DetectPlatformErrors(self, results): 366 if not self._results['results']['pass']: 367 crash_msg = None 368 for line in self._results['results']['output'].splitlines(): 369 m = _LONG_MSG_RE.search(line) 370 if m: 371 crash_msg = m.group(1) 372 break 373 m = _SHORT_MSG_RE.search(line) 374 if m: 375 crash_msg = m.group(1) 376 if crash_msg: 377 self._LogLogcat() 378 results.AddResult(base_test_result.BaseTestResult( 379 crash_msg, base_test_result.ResultType.CRASH)) 380 elif self._DidDeviceGoOffline(): 381 self._LogLogcat() 382 self._LogAdbTraceLog() 383 raise remote_device_helper.RemoteDeviceError( 384 'Remote service unable to reach device.', is_infra_error=True) 385 else: 386 # Remote service is reporting a failure, but no failure in results obj. 387 if results.DidRunPass(): 388 results.AddResult(base_test_result.BaseTestResult( 389 'Remote service detected error.', 390 base_test_result.ResultType.UNKNOWN)) 391
