1 # Copyright 2014 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import collections 6 import itertools 7 import logging 8 import os 9 import posixpath 10 11 from devil.android import device_errors 12 from devil.android import device_temp_file 13 from devil.android import ports 14 from devil.utils import reraiser_thread 15 from pylib import constants 16 from pylib.base import base_test_result 17 from pylib.gtest import gtest_test_instance 18 from pylib.local import local_test_server_spawner 19 from pylib.local.device import local_device_environment 20 from pylib.local.device import local_device_test_run 21 22 _COMMAND_LINE_FLAGS_SUPPORTED = True 23 24 _MAX_INLINE_FLAGS_LENGTH = 50 # Arbitrarily chosen. 25 _EXTRA_COMMAND_LINE_FILE = ( 26 'org.chromium.native_test.NativeTestActivity.CommandLineFile') 27 _EXTRA_COMMAND_LINE_FLAGS = ( 28 'org.chromium.native_test.NativeTestActivity.CommandLineFlags') 29 _EXTRA_TEST_LIST = ( 30 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 31 '.TestList') 32 _EXTRA_TEST = ( 33 'org.chromium.native_test.NativeTestInstrumentationTestRunner' 34 '.Test') 35 36 _MAX_SHARD_SIZE = 256 37 _SECONDS_TO_NANOS = int(1e9) 38 39 # The amount of time a test executable may run before it gets killed. 40 _TEST_TIMEOUT_SECONDS = 30*60 41 42 # TODO(jbudorick): Move this up to the test instance if the net test server is 43 # handled outside of the APK for the remote_device environment. 44 _SUITE_REQUIRES_TEST_SERVER_SPAWNER = [ 45 'components_browsertests', 'content_unittests', 'content_browsertests', 46 'net_unittests', 'unit_tests' 47 ] 48 49 # No-op context manager. If we used Python 3, we could change this to 50 # contextlib.ExitStack() 51 class _NullContextManager(object): 52 def __enter__(self): 53 pass 54 def __exit__(self, *args): 55 pass 56 57 58 # TODO(jbudorick): Move this inside _ApkDelegate once TestPackageApk is gone. 59 def PullAppFilesImpl(device, package, files, directory): 60 device_dir = device.GetApplicationDataDirectory(package) 61 host_dir = os.path.join(directory, str(device)) 62 for f in files: 63 device_file = posixpath.join(device_dir, f) 64 host_file = os.path.join(host_dir, *f.split(posixpath.sep)) 65 host_file_base, ext = os.path.splitext(host_file) 66 for i in itertools.count(): 67 host_file = '%s_%d%s' % (host_file_base, i, ext) 68 if not os.path.exists(host_file): 69 break 70 device.PullFile(device_file, host_file) 71 72 73 def _ExtractTestsFromFilter(gtest_filter): 74 """Returns the list of tests specified by the given filter. 75 76 Returns: 77 None if the device should be queried for the test list instead. 78 """ 79 # Empty means all tests, - means exclude filter. 80 if not gtest_filter or '-' in gtest_filter: 81 return None 82 83 patterns = gtest_filter.split(':') 84 # For a single pattern, allow it even if it has a wildcard so long as the 85 # wildcard comes at the end and there is at least one . to prove the scope is 86 # not too large. 87 # This heuristic is not necessarily faster, but normally is. 88 if len(patterns) == 1 and patterns[0].endswith('*'): 89 no_suffix = patterns[0].rstrip('*') 90 if '*' not in no_suffix and '.' in no_suffix: 91 return patterns 92 93 if '*' in gtest_filter: 94 return None 95 return patterns 96 97 98 class _ApkDelegate(object): 99 def __init__(self, test_instance): 100 self._activity = test_instance.activity 101 self._apk_helper = test_instance.apk_helper 102 self._test_apk_incremental_install_script = ( 103 test_instance.test_apk_incremental_install_script) 104 self._package = test_instance.package 105 self._runner = test_instance.runner 106 self._permissions = test_instance.permissions 107 self._suite = test_instance.suite 108 self._component = '%s/%s' % (self._package, self._runner) 109 self._extras = test_instance.extras 110 111 def Install(self, device): 112 if self._test_apk_incremental_install_script: 113 local_device_test_run.IncrementalInstall(device, self._apk_helper, 114 self._test_apk_incremental_install_script) 115 else: 116 device.Install(self._apk_helper, reinstall=True, 117 permissions=self._permissions) 118 119 def Run(self, test, device, flags=None, **kwargs): 120 extras = dict(self._extras) 121 122 if ('timeout' in kwargs 123 and gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras): 124 # Make sure the instrumentation doesn't kill the test before the 125 # scripts do. The provided timeout value is in seconds, but the 126 # instrumentation deals with nanoseconds because that's how Android 127 # handles time. 128 extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int( 129 kwargs['timeout'] * _SECONDS_TO_NANOS) 130 131 command_line_file = _NullContextManager() 132 if flags: 133 if len(flags) > _MAX_INLINE_FLAGS_LENGTH: 134 command_line_file = device_temp_file.DeviceTempFile(device.adb) 135 device.WriteFile(command_line_file.name, '_ %s' % flags) 136 extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name 137 else: 138 extras[_EXTRA_COMMAND_LINE_FLAGS] = flags 139 140 test_list_file = _NullContextManager() 141 if test: 142 if len(test) > 1: 143 test_list_file = device_temp_file.DeviceTempFile(device.adb) 144 device.WriteFile(test_list_file.name, '\n'.join(test)) 145 extras[_EXTRA_TEST_LIST] = test_list_file.name 146 else: 147 extras[_EXTRA_TEST] = test[0] 148 149 with command_line_file, test_list_file: 150 try: 151 return device.StartInstrumentation( 152 self._component, extras=extras, raw=False, **kwargs) 153 except Exception: 154 device.ForceStop(self._package) 155 raise 156 157 def PullAppFiles(self, device, files, directory): 158 PullAppFilesImpl(device, self._package, files, directory) 159 160 def Clear(self, device): 161 device.ClearApplicationState(self._package, permissions=self._permissions) 162 163 164 class _ExeDelegate(object): 165 def __init__(self, tr, dist_dir): 166 self._host_dist_dir = dist_dir 167 self._exe_file_name = os.path.basename(dist_dir)[:-len('__dist')] 168 self._device_dist_dir = posixpath.join( 169 constants.TEST_EXECUTABLE_DIR, os.path.basename(dist_dir)) 170 self._test_run = tr 171 172 def Install(self, device): 173 # TODO(jbudorick): Look into merging this with normal data deps pushing if 174 # executables become supported on nonlocal environments. 175 device.PushChangedFiles([(self._host_dist_dir, self._device_dist_dir)], 176 delete_device_stale=True) 177 178 def Run(self, test, device, flags=None, **kwargs): 179 tool = self._test_run.GetTool(device).GetTestWrapper() 180 if tool: 181 cmd = [tool] 182 else: 183 cmd = [] 184 cmd.append(posixpath.join(self._device_dist_dir, self._exe_file_name)) 185 186 if test: 187 cmd.append('--gtest_filter=%s' % ':'.join(test)) 188 if flags: 189 # TODO(agrieve): This won't work if multiple flags are passed. 190 cmd.append(flags) 191 cwd = constants.TEST_EXECUTABLE_DIR 192 193 env = { 194 'LD_LIBRARY_PATH': self._device_dist_dir 195 } 196 try: 197 gcov_strip_depth = os.environ['NATIVE_COVERAGE_DEPTH_STRIP'] 198 external = device.GetExternalStoragePath() 199 env['GCOV_PREFIX'] = '%s/gcov' % external 200 env['GCOV_PREFIX_STRIP'] = gcov_strip_depth 201 except (device_errors.CommandFailedError, KeyError): 202 pass 203 204 output = device.RunShellCommand( 205 cmd, cwd=cwd, env=env, check_return=True, large_output=True, **kwargs) 206 return output 207 208 def PullAppFiles(self, device, files, directory): 209 pass 210 211 def Clear(self, device): 212 device.KillAll(self._exe_file_name, blocking=True, timeout=30, quiet=True) 213 214 215 class LocalDeviceGtestRun(local_device_test_run.LocalDeviceTestRun): 216 217 def __init__(self, env, test_instance): 218 assert isinstance(env, local_device_environment.LocalDeviceEnvironment) 219 assert isinstance(test_instance, gtest_test_instance.GtestTestInstance) 220 super(LocalDeviceGtestRun, self).__init__(env, test_instance) 221 222 if self._test_instance.apk: 223 self._delegate = _ApkDelegate(self._test_instance) 224 elif self._test_instance.exe_dist_dir: 225 self._delegate = _ExeDelegate(self, self._test_instance.exe_dist_dir) 226 self._crashes = set() 227 self._servers = collections.defaultdict(list) 228 229 #override 230 def TestPackage(self): 231 return self._test_instance.suite 232 233 #override 234 def SetUp(self): 235 @local_device_test_run.handle_shard_failures_with( 236 on_failure=self._env.BlacklistDevice) 237 def individual_device_set_up(dev): 238 def install_apk(): 239 # Install test APK. 240 self._delegate.Install(dev) 241 242 def push_test_data(): 243 # Push data dependencies. 244 external_storage = dev.GetExternalStoragePath() 245 data_deps = self._test_instance.GetDataDependencies() 246 host_device_tuples = [ 247 (h, d if d is not None else external_storage) 248 for h, d in data_deps] 249 dev.PushChangedFiles(host_device_tuples) 250 251 def init_tool_and_start_servers(): 252 tool = self.GetTool(dev) 253 tool.CopyFiles(dev) 254 tool.SetupEnvironment() 255 256 self._servers[str(dev)] = [] 257 if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER: 258 self._servers[str(dev)].append( 259 local_test_server_spawner.LocalTestServerSpawner( 260 ports.AllocateTestServerPort(), dev, tool)) 261 262 for s in self._servers[str(dev)]: 263 s.SetUp() 264 265 steps = (install_apk, push_test_data, init_tool_and_start_servers) 266 if self._env.concurrent_adb: 267 reraiser_thread.RunAsync(steps) 268 else: 269 for step in steps: 270 step() 271 272 self._env.parallel_devices.pMap(individual_device_set_up) 273 274 #override 275 def _ShouldShard(self): 276 return True 277 278 #override 279 def _CreateShards(self, tests): 280 # _crashes are tests that might crash and make the tests in the same shard 281 # following the crashed testcase not run. 282 # Thus we need to create separate shards for each crashed testcase, 283 # so that other tests can be run. 284 device_count = len(self._env.devices) 285 shards = [] 286 287 # Add shards with only one suspect testcase. 288 shards += [[crash] for crash in self._crashes if crash in tests] 289 290 # Delete suspect testcase from tests. 291 tests = [test for test in tests if not test in self._crashes] 292 293 for i in xrange(0, device_count): 294 unbounded_shard = tests[i::device_count] 295 shards += [unbounded_shard[j:j+_MAX_SHARD_SIZE] 296 for j in xrange(0, len(unbounded_shard), _MAX_SHARD_SIZE)] 297 return shards 298 299 #override 300 def _GetTests(self): 301 if self._test_instance.extract_test_list_from_filter: 302 # When the exact list of tests to run is given via command-line (e.g. when 303 # locally iterating on a specific test), skip querying the device (which 304 # takes ~3 seconds). 305 tests = _ExtractTestsFromFilter(self._test_instance.gtest_filter) 306 if tests: 307 return tests 308 309 # Even when there's only one device, it still makes sense to retrieve the 310 # test list so that tests can be split up and run in batches rather than all 311 # at once (since test output is not streamed). 312 @local_device_test_run.handle_shard_failures_with( 313 on_failure=self._env.BlacklistDevice) 314 def list_tests(dev): 315 raw_test_list = self._delegate.Run( 316 None, dev, flags='--gtest_list_tests', timeout=30) 317 tests = gtest_test_instance.ParseGTestListTests(raw_test_list) 318 if not tests: 319 logging.info('No tests found. Output:') 320 for l in raw_test_list: 321 logging.info(' %s', l) 322 tests = self._test_instance.FilterTests(tests) 323 return tests 324 325 # Query all devices in case one fails. 326 test_lists = self._env.parallel_devices.pMap(list_tests).pGet(None) 327 328 # If all devices failed to list tests, raise an exception. 329 # Check that tl is not None and is not empty. 330 if all(not tl for tl in test_lists): 331 raise device_errors.CommandFailedError( 332 'Failed to list tests on any device') 333 return list(sorted(set().union(*[set(tl) for tl in test_lists if tl]))) 334 335 #override 336 def _RunTest(self, device, test): 337 # Run the test. 338 timeout = (self._test_instance.shard_timeout 339 * self.GetTool(device).GetTimeoutScale()) 340 output = self._delegate.Run( 341 test, device, flags=self._test_instance.test_arguments, 342 timeout=timeout, retries=0) 343 for s in self._servers[str(device)]: 344 s.Reset() 345 if self._test_instance.app_files: 346 self._delegate.PullAppFiles(device, self._test_instance.app_files, 347 self._test_instance.app_file_dir) 348 if not self._env.skip_clear_data: 349 self._delegate.Clear(device) 350 351 # Parse the output. 352 # TODO(jbudorick): Transition test scripts away from parsing stdout. 353 results = self._test_instance.ParseGTestOutput(output) 354 355 # Check whether there are any crashed testcases. 356 self._crashes.update(r.GetName() for r in results 357 if r.GetType() == base_test_result.ResultType.CRASH) 358 return results 359 360 #override 361 def TearDown(self): 362 @local_device_test_run.handle_shard_failures 363 def individual_device_tear_down(dev): 364 for s in self._servers.get(str(dev), []): 365 s.TearDown() 366 367 tool = self.GetTool(dev) 368 tool.CleanUpEnvironment() 369 370 self._env.parallel_devices.pMap(individual_device_tear_down) 371