1 # Copyright 2013 The Chromium Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 """Generates test runner factory and tests for GTests.""" 6 # pylint: disable=W0212 7 8 import fnmatch 9 import glob 10 import logging 11 import os 12 import shutil 13 import sys 14 15 from pylib import cmd_helper 16 from pylib import constants 17 18 from pylib.base import base_test_result 19 from pylib.base import test_dispatcher 20 from pylib.gtest import test_package_apk 21 from pylib.gtest import test_package_exe 22 from pylib.gtest import test_runner 23 24 sys.path.insert(0, 25 os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib', 26 'common')) 27 import unittest_util # pylint: disable=F0401 28 29 30 _ISOLATE_FILE_PATHS = { 31 'base_unittests': 'base/base_unittests.isolate', 32 'blink_heap_unittests': 33 'third_party/WebKit/Source/platform/heap/BlinkHeapUnitTests.isolate', 34 'breakpad_unittests': 'breakpad/breakpad_unittests.isolate', 35 'cc_perftests': 'cc/cc_perftests.isolate', 36 'components_unittests': 'components/components_unittests.isolate', 37 'content_browsertests': 'content/content_browsertests.isolate', 38 'content_unittests': 'content/content_unittests.isolate', 39 'media_perftests': 'media/media_perftests.isolate', 40 'media_unittests': 'media/media_unittests.isolate', 41 'net_unittests': 'net/net_unittests.isolate', 42 'ui_unittests': 'ui/ui_unittests.isolate', 43 'unit_tests': 'chrome/unit_tests.isolate', 44 'webkit_unit_tests': 45 'third_party/WebKit/Source/web/WebKitUnitTests.isolate', 46 } 47 48 # Paths relative to third_party/webrtc/ (kept separate for readability). 49 _WEBRTC_ISOLATE_FILE_PATHS = { 50 'audio_decoder_unittests': 51 'modules/audio_coding/neteq/audio_decoder_unittests.isolate', 52 'common_audio_unittests': 'common_audio/common_audio_unittests.isolate', 53 'common_video_unittests': 'common_video/common_video_unittests.isolate', 54 'modules_tests': 'modules/modules_tests.isolate', 55 'modules_unittests': 'modules/modules_unittests.isolate', 56 'system_wrappers_unittests': 57 'system_wrappers/source/system_wrappers_unittests.isolate', 58 'test_support_unittests': 'test/test_support_unittests.isolate', 59 'tools_unittests': 'tools/tools_unittests.isolate', 60 'video_engine_tests': 'video_engine_tests.isolate', 61 'video_engine_core_unittests': 62 'video_engine/video_engine_core_unittests.isolate', 63 'voice_engine_unittests': 'voice_engine/voice_engine_unittests.isolate', 64 'webrtc_perf_tests': 'webrtc_perf_tests.isolate', 65 } 66 67 # Append the WebRTC tests with the full path from Chromium's src/ root. 68 for webrtc_test, isolate_path in _WEBRTC_ISOLATE_FILE_PATHS.items(): 69 _ISOLATE_FILE_PATHS[webrtc_test] = 'third_party/webrtc/%s' % isolate_path 70 71 # Used for filtering large data deps at a finer grain than what's allowed in 72 # isolate files since pushing deps to devices is expensive. 73 # Wildcards are allowed. 74 _DEPS_EXCLUSION_LIST = [ 75 'chrome/test/data/extensions/api_test', 76 'chrome/test/data/extensions/secure_shell', 77 'chrome/test/data/firefox*', 78 'chrome/test/data/gpu', 79 'chrome/test/data/image_decoding', 80 'chrome/test/data/import', 81 'chrome/test/data/page_cycler', 82 'chrome/test/data/perf', 83 'chrome/test/data/pyauto_private', 84 'chrome/test/data/safari_import', 85 'chrome/test/data/scroll', 86 'chrome/test/data/third_party', 87 'third_party/hunspell_dictionaries/*.dic', 88 # crbug.com/258690 89 'webkit/data/bmp_decoder', 90 'webkit/data/ico_decoder', 91 ] 92 93 _ISOLATE_SCRIPT = os.path.join( 94 constants.DIR_SOURCE_ROOT, 'tools', 'swarming_client', 'isolate.py') 95 96 97 def _GenerateDepsDirUsingIsolate(suite_name, isolate_file_path=None): 98 """Generate the dependency dir for the test suite using isolate. 99 100 Args: 101 suite_name: Name of the test suite (e.g. base_unittests). 102 isolate_file_path: .isolate file path to use. If there is a default .isolate 103 file path for the suite_name, this will override it. 104 """ 105 if os.path.isdir(constants.ISOLATE_DEPS_DIR): 106 shutil.rmtree(constants.ISOLATE_DEPS_DIR) 107 108 if isolate_file_path: 109 if os.path.isabs(isolate_file_path): 110 isolate_abs_path = isolate_file_path 111 else: 112 isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, 113 isolate_file_path) 114 else: 115 isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name) 116 if not isolate_rel_path: 117 logging.info('Did not find an isolate file for the test suite.') 118 return 119 isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path) 120 121 isolated_abs_path = os.path.join( 122 constants.GetOutDirectory(), '%s.isolated' % suite_name) 123 assert os.path.exists(isolate_abs_path) 124 # This needs to be kept in sync with the cmd line options for isolate.py 125 # in src/build/isolate.gypi. 126 isolate_cmd = [ 127 'python', _ISOLATE_SCRIPT, 128 'remap', 129 '--isolate', isolate_abs_path, 130 '--isolated', isolated_abs_path, 131 '--outdir', constants.ISOLATE_DEPS_DIR, 132 133 '--path-variable', 'DEPTH', constants.DIR_SOURCE_ROOT, 134 '--path-variable', 'PRODUCT_DIR', constants.GetOutDirectory(), 135 136 '--config-variable', 'OS', 'android', 137 '--config-variable', 'chromeos', '0', 138 '--config-variable', 'component', 'static_library', 139 '--config-variable', 'icu_use_data_file_flag', '1', 140 '--config-variable', 'use_openssl', '0', 141 ] 142 assert not cmd_helper.RunCmd(isolate_cmd) 143 144 # We're relying on the fact that timestamps are preserved 145 # by the remap command (hardlinked). Otherwise, all the data 146 # will be pushed to the device once we move to using time diff 147 # instead of md5sum. Perform a sanity check here. 148 for root, _, filenames in os.walk(constants.ISOLATE_DEPS_DIR): 149 if filenames: 150 linked_file = os.path.join(root, filenames[0]) 151 orig_file = os.path.join( 152 constants.DIR_SOURCE_ROOT, 153 os.path.relpath(linked_file, constants.ISOLATE_DEPS_DIR)) 154 if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino: 155 break 156 else: 157 raise Exception('isolate remap command did not use hardlinks.') 158 159 # Delete excluded files as defined by _DEPS_EXCLUSION_LIST. 160 old_cwd = os.getcwd() 161 try: 162 os.chdir(constants.ISOLATE_DEPS_DIR) 163 excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)] 164 if excluded_paths: 165 logging.info('Excluding the following from dependency list: %s', 166 excluded_paths) 167 for p in excluded_paths: 168 if os.path.isdir(p): 169 shutil.rmtree(p) 170 else: 171 os.remove(p) 172 finally: 173 os.chdir(old_cwd) 174 175 # On Android, all pak files need to be in the top-level 'paks' directory. 176 paks_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'paks') 177 os.mkdir(paks_dir) 178 179 deps_out_dir = os.path.join( 180 constants.ISOLATE_DEPS_DIR, 181 os.path.relpath(os.path.join(constants.GetOutDirectory(), os.pardir), 182 constants.DIR_SOURCE_ROOT)) 183 for root, _, filenames in os.walk(deps_out_dir): 184 for filename in fnmatch.filter(filenames, '*.pak'): 185 shutil.move(os.path.join(root, filename), paks_dir) 186 187 # Move everything in PRODUCT_DIR to top level. 188 deps_product_dir = os.path.join(deps_out_dir, constants.GetBuildType()) 189 if os.path.isdir(deps_product_dir): 190 for p in os.listdir(deps_product_dir): 191 shutil.move(os.path.join(deps_product_dir, p), constants.ISOLATE_DEPS_DIR) 192 os.rmdir(deps_product_dir) 193 os.rmdir(deps_out_dir) 194 195 196 def _GetDisabledTestsFilterFromFile(suite_name): 197 """Returns a gtest filter based on the *_disabled file. 198 199 Args: 200 suite_name: Name of the test suite (e.g. base_unittests). 201 202 Returns: 203 A gtest filter which excludes disabled tests. 204 Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc' 205 """ 206 filter_file_path = os.path.join( 207 os.path.abspath(os.path.dirname(__file__)), 208 'filter', '%s_disabled' % suite_name) 209 210 if not filter_file_path or not os.path.exists(filter_file_path): 211 logging.info('No filter file found at %s', filter_file_path) 212 return '*' 213 214 filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()] 215 if x and x[0] != '#'] 216 disabled_filter = '*-%s' % ':'.join(filters) 217 logging.info('Applying filter "%s" obtained from %s', 218 disabled_filter, filter_file_path) 219 return disabled_filter 220 221 222 def _GetTests(test_options, test_package, devices): 223 """Get a list of tests. 224 225 Args: 226 test_options: A GTestOptions object. 227 test_package: A TestPackageApk object. 228 devices: A list of attached devices. 229 230 Returns: 231 A list of all the tests in the test suite. 232 """ 233 def TestListerRunnerFactory(device, _shard_index): 234 class TestListerRunner(test_runner.TestRunner): 235 def RunTest(self, _test): 236 result = base_test_result.BaseTestResult( 237 'gtest_list_tests', base_test_result.ResultType.PASS) 238 self.test_package.Install(self.device) 239 result.test_list = self.test_package.GetAllTests(self.device) 240 results = base_test_result.TestRunResults() 241 results.AddResult(result) 242 return results, None 243 return TestListerRunner(test_options, device, test_package) 244 245 results, _no_retry = test_dispatcher.RunTests( 246 ['gtest_list_tests'], TestListerRunnerFactory, devices) 247 tests = [] 248 for r in results.GetAll(): 249 tests.extend(r.test_list) 250 return tests 251 252 253 def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False): 254 """Removes tests with disabled prefixes. 255 256 Args: 257 all_tests: List of tests to filter. 258 pre: If True, include tests with PRE_ prefix. 259 manual: If True, include tests with MANUAL_ prefix. 260 261 Returns: 262 List of tests remaining. 263 """ 264 filtered_tests = [] 265 filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_'] 266 267 if not pre: 268 filter_prefixes.append('PRE_') 269 270 if not manual: 271 filter_prefixes.append('MANUAL_') 272 273 for t in all_tests: 274 test_case, test = t.split('.', 1) 275 if not any([test_case.startswith(prefix) or test.startswith(prefix) for 276 prefix in filter_prefixes]): 277 filtered_tests.append(t) 278 return filtered_tests 279 280 281 def _FilterDisabledTests(tests, suite_name, has_gtest_filter): 282 """Removes disabled tests from |tests|. 283 284 Applies the following filters in order: 285 1. Remove tests with disabled prefixes. 286 2. Remove tests specified in the *_disabled files in the 'filter' dir 287 288 Args: 289 tests: List of tests. 290 suite_name: Name of the test suite (e.g. base_unittests). 291 has_gtest_filter: Whether a gtest_filter is provided. 292 293 Returns: 294 List of tests remaining. 295 """ 296 tests = _FilterTestsUsingPrefixes( 297 tests, has_gtest_filter, has_gtest_filter) 298 tests = unittest_util.FilterTestNames( 299 tests, _GetDisabledTestsFilterFromFile(suite_name)) 300 301 return tests 302 303 304 def Setup(test_options, devices): 305 """Create the test runner factory and tests. 306 307 Args: 308 test_options: A GTestOptions object. 309 devices: A list of attached devices. 310 311 Returns: 312 A tuple of (TestRunnerFactory, tests). 313 """ 314 test_package = test_package_apk.TestPackageApk(test_options.suite_name) 315 if not os.path.exists(test_package.suite_path): 316 test_package = test_package_exe.TestPackageExecutable( 317 test_options.suite_name) 318 if not os.path.exists(test_package.suite_path): 319 raise Exception( 320 'Did not find %s target. Ensure it has been built.' 321 % test_options.suite_name) 322 logging.warning('Found target %s', test_package.suite_path) 323 324 _GenerateDepsDirUsingIsolate(test_options.suite_name, 325 test_options.isolate_file_path) 326 327 tests = _GetTests(test_options, test_package, devices) 328 329 # Constructs a new TestRunner with the current options. 330 def TestRunnerFactory(device, _shard_index): 331 return test_runner.TestRunner( 332 test_options, 333 device, 334 test_package) 335 336 if test_options.run_disabled: 337 test_options = test_options._replace( 338 test_arguments=('%s --gtest_also_run_disabled_tests' % 339 test_options.test_arguments)) 340 else: 341 tests = _FilterDisabledTests(tests, test_options.suite_name, 342 bool(test_options.gtest_filter)) 343 if test_options.gtest_filter: 344 tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter) 345 346 # Coalesce unit tests into a single test per device 347 if test_options.suite_name != 'content_browsertests': 348 num_devices = len(devices) 349 tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)] 350 tests = [t for t in tests if t] 351 352 return (TestRunnerFactory, tests) 353
