1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2015 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 """Runs all NDK tests. 18 19 TODO: Handle default ABI case. 20 The old test script would build and test all ABIs listed in APP_ABI in a single 21 invocation if an explicit ABI was not given. Currently this will fall down over 22 that case. I've written most of the code to handle this, but I decided to 23 factor that for-each up several levels because it was going to make the device 24 tests rather messy and I haven't finished doing that yet. 25 26 TODO: Handle explicit test lists from command line. 27 The old test runner allowed specifying an exact list of tests to run with 28 --tests. That seems like a useful thing to keep around, but I haven't ported it 29 yet. 30 """ 31 from __future__ import print_function 32 33 import argparse 34 import atexit 35 import distutils.spawn 36 import inspect 37 import os 38 import re 39 import shutil 40 import subprocess 41 import sys 42 import tempfile 43 44 import adb 45 import filters 46 import ndk 47 import printers 48 import tests 49 50 from tests import AwkTest, BuildTest, DeviceTest 51 52 53 SUPPORTED_ABIS = ( 54 'armeabi', 55 'armeabi-v7a', 56 'armeabi-v7a-hard', 57 'arm64-v8a', 58 'mips', 59 'mips64', 60 'x86', 61 'x86_64', 62 ) 63 64 65 # TODO(danalbert): How much time do we actually save by not running these? 66 LONG_TESTS = ( 67 'prebuild-stlport', 68 'test-stlport', 69 'test-gnustl-full', 70 'test-stlport_shared-exception', 71 'test-stlport_static-exception', 72 'test-gnustl_shared-exception-full', 73 'test-gnustl_static-exception-full', 74 'test-googletest-full', 75 'test-libc++-shared-full', 76 'test-libc++-static-full', 77 ) 78 79 80 def get_test_device(): 81 if distutils.spawn.find_executable('adb') is None: 82 raise RuntimeError('Could not find adb.') 83 84 p = subprocess.Popen(['adb', 'devices'], stdout=subprocess.PIPE) 85 out, _ = p.communicate() 86 if p.returncode != 0: 87 raise RuntimeError('Failed to get list of devices from adb.') 88 89 # The first line of `adb devices` just says "List of attached devices", so 90 # skip that. 91 devices = [] 92 for line in out.split('\n')[1:]: 93 if not line.strip(): 94 continue 95 if 'offline' in line: 96 continue 97 98 serial, _ = re.split(r'\s+', line, maxsplit=1) 99 devices.append(serial) 100 101 if len(devices) == 0: 102 raise RuntimeError('No devices detected.') 103 104 device = os.getenv('ANDROID_SERIAL') 105 if device is None and len(devices) == 1: 106 device = devices[0] 107 108 if device is not None and device not in devices: 109 raise RuntimeError('Device {} is not available.'.format(device)) 110 111 # TODO(danalbert): Handle running against multiple devices in one pass. 112 if len(devices) > 1 and device is None: 113 raise RuntimeError('Multiple devices detected and ANDROID_SERIAL not ' 114 'set. Cannot continue.') 115 116 return device 117 118 119 def get_device_abis(): 120 # 64-bit devices list their ABIs differently than 32-bit devices. Check all 121 # the possible places for stashing ABI info and merge them. 122 abi_properties = [ 123 'ro.product.cpu.abi', 124 'ro.product.cpu.abi2', 125 'ro.product.cpu.abilist', 126 ] 127 abis = set() 128 for abi_prop in abi_properties: 129 prop = adb.get_prop(abi_prop) 130 if prop is not None: 131 abis.update(prop.split(',')) 132 133 if 'armeabi-v7a' in abis: 134 abis.add('armeabi-v7a-hard') 135 return sorted(list(abis)) 136 137 138 def check_adb_works_or_die(abi): 139 # TODO(danalbert): Check that we can do anything with the device. 140 try: 141 device = get_test_device() 142 except RuntimeError as ex: 143 sys.exit('Error: {}'.format(ex)) 144 145 supported_abis = get_device_abis() 146 if abi is not None and abi not in supported_abis: 147 msg = ('The test device ({}) does not support the requested ABI ' 148 '({}).\nSupported ABIs: {}'.format(device, abi, 149 ', '.join(supported_abis))) 150 sys.exit(msg) 151 152 153 def can_use_asan(abi, api, toolchain): 154 # ASAN is currently only supported for 32-bit ARM and x86... 155 if not abi.startswith('armeabi') and not abi == 'x86': 156 return False 157 158 # On KitKat and newer... 159 if api < 19: 160 return False 161 162 # When using clang... 163 if toolchain != 'clang': 164 return False 165 166 # On rooted devices. 167 if int(adb.get_prop('ro.debuggable')) == 0: 168 return False 169 170 return True 171 172 173 def asan_device_setup(): 174 path = os.path.join( 175 os.environ['NDK'], 'toolchains', 'llvm', 'prebuilt', 176 ndk.get_host_tag(), 'bin', 'asan_device_setup') 177 subprocess.check_call([path]) 178 179 180 def is_valid_platform_version(version_string): 181 match = re.match(r'^android-(\d+)$', version_string) 182 if not match: 183 return False 184 185 # We don't support anything before Gingerbread. 186 version = int(match.group(1)) 187 return version >= 9 188 189 190 def android_platform_version(version_string): 191 if is_valid_platform_version(version_string): 192 return version_string 193 else: 194 raise argparse.ArgumentTypeError( 195 'Platform version must match the format "android-VERSION", where ' 196 'VERSION >= 9.') 197 198 199 class ArgParser(argparse.ArgumentParser): 200 def __init__(self): 201 super(ArgParser, self).__init__( 202 description=inspect.getdoc(sys.modules[__name__])) 203 204 self.add_argument( 205 '--abi', default=None, choices=SUPPORTED_ABIS, 206 help=('Run tests against the specified ABI. Defaults to the ' 207 'contents of APP_ABI in jni/Application.mk')) 208 self.add_argument( 209 '--platform', default=None, type=android_platform_version, 210 help=('Run tests against the specified platform version. Defaults ' 211 'to the contents of APP_PLATFORM in jni/Application.mk')) 212 self.add_argument( 213 '--toolchain', default='clang', choices=('4.9', 'clang'), 214 help='Toolchain for building tests. Defaults to clang.') 215 216 self.add_argument( 217 '--show-commands', action='store_true', 218 help='Show build commands for each test.') 219 self.add_argument( 220 '--suite', default=None, 221 choices=('awk', 'build', 'device'), 222 help=('Run only the chosen test suite.')) 223 224 self.add_argument( 225 '--filter', help='Only run tests that match the given patterns.') 226 self.add_argument( 227 '--quick', action='store_true', help='Skip long running tests.') 228 self.add_argument( 229 '--show-all', action='store_true', 230 help='Show all test results, not just failures.') 231 232 self.add_argument( 233 '--out-dir', 234 help='Path to build tests to. Will not be removed upon exit.') 235 236 237 class ResultStats(object): 238 def __init__(self, suites, results): 239 self.num_tests = sum(len(s) for s in results.values()) 240 241 zero_stats = {'pass': 0, 'skip': 0, 'fail': 0} 242 self.global_stats = dict(zero_stats) 243 self.suite_stats = {suite: dict(zero_stats) for suite in suites} 244 self._analyze_results(results) 245 246 def _analyze_results(self, results): 247 for suite, test_results in results.items(): 248 for result in test_results: 249 if result.failed(): 250 self.suite_stats[suite]['fail'] += 1 251 self.global_stats['fail'] += 1 252 elif result.passed(): 253 self.suite_stats[suite]['pass'] += 1 254 self.global_stats['pass'] += 1 255 else: 256 self.suite_stats[suite]['skip'] += 1 257 self.global_stats['skip'] += 1 258 259 260 def main(): 261 orig_cwd = os.getcwd() 262 os.chdir(os.path.dirname(os.path.realpath(__file__))) 263 264 # Defining _NDK_TESTING_ALL_=yes to put armeabi-v7a-hard in its own 265 # libs/armeabi-v7a-hard directory and tested separately from armeabi-v7a. 266 # Some tests are now compiled with both APP_ABI=armeabi-v7a and 267 # APP_ABI=armeabi-v7a-hard. Without _NDK_TESTING_ALL_=yes, tests may fail 268 # to install due to race condition on the same libs/armeabi-v7a 269 if '_NDK_TESTING_ALL_' not in os.environ: 270 os.environ['_NDK_TESTING_ALL_'] = 'all' 271 272 if 'NDK' not in os.environ: 273 os.environ['NDK'] = os.path.dirname(os.getcwd()) 274 275 args = ArgParser().parse_args() 276 ndk_build_flags = [] 277 if args.abi is not None: 278 ndk_build_flags.append('APP_ABI={}'.format(args.abi)) 279 if args.platform is not None: 280 ndk_build_flags.append('APP_PLATFORM={}'.format(args.platform)) 281 if args.show_commands: 282 ndk_build_flags.append('V=1') 283 284 if not os.path.exists(os.path.join('../build/tools/prebuilt-common.sh')): 285 sys.exit('Error: Not run from a valid NDK.') 286 287 out_dir = args.out_dir 288 if out_dir is not None: 289 if not os.path.isabs(out_dir): 290 out_dir = os.path.join(orig_cwd, out_dir) 291 if os.path.exists(out_dir): 292 shutil.rmtree(out_dir) 293 os.makedirs(out_dir) 294 else: 295 out_dir = tempfile.mkdtemp() 296 atexit.register(lambda: shutil.rmtree(out_dir)) 297 298 suites = ['awk', 'build', 'device'] 299 if args.suite: 300 suites = [args.suite] 301 302 # Do this early so we find any device issues now rather than after we've 303 # run all the build tests. 304 if 'device' in suites: 305 check_adb_works_or_die(args.abi) 306 api_level = int(adb.get_prop('ro.build.version.sdk')) 307 308 # PIE is required in L. All of the device tests are written toward the 309 # ndk-build defaults, so we need to inform the build that we need PIE 310 # if we're running on a newer device. 311 if api_level >= 21: 312 ndk_build_flags.append('APP_PIE=true') 313 314 os.environ['ANDROID_SERIAL'] = get_test_device() 315 316 if can_use_asan(args.abi, api_level, args.toolchain): 317 asan_device_setup() 318 319 # Do this as part of initialization rather than with a `mkdir -p` later 320 # because Gingerbread didn't actually support -p :( 321 adb.shell('rm -r /data/local/tmp/ndk-tests') 322 adb.shell('mkdir /data/local/tmp/ndk-tests') 323 324 runner = tests.TestRunner() 325 if 'awk' in suites: 326 runner.add_suite('awk', 'awk', AwkTest) 327 if 'build' in suites: 328 runner.add_suite('build', 'build', BuildTest, args.abi, args.platform, 329 args.toolchain, ndk_build_flags) 330 if 'device' in suites: 331 runner.add_suite('device', 'device', DeviceTest, args.abi, 332 args.platform, api_level, args.toolchain, 333 ndk_build_flags) 334 335 test_filters = filters.TestFilter.from_string(args.filter) 336 results = runner.run(out_dir, test_filters) 337 338 stats = ResultStats(suites, results) 339 340 use_color = sys.stdin.isatty() and os.name != 'nt' 341 printer = printers.StdoutPrinter(use_color=use_color, 342 show_all=args.show_all) 343 printer.print_results(results, stats) 344 sys.exit(stats.global_stats['fail'] > 0) 345 346 347 if __name__ == '__main__': 348 main() 349
