1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2015 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 """Verifies that the build is sane. 18 19 Cleans old build artifacts, configures the required environment, determines 20 build goals, and invokes the build scripts. 21 """ 22 from __future__ import print_function 23 24 import argparse 25 import collections 26 import datetime 27 import inspect 28 import os 29 import shutil 30 import site 31 import subprocess 32 import sys 33 import tempfile 34 import textwrap 35 36 site.addsitedir(os.path.join(os.path.dirname(__file__), 'build/lib')) 37 38 import build_support # pylint: disable=import-error 39 40 41 ALL_MODULES = { 42 'binutils', 43 'build', 44 'clang', 45 'cpufeatures', 46 'gabi++', 47 'gcc', 48 'gcclibs', 49 'gdbserver', 50 'gnustl', 51 'gtest', 52 'host-tools', 53 'libandroid_support', 54 'libc++', 55 'libc++abi', 56 'native_app_glue', 57 'ndk_helper', 58 'platforms', 59 'python-packages', 60 'stlport', 61 'system-stl', 62 } 63 64 65 class ArgParser(argparse.ArgumentParser): 66 def __init__(self): 67 super(ArgParser, self).__init__( 68 description=inspect.getdoc(sys.modules[__name__])) 69 70 self.add_argument( 71 '--arch', 72 choices=('arm', 'arm64', 'mips', 'mips64', 'x86', 'x86_64'), 73 help='Build for the given architecture. Build all by default.') 74 75 package_group = self.add_mutually_exclusive_group() 76 package_group.add_argument( 77 '--package', action='store_true', dest='package', default=True, 78 help='Package the NDK when done building (default).') 79 package_group.add_argument( 80 '--no-package', action='store_false', dest='package', 81 help='Do not package the NDK when done building.') 82 83 test_group = self.add_mutually_exclusive_group() 84 test_group.add_argument( 85 '--test', action='store_true', dest='test', default=True, 86 help=textwrap.dedent("""\ 87 Run host tests when finished. --package is required. Not supported 88 when targeting Windows. 89 """)) 90 test_group.add_argument( 91 '--no-test', action='store_false', dest='test', 92 help='Do not run host tests when finished.') 93 94 self.add_argument( 95 '--release', 96 help='Release name. Package will be named android-ndk-RELEASE.') 97 98 self.add_argument( 99 '--system', choices=('darwin', 'linux', 'windows', 'windows64'), 100 default=build_support.get_default_host(), 101 help='Build for the given OS.') 102 103 module_group = self.add_mutually_exclusive_group() 104 105 module_group.add_argument( 106 '--module', choices=sorted(ALL_MODULES), 107 help='NDK modules to build.') 108 109 module_group.add_argument( 110 '--host-only', action='store_true', 111 help='Skip building target components.') 112 113 114 def _invoke_build(script, args): 115 if args is None: 116 args = [] 117 subprocess.check_call([build_support.android_path(script)] + args) 118 119 120 def invoke_build(script, args=None): 121 script_path = os.path.join('build/tools', script) 122 _invoke_build(build_support.ndk_path(script_path), args) 123 124 125 def invoke_external_build(script, args=None): 126 _invoke_build(build_support.android_path(script), args) 127 128 129 def package_ndk(out_dir, dist_dir, args): 130 package_args = common_build_args(out_dir, dist_dir, args) 131 package_args.append(dist_dir) 132 133 if args.release is not None: 134 package_args.append('--release={}'.format(args.release)) 135 136 if args.arch is not None: 137 package_args.append('--arch={}'.format(args.arch)) 138 139 invoke_build('package.py', package_args) 140 141 142 def test_ndk(out_dir, args): 143 release = args.release 144 if args.release is None: 145 release = datetime.date.today().strftime('%Y%m%d') 146 147 # The packaging step extracts all the modules to a known directory for 148 # packaging. This directory is not cleaned up after packaging, so we can 149 # reuse that for testing. 150 test_dir = os.path.join(out_dir, 'android-ndk-{}'.format(release)) 151 152 test_env = dict(os.environ) 153 test_env['NDK'] = test_dir 154 155 abis = build_support.ALL_ABIS 156 if args.arch is not None: 157 abis = build_support.arch_to_abis(args.arch) 158 159 results = {} 160 for abi in abis: 161 cmd = [ 162 'python', build_support.ndk_path('tests/run-all.py'), 163 '--abi', abi, '--suite', 'build' 164 ] 165 print('Running tests: {}'.format(' '.join(cmd))) 166 result = subprocess.call(cmd, env=test_env) 167 results[abi] = result == 0 168 169 print('Results:') 170 for abi, result in results.iteritems(): 171 print('{}: {}'.format(abi, 'PASS' if result else 'FAIL')) 172 return all(results.values()) 173 174 175 def common_build_args(out_dir, dist_dir, args): 176 build_args = ['--out-dir={}'.format(out_dir)] 177 build_args = ['--dist-dir={}'.format(dist_dir)] 178 build_args.append('--host={}'.format(args.system)) 179 return build_args 180 181 182 def fixup_toolchain_triple(toolchain): 183 """Maps toolchain names to their proper triple. 184 185 The x86 toolchains are named stupidly and aren't a proper triple. 186 """ 187 return { 188 'x86': 'i686-linux-android', 189 'x86_64': 'x86_64-linux-android', 190 }.get(toolchain, toolchain) 191 192 193 def get_binutils_files(triple, has_gold, is_windows): 194 files = [ 195 'ld.bfd', 196 'nm', 197 'as', 198 'objcopy', 199 'strip', 200 'objdump', 201 'ld', 202 'ar', 203 'ranlib', 204 ] 205 206 if has_gold: 207 files.append('ld.gold') 208 209 if is_windows: 210 files = [f + '.exe' for f in files] 211 212 # binutils programs get installed to two locations: 213 # 1: $INSTALL_DIR/bin/$TRIPLE-$PROGRAM 214 # 2: $INSTALL_DIR/$TRIPLE/bin/$PROGRAM 215 # 216 # We need to copy both. 217 218 prefixed_files = [] 219 for file_name in files: 220 prefixed_name = '-'.join([triple, file_name]) 221 prefixed_files.append(os.path.join('bin', prefixed_name)) 222 223 dir_prefixed_files = [] 224 for file_name in files: 225 dir_prefixed_files.append(os.path.join(triple, 'bin', file_name)) 226 227 ldscripts_dir = os.path.join(triple, 'lib/ldscripts') 228 return prefixed_files + dir_prefixed_files + [ldscripts_dir] 229 230 231 def install_file(file_name, src_dir, dst_dir): 232 src_file = os.path.join(src_dir, file_name) 233 dst_file = os.path.join(dst_dir, file_name) 234 235 print('Copying {} to {}...'.format(src_file, dst_file)) 236 if os.path.isdir(src_file): 237 _install_dir(src_file, dst_file) 238 elif os.path.islink(src_file): 239 _install_symlink(src_file, dst_file) 240 else: 241 _install_file(src_file, dst_file) 242 243 244 def _install_dir(src_dir, dst_dir): 245 parent_dir = os.path.normpath(os.path.join(dst_dir, '..')) 246 if not os.path.exists(parent_dir): 247 os.makedirs(parent_dir) 248 shutil.copytree(src_dir, dst_dir, symlinks=True) 249 250 251 def _install_symlink(src_file, dst_file): 252 dirname = os.path.dirname(dst_file) 253 if not os.path.exists(dirname): 254 os.makedirs(dirname) 255 link_target = os.readlink(src_file) 256 os.symlink(link_target, dst_file) 257 258 259 def _install_file(src_file, dst_file): 260 dirname = os.path.dirname(dst_file) 261 if not os.path.exists(dirname): 262 os.makedirs(dirname) 263 # copy2 is just copy followed by copystat (preserves file metadata). 264 shutil.copy2(src_file, dst_file) 265 266 267 def pack_binutils(arch, host_tag, out_dir, binutils_path): 268 archive_name = '-'.join(['binutils', arch, host_tag]) 269 build_support.make_package(archive_name, binutils_path, out_dir) 270 271 272 def get_prebuilt_gcc(host, arch): 273 tag = build_support.host_to_tag(host) 274 system_subdir = 'prebuilts/ndk/current/toolchains/{}'.format(tag) 275 system_path = build_support.android_path(system_subdir) 276 toolchain = build_support.arch_to_toolchain(arch) 277 toolchain_dir = toolchain + '-4.9' 278 return os.path.join(system_path, toolchain_dir) 279 280 281 def build_binutils(out_dir, dist_dir, args): 282 print('Extracting binutils package from GCC...') 283 284 arches = build_support.ALL_ARCHITECTURES 285 if args.arch is not None: 286 arches = [args.arch] 287 288 host_tag = build_support.host_to_tag(args.system) 289 290 for arch in arches: 291 toolchain = build_support.arch_to_toolchain(arch) 292 toolchain_path = get_prebuilt_gcc(args.system, arch) 293 294 triple = fixup_toolchain_triple(toolchain) 295 296 install_dir = os.path.join(out_dir, 'binutils', triple) 297 if os.path.exists(install_dir): 298 shutil.rmtree(install_dir) 299 os.makedirs(install_dir) 300 301 has_gold = True 302 if host_tag == 'windows': 303 # Note: 64-bit Windows is fine. 304 has_gold = False 305 if arch in ('mips', 'mips64'): 306 has_gold = False 307 308 is_windows = host_tag.startswith('windows') 309 for file_name in get_binutils_files(triple, has_gold, is_windows): 310 install_file(file_name, toolchain_path, install_dir) 311 312 license_path = build_support.android_path( 313 'toolchain/binutils/binutils-2.25/COPYING') 314 shutil.copy2(license_path, os.path.join(install_dir, 'NOTICE')) 315 316 pack_binutils(arch, host_tag, dist_dir, install_dir) 317 318 319 def build_clang(out_dir, dist_dir, args): 320 print('Building Clang...') 321 invoke_build('build-llvm.py', common_build_args(out_dir, dist_dir, args)) 322 323 324 def build_gcc(out_dir, dist_dir, args): 325 print('Building GCC...') 326 build_args = common_build_args(out_dir, dist_dir, args) 327 if args.arch is not None: 328 build_args.append('--arch={}'.format(args.arch)) 329 invoke_build('build-gcc.py', build_args) 330 331 332 def build_gcc_libs(out_dir, dist_dir, args): 333 print('Packaging GCC libs...') 334 335 arches = build_support.ALL_ARCHITECTURES 336 if args.arch is not None: 337 arches = [args.arch] 338 339 for arch in arches: 340 toolchain = build_support.arch_to_toolchain(arch) 341 triple = fixup_toolchain_triple(toolchain) 342 libgcc_subdir = 'lib/gcc/{}/4.9'.format(triple) 343 is64 = arch.endswith('64') 344 libatomic_subdir = '{}/lib{}'.format(triple, '64' if is64 else '') 345 346 lib_names = [ 347 (libatomic_subdir, 'libatomic.a'), 348 (libgcc_subdir, 'libgcc.a'), 349 ] 350 351 lib_dirs = [''] 352 if arch == 'arm': 353 lib_dirs += [ 354 'armv7-a', 355 'armv7-a/hard', 356 'armv7-a/thumb', 357 'armv7-a/thumb/hard', 358 'thumb', 359 ] 360 361 libs = [] 362 for lib_dir in lib_dirs: 363 for subdir, lib in lib_names: 364 libs.append((subdir, os.path.join(lib_dir, lib))) 365 366 install_dir = os.path.join(out_dir, 'gcclibs', triple) 367 if os.path.exists(install_dir): 368 shutil.rmtree(install_dir) 369 os.makedirs(install_dir) 370 371 # These are target libraries, so the OS we use here is not 372 # important. We explicitly use Linux because for whatever reason 373 # the Windows aarch64 toolchain doesn't include libatomic. 374 gcc_path = get_prebuilt_gcc('linux', arch) 375 for gcc_subdir, lib in libs: 376 src = os.path.join(gcc_path, gcc_subdir, lib) 377 dst = os.path.join(install_dir, lib) 378 dst_dir = os.path.dirname(dst) 379 if not os.path.exists(dst_dir): 380 os.makedirs(dst_dir) 381 shutil.copy2(src, dst) 382 383 shutil.copy2( 384 os.path.join(gcc_path, 'NOTICE'), 385 os.path.join(install_dir, 'NOTICE')) 386 387 archive_name = os.path.join('gcclibs-' + arch) 388 build_support.make_package(archive_name, install_dir, dist_dir) 389 390 391 def build_host_tools(out_dir, dist_dir, args): 392 build_args = common_build_args(out_dir, dist_dir, args) 393 394 print('Building ndk-stack...') 395 invoke_external_build( 396 'ndk/sources/host-tools/ndk-stack/build.py', build_args) 397 398 print('Building ndk-depends...') 399 invoke_external_build( 400 'ndk/sources/host-tools/ndk-depends/build.py', build_args) 401 402 print('Building awk...') 403 invoke_external_build( 404 'ndk/sources/host-tools/nawk-20071023/build.py', build_args) 405 406 print('Building make...') 407 invoke_external_build( 408 'ndk/sources/host-tools/make-3.81/build.py', build_args) 409 410 if args.system in ('windows', 'windows64'): 411 print('Building toolbox...') 412 invoke_external_build( 413 'ndk/sources/host-tools/toolbox/build.py', build_args) 414 415 print('Building Python...') 416 invoke_external_build('toolchain/python/build.py', build_args) 417 418 print('Building GDB...') 419 invoke_external_build('toolchain/gdb/build.py', build_args) 420 421 print('Building YASM...') 422 invoke_external_build('toolchain/yasm/build.py', build_args) 423 424 package_host_tools(out_dir, dist_dir, args.system) 425 426 427 def merge_license_files(output_path, files): 428 licenses = [] 429 for license_path in files: 430 with open(license_path) as license_file: 431 licenses.append(license_file.read()) 432 433 with open(output_path, 'w') as output_file: 434 output_file.write('\n'.join(licenses)) 435 436 437 def package_host_tools(out_dir, dist_dir, host): 438 packages = [ 439 'gdb-multiarch-7.10', 440 'ndk-awk', 441 'ndk-depends', 442 'ndk-make', 443 'ndk-python', 444 'ndk-stack', 445 'ndk-yasm', 446 ] 447 448 files = [ 449 'ndk-gdb', 450 'ndk-gdb.cmd', 451 'ndk-gdb.py', 452 ] 453 454 if host in ('windows', 'windows64'): 455 packages.append('toolbox') 456 457 host_tag = build_support.host_to_tag(host) 458 459 package_names = [p + '-' + host_tag + '.tar.bz2' for p in packages] 460 for package_name in package_names: 461 package_path = os.path.join(out_dir, package_name) 462 subprocess.check_call(['tar', 'xf', package_path, '-C', out_dir]) 463 464 for f in files: 465 shutil.copy2(f, os.path.join(out_dir, 'host-tools/bin')) 466 467 merge_license_files(os.path.join(out_dir, 'host-tools/NOTICE'), [ 468 build_support.android_path('toolchain/gdb/gdb-7.10/COPYING'), 469 build_support.ndk_path('sources/host-tools/nawk-20071023/NOTICE'), 470 build_support.ndk_path('sources/host-tools/ndk-depends/NOTICE'), 471 build_support.ndk_path('sources/host-tools/make-3.81/COPYING'), 472 build_support.android_path( 473 'toolchain/python/Python-2.7.5/LICENSE'), 474 build_support.ndk_path('sources/host-tools/ndk-stack/NOTICE'), 475 build_support.ndk_path('sources/host-tools/toolbox/NOTICE'), 476 build_support.android_path('toolchain/yasm/COPYING'), 477 build_support.android_path('toolchain/yasm/BSD.txt'), 478 build_support.android_path('toolchain/yasm/Artistic.txt'), 479 build_support.android_path('toolchain/yasm/GNU_GPL-2.0'), 480 build_support.android_path('toolchain/yasm/GNU_LGPL-2.0'), 481 ]) 482 483 package_name = 'host-tools-' + host_tag 484 path = os.path.join(out_dir, 'host-tools') 485 build_support.make_package(package_name, path, dist_dir) 486 487 488 def build_gdbserver(out_dir, dist_dir, args): 489 print('Building gdbserver...') 490 build_args = common_build_args(out_dir, dist_dir, args) 491 if args.arch is not None: 492 build_args.append('--arch={}'.format(args.arch)) 493 invoke_build('build-gdbserver.py', build_args) 494 495 496 def _build_stl(out_dir, dist_dir, args, stl): 497 build_args = common_build_args(out_dir, dist_dir, args) 498 if args.arch is not None: 499 build_args.append('--arch={}'.format(args.arch)) 500 script = 'ndk/sources/cxx-stl/{}/build.py'.format(stl) 501 invoke_external_build(script, build_args) 502 503 504 def build_gnustl(out_dir, dist_dir, args): 505 print('Building gnustl...') 506 _build_stl(out_dir, dist_dir, args, 'gnu-libstdc++') 507 508 509 def build_libcxx(out_dir, dist_dir, args): 510 print('Building libc++...') 511 _build_stl(out_dir, dist_dir, args, 'llvm-libc++') 512 513 514 def build_stlport(out_dir, dist_dir, args): 515 print('Building stlport...') 516 _build_stl(out_dir, dist_dir, args, 'stlport') 517 518 519 def build_platforms(out_dir, dist_dir, args): 520 print('Building platforms...') 521 build_args = common_build_args(out_dir, dist_dir, args) 522 invoke_build('build-platforms.py', build_args) 523 524 525 def build_cpufeatures(_, dist_dir, __): 526 path = build_support.ndk_path('sources/android/cpufeatures') 527 build_support.make_package('cpufeatures', path, dist_dir) 528 529 530 def build_native_app_glue(_, dist_dir, __): 531 path = build_support.android_path( 532 'development/ndk/sources/android/native_app_glue') 533 build_support.make_package('native_app_glue', path, dist_dir) 534 535 536 def build_ndk_helper(_, dist_dir, __): 537 path = build_support.android_path( 538 'development/ndk/sources/android/ndk_helper') 539 build_support.make_package('ndk_helper', path, dist_dir) 540 541 542 def build_gtest(_, dist_dir, __): 543 path = build_support.ndk_path('sources/third_party/googletest') 544 build_support.make_package('gtest', path, dist_dir) 545 546 547 def build_build(_, dist_dir, __): 548 path = build_support.ndk_path('build') 549 build_support.make_package('build', path, dist_dir) 550 551 552 def build_python_packages(_, dist_dir, __): 553 # Stage the files in a temporary directory to make things easier. 554 temp_dir = tempfile.mkdtemp() 555 try: 556 path = os.path.join(temp_dir, 'python-packages') 557 shutil.copytree( 558 build_support.android_path('development/python-packages'), path) 559 build_support.make_package('python-packages', path, dist_dir) 560 finally: 561 shutil.rmtree(temp_dir) 562 563 564 def build_gabixx(_out_dir, dist_dir, _args): 565 print('Building gabi++...') 566 path = build_support.ndk_path('sources/cxx-stl/gabi++') 567 build_support.make_package('gabixx', path, dist_dir) 568 569 570 def build_system_stl(_out_dir, dist_dir, _args): 571 print('Building system-stl...') 572 path = build_support.ndk_path('sources/cxx-stl/system') 573 build_support.make_package('system-stl', path, dist_dir) 574 575 576 def build_libandroid_support(_out_dir, dist_dir, _args): 577 print('Building libandroid_support...') 578 path = build_support.ndk_path('sources/android/support') 579 build_support.make_package('libandroid_support', path, dist_dir) 580 581 582 def build_libcxxabi(_out_dir, dist_dir, _args): 583 print('Building libc++abi...') 584 path = build_support.ndk_path('sources/cxx-stl/llvm-libc++abi') 585 build_support.make_package('libcxxabi', path, dist_dir) 586 587 588 def main(): 589 parser = ArgParser() 590 args = parser.parse_args() 591 592 if args.module is None: 593 modules = ALL_MODULES 594 else: 595 modules = {args.module} 596 597 if args.host_only: 598 modules = { 599 'clang', 600 'gcc', 601 'host-tools', 602 } 603 604 required_package_modules = ALL_MODULES 605 if args.package and required_package_modules <= modules: 606 do_package = True 607 else: 608 do_package = False 609 610 # TODO(danalbert): wine? 611 # We're building the Windows packages from Linux, so we can't actually run 612 # any of the tests from here. 613 if args.system.startswith('windows') or not do_package: 614 args.test = False 615 616 # Disable buffering on stdout so the build output doesn't hide all of our 617 # "Building..." messages. 618 sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) 619 620 os.chdir(os.path.dirname(os.path.realpath(__file__))) 621 622 # Set ANDROID_BUILD_TOP. 623 if 'ANDROID_BUILD_TOP' not in os.environ: 624 os.environ['ANDROID_BUILD_TOP'] = os.path.realpath('..') 625 626 out_dir = build_support.get_out_dir() 627 dist_dir = build_support.get_dist_dir(out_dir) 628 629 print('Cleaning up...') 630 invoke_build('dev-cleanup.sh') 631 632 module_builds = collections.OrderedDict([ 633 ('binutils', build_binutils), 634 ('build', build_build), 635 ('clang', build_clang), 636 ('cpufeatures', build_cpufeatures), 637 ('gabi++', build_gabixx), 638 ('gcc', build_gcc), 639 ('gcclibs', build_gcc_libs), 640 ('gdbserver', build_gdbserver), 641 ('gnustl', build_gnustl), 642 ('gtest', build_gtest), 643 ('host-tools', build_host_tools), 644 ('libandroid_support', build_libandroid_support), 645 ('libc++', build_libcxx), 646 ('libc++abi', build_libcxxabi), 647 ('native_app_glue', build_native_app_glue), 648 ('ndk_helper', build_ndk_helper), 649 ('platforms', build_platforms), 650 ('python-packages', build_python_packages), 651 ('stlport', build_stlport), 652 ('system-stl', build_system_stl), 653 ]) 654 655 print('Building modules: {}'.format(' '.join(modules))) 656 for module in modules: 657 module_builds[module](out_dir, dist_dir, args) 658 659 if do_package: 660 package_ndk(out_dir, dist_dir, args) 661 662 if args.test: 663 result = test_ndk(out_dir, args) 664 sys.exit(0 if result else 1) 665 666 667 if __name__ == '__main__': 668 main() 669
