1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2015 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 from __future__ import print_function 18 19 import argparse 20 import glob 21 import multiprocessing 22 import os 23 import shutil 24 import subprocess 25 import sys 26 27 import version 28 29 30 THIS_DIR = os.path.realpath(os.path.dirname(__file__)) 31 ORIG_ENV = dict(os.environ) 32 33 34 def android_path(*args): 35 return os.path.realpath(os.path.join(THIS_DIR, '../..', *args)) 36 37 38 def build_path(*args): 39 # Our multistage build directories will be placed under OUT_DIR if it is in 40 # the environment. By default they will be placed under 41 # $ANDROID_BUILD_TOP/out. 42 top_out = ORIG_ENV.get('OUT_DIR', android_path('out')) 43 if not os.path.isabs(top_out): 44 top_out = os.path.realpath(top_out) 45 return os.path.join(top_out, *args) 46 47 48 def short_version(): 49 return '.'.join([version.major, version.minor]) 50 51 52 def long_version(): 53 return '.'.join([version.major, version.minor, version.patch]) 54 55 56 def install_file(src, dst): 57 print('Copying ' + src) 58 shutil.copy2(src, dst) 59 60 61 def install_directory(src, dst): 62 print('Copying ' + src) 63 shutil.copytree(src, dst) 64 65 66 def build(out_dir, prebuilts_path=None, prebuilts_version=None, 67 build_all_llvm_tools=None): 68 products = ( 69 'aosp_arm', 70 'aosp_arm64', 71 'aosp_mips', 72 'aosp_mips64', 73 'aosp_x86', 74 'aosp_x86_64', 75 ) 76 for product in products: 77 build_product(out_dir, product, prebuilts_path, prebuilts_version, 78 build_all_llvm_tools) 79 80 81 def build_product(out_dir, product, prebuilts_path, prebuilts_version, 82 build_all_llvm_tools): 83 env = dict(ORIG_ENV) 84 env['DISABLE_LLVM_DEVICE_BUILDS'] = 'true' 85 env['DISABLE_RELOCATION_PACKER'] = 'true' 86 env['FORCE_BUILD_LLVM_COMPONENTS'] = 'true' 87 env['FORCE_BUILD_SANITIZER_SHARED_OBJECTS'] = 'true' 88 env['OUT_DIR'] = out_dir 89 env['SKIP_LLVM_TESTS'] = 'true' 90 env['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true' 91 env['TARGET_BUILD_VARIANT'] = 'userdebug' 92 env['TARGET_PRODUCT'] = product 93 94 overrides = [] 95 if prebuilts_path is not None: 96 overrides.append('LLVM_PREBUILTS_BASE={}'.format(prebuilts_path)) 97 if prebuilts_version is not None: 98 overrides.append('LLVM_PREBUILTS_VERSION={}'.format(prebuilts_version)) 99 100 jobs_arg = '-j{}'.format(multiprocessing.cpu_count()) 101 targets = ['clang-toolchain'] 102 if build_all_llvm_tools: 103 targets += ['llvm-tools'] 104 subprocess.check_call( 105 ['make', jobs_arg] + overrides + targets, cwd=android_path(), env=env) 106 107 108 def package_toolchain(build_dir, build_name, host, dist_dir): 109 package_name = 'clang-' + build_name 110 install_host_dir = build_path('install', host) 111 install_dir = os.path.join(install_host_dir, package_name) 112 113 # Remove any previously installed toolchain so it doesn't pollute the 114 # build. 115 if os.path.exists(install_host_dir): 116 shutil.rmtree(install_host_dir) 117 118 install_toolchain(build_dir, install_dir, host) 119 120 version_file_path = os.path.join(install_dir, 'AndroidVersion.txt') 121 with open(version_file_path, 'w') as version_file: 122 version_file.write('{}.{}.{}\n'.format( 123 version.major, version.minor, version.patch)) 124 125 tarball_name = package_name + '-' + host 126 package_path = os.path.join(dist_dir, tarball_name) + '.tar.bz2' 127 print('Packaging ' + package_path) 128 args = [ 129 'tar', '-cjC', install_host_dir, '-f', package_path, package_name 130 ] 131 subprocess.check_call(args) 132 133 134 def install_toolchain(build_dir, install_dir, host): 135 install_built_host_files(build_dir, install_dir, host) 136 install_sanitizer_scripts(install_dir) 137 install_scan_scripts(install_dir) 138 install_analyzer_scripts(install_dir) 139 install_headers(build_dir, install_dir, host) 140 install_profile_rt(build_dir, install_dir, host) 141 install_sanitizers(build_dir, install_dir, host) 142 install_license_files(install_dir) 143 install_repo_prop(install_dir) 144 145 146 def install_built_host_files(build_dir, install_dir, host): 147 is_windows = host.startswith('windows') 148 is_darwin = host.startswith('darwin-x86') 149 bin_ext = '.exe' if is_windows else '' 150 151 if is_windows: 152 lib_ext = '.dll' 153 elif is_darwin: 154 lib_ext = '.dylib' 155 else: 156 lib_ext = '.so' 157 158 built_files = [ 159 'bin/clang' + bin_ext, 160 'bin/clang++' + bin_ext, 161 ] 162 if host != 'windows-x86': 163 built_files.extend([ 164 'bin/FileCheck' + bin_ext, 165 'bin/llvm-as' + bin_ext, 166 'bin/llvm-dis' + bin_ext, 167 'bin/llvm-link' + bin_ext, 168 'lib64/libc++' + lib_ext, 169 'lib64/libLLVM' + lib_ext, 170 'lib64/LLVMgold' + lib_ext, 171 ]) 172 173 for built_file in built_files: 174 dirname = os.path.dirname(built_file) 175 install_path = os.path.join(install_dir, dirname) 176 if not os.path.exists(install_path): 177 os.makedirs(install_path) 178 179 built_path = os.path.join(build_dir, 'host', host, built_file) 180 install_file(built_path, install_path) 181 182 file_name = os.path.basename(built_file) 183 184 # Only strip bin files (not libs) on darwin. 185 if not is_darwin or built_file.startswith('bin/'): 186 subprocess.check_call( 187 ['strip', os.path.join(install_path, file_name)]) 188 189 190 def install_sanitizer_scripts(install_dir): 191 script_path = android_path( 192 'external/compiler-rt/lib/asan/scripts/asan_device_setup') 193 shutil.copy2(script_path, os.path.join(install_dir, 'bin')) 194 195 196 def install_analyzer_scripts(install_dir): 197 """Create and install bash scripts for invoking Clang for analysis.""" 198 analyzer_text = ( 199 '#!/bin/bash\n' 200 'if [ "$1" != "-cc1" ]; then\n' 201 ' `dirname $0`/../clang{clang_suffix} -target {target} "$@"\n' 202 'else\n' 203 ' # target/triple already spelled out.\n' 204 ' `dirname $0`/../clang{clang_suffix} "$@"\n' 205 'fi\n' 206 ) 207 208 arch_target_pairs = ( 209 ('arm64-v8a', 'aarch64-none-linux-android'), 210 ('armeabi', 'armv5te-none-linux-androideabi'), 211 ('armeabi-v7a', 'armv7-none-linux-androideabi'), 212 ('armeabi-v7a-hard', 'armv7-none-linux-androideabi'), 213 ('mips', 'mipsel-none-linux-android'), 214 ('mips64', 'mips64el-none-linux-android'), 215 ('x86', 'i686-none-linux-android'), 216 ('x86_64', 'x86_64-none-linux-android'), 217 ) 218 219 for arch, target in arch_target_pairs: 220 arch_path = os.path.join(install_dir, 'bin', arch) 221 os.makedirs(arch_path) 222 223 analyzer_file_path = os.path.join(arch_path, 'analyzer') 224 print('Creating ' + analyzer_file_path) 225 with open(analyzer_file_path, 'w') as analyzer_file: 226 analyzer_file.write( 227 analyzer_text.format(clang_suffix='', target=target)) 228 229 analyzerpp_file_path = os.path.join(arch_path, 'analyzer++') 230 print('Creating ' + analyzerpp_file_path) 231 with open(analyzerpp_file_path, 'w') as analyzerpp_file: 232 analyzerpp_file.write( 233 analyzer_text.format(clang_suffix='++', target=target)) 234 235 236 def install_scan_scripts(install_dir): 237 tools_install_dir = os.path.join(install_dir, 'tools') 238 os.makedirs(tools_install_dir) 239 tools = ('scan-build', 'scan-view') 240 tools_dir = android_path('external/clang/tools') 241 for tool in tools: 242 tool_path = os.path.join(tools_dir, tool) 243 install_path = os.path.join(install_dir, 'tools', tool) 244 install_directory(tool_path, install_path) 245 246 247 def install_headers(build_dir, install_dir, host): 248 def should_copy(path): 249 if os.path.basename(path) in ('Makefile', 'CMakeLists.txt'): 250 return False 251 _, ext = os.path.splitext(path) 252 if ext == '.mk': 253 return False 254 return True 255 256 headers_src = android_path('external/clang/lib/Headers') 257 headers_dst = os.path.join( 258 install_dir, 'lib64/clang', short_version(), 'include') 259 os.makedirs(headers_dst) 260 for header in os.listdir(headers_src): 261 if not should_copy(header): 262 continue 263 install_file(os.path.join(headers_src, header), headers_dst) 264 265 install_file(android_path('bionic/libc/include/stdatomic.h'), headers_dst) 266 267 # arm_neon.h gets produced as part of external/clang/lib/Basic/Android.mk. 268 # We must bundle the resulting file as part of the official Clang headers. 269 arm_neon_h = os.path.join( 270 build_dir, 'host', host, 'obj/STATIC_LIBRARIES/' 271 'libclangBasic_intermediates/include/clang/Basic/arm_neon.h') 272 install_file(arm_neon_h, headers_dst) 273 274 os.symlink(short_version(), 275 os.path.join(install_dir, 'lib64/clang', long_version())) 276 277 278 def install_profile_rt(build_dir, install_dir, host): 279 lib_dir = os.path.join( 280 install_dir, 'lib64/clang', short_version(), 'lib/linux') 281 os.makedirs(lib_dir) 282 283 install_target_profile_rt(build_dir, lib_dir) 284 285 # We only support profiling libs for Linux and Android. 286 if host == 'linux-x86': 287 install_host_profile_rt(build_dir, host, lib_dir) 288 289 290 def install_target_profile_rt(build_dir, lib_dir): 291 product_to_arch = { 292 'generic': 'arm', 293 'generic_arm64': 'aarch64', 294 'generic_mips': 'mipsel', 295 'generic_mips64': 'mips64el', 296 'generic_x86': 'i686', 297 'generic_x86_64': 'x86_64', 298 } 299 300 for product, arch in product_to_arch.items(): 301 product_dir = os.path.join(build_dir, 'target/product', product) 302 static_libs = os.path.join(product_dir, 'obj/STATIC_LIBRARIES') 303 built_lib = os.path.join( 304 static_libs, 'libprofile_rt_intermediates/libprofile_rt.a') 305 lib_name = 'libclang_rt.profile-{}-android.a'.format(arch) 306 install_file(built_lib, os.path.join(lib_dir, lib_name)) 307 308 309 def install_host_profile_rt(build_dir, host, lib_dir): 310 arch_to_obj_dir = { 311 'i686': 'obj32', 312 'x86_64': 'obj', 313 } 314 315 for arch, obj_dir in arch_to_obj_dir.items(): 316 static_libs = os.path.join( 317 build_dir, 'host', host, obj_dir, 'STATIC_LIBRARIES') 318 built_lib = os.path.join( 319 static_libs, 'libprofile_rt_intermediates/libprofile_rt.a') 320 lib_name = 'libclang_rt.profile-{}.a'.format(arch) 321 install_file(built_lib, os.path.join(lib_dir, lib_name)) 322 323 324 def install_sanitizers(build_dir, install_dir, host): 325 headers_src = android_path('external/compiler-rt/include/sanitizer') 326 clang_lib = os.path.join(install_dir, 'lib64/clang', short_version()) 327 headers_dst = os.path.join(clang_lib, 'include/sanitizer') 328 lib_dst = os.path.join(clang_lib, 'lib/linux') 329 install_directory(headers_src, headers_dst) 330 331 if host == 'linux-x86': 332 install_host_sanitizers(build_dir, host, lib_dst) 333 334 # Tuples of (product, arch, libdir) 335 product_to_arch = ( 336 ('generic', 'arm', 'lib'), 337 ('generic_arm64', 'aarch64', 'lib64'), 338 ('generic_x86', 'i686', 'lib'), 339 ) 340 341 for product, arch, libdir in product_to_arch: 342 product_dir = os.path.join(build_dir, 'target/product', product) 343 system_dir = os.path.join(product_dir, 'system') 344 system_lib_dir = os.path.join(system_dir, libdir) 345 lib_name = 'libclang_rt.asan-{}-android.so'.format(arch) 346 built_lib = os.path.join(system_lib_dir, lib_name) 347 install_file(built_lib, lib_dst) 348 349 350 def install_host_sanitizers(build_dir, host, lib_dst): 351 # Tuples of (name, multilib). 352 libs = ( 353 ('asan', True), 354 ('asan_cxx', True), 355 ('ubsan_standalone', True), 356 ('ubsan_standalone_cxx', True), 357 ('tsan', False), 358 ('tsan_cxx', False), 359 ) 360 361 obj32 = os.path.join(build_dir, 'host', host, 'obj32/STATIC_LIBRARIES') 362 obj64 = os.path.join(build_dir, 'host', host, 'obj/STATIC_LIBRARIES') 363 for lib, is_multilib in libs: 364 built_lib_name = 'lib{}.a'.format(lib) 365 366 obj64_dir = os.path.join(obj64, 'lib{}_intermediates'.format(lib)) 367 lib64_name = 'libclang_rt.{}-x86_64.a'.format(lib) 368 built_lib64 = os.path.join(obj64_dir, built_lib_name) 369 install_file(built_lib64, os.path.join(lib_dst, lib64_name)) 370 if is_multilib: 371 obj32_dir = os.path.join(obj32, 'lib{}_intermediates'.format(lib)) 372 lib32_name = 'libclang_rt.{}-i686.a'.format(lib) 373 built_lib32 = os.path.join(obj32_dir, built_lib_name) 374 install_file(built_lib32, os.path.join(lib_dst, lib32_name)) 375 376 377 def install_license_files(install_dir): 378 projects = ( 379 'clang', 380 'compiler-rt', 381 'libcxx', 382 'libcxxabi', 383 'libunwind_llvm', 384 'llvm', 385 ) 386 387 notices = [] 388 for project in projects: 389 project_path = android_path('external', project) 390 license_pattern = os.path.join(project_path, 'MODULE_LICENSE_*') 391 for license_file in glob.glob(license_pattern): 392 install_file(license_file, install_dir) 393 with open(os.path.join(project_path, 'NOTICE')) as notice_file: 394 notices.append(notice_file.read()) 395 with open(os.path.join(install_dir, 'NOTICE'), 'w') as notice_file: 396 notice_file.write('\n'.join(notices)) 397 398 399 def install_repo_prop(install_dir): 400 file_name = 'repo.prop' 401 402 dist_dir = os.environ.get('DIST_DIR') 403 if dist_dir is not None: 404 dist_repo_prop = os.path.join(dist_dir, file_name) 405 shutil.copy(dist_repo_prop, install_dir) 406 else: 407 out_file = os.path.join(install_dir, file_name) 408 with open(out_file, 'w') as prop_file: 409 cmd = [ 410 'repo', 'forall', '-c', 411 'echo $REPO_PROJECT $(git rev-parse HEAD)', 412 ] 413 subprocess.check_call(cmd, stdout=prop_file) 414 415 416 def parse_args(): 417 parser = argparse.ArgumentParser() 418 419 parser.add_argument( 420 '--build-name', default='dev', help='Release name for the package.') 421 422 multi_stage_group = parser.add_mutually_exclusive_group() 423 multi_stage_group.add_argument( 424 '--multi-stage', action='store_true', default=True, 425 help='Perform multi-stage build (enabled by default).') 426 multi_stage_group.add_argument( 427 '--no-multi-stage', action='store_false', dest='multi_stage', 428 help='Do not perform multi-stage build.') 429 430 parser.add_argument( 431 '--build-all-llvm-tools', action='store_true', default=True, 432 help='Build all the LLVM tools/utilities.') 433 434 parser.add_argument( 435 '--no-build-all-llvm-tools', action='store_false', 436 dest='build_all_llvm_tools', 437 help='Build all the LLVM tools/utilities.') 438 439 return parser.parse_args() 440 441 442 def main(): 443 args = parse_args() 444 445 if sys.platform.startswith('linux'): 446 hosts = ['linux-x86', 'windows-x86'] 447 elif sys.platform == 'darwin': 448 hosts = ['darwin-x86'] 449 else: 450 raise RuntimeError('Unsupported host: {}'.format(sys.platform)) 451 452 stage_1_out_dir = build_path('stage1') 453 build(out_dir=stage_1_out_dir) 454 final_out_dir = stage_1_out_dir 455 if args.multi_stage: 456 stage_1_install_dir = build_path('stage1-install') 457 for host in hosts: 458 package_name = 'clang-' + args.build_name 459 install_host_dir = os.path.join(stage_1_install_dir, host) 460 install_dir = os.path.join(install_host_dir, package_name) 461 462 # Remove any previously installed toolchain so it doesn't pollute 463 # the build. 464 if os.path.exists(install_host_dir): 465 shutil.rmtree(install_host_dir) 466 467 install_toolchain(stage_1_out_dir, install_dir, host) 468 469 stage_2_out_dir = build_path('stage2') 470 build(out_dir=stage_2_out_dir, prebuilts_path=stage_1_install_dir, 471 prebuilts_version=package_name, 472 build_all_llvm_tools=args.build_all_llvm_tools) 473 final_out_dir = stage_2_out_dir 474 475 dist_dir = ORIG_ENV.get('DIST_DIR', final_out_dir) 476 for host in hosts: 477 package_toolchain(final_out_dir, args.build_name, host, dist_dir) 478 479 480 if __name__ == '__main__': 481 main() 482
