1 #!/usr/bin/env python 2 # 3 # Copyright (C) 2015 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 """Makes an old style monolithic NDK package out of individual modules.""" 18 from __future__ import print_function 19 20 import argparse 21 import datetime 22 import os 23 import shutil 24 import site 25 import stat 26 import subprocess 27 import sys 28 import tempfile 29 import zipfile 30 31 32 site.addsitedir(os.path.join(os.path.dirname(__file__), '../lib')) 33 import build_support # pylint: disable=import-error 34 35 36 THIS_DIR = os.path.dirname(__file__) 37 ANDROID_TOP = os.path.realpath(os.path.join(THIS_DIR, '../../..')) 38 39 # Note that we can't actually support creating both layouts from the same 40 # sources because changes are needed in gnustl's Android.mk and in the build 41 # system. The various "USE_NEW_LAYOUT" blocks are so we can more easily undo 42 # this change when we finalize a new layout. 43 USE_NEW_LAYOUT = False 44 45 46 def expand_packages(package, host, arches): 47 """Expands package definition tuple into list of full package names. 48 49 >>> expand_packages('gcc-{arch}-{host}', 'linux', ['arm64', 'x86_64']) 50 ['gcc-arm64-linux-x86_64', 'gcc-x86_64-linux-x86_64'] 51 52 >>> expand_packages('gcclibs-{arch}', 'linux', ['arm64', 'x86_64']) 53 ['gcclibs-arm64', 'gcclibs-x86_64'] 54 55 >>> expand_packages('llvm-{host}', 'linux', ['arm']) 56 ['llvm-linux-x86_64'] 57 58 >>> expand_packages('platforms', 'linux', ['arm']) 59 ['platforms'] 60 61 >>> expand_packages('libc++-{abi}', 'linux', ['arm']) 62 ['libc++-armeabi', 'libc++-armeabi-v7a', 'libc++-armeabi-v7a-hard'] 63 64 >>> expand_packages('binutils/{triple}', 'linux', ['arm', 'x86_64']) 65 ['binutils/arm-linux-androideabi', 'binutils/x86_64-linux-android'] 66 67 >> expand_packages('toolchains/{toolchain}-4.9', 'linux', ['arm', 'x86']) 68 ['toolchains/arm-linux-androideabi-4.9', 'toolchains/x86-4.9'] 69 """ 70 host_tag = build_support.host_to_tag(host) 71 seen_packages = set() 72 packages = [] 73 for arch in arches: 74 triple = build_support.arch_to_triple(arch) 75 toolchain = build_support.arch_to_toolchain(arch) 76 for abi in build_support.arch_to_abis(arch): 77 expanded = package.format( 78 abi=abi, arch=arch, host=host_tag, triple=triple, 79 toolchain=toolchain) 80 if expanded not in seen_packages: 81 packages.append(expanded) 82 seen_packages.add(expanded) 83 return packages 84 85 86 def get_all_packages(host, arches): 87 new_layout = [ 88 ('binutils-{arch}-{host}', 'binutils/{triple}'), 89 ('build', 'build'), 90 ('cpufeatures', 'sources/android/cpufeatures'), 91 ('gabixx', 'sources/cxx-stl/gabi++'), 92 ('gcc-{arch}-{host}', 'toolchains/{toolchain}-4.9'), 93 ('gcclibs-{arch}', 'gcclibs/{triple}'), 94 ('gdbserver-{arch}', 'gdbserver/{arch}'), 95 ('gnustl-4.9', 'sources/cxx-stl/gnu-libstdc++'), 96 ('gtest', 'sources/third_party/googletest'), 97 ('host-tools-{host}', 'host-tools'), 98 ('libandroid_support', 'sources/android/support'), 99 ('libcxx', 'sources/cxx-stl/llvm-libc++'), 100 ('libcxxabi', 'sources/cxx-stl/llvm-libc++abi'), 101 ('llvm-{host}', 'toolchains/llvm'), 102 ('native_app_glue', 'sources/android/native_app_glue'), 103 ('ndk_helper', 'sources/android/ndk_helper'), 104 ('python-packages', 'python-packages'), 105 ('stlport', 'sources/cxx-stl/stlport'), 106 ('system-stl', 'sources/cxx-stl/system'), 107 ] 108 109 old_layout = [ 110 ('build', 'build'), 111 ('cpufeatures', 'sources/android/cpufeatures'), 112 ('gabixx', 'sources/cxx-stl/gabi++'), 113 ('gcc-{arch}-{host}', 'toolchains/{toolchain}-4.9/prebuilt/{host}'), 114 ('gdbserver-{arch}', 'prebuilt/android-{arch}/gdbserver'), 115 ('gnustl-4.9', 'sources/cxx-stl/gnu-libstdc++/4.9'), 116 ('gtest', 'sources/third_party/googletest'), 117 ('host-tools-{host}', 'prebuilt/{host}'), 118 ('libandroid_support', 'sources/android/support'), 119 ('libcxx', 'sources/cxx-stl/llvm-libc++'), 120 ('libcxxabi', 'sources/cxx-stl/llvm-libc++abi'), 121 ('llvm-{host}', 'toolchains/llvm/prebuilt/{host}'), 122 ('native_app_glue', 'sources/android/native_app_glue'), 123 ('ndk_helper', 'sources/android/ndk_helper'), 124 ('python-packages', 'python-packages'), 125 ('stlport', 'sources/cxx-stl/stlport'), 126 ('system-stl', 'sources/cxx-stl/system'), 127 ] 128 129 if USE_NEW_LAYOUT: 130 packages = new_layout 131 else: 132 packages = old_layout 133 134 platforms_path = 'development/ndk/platforms' 135 for platform_dir in os.listdir(build_support.android_path(platforms_path)): 136 if not platform_dir.startswith('android-'): 137 continue 138 _, platform_str = platform_dir.split('-') 139 package_name = 'platform-' + platform_str 140 install_path = 'platforms/android-' + platform_str 141 packages.append((package_name, install_path)) 142 143 expanded = [] 144 for package, extract_path in packages: 145 package_names = expand_packages(package, host, arches) 146 extract_paths = expand_packages(extract_path, host, arches) 147 expanded.extend(zip(package_names, extract_paths)) 148 return expanded 149 150 151 def check_packages(path, packages): 152 for package, _ in packages: 153 print('Checking ' + package) 154 package_path = os.path.join(path, package + '.zip') 155 if not os.path.exists(package_path): 156 raise RuntimeError('Missing package: ' + package_path) 157 158 top_level_files = [] 159 with zipfile.ZipFile(package_path, 'r') as zip_file: 160 for f in zip_file.namelist(): 161 components = os.path.split(f) 162 if len(components) == 2: 163 top_level_files.append(components[1]) 164 165 if 'repo.prop' not in top_level_files: 166 msg = 'Package does not contain a repo.prop: ' + package_path 167 raise RuntimeError(msg) 168 169 if 'NOTICE' not in top_level_files: 170 msg = 'Package does not contain a NOTICE: ' + package_path 171 raise RuntimeError(msg) 172 173 174 def extract_all(path, packages, out_dir): 175 os.makedirs(out_dir) 176 for package, extract_path in packages: 177 print('Unpacking ' + package) 178 package_path = os.path.join(path, package + '.zip') 179 install_dir = os.path.join(out_dir, extract_path) 180 181 if os.path.exists(install_dir): 182 raise RuntimeError('Install path already exists: ' + install_dir) 183 184 if extract_path == '.': 185 raise RuntimeError('Found old style package: ' + package) 186 187 extract_dir = tempfile.mkdtemp() 188 try: 189 subprocess.check_call( 190 ['unzip', '-q', package_path, '-d', extract_dir]) 191 dirs = os.listdir(extract_dir) 192 if len(dirs) > 1: 193 msg = 'Package has more than one root directory: ' + package 194 raise RuntimeError(msg) 195 elif len(dirs) == 0: 196 raise RuntimeError('Package was empty: ' + package) 197 parent_dir = os.path.dirname(install_dir) 198 if not os.path.exists(parent_dir): 199 os.makedirs(parent_dir) 200 shutil.move(os.path.join(extract_dir, dirs[0]), install_dir) 201 finally: 202 shutil.rmtree(extract_dir) 203 204 if not USE_NEW_LAYOUT: 205 # FIXME(danalbert): OMG HACK 206 # The old package layout had libstdc++'s Android.mk at 207 # sources/cxx-stl/gnu-libstdc++/Android.mk. The gnustl package doesn't 208 # include the version in the path. To mimic the old package layout, we 209 # extract the gnustl package to sources/cxx-stl/gnu-libstdc++/4.9. As 210 # such, the Android.mk ends up in the 4.9 directory. We need to pull it 211 # up a directory. 212 gnustl_path = os.path.join(out_dir, 'sources/cxx-stl/gnu-libstdc++') 213 shutil.move(os.path.join(gnustl_path, '4.9/Android.mk'), 214 os.path.join(gnustl_path, 'Android.mk')) 215 216 217 def make_ndk_build_shortcut(out_dir, host): 218 if host.startswith('windows'): 219 make_ndk_build_cmd_helper(out_dir) 220 else: 221 make_ndk_build_sh_helper(out_dir) 222 223 224 def make_ndk_build_cmd_helper(out_dir): 225 with open(os.path.join(out_dir, 'ndk-build.cmd'), 'w') as helper: 226 helper.writelines([ 227 '@echo off\n', 228 r'%~dp0\build\ndk-build.cmd %*', 229 ]) 230 231 232 def make_ndk_build_sh_helper(out_dir): 233 file_path = os.path.join(out_dir, 'ndk-build') 234 with open(file_path, 'w') as helper: 235 helper.writelines([ 236 '#!/bin/sh\n', 237 'DIR="$(cd "$(dirname "$0")" && pwd)"\n', 238 '$DIR/build/ndk-build "$@"', 239 ]) 240 mode = os.stat(file_path).st_mode 241 os.chmod(file_path, mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) 242 243 244 def make_source_properties(out_dir): 245 path = os.path.join(out_dir, 'source.properties') 246 with open(path, 'w') as source_properties: 247 source_properties.write('\n'.join([ 248 'Pkg.Desc = Android NDK', 249 'Pkg.Revision = 11.0.0', 250 ])) 251 252 253 def copy_changelog(out_dir): 254 changelog_path = build_support.ndk_path('CHANGELOG.md') 255 shutil.copy2(changelog_path, out_dir) 256 257 258 def make_package(release, package_dir, packages, host, out_dir, temp_dir): 259 release_name = 'android-ndk-{}'.format(release) 260 extract_dir = os.path.join(temp_dir, release_name) 261 if os.path.exists(extract_dir): 262 shutil.rmtree(extract_dir) 263 extract_all(package_dir, packages, extract_dir) 264 265 make_ndk_build_shortcut(extract_dir, host) 266 make_source_properties(extract_dir) 267 copy_changelog(extract_dir) 268 269 host_tag = build_support.host_to_tag(host) 270 package_name = '{}-{}'.format(release_name, host_tag) 271 package_path = os.path.join(out_dir, package_name) 272 print('Packaging ' + package_name) 273 files = os.path.relpath(extract_dir, temp_dir) 274 275 if host.startswith('windows'): 276 _make_zip_package(package_path, temp_dir, files) 277 else: 278 _make_tar_package(package_path, temp_dir, files) 279 280 281 def _make_tar_package(package_path, base_dir, files): 282 subprocess.check_call( 283 ['tar', 'cjf', package_path + '.tar.bz2', '-C', base_dir, files]) 284 285 286 def _make_zip_package(package_path, base_dir, files): 287 cwd = os.getcwd() 288 package_path = os.path.realpath(package_path) 289 os.chdir(base_dir) 290 try: 291 subprocess.check_call(['zip', '-9qr', package_path + '.zip', files]) 292 finally: 293 os.chdir(cwd) 294 295 296 class ArgParser(argparse.ArgumentParser): 297 def __init__(self): 298 super(ArgParser, self).__init__( 299 description='Repackages NDK modules as a monolithic package. If ' 300 '--unpack is used, instead installs the monolithic ' 301 'package to a directory.') 302 303 self.add_argument( 304 '--arch', choices=build_support.ALL_ARCHITECTURES, 305 help='Bundle only the given architecture.') 306 self.add_argument( 307 '--host', choices=('darwin', 'linux', 'windows', 'windows64'), 308 default=build_support.get_default_host(), 309 help='Package binaries for given OS (e.g. linux).') 310 self.add_argument( 311 '--release', default=datetime.date.today().strftime('%Y%m%d'), 312 help='Release name for the package.') 313 314 self.add_argument( 315 '-f', '--force', dest='force', action='store_true', 316 help='Clobber out directory if it exists.') 317 self.add_argument( 318 '--dist-dir', type=os.path.realpath, 319 default=build_support.get_dist_dir(build_support.get_out_dir()), 320 help='Directory containing NDK modules.') 321 self.add_argument( 322 '--unpack', action='store_true', 323 help='Unpack the NDK to a directory rather than make a tarball.') 324 self.add_argument( 325 'out_dir', metavar='OUT_DIR', 326 help='Directory to install bundle or tarball to.') 327 328 329 def main(): 330 if 'ANDROID_BUILD_TOP' not in os.environ: 331 os.environ['ANDROID_BUILD_TOP'] = ANDROID_TOP 332 333 args = ArgParser().parse_args() 334 arches = build_support.ALL_ARCHITECTURES 335 if args.arch is not None: 336 arches = [args.arch] 337 338 if os.path.exists(args.out_dir) and args.unpack: 339 if args.force: 340 shutil.rmtree(args.out_dir) 341 else: 342 sys.exit(args.out_dir + ' already exists. Use -f to overwrite.') 343 344 packages = get_all_packages(args.host, arches) 345 check_packages(args.dist_dir, packages) 346 347 if args.unpack: 348 extract_all(args.dist_dir, packages, args.out_dir) 349 make_ndk_build_shortcut(args.out_dir, args.host) 350 else: 351 make_package(args.release, args.dist_dir, packages, args.host, 352 args.out_dir, build_support.get_out_dir()) 353 354 355 if __name__ == '__main__': 356 main() 357
