1 #!/usr/bin/env python 2 # Copyright (C) 2017 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 import argparse 17 import hashlib 18 import logging 19 import os 20 import shutil 21 import subprocess 22 import sys 23 import urllib 24 import zipfile 25 26 from collections import namedtuple 27 28 # When adding a new git dependency here please also add a corresponding entry in 29 # .travis.yml under the "cache:" section. 30 31 # The format for the deps below is the following: 32 # (target_folder, source_url, sha1, target_platform) 33 # |source_url| can be either a git repo or a http url. 34 # If a git repo, |sha1| is the committish that will be checked out. 35 # If a http url, |sha1| is the shasum of the original file. 36 # If the url is a .zip or .tgz file it will be automatically deflated under 37 # |target_folder|, taking care of stripping the root folder if it's a single 38 # root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have 39 # instead just buildtools/protobuf). 40 # |target_platform| is either 'darwin', 'linux2' or 'all' and applies the dep 41 # only on the given platform (ask python why linux2 and not just linux). 42 43 # Dependencies required to build code on the host or when targeting desktop OS. 44 BUILD_DEPS_HOST = [ 45 # GN 46 ('buildtools/mac/gn', 47 'https://storage.googleapis.com/chromium-gn/c2c934d4dda1f470a6511b1015dda9a9fb1ce50b', 48 'c2c934d4dda1f470a6511b1015dda9a9fb1ce50b', 49 'darwin' 50 ), 51 ('buildtools/linux64/gn', 52 'https://storage.googleapis.com/chromium-gn/b53fa13e950948c6f9a062189b76b34a9610281f', 53 'b53fa13e950948c6f9a062189b76b34a9610281f', 54 'linux2' 55 ), 56 57 # clang-format 58 ('buildtools/mac/clang-format', 59 'https://storage.googleapis.com/chromium-clang-format/0679b295e2ce2fce7919d1e8d003e497475f24a3', 60 '0679b295e2ce2fce7919d1e8d003e497475f24a3', 61 'darwin' 62 ), 63 ('buildtools/linux64/clang-format', 64 'https://storage.googleapis.com/chromium-clang-format/5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', 65 '5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', 66 'linux2' 67 ), 68 # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS. 69 ('buildtools/clang_format/script', 70 'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git', 71 '0653eee0c81ea04715c635dd0885e8096ff6ba6d', 72 'all' 73 ), 74 75 # Ninja 76 ('buildtools/mac/ninja', 77 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/mac/a1db595e824c50cf565fbf0af2437fd91b7babf4', 78 'a1db595e824c50cf565fbf0af2437fd91b7babf4', 79 'darwin' 80 ), 81 ('buildtools/linux64/ninja', 82 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/linux64/d35b36c84a09f7e38b25947cafada10e8bf835bc', 83 'd35b36c84a09f7e38b25947cafada10e8bf835bc', 84 'linux2' 85 ), 86 87 # Keep in sync with Android's //external/googletest/README.version. 88 ('buildtools/googletest.zip', 89 'https://github.com/google/googletest/archive/ff07a5de0e81580547f1685e101194ed1a4fcd56.zip', 90 'c7edec7d7e6db1fc37a20710de9c4d89e3a3893b', 91 'all' 92 ), 93 94 # Keep in sync with Android's //external/protobuf/README.version. 95 ('buildtools/protobuf.zip', 96 'https://github.com/google/protobuf/releases/download/v3.0.0-beta-3/protobuf-cpp-3.0.0-beta-3.zip', 97 '3caec60aa9d8eefc8c3c3201b6b8ca19935edb89', 98 'all' 99 ), 100 101 # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++ 102 # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS. 103 ('buildtools/libcxx', 104 'https://chromium.googlesource.com/chromium/llvm-project/libcxx.git', 105 '3a07dd740be63878167a0ea19fe81869954badd7', 106 'all' 107 ), 108 ('buildtools/libcxxabi', 109 'https://chromium.googlesource.com/chromium/llvm-project/libcxxabi.git', 110 '4072e8fd76febee37f60aeda76d6d9f5e3791daa', 111 'all' 112 ), 113 ('buildtools/libunwind', 114 'https://chromium.googlesource.com/external/llvm.org/libunwind.git', 115 '41f982e5887185b904a456e20dfcd58e6be6cc19', 116 'all' 117 ), 118 119 # Keep the revision in sync with Chrome's CLANG_REVISION in 120 # tools/clang/scripts/update.py. 121 ('buildtools/clang.tgz', 122 'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-321529-1.tgz', 123 '7ca04034ac7e4a956b0084a8bc604bacab94af26', 124 'linux2' 125 ), 126 127 # Benchmarking tool. 128 ('buildtools/benchmark.zip', 129 'https://github.com/google/benchmark/archive/v1.3.0.zip', 130 'f387e0df37d54bfd5be239e8d0d3ea2e2c3e34f4', 131 'all' 132 ), 133 134 # Libbacktrace, for stacktraces in Linux/Android debug builds. 135 ('buildtools/libbacktrace.zip', 136 'https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip', 137 'b723fe9d671d1ab54df1297f6afbf2893a41c3ea', 138 'all' 139 ), 140 ] 141 142 # Dependencies required to build Android code. 143 # URLs and SHA1s taken from: 144 # - https://dl.google.com/android/repository/repository-11.xml 145 # - https://dl.google.com/android/repository/sys-img/android/sys-img.xml 146 BUILD_DEPS_ANDROID = [ 147 # Android NDK 148 ('buildtools/ndk.zip', 149 'https://dl.google.com/android/repository/android-ndk-r15c-darwin-x86_64.zip', 150 'ea4b5d76475db84745aa8828000d009625fc1f98', 151 'darwin' 152 ), 153 ('buildtools/ndk.zip', 154 'https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip', 155 '0bf02d4e8b85fd770fd7b9b2cdec57f9441f27a2', 156 'linux2' 157 ), 158 ] 159 160 # Dependencies required to run Android tests. 161 TEST_DEPS_ANDROID = [ 162 # Android emulator images. 163 ('buildtools/aosp-arm.zip', 164 'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip', 165 'a480d5e7d3ca888b0a58fe15ce76b1791537429a', 166 'all' 167 ), 168 169 # platform-tools.zip contains adb binaries. 170 ('buildtools/android_sdk/platform-tools.zip', 171 'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip', 172 'e75b6137dc444f777eb02f44a6d9819b3aabff82', 173 'darwin' 174 ), 175 ('buildtools/android_sdk/platform-tools.zip', 176 'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip', 177 '00de8a6631405b617c10f68cd11ff2e1cd528e23', 178 'linux2' 179 ), 180 181 # Android emulator binaries. 182 ('buildtools/emulator', 183 'https://android.googlesource.com/platform/prebuilts/android-emulator.git', 184 '4b260028dc27bc92c39bee9129cb2ba839970956', 185 'all' 186 ), 187 ] 188 189 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 190 191 192 def ReadFile(path): 193 if not os.path.exists(path): 194 return None 195 with open(path) as f: 196 return f.read().strip() 197 198 199 def MkdirRecursive(path): 200 # Works with both relative and absolute paths 201 cwd = '/' if path.startswith('/') else ROOT_DIR 202 for part in path.split('/'): 203 cwd = os.path.join(cwd, part) 204 if not os.path.exists(cwd): 205 os.makedirs(cwd) 206 else: 207 assert(os.path.isdir(cwd)) 208 209 210 def HashLocalFile(path): 211 if not os.path.exists(path): 212 return None 213 with open(path, 'rb') as f: 214 return hashlib.sha1(f.read()).hexdigest() 215 216 217 def ExtractZipfilePreservePermissions(zf, info, path): 218 zf.extract(info.filename, path=path) 219 target_path = os.path.join(path, info.filename) 220 min_acls = 0o755 if info.filename.endswith('/') else 0o644 221 os.chmod(target_path, (info.external_attr >> 16L) | min_acls) 222 223 224 def IsGitRepoCheckoutOutAtRevision(path, revision): 225 return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision 226 227 228 def CheckoutGitRepo(path, git_url, revision): 229 if IsGitRepoCheckoutOutAtRevision(path, revision): 230 return 231 if os.path.exists(path): 232 shutil.rmtree(path) 233 MkdirRecursive(path) 234 logging.info('Fetching %s @ %s into %s', git_url, revision, path) 235 subprocess.check_call(['git', 'init', path], cwd=path) 236 subprocess.check_call( 237 ['git', 'fetch', '--depth', '1', git_url, revision], cwd=path) 238 subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path) 239 assert(IsGitRepoCheckoutOutAtRevision(path, revision)) 240 241 242 def Main(): 243 parser = argparse.ArgumentParser() 244 parser.add_argument('--no-android', action='store_true') 245 args = parser.parse_args() 246 deps = BUILD_DEPS_HOST 247 if not args.no_android: 248 deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID 249 for rel_path, url, expected_sha1, platform in deps: 250 if (platform != 'all' and platform != sys.platform): 251 continue 252 local_path = os.path.join(ROOT_DIR, rel_path) 253 if url.endswith('.git'): 254 CheckoutGitRepo(local_path, url, expected_sha1) 255 continue 256 is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz') 257 zip_target_dir = local_path[:-4] if is_zip else None 258 zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None 259 260 if ((not is_zip and HashLocalFile(local_path) == expected_sha1) or 261 (is_zip and ReadFile(zip_dir_stamp) == expected_sha1)): 262 continue 263 MkdirRecursive(os.path.dirname(rel_path)) 264 if HashLocalFile(local_path) != expected_sha1: 265 download_path = local_path + '.tmp' 266 logging.info('Downloading %s from %s', local_path, url) 267 urllib.urlretrieve(url, download_path) 268 os.chmod(download_path, 0o755) 269 if (HashLocalFile(download_path) != expected_sha1): 270 os.remove(download_path) 271 logging.fatal('SHA1 mismatch for %s', download_path) 272 return 1 273 os.rename(download_path, local_path) 274 assert(HashLocalFile(local_path) == expected_sha1) 275 276 if is_zip: 277 logging.info('Extracting %s into %s' % (local_path, zip_target_dir)) 278 assert(os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR) 279 if os.path.exists(zip_target_dir): 280 logging.info('Deleting stale dir %s' % zip_target_dir) 281 shutil.rmtree(zip_target_dir) 282 283 # Decompress the archive. 284 if local_path.endswith('.tgz'): 285 MkdirRecursive(zip_target_dir) 286 subprocess.check_call(['tar', '-zxf', local_path], cwd=zip_target_dir) 287 elif local_path.endswith('.zip'): 288 with zipfile.ZipFile(local_path, 'r') as zf: 289 for info in zf.infolist(): 290 ExtractZipfilePreservePermissions(zf, info, zip_target_dir) 291 292 # If the zip contains one root folder, rebase one level up moving all 293 # its sub files and folders inside |target_dir|. 294 subdir = os.listdir(zip_target_dir) 295 if len(subdir) == 1: 296 subdir = os.path.join(zip_target_dir, subdir[0]) 297 if os.path.isdir(subdir): 298 for subf in os.listdir(subdir): 299 shutil.move(os.path.join(subdir,subf), zip_target_dir) 300 os.rmdir(subdir) 301 302 # Create stamp and remove the archive. 303 with open(zip_dir_stamp, 'w') as stamp_file: 304 stamp_file.write(expected_sha1) 305 os.remove(local_path) 306 307 308 if __name__ == '__main__': 309 logging.basicConfig(level=logging.INFO) 310 sys.exit(Main()) 311
