1 #!/usr/bin/env python 2 # Copyright (C) 2017 The Android Open Source Project 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 import argparse 17 import hashlib 18 import logging 19 import os 20 import shutil 21 import subprocess 22 import sys 23 import urllib 24 import zipfile 25 26 from collections import namedtuple 27 28 # When adding a new git dependency here please also add a corresponding entry in 29 # .travis.yml under the "cache:" section. 30 31 # The format for the deps below is the following: 32 # (target_folder, source_url, sha1, target_platform) 33 # |source_url| can be either a git repo or a http url. 34 # If a git repo, |sha1| is the committish that will be checked out. 35 # If a http url, |sha1| is the shasum of the original file. 36 # If the url is a .zip or .tgz file it will be automatically deflated under 37 # |target_folder|, taking care of stripping the root folder if it's a single 38 # root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have 39 # instead just buildtools/protobuf). 40 # |target_platform| is either 'darwin', 'linux2' or 'all' and applies the dep 41 # only on the given platform (ask python why linux2 and not just linux). 42 43 # Dependencies required to build code on the host or when targeting desktop OS. 44 BUILD_DEPS_HOST = [ 45 # GN 46 ('buildtools/mac/gn', 47 'https://storage.googleapis.com/chromium-gn/9be792dd9010ce303a9c3a497a67bcc5ac8c7666', 48 '9be792dd9010ce303a9c3a497a67bcc5ac8c7666', 49 'darwin' 50 ), 51 ('buildtools/linux64/gn', 52 'https://storage.googleapis.com/chromium-gn/2f27ff0b6118e5886df976da5effa6003d19d1ce', 53 '2f27ff0b6118e5886df976da5effa6003d19d1ce', 54 'linux2' 55 ), 56 57 # clang-format 58 ('buildtools/mac/clang-format', 59 'https://storage.googleapis.com/chromium-clang-format/0679b295e2ce2fce7919d1e8d003e497475f24a3', 60 '0679b295e2ce2fce7919d1e8d003e497475f24a3', 61 'darwin' 62 ), 63 ('buildtools/linux64/clang-format', 64 'https://storage.googleapis.com/chromium-clang-format/5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', 65 '5349d1954e17f6ccafb6e6663b0f13cdb2bb33c8', 66 'linux2' 67 ), 68 # Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS. 69 ('buildtools/clang_format/script', 70 'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git', 71 '0653eee0c81ea04715c635dd0885e8096ff6ba6d', 72 'all' 73 ), 74 75 # Ninja 76 ('buildtools/mac/ninja', 77 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/mac/a1db595e824c50cf565fbf0af2437fd91b7babf4', 78 'a1db595e824c50cf565fbf0af2437fd91b7babf4', 79 'darwin' 80 ), 81 ('buildtools/linux64/ninja', 82 'https://storage.googleapis.com/fuchsia-build/fuchsia/ninja/linux64/d35b36c84a09f7e38b25947cafada10e8bf835bc', 83 'd35b36c84a09f7e38b25947cafada10e8bf835bc', 84 'linux2' 85 ), 86 87 # Keep in sync with Android's //external/googletest/README.version. 88 ('buildtools/googletest.zip', 89 'https://github.com/google/googletest/archive/ff07a5de0e81580547f1685e101194ed1a4fcd56.zip', 90 'c7edec7d7e6db1fc37a20710de9c4d89e3a3893b', 91 'all' 92 ), 93 94 # Keep in sync with Android's //external/protobuf/README.version. 95 ('buildtools/protobuf.zip', 96 'https://github.com/google/protobuf/releases/download/v3.0.0-beta-3/protobuf-cpp-3.0.0-beta-3.zip', 97 '3caec60aa9d8eefc8c3c3201b6b8ca19935edb89', 98 'all' 99 ), 100 101 # libc++, libc++abi and libunwind for Linux where we need to rebuild the C++ 102 # lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS. 103 ('buildtools/libcxx', 104 'https://chromium.googlesource.com/chromium/llvm-project/libcxx.git', 105 '2199647acb904b91eea0a5e045f5b227c87d6e85', 106 'all' 107 ), 108 ('buildtools/libcxxabi', 109 'https://chromium.googlesource.com/chromium/llvm-project/libcxxabi.git', 110 'c3f4753f7139c73063304235781e4f7788a94c06', 111 'all' 112 ), 113 ('buildtools/libunwind', 114 'https://chromium.googlesource.com/external/llvm.org/libunwind.git', 115 '317087cfd8e608bd24e53934d59b5b85e0a9ded6', 116 'all' 117 ), 118 119 # Keep the revision in sync with Chrome's CLANG_REVISION in 120 # tools/clang/scripts/update.py. 121 ('buildtools/clang.tgz', 122 'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-346388-1.tgz', 123 'c2998d67a9c623fe12e01a33e8b7cf437b396099', 124 'linux2' 125 ), 126 127 # Keep in sync with chromium DEPS. 128 ('buildtools/libfuzzer', 129 'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git', 130 '2a53098584c48af50aec3fb51febe5e651489774', 131 'linux2' 132 ), 133 134 # Benchmarking tool. 135 ('buildtools/benchmark.zip', 136 'https://github.com/google/benchmark/archive/v1.3.0.zip', 137 'f387e0df37d54bfd5be239e8d0d3ea2e2c3e34f4', 138 'all' 139 ), 140 141 # Libbacktrace, for stacktraces in Linux/Android debug builds. 142 ('buildtools/libbacktrace.zip', 143 'https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip', 144 'b723fe9d671d1ab54df1297f6afbf2893a41c3ea', 145 'all' 146 ), 147 148 # Sqlite for the trace processing library. 149 # This is the amalgamated source whose compiled output is meant to be faster. 150 # We still pull the full source for the extensions (not amalgamated). 151 ('buildtools/sqlite.zip', 152 'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3250300.zip', 153 'b78c2cb0d2c9182686c582312479f96a82bf5380', 154 'all' 155 ), 156 ('buildtools/sqlite_src.zip', 157 'https://storage.googleapis.com/perfetto/sqlite-src-3250300.zip', 158 'd1af2883bb800852946f9bf8ab6055e7698e18ee', 159 'all' 160 ), 161 162 # JsonCpp for legacy json import. Used only by the trace processor in 163 # standalone builds. 164 ('buildtools/jsoncpp.zip', 165 'https://github.com/open-source-parsers/jsoncpp/archive/1.0.0.zip', 166 '3219e26f2e249bb46b7d688478208c7ec138fea4', 167 'all' 168 ), 169 170 # These dependencies are for libunwindstack, which is used by src/profiling. 171 ('buildtools/android-core', 172 'https://android.googlesource.com/platform/system/core.git', 173 '9d3310c019839ec342b5c0712f3ba59cfd5ca4a0', 174 'all' 175 ), 176 177 ('buildtools/lzma', 178 'https://android.googlesource.com/platform/external/lzma.git', 179 '7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 180 'all' 181 ), 182 183 ('buildtools/zlib', 184 'https://android.googlesource.com/platform/external/zlib.git', 185 'dfa0646a03b4e1707469e04dc931b09774968fe6', 186 'all' 187 ), 188 189 ('buildtools/bionic', 190 'https://android.googlesource.com/platform/bionic.git', 191 'a60488109cda997dfd83832731c8527feaa2825e', 192 'all' 193 ), 194 195 # Example traces for regression tests. 196 ('buildtools/test_data.zip', 197 'https://storage.googleapis.com/perfetto/test-data-20190423-131328.zip', 198 '263db97612203fd0dd047edd54eaa7007e32bf91', 199 'all', 200 ), 201 202 # Linenoise, used only by trace_processor in standalone builds. 203 ('buildtools/linenoise', 204 'https://fuchsia.googlesource.com/third_party/linenoise.git', 205 'c894b9e59f02203dbe4e2be657572cf88c4230c3', 206 'all' 207 ), 208 ] 209 210 # Dependencies required to build Android code. 211 # URLs and SHA1s taken from: 212 # - https://dl.google.com/android/repository/repository-11.xml 213 # - https://dl.google.com/android/repository/sys-img/android/sys-img.xml 214 BUILD_DEPS_ANDROID = [ 215 # Android NDK 216 ('buildtools/ndk.zip', 217 'https://dl.google.com/android/repository/android-ndk-r17b-darwin-x86_64.zip', 218 'f990aafaffec0b583d2c5420bfa622e52ac14248', 219 'darwin' 220 ), 221 ('buildtools/ndk.zip', 222 'https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip', 223 'dd5762ee7ef4995ad04fe0c45a608c344d99ca9f', 224 'linux2' 225 ), 226 ] 227 228 # Dependencies required to run Android tests. 229 TEST_DEPS_ANDROID = [ 230 # Android emulator images. 231 ('buildtools/aosp-arm.zip', 232 'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip', 233 'a480d5e7d3ca888b0a58fe15ce76b1791537429a', 234 'all' 235 ), 236 237 # platform-tools.zip contains adb binaries. 238 ('buildtools/android_sdk/platform-tools.zip', 239 'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip', 240 'e75b6137dc444f777eb02f44a6d9819b3aabff82', 241 'darwin' 242 ), 243 ('buildtools/android_sdk/platform-tools.zip', 244 'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip', 245 '00de8a6631405b617c10f68cd11ff2e1cd528e23', 246 'linux2' 247 ), 248 249 # Android emulator binaries. 250 ('buildtools/emulator', 251 'https://android.googlesource.com/platform/prebuilts/android-emulator.git', 252 '4b260028dc27bc92c39bee9129cb2ba839970956', 253 'all' 254 ), 255 ] 256 257 # This variable is updated by tools/roll-catapult-trace-viewer. 258 CATAPULT_SHA1 = 'ff5d8fd7244680b4d4456c25d5fdc04c76f9ef66' 259 260 TYPEFACES_SHA1 = '756b0a015b8f99f5718f7fdf967d052c1ec55ab3' 261 262 UI_DEPS = [ 263 ('buildtools/nodejs.tgz', 264 'https://storage.googleapis.com/perfetto/node-v10.3.0-darwin-x64.tar.gz', 265 '6d9a122785f38c256add3b25f74adf125497861a', 266 'darwin' 267 ), 268 ('buildtools/nodejs.tgz', 269 'https://storage.googleapis.com/perfetto/node-v10.3.0-linux-x64.tar.xz', 270 '118f6ea19f75089b3f12ac2ddfce357bff872b5e', 271 'linux2' 272 ), 273 ('buildtools/emsdk/emscripten.tgz', 274 'https://storage.googleapis.com/perfetto/emscripten-1.37.40.tar.gz', 275 '588c28221321ebbdfc8e3a6f47ea6106f589669b', 276 'all' 277 ), 278 ('buildtools/emsdk/llvm.tgz', 279 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-darwin.tar.gz', 280 '7a894ef0a52821c62f6abaac552dc4ce5d424607', 281 'darwin' 282 ), 283 ('buildtools/emsdk/llvm.tgz', 284 'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-static-linux.tar.gz', 285 '478501b9b7a14884e546c84efe209a90052cbb07', 286 'linux2' 287 ), 288 ('buildtools/d8.tgz', 289 'https://storage.googleapis.com/perfetto/d8-linux2-5.7.492.65.tar.gz', 290 '95e82ad7faf0a6f74d950c2aa65e3858b7bdb6c6', 291 'linux2' 292 ), 293 ('buildtools/d8.tgz', 294 'https://storage.googleapis.com/perfetto/d8-darwin-6.6.346.32.tar.gz', 295 '1abd630619bb1977ab62095570a113d782a1545d', 296 'darwin' 297 ), 298 ('buildtools/catapult_trace_viewer.tgz', 299 'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz' % CATAPULT_SHA1, 300 CATAPULT_SHA1, 301 'all' 302 ), 303 ('buildtools/typefaces.tgz', 304 'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' % TYPEFACES_SHA1, 305 TYPEFACES_SHA1, 306 'all' 307 ) 308 ] 309 310 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 311 312 313 def ReadFile(path): 314 if not os.path.exists(path): 315 return None 316 with open(path) as f: 317 return f.read().strip() 318 319 320 def MkdirRecursive(path): 321 # Works with both relative and absolute paths 322 cwd = '/' if path.startswith('/') else ROOT_DIR 323 for part in path.split('/'): 324 cwd = os.path.join(cwd, part) 325 if not os.path.exists(cwd): 326 os.makedirs(cwd) 327 else: 328 assert(os.path.isdir(cwd)) 329 330 331 def HashLocalFile(path): 332 if not os.path.exists(path): 333 return None 334 with open(path, 'rb') as f: 335 return hashlib.sha1(f.read()).hexdigest() 336 337 338 def ExtractZipfilePreservePermissions(zf, info, path): 339 zf.extract(info.filename, path=path) 340 target_path = os.path.join(path, info.filename) 341 min_acls = 0o755 if info.filename.endswith('/') else 0o644 342 os.chmod(target_path, (info.external_attr >> 16L) | min_acls) 343 344 345 def IsGitRepoCheckoutOutAtRevision(path, revision): 346 return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision 347 348 349 def CheckoutGitRepo(path, git_url, revision): 350 if IsGitRepoCheckoutOutAtRevision(path, revision): 351 return False 352 if os.path.exists(path): 353 shutil.rmtree(path) 354 MkdirRecursive(path) 355 logging.info('Fetching %s @ %s into %s', git_url, revision, path) 356 subprocess.check_call(['git', 'init', path], cwd=path) 357 subprocess.check_call( 358 ['git', 'fetch', '--quiet', '--depth', '1', git_url, revision], cwd=path) 359 subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path) 360 assert(IsGitRepoCheckoutOutAtRevision(path, revision)) 361 return True 362 363 def InstallNodeModules(): 364 ui_dir = os.path.join(ROOT_DIR, 'ui') 365 logging.info("Running npm install in {0}".format(ui_dir)) 366 subprocess.check_call( 367 [os.path.join(ui_dir, 'npm'), 'install', '--no-save'], 368 cwd=os.path.join(ROOT_DIR, 'ui')) 369 370 def Main(): 371 parser = argparse.ArgumentParser() 372 parser.add_argument('--no-android', action='store_true') 373 parser.add_argument('--ui', action='store_true') 374 args = parser.parse_args() 375 deps = BUILD_DEPS_HOST 376 if not args.no_android: 377 deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID 378 if args.ui: 379 deps += UI_DEPS 380 deps_updated = False 381 for rel_path, url, expected_sha1, platform in deps: 382 if (platform != 'all' and platform != sys.platform): 383 continue 384 local_path = os.path.join(ROOT_DIR, rel_path) 385 if url.endswith('.git'): 386 deps_updated |= CheckoutGitRepo(local_path, url, expected_sha1) 387 continue 388 is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz') 389 zip_target_dir = local_path[:-4] if is_zip else None 390 zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None 391 392 if ((not is_zip and HashLocalFile(local_path) == expected_sha1) or 393 (is_zip and ReadFile(zip_dir_stamp) == expected_sha1)): 394 continue 395 deps_updated = True 396 MkdirRecursive(os.path.dirname(rel_path)) 397 if HashLocalFile(local_path) != expected_sha1: 398 download_path = local_path + '.tmp' 399 logging.info('Downloading %s from %s', local_path, url) 400 urllib.urlretrieve(url, download_path) 401 os.chmod(download_path, 0o755) 402 actual_sha1 = HashLocalFile(download_path) 403 if (actual_sha1 != expected_sha1): 404 os.remove(download_path) 405 logging.fatal('SHA1 mismatch for {} expected {} was {}'.format( 406 download_path, expected_sha1, actual_sha1)) 407 return 1 408 os.rename(download_path, local_path) 409 assert(HashLocalFile(local_path) == expected_sha1) 410 411 if is_zip: 412 logging.info('Extracting %s into %s' % (local_path, zip_target_dir)) 413 assert(os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR) 414 if os.path.exists(zip_target_dir): 415 logging.info('Deleting stale dir %s' % zip_target_dir) 416 shutil.rmtree(zip_target_dir) 417 418 # Decompress the archive. 419 if local_path.endswith('.tgz'): 420 MkdirRecursive(zip_target_dir) 421 subprocess.check_call(['tar', '-xf', local_path], cwd=zip_target_dir) 422 elif local_path.endswith('.zip'): 423 with zipfile.ZipFile(local_path, 'r') as zf: 424 for info in zf.infolist(): 425 ExtractZipfilePreservePermissions(zf, info, zip_target_dir) 426 427 # If the zip contains one root folder, rebase one level up moving all 428 # its sub files and folders inside |target_dir|. 429 subdir = os.listdir(zip_target_dir) 430 if len(subdir) == 1: 431 subdir = os.path.join(zip_target_dir, subdir[0]) 432 if os.path.isdir(subdir): 433 for subf in os.listdir(subdir): 434 shutil.move(os.path.join(subdir,subf), zip_target_dir) 435 os.rmdir(subdir) 436 437 # Create stamp and remove the archive. 438 with open(zip_dir_stamp, 'w') as stamp_file: 439 stamp_file.write(expected_sha1) 440 os.remove(local_path) 441 442 if args.ui: 443 # Needs to happen after nodejs is installed above. 444 InstallNodeModules() 445 446 if deps_updated: 447 # Stale binary files may be compiled against old sysroot headers that aren't 448 # tracked by gn. 449 logging.warn('Remember to run "gn clean <output_directory>" ' + 450 'to avoid stale binary files.') 451 452 if __name__ == '__main__': 453 logging.basicConfig(level=logging.INFO) 454 sys.exit(Main()) 455
