1 #!/usr/bin/env python 2 # Copyright (c) 2015 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """This script will check out llvm and clang, and then package the results up 7 to a tgz file.""" 8 9 import argparse 10 import fnmatch 11 import itertools 12 import os 13 import shutil 14 import subprocess 15 import sys 16 import tarfile 17 18 # Path constants. 19 THIS_DIR = os.path.dirname(__file__) 20 CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) 21 THIRD_PARTY_DIR = os.path.join(THIS_DIR, '..', '..', '..', 'third_party') 22 LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') 23 LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') 24 LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, 25 'llvm-bootstrap-install') 26 LLVM_BUILD_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-build') 27 LLVM_RELEASE_DIR = os.path.join(LLVM_BUILD_DIR, 'Release+Asserts') 28 LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin') 29 STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') 30 31 32 def Tee(output, logfile): 33 logfile.write(output) 34 print output, 35 36 37 def TeeCmd(cmd, logfile, fail_hard=True): 38 """Runs cmd and writes the output to both stdout and logfile.""" 39 # Reading from PIPE can deadlock if one buffer is full but we wait on a 40 # different one. To work around this, pipe the subprocess's stderr to 41 # its stdout buffer and don't give it a stdin. 42 # shell=True is required in cmd.exe since depot_tools has an svn.bat, and 43 # bat files only work with shell=True set. 44 proc = subprocess.Popen(cmd, bufsize=1, shell=sys.platform == 'win32', 45 stdin=open(os.devnull), stdout=subprocess.PIPE, 46 stderr=subprocess.STDOUT) 47 for line in iter(proc.stdout.readline,''): 48 Tee(line, logfile) 49 if proc.poll() is not None: 50 break 51 exit_code = proc.wait() 52 if exit_code != 0 and fail_hard: 53 print 'Failed:', cmd 54 sys.exit(1) 55 56 57 def PrintTarProgress(tarinfo): 58 print 'Adding', tarinfo.name 59 return tarinfo 60 61 62 def GetExpectedStamp(): 63 rev_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), 64 '--print-revision'] 65 return subprocess.check_output(rev_cmd).rstrip() 66 67 68 def GetGsutilPath(): 69 if not 'find_depot_tools' in sys.modules: 70 sys.path.insert(0, os.path.join(CHROMIUM_DIR, 'build')) 71 global find_depot_tools 72 import find_depot_tools 73 depot_path = find_depot_tools.add_depot_tools_to_path() 74 if depot_path is None: 75 print ('depot_tools are not found in PATH. ' 76 'Follow the instructions in this document ' 77 'http://dev.chromium.org/developers/how-tos/install-depot-tools' 78 ' to install depot_tools and then try again.') 79 sys.exit(1) 80 gsutil_path = os.path.join(depot_path, 'gsutil.py') 81 return gsutil_path 82 83 84 def RunGsutil(args): 85 return subprocess.call([sys.executable, GetGsutilPath()] + args) 86 87 88 def GsutilArchiveExists(archive_name, platform): 89 gsutil_args = ['-q', 'stat', 90 'gs://chromium-browser-clang/%s/%s.tgz' % 91 (platform, archive_name)] 92 return RunGsutil(gsutil_args) == 0 93 94 95 def MaybeUpload(args, archive_name, platform): 96 # We don't want to rewrite the file, if it already exists on the server, 97 # so -n option to gsutil is used. It will warn, if the upload was aborted. 98 gsutil_args = ['cp', '-n', '-a', 'public-read', 99 '%s.tgz' % archive_name, 100 'gs://chromium-browser-clang/%s/%s.tgz' % 101 (platform, archive_name)] 102 if args.upload: 103 print 'Uploading %s to Google Cloud Storage...' % archive_name 104 exit_code = RunGsutil(gsutil_args) 105 if exit_code != 0: 106 print "gsutil failed, exit_code: %s" % exit_code 107 os.exit(exit_code) 108 else: 109 print 'To upload, run:' 110 print ('gsutil %s' % ' '.join(gsutil_args)) 111 112 113 def main(): 114 parser = argparse.ArgumentParser(description='build and package clang') 115 parser.add_argument('--upload', action='store_true', 116 help='Upload the target archive to Google Cloud Storage.') 117 args = parser.parse_args() 118 119 # Check that the script is not going to upload a toolchain built from HEAD. 120 use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ 121 if args.upload and use_head_revision: 122 print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used " 123 "at the same time.") 124 return 1 125 126 expected_stamp = GetExpectedStamp() 127 pdir = 'clang-' + expected_stamp 128 golddir = 'llvmgold-' + expected_stamp 129 print pdir 130 131 if sys.platform == 'darwin': 132 platform = 'Mac' 133 elif sys.platform == 'win32': 134 platform = 'Win' 135 else: 136 platform = 'Linux_x64' 137 138 # Check if Google Cloud Storage already has the artifacts we want to build. 139 if (args.upload and GsutilArchiveExists(pdir, platform) and 140 not sys.platform.startswith('linux') or 141 GsutilArchiveExists(golddir, platform)): 142 print ('Desired toolchain revision %s is already available ' 143 'in Google Cloud Storage:') % expected_stamp 144 print 'gs://chromium-browser-clang/%s/%s.tgz' % (platform, pdir) 145 if sys.platform.startswith('linux'): 146 print 'gs://chromium-browser-clang/%s/%s.tgz' % (platform, golddir) 147 return 0 148 149 with open('buildlog.txt', 'w') as log: 150 Tee('Diff in llvm:\n', log) 151 TeeCmd(['svn', 'stat', LLVM_DIR], log, fail_hard=False) 152 TeeCmd(['svn', 'diff', LLVM_DIR], log, fail_hard=False) 153 Tee('Diff in llvm/tools/clang:\n', log) 154 TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'tools', 'clang')], 155 log, fail_hard=False) 156 TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'tools', 'clang')], 157 log, fail_hard=False) 158 # TODO(thakis): compiler-rt is in projects/compiler-rt on Windows but 159 # llvm/compiler-rt elsewhere. So this diff call is currently only right on 160 # Windows. 161 Tee('Diff in llvm/compiler-rt:\n', log) 162 TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')], 163 log, fail_hard=False) 164 TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'compiler-rt')], 165 log, fail_hard=False) 166 Tee('Diff in llvm/projects/libcxx:\n', log) 167 TeeCmd(['svn', 'stat', os.path.join(LLVM_DIR, 'projects', 'libcxx')], 168 log, fail_hard=False) 169 TeeCmd(['svn', 'diff', os.path.join(LLVM_DIR, 'projects', 'libcxx')], 170 log, fail_hard=False) 171 172 Tee('Starting build\n', log) 173 174 # Do a clobber build. 175 shutil.rmtree(LLVM_BOOTSTRAP_DIR, ignore_errors=True) 176 shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True) 177 shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True) 178 179 opt_flags = [] 180 if sys.platform.startswith('linux'): 181 opt_flags += ['--lto-gold-plugin'] 182 build_cmd = [sys.executable, os.path.join(THIS_DIR, 'update.py'), 183 '--bootstrap', '--force-local-build', 184 '--run-tests'] + opt_flags 185 TeeCmd(build_cmd, log) 186 187 stamp = open(STAMP_FILE).read().rstrip() 188 if stamp != expected_stamp: 189 print 'Actual stamp (%s) != expected stamp (%s).' % (stamp, expected_stamp) 190 return 1 191 192 shutil.rmtree(pdir, ignore_errors=True) 193 194 # Copy a whitelist of files to the directory we're going to tar up. 195 # This supports the same patterns that the fnmatch module understands. 196 exe_ext = '.exe' if sys.platform == 'win32' else '' 197 want = ['bin/llvm-symbolizer' + exe_ext, 198 'bin/sancov' + exe_ext, 199 'lib/clang/*/asan_blacklist.txt', 200 'lib/clang/*/cfi_blacklist.txt', 201 # Copy built-in headers (lib/clang/3.x.y/include). 202 'lib/clang/*/include/*', 203 ] 204 if sys.platform == 'win32': 205 want.append('bin/clang-cl.exe') 206 want.append('bin/lld-link.exe') 207 else: 208 so_ext = 'dylib' if sys.platform == 'darwin' else 'so' 209 want.extend(['bin/clang', 210 'lib/libFindBadConstructs.' + so_ext, 211 'lib/libBlinkGCPlugin.' + so_ext, 212 ]) 213 if sys.platform == 'darwin': 214 want.extend([# Copy only the OSX and iossim (ASan and profile) runtime 215 # libraries: 216 'lib/clang/*/lib/darwin/*asan_osx*', 217 'lib/clang/*/lib/darwin/*asan_iossim*', 218 'lib/clang/*/lib/darwin/*profile_osx*', 219 'lib/clang/*/lib/darwin/*profile_iossim*', 220 ]) 221 elif sys.platform.startswith('linux'): 222 # Copy the libstdc++.so.6 we linked Clang against so it can run. 223 want.append('lib/libstdc++.so.6') 224 # Copy only 225 # lib/clang/*/lib/linux/libclang_rt.{[atm]san,san,ubsan,profile}-*.a , 226 # but not dfsan. 227 want.extend(['lib/clang/*/lib/linux/*[atm]san*', 228 'lib/clang/*/lib/linux/*ubsan*', 229 'lib/clang/*/lib/linux/*libclang_rt.san*', 230 'lib/clang/*/lib/linux/*profile*', 231 'lib/clang/*/msan_blacklist.txt', 232 ]) 233 elif sys.platform == 'win32': 234 want.extend(['lib/clang/*/lib/windows/clang_rt.asan*.dll', 235 'lib/clang/*/lib/windows/clang_rt.asan*.lib', 236 'lib/clang/*/include_sanitizer/*', 237 ]) 238 239 for root, dirs, files in os.walk(LLVM_RELEASE_DIR): 240 # root: third_party/llvm-build/Release+Asserts/lib/..., rel_root: lib/... 241 rel_root = root[len(LLVM_RELEASE_DIR)+1:] 242 rel_files = [os.path.join(rel_root, f) for f in files] 243 wanted_files = list(set(itertools.chain.from_iterable( 244 fnmatch.filter(rel_files, p) for p in want))) 245 if wanted_files: 246 # Guaranteed to not yet exist at this point: 247 os.makedirs(os.path.join(pdir, rel_root)) 248 for f in wanted_files: 249 src = os.path.join(LLVM_RELEASE_DIR, f) 250 dest = os.path.join(pdir, f) 251 shutil.copy(src, dest) 252 # Strip libraries. 253 if sys.platform == 'darwin' and f.endswith('.dylib'): 254 subprocess.call(['strip', '-x', dest]) 255 elif (sys.platform.startswith('linux') and 256 os.path.splitext(f)[1] in ['.so', '.a']): 257 subprocess.call(['strip', '-g', dest]) 258 259 # Set up symlinks. 260 if sys.platform != 'win32': 261 os.symlink('clang', os.path.join(pdir, 'bin', 'clang++')) 262 os.symlink('clang', os.path.join(pdir, 'bin', 'clang-cl')) 263 264 # Copy libc++ headers. 265 if sys.platform == 'darwin': 266 shutil.copytree(os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'include', 'c++'), 267 os.path.join(pdir, 'include', 'c++')) 268 269 # Copy buildlog over. 270 shutil.copy('buildlog.txt', pdir) 271 272 # Create archive. 273 tar_entries = ['bin', 'lib', 'buildlog.txt'] 274 if sys.platform == 'darwin': 275 tar_entries += ['include'] 276 with tarfile.open(pdir + '.tgz', 'w:gz') as tar: 277 for entry in tar_entries: 278 tar.add(os.path.join(pdir, entry), arcname=entry, filter=PrintTarProgress) 279 280 MaybeUpload(args, pdir, platform) 281 282 # Zip up gold plugin on Linux. 283 if sys.platform.startswith('linux'): 284 shutil.rmtree(golddir, ignore_errors=True) 285 os.makedirs(os.path.join(golddir, 'lib')) 286 shutil.copy(os.path.join(LLVM_LTO_GOLD_PLUGIN_DIR, 'lib', 'LLVMgold.so'), 287 os.path.join(golddir, 'lib')) 288 with tarfile.open(golddir + '.tgz', 'w:gz') as tar: 289 tar.add(os.path.join(golddir, 'lib'), arcname='lib', 290 filter=PrintTarProgress) 291 MaybeUpload(args, golddir, platform) 292 293 # Zip up llvm-objdump for sanitizer coverage. 294 objdumpdir = 'llvmobjdump-' + stamp 295 shutil.rmtree(objdumpdir, ignore_errors=True) 296 os.makedirs(os.path.join(objdumpdir, 'bin')) 297 shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-objdump' + exe_ext), 298 os.path.join(objdumpdir, 'bin')) 299 with tarfile.open(objdumpdir + '.tgz', 'w:gz') as tar: 300 tar.add(os.path.join(objdumpdir, 'bin'), arcname='bin', 301 filter=PrintTarProgress) 302 MaybeUpload(args, objdumpdir, platform) 303 304 # FIXME: Warn if the file already exists on the server. 305 306 307 if __name__ == '__main__': 308 sys.exit(main()) 309
