1 #!/usr/bin/env python 2 # 3 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========# 4 # 5 # The LLVM Compiler Infrastructure 6 # 7 # This file is distributed under the University of Illinois Open Source 8 # License. See LICENSE.TXT for details. 9 # 10 # ==------------------------------------------------------------------------==# 11 12 """ 13 git-llvm integration 14 ==================== 15 16 This file provides integration for git. 17 """ 18 19 from __future__ import print_function 20 import argparse 21 import collections 22 import contextlib 23 import errno 24 import os 25 import re 26 import subprocess 27 import sys 28 import tempfile 29 import time 30 assert sys.version_info >= (2, 7) 31 32 33 # It's *almost* a straightforward mapping from the monorepo to svn... 34 GIT_TO_SVN_DIR = { 35 d: (d + '/trunk') 36 for d in [ 37 'clang-tools-extra', 38 'compiler-rt', 39 'debuginfo-tests', 40 'dragonegg', 41 'klee', 42 'libclc', 43 'libcxx', 44 'libcxxabi', 45 'libunwind', 46 'lld', 47 'lldb', 48 'llgo', 49 'llvm', 50 'openmp', 51 'parallel-libs', 52 'polly', 53 ] 54 } 55 GIT_TO_SVN_DIR.update({'clang': 'cfe/trunk'}) 56 57 VERBOSE = False 58 QUIET = False 59 dev_null_fd = None 60 61 62 def eprint(*args, **kwargs): 63 print(*args, file=sys.stderr, **kwargs) 64 65 66 def log(*args, **kwargs): 67 if QUIET: 68 return 69 print(*args, **kwargs) 70 71 72 def log_verbose(*args, **kwargs): 73 if not VERBOSE: 74 return 75 print(*args, **kwargs) 76 77 78 def die(msg): 79 eprint(msg) 80 sys.exit(1) 81 82 83 def first_dirname(d): 84 while True: 85 (head, tail) = os.path.split(d) 86 if not head or head == '/': 87 return tail 88 d = head 89 90 91 def get_dev_null(): 92 """Lazily create a /dev/null fd for use in shell()""" 93 global dev_null_fd 94 if dev_null_fd is None: 95 dev_null_fd = open(os.devnull, 'w') 96 return dev_null_fd 97 98 99 def shell(cmd, strip=True, cwd=None, stdin=None, die_on_failure=True, 100 ignore_errors=False): 101 log_verbose('Running: %s' % ' '.join(cmd)) 102 103 err_pipe = subprocess.PIPE 104 if ignore_errors: 105 # Silence errors if requested. 106 err_pipe = get_dev_null() 107 108 start = time.time() 109 p = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=err_pipe, 110 stdin=subprocess.PIPE, universal_newlines=True) 111 stdout, stderr = p.communicate(input=stdin) 112 elapsed = time.time() - start 113 114 log_verbose('Command took %0.1fs' % elapsed) 115 116 if p.returncode == 0 or ignore_errors: 117 if stderr and not ignore_errors: 118 eprint('`%s` printed to stderr:' % ' '.join(cmd)) 119 eprint(stderr.rstrip()) 120 if strip: 121 stdout = stdout.rstrip('\r\n') 122 return stdout 123 err_msg = '`%s` returned %s' % (' '.join(cmd), p.returncode) 124 eprint(err_msg) 125 if stderr: 126 eprint(stderr.rstrip()) 127 if die_on_failure: 128 sys.exit(2) 129 raise RuntimeError(err_msg) 130 131 132 def git(*cmd, **kwargs): 133 return shell(['git'] + list(cmd), kwargs.get('strip', True)) 134 135 136 def svn(cwd, *cmd, **kwargs): 137 # TODO: Better way to do default arg when we have *cmd? 138 return shell(['svn'] + list(cmd), cwd=cwd, stdin=kwargs.get('stdin', None), 139 ignore_errors=kwargs.get('ignore_errors', None)) 140 141 def program_exists(cmd): 142 if sys.platform == 'win32' and not cmd.endswith('.exe'): 143 cmd += '.exe' 144 for path in os.environ["PATH"].split(os.pathsep): 145 if os.access(os.path.join(path, cmd), os.X_OK): 146 return True 147 return False 148 149 def get_default_rev_range(): 150 # Get the branch tracked by the current branch, as set by 151 # git branch --set-upstream-to See http://serverfault.com/a/352236/38694. 152 cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD') 153 upstream_branch = git('for-each-ref', '--format=%(upstream:short)', 154 cur_branch) 155 if not upstream_branch: 156 upstream_branch = 'origin/master' 157 158 # Get the newest common ancestor between HEAD and our upstream branch. 159 upstream_rev = git('merge-base', 'HEAD', upstream_branch) 160 return '%s..' % upstream_rev 161 162 163 def get_revs_to_push(rev_range): 164 if not rev_range: 165 rev_range = get_default_rev_range() 166 # Use git show rather than some plumbing command to figure out which revs 167 # are in rev_range because it handles single revs (HEAD^) and ranges 168 # (foo..bar) like we want. 169 revs = git('show', '--reverse', '--quiet', 170 '--pretty=%h', rev_range).splitlines() 171 if not revs: 172 die('Nothing to push: No revs in range %s.' % rev_range) 173 return revs 174 175 176 def clean_and_update_svn(svn_repo): 177 svn(svn_repo, 'revert', '-R', '.') 178 179 # Unfortunately it appears there's no svn equivalent for git clean, so we 180 # have to do it ourselves. 181 for line in svn(svn_repo, 'status', '--no-ignore').split('\n'): 182 if not line.startswith('?'): 183 continue 184 filename = line[1:].strip() 185 os.remove(os.path.join(svn_repo, filename)) 186 187 svn(svn_repo, 'update', *list(GIT_TO_SVN_DIR.values())) 188 189 190 def svn_init(svn_root): 191 if not os.path.exists(svn_root): 192 log('Creating svn staging directory: (%s)' % (svn_root)) 193 os.makedirs(svn_root) 194 log('This is a one-time initialization, please be patient for a few' 195 ' minutes...') 196 svn(svn_root, 'checkout', '--depth=immediates', 197 'https://llvm.org/svn/llvm-project/', '.') 198 svn(svn_root, 'update', *list(GIT_TO_SVN_DIR.values())) 199 log("svn staging area ready in '%s'" % svn_root) 200 if not os.path.isdir(svn_root): 201 die("Can't initialize svn staging dir (%s)" % svn_root) 202 203 204 def fix_eol_style_native(rev, sr, svn_sr_path): 205 """Fix line endings before applying patches with Unix endings 206 207 SVN on Windows will check out files with CRLF for files with the 208 svn:eol-style property set to "native". This breaks `git apply`, which 209 typically works with Unix-line ending patches. Work around the problem here 210 by doing a dos2unix up front for files with svn:eol-style set to "native". 211 SVN will not commit a mass line ending re-doing because it detects the line 212 ending format for files with this property. 213 """ 214 files = git('diff-tree', '--no-commit-id', '--name-only', '-r', rev, '--', 215 sr).split('\n') 216 files = [f.split('/', 1)[1] for f in files] 217 # Skip files that don't exist in SVN yet. 218 files = [f for f in files if os.path.exists(os.path.join(svn_sr_path, f))] 219 # Use ignore_errors because 'svn propget' prints errors if the file doesn't 220 # have the named property. There doesn't seem to be a way to suppress that. 221 eol_props = svn(svn_sr_path, 'propget', 'svn:eol-style', *files, 222 ignore_errors=True) 223 crlf_files = [] 224 if len(files) == 1: 225 # No need to split propget output on ' - ' when we have one file. 226 if eol_props.strip() == 'native': 227 crlf_files = files 228 else: 229 for eol_prop in eol_props.split('\n'): 230 # Remove spare CR. 231 eol_prop = eol_prop.strip('\r') 232 if not eol_prop: 233 continue 234 prop_parts = eol_prop.rsplit(' - ', 1) 235 if len(prop_parts) != 2: 236 eprint("unable to parse svn propget line:") 237 eprint(eol_prop) 238 continue 239 (f, eol_style) = prop_parts 240 if eol_style == 'native': 241 crlf_files.append(f) 242 # Reformat all files with native SVN line endings to Unix format. SVN knows 243 # files with native line endings are text files. It will commit just the 244 # diff, and not a mass line ending change. 245 shell(['dos2unix', '-q'] + crlf_files, cwd=svn_sr_path) 246 247 248 def svn_push_one_rev(svn_repo, rev, dry_run): 249 files = git('diff-tree', '--no-commit-id', '--name-only', '-r', 250 rev).split('\n') 251 subrepos = {first_dirname(f) for f in files} 252 if not subrepos: 253 raise RuntimeError('Empty diff for rev %s?' % rev) 254 255 status = svn(svn_repo, 'status', '--no-ignore') 256 if status: 257 die("Can't push git rev %s because svn status is not empty:\n%s" % 258 (rev, status)) 259 260 for sr in subrepos: 261 svn_sr_path = os.path.join(svn_repo, GIT_TO_SVN_DIR[sr]) 262 if os.name == 'nt': 263 fix_eol_style_native(rev, sr, svn_sr_path) 264 diff = git('show', '--binary', rev, '--', sr, strip=False) 265 # git is the only thing that can handle its own patches... 266 log_verbose('Apply patch: %s' % diff) 267 try: 268 shell(['git', 'apply', '-p2', '-'], cwd=svn_sr_path, stdin=diff, 269 die_on_failure=False) 270 except RuntimeError as e: 271 eprint("Patch doesn't apply: maybe you should try `git pull -r` " 272 "first?") 273 sys.exit(2) 274 275 status_lines = svn(svn_repo, 'status', '--no-ignore').split('\n') 276 277 for l in (l for l in status_lines if (l.startswith('?') or 278 l.startswith('I'))): 279 svn(svn_repo, 'add', '--no-ignore', l[1:].strip()) 280 for l in (l for l in status_lines if l.startswith('!')): 281 svn(svn_repo, 'remove', l[1:].strip()) 282 283 # Now we're ready to commit. 284 commit_msg = git('show', '--pretty=%B', '--quiet', rev) 285 if not dry_run: 286 log(svn(svn_repo, 'commit', '-m', commit_msg, '--force-interactive')) 287 log('Committed %s to svn.' % rev) 288 else: 289 log("Would have committed %s to svn, if this weren't a dry run." % rev) 290 291 292 def cmd_push(args): 293 '''Push changes back to SVN: this is extracted from Justin Lebar's script 294 available here: https://github.com/jlebar/llvm-repo-tools/ 295 296 Note: a current limitation is that git does not track file rename, so they 297 will show up in SVN as delete+add. 298 ''' 299 # Get the git root 300 git_root = git('rev-parse', '--show-toplevel') 301 if not os.path.isdir(git_root): 302 die("Can't find git root dir") 303 304 # Push from the root of the git repo 305 os.chdir(git_root) 306 307 # We need a staging area for SVN, let's hide it in the .git directory. 308 dot_git_dir = git('rev-parse', '--git-common-dir') 309 svn_root = os.path.join(dot_git_dir, 'llvm-upstream-svn') 310 svn_init(svn_root) 311 312 rev_range = args.rev_range 313 dry_run = args.dry_run 314 revs = get_revs_to_push(rev_range) 315 log('Pushing %d commit%s:\n%s' % 316 (len(revs), 's' if len(revs) != 1 317 else '', '\n'.join(' ' + git('show', '--oneline', '--quiet', c) 318 for c in revs))) 319 for r in revs: 320 clean_and_update_svn(svn_root) 321 svn_push_one_rev(svn_root, r, dry_run) 322 323 324 if __name__ == '__main__': 325 if not program_exists('svn'): 326 die('error: git-llvm needs svn command, but svn is not installed.') 327 328 argv = sys.argv[1:] 329 p = argparse.ArgumentParser( 330 prog='git llvm', formatter_class=argparse.RawDescriptionHelpFormatter, 331 description=__doc__) 332 subcommands = p.add_subparsers(title='subcommands', 333 description='valid subcommands', 334 help='additional help') 335 verbosity_group = p.add_mutually_exclusive_group() 336 verbosity_group.add_argument('-q', '--quiet', action='store_true', 337 help='print less information') 338 verbosity_group.add_argument('-v', '--verbose', action='store_true', 339 help='print more information') 340 341 parser_push = subcommands.add_parser( 342 'push', description=cmd_push.__doc__, 343 help='push changes back to the LLVM SVN repository') 344 parser_push.add_argument( 345 '-n', 346 '--dry-run', 347 dest='dry_run', 348 action='store_true', 349 help='Do everything other than commit to svn. Leaves junk in the svn ' 350 'repo, so probably will not work well if you try to commit more ' 351 'than one rev.') 352 parser_push.add_argument( 353 'rev_range', 354 metavar='GIT_REVS', 355 type=str, 356 nargs='?', 357 help="revs to push (default: everything not in the branch's " 358 'upstream, or not in origin/master if the branch lacks ' 359 'an explicit upstream)') 360 parser_push.set_defaults(func=cmd_push) 361 args = p.parse_args(argv) 362 VERBOSE = args.verbose 363 QUIET = args.quiet 364 365 # Dispatch to the right subcommand 366 args.func(args) 367
