1 #!/usr/bin/python 2 # Copyright 2014 Google Inc. 3 # 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """Parse a DEPS file and git checkout all of the dependencies. 9 10 Args: 11 An optional list of deps_os values. 12 13 Environment Variables: 14 GIT_EXECUTABLE: path to "git" binary; if unset, will look for one of 15 ['git', 'git.exe', 'git.bat'] in your default path. 16 17 GIT_SYNC_DEPS_PATH: file to get the dependency list from; if unset, 18 will use the file ../DEPS relative to this script's directory. 19 20 GIT_SYNC_DEPS_QUIET: if set to non-empty string, suppress messages. 21 22 Git Config: 23 To disable syncing of a single repository: 24 cd path/to/repository 25 git config sync-deps.disable true 26 27 To re-enable sync: 28 cd path/to/repository 29 git config --unset sync-deps.disable 30 """ 31 32 33 import os 34 import subprocess 35 import sys 36 import threading 37 38 39 def git_executable(): 40 """Find the git executable. 41 42 Returns: 43 A string suitable for passing to subprocess functions, or None. 44 """ 45 envgit = os.environ.get('GIT_EXECUTABLE') 46 searchlist = ['git', 'git.exe', 'git.bat'] 47 if envgit: 48 searchlist.insert(0, envgit) 49 with open(os.devnull, 'w') as devnull: 50 for git in searchlist: 51 try: 52 subprocess.call([git, '--version'], stdout=devnull) 53 except (OSError,): 54 continue 55 return git 56 return None 57 58 59 DEFAULT_DEPS_PATH = os.path.normpath( 60 os.path.join(os.path.dirname(__file__), os.pardir, 'DEPS')) 61 62 63 def usage(deps_file_path = None): 64 sys.stderr.write( 65 'Usage: run to grab dependencies, with optional platform support:\n') 66 sys.stderr.write(' python %s' % __file__) 67 if deps_file_path: 68 for deps_os in parse_file_to_dict(deps_file_path)['deps_os']: 69 sys.stderr.write(' [%s]' % deps_os) 70 else: 71 sys.stderr.write(' [DEPS_OS...]') 72 sys.stderr.write('\n\n') 73 sys.stderr.write(__doc__) 74 75 76 def git_repository_sync_is_disabled(git, directory): 77 try: 78 disable = subprocess.check_output( 79 [git, 'config', 'sync-deps.disable'], cwd=directory) 80 return disable.lower().strip() in ['true', '1', 'yes', 'on'] 81 except subprocess.CalledProcessError: 82 return False 83 84 85 def is_git_toplevel(git, directory): 86 """Return true iff the directory is the top level of a Git repository. 87 88 Args: 89 git (string) the git executable 90 91 directory (string) the path into which the repository 92 is expected to be checked out. 93 """ 94 try: 95 toplevel = subprocess.check_output( 96 [git, 'rev-parse', '--show-toplevel'], cwd=directory).strip() 97 return os.path.realpath(directory) == os.path.realpath(toplevel) 98 except subprocess.CalledProcessError: 99 return False 100 101 102 def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): 103 """Checkout (and clone if needed) a Git repository. 104 105 Args: 106 git (string) the git executable 107 108 repo (string) the location of the repository, suitable 109 for passing to `git clone`. 110 111 checkoutable (string) a tag, branch, or commit, suitable for 112 passing to `git checkout` 113 114 directory (string) the path into which the repository 115 should be checked out. 116 117 verbose (boolean) 118 119 Raises an exception if any calls to git fail. 120 """ 121 if not os.path.isdir(directory): 122 subprocess.check_call( 123 [git, 'clone', '--quiet', repo, directory]) 124 125 if not is_git_toplevel(git, directory): 126 # if the directory exists, but isn't a git repo, you will modify 127 # the parent repostory, which isn't what you want. 128 sys.stdout.write('%s\n IS NOT TOP-LEVEL GIT DIRECTORY.\n' % directory) 129 return 130 131 # Check to see if this repo is disabled. Quick return. 132 if git_repository_sync_is_disabled(git, directory): 133 sys.stdout.write('%s\n SYNC IS DISABLED.\n' % directory) 134 return 135 136 if 0 == subprocess.call( 137 [git, 'checkout', '--quiet', checkoutable], cwd=directory): 138 # if this succeeds, skip slow `git fetch`. 139 if verbose: 140 sys.stdout.write('%s\n @ %s\n' % (directory, checkoutable)) 141 return 142 143 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 144 145 if 0 != subprocess.call( 146 [git, 'checkout', '--quiet', checkoutable], cwd=directory): 147 subprocess.check_call( 148 [git, 'remote', 'set-url', 'origin', repo], cwd=directory) 149 subprocess.check_call([git, 'fetch', '--quiet'], cwd=directory) 150 subprocess.check_call([git, 'checkout', '--quiet'], cwd=directory) 151 152 if verbose: 153 sys.stdout.write('%s\n @ %s\n' % (directory, checkoutable)) # Success. 154 155 156 def parse_file_to_dict(path): 157 dictionary = {} 158 execfile(path, dictionary) 159 return dictionary 160 161 162 def git_sync_deps(deps_file_path, command_line_os_requests, verbose): 163 """Grab dependencies, with optional platform support. 164 165 Args: 166 deps_file_path (string) Path to the DEPS file. 167 168 command_line_os_requests (list of strings) Can be empty list. 169 List of strings that should each be a key in the deps_os 170 dictionary in the DEPS file. 171 172 Raises git Exceptions. 173 """ 174 git = git_executable() 175 assert git 176 177 deps_file_directory = os.path.dirname(deps_file_path) 178 deps_file = parse_file_to_dict(deps_file_path) 179 dependencies = deps_file['deps'].copy() 180 os_specific_dependencies = deps_file.get('deps_os', []) 181 for os_name in command_line_os_requests: 182 # Add OS-specific dependencies 183 if os_name in os_specific_dependencies: 184 dependencies.update(os_specific_dependencies[os_name]) 185 list_of_arg_lists = [] 186 for directory in dependencies: 187 if '@' in dependencies[directory]: 188 repo, checkoutable = dependencies[directory].split('@', 1) 189 else: 190 raise Exception("please specify commit or tag") 191 192 relative_directory = os.path.join(deps_file_directory, directory) 193 194 list_of_arg_lists.append( 195 (git, repo, checkoutable, relative_directory, verbose)) 196 197 multithread(git_checkout_to_directory, list_of_arg_lists) 198 199 for directory in deps_file.get('recursedeps', []): 200 recursive_path = os.path.join(deps_file_directory, directory, 'DEPS') 201 git_sync_deps(recursive_path, command_line_os_requests, verbose) 202 203 204 def multithread(function, list_of_arg_lists): 205 # for args in list_of_arg_lists: 206 # function(*args) 207 # return 208 threads = [] 209 for args in list_of_arg_lists: 210 thread = threading.Thread(None, function, None, args) 211 thread.start() 212 threads.append(thread) 213 for thread in threads: 214 thread.join() 215 216 217 def main(argv): 218 deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) 219 verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) 220 221 if '--help' in argv or '-h' in argv: 222 usage(deps_file_path) 223 return 1 224 225 git_sync_deps(deps_file_path, argv, verbose) 226 return 0 227 228 229 if __name__ == '__main__': 230 exit(main(sys.argv[1:])) 231
