1 #!/usr/bin/python2 2 # 3 # Copyright 2010 Google Inc. All Rights Reserved. 4 """Module for transferring files between various types of repositories.""" 5 6 from __future__ import print_function 7 8 __author__ = 'asharif (at] google.com (Ahmad Sharif)' 9 10 import argparse 11 import datetime 12 import json 13 import os 14 import re 15 import socket 16 import sys 17 import tempfile 18 19 from automation.clients.helper import perforce 20 from cros_utils import command_executer 21 from cros_utils import logger 22 from cros_utils import misc 23 24 # pylint: disable=anomalous-backslash-in-string 25 26 def GetCanonicalMappings(mappings): 27 canonical_mappings = [] 28 for mapping in mappings: 29 remote_path, local_path = mapping.split() 30 if local_path.endswith('/') and not remote_path.endswith('/'): 31 local_path = os.path.join(local_path, os.path.basename(remote_path)) 32 remote_path = remote_path.lstrip('/').split('/', 1)[1] 33 canonical_mappings.append(perforce.PathMapping(remote_path, local_path)) 34 return canonical_mappings 35 36 37 def SplitMapping(mapping): 38 parts = mapping.split() 39 assert len(parts) <= 2, 'Mapping %s invalid' % mapping 40 remote_path = parts[0] 41 if len(parts) == 2: 42 local_path = parts[1] 43 else: 44 local_path = '.' 45 return remote_path, local_path 46 47 48 class Repo(object): 49 """Basic repository base class.""" 50 51 def __init__(self, no_create_tmp_dir=False): 52 self.repo_type = None 53 self.address = None 54 self.mappings = None 55 self.revision = None 56 self.ignores = ['.gitignore', '.p4config', 'README.google'] 57 if no_create_tmp_dir: 58 self._root_dir = None 59 else: 60 self._root_dir = tempfile.mkdtemp() 61 self._ce = command_executer.GetCommandExecuter() 62 self._logger = logger.GetLogger() 63 64 def PullSources(self): 65 """Pull all sources into an internal dir.""" 66 pass 67 68 def SetupForPush(self): 69 """Setup a repository for pushing later.""" 70 pass 71 72 def PushSources(self, commit_message=None, dry_run=False, message_file=None): 73 """Push to the external repo with the commit message.""" 74 pass 75 76 def _RsyncExcludingRepoDirs(self, source_dir, dest_dir): 77 for f in os.listdir(source_dir): 78 if f in ['.git', '.svn', '.p4config']: 79 continue 80 dest_file = os.path.join(dest_dir, f) 81 source_file = os.path.join(source_dir, f) 82 if os.path.exists(dest_file): 83 command = 'rm -rf %s' % dest_file 84 self._ce.RunCommand(command) 85 command = 'rsync -a %s %s' % (source_file, dest_dir) 86 self._ce.RunCommand(command) 87 return 0 88 89 def MapSources(self, dest_dir): 90 """Copy sources from the internal dir to root_dir.""" 91 return self._RsyncExcludingRepoDirs(self._root_dir, dest_dir) 92 93 def GetRoot(self): 94 return self._root_dir 95 96 def SetRoot(self, directory): 97 self._root_dir = directory 98 99 def CleanupRoot(self): 100 command = 'rm -rf %s' % self._root_dir 101 return self._ce.RunCommand(command) 102 103 def __str__(self): 104 return '\n'.join(str(s) 105 for s in [self.repo_type, self.address, self.mappings]) 106 107 108 # Note - this type of repo is used only for "readonly", in other words, this 109 # only serves as a incoming repo. 110 class FileRepo(Repo): 111 """Class for file repositories.""" 112 113 def __init__(self, address, ignores=None): 114 Repo.__init__(self, no_create_tmp_dir=True) 115 self.repo_type = 'file' 116 self.address = address 117 self.mappings = None 118 self.branch = None 119 self.revision = '{0} (as of "{1}")'.format(address, datetime.datetime.now()) 120 self.gerrit = None 121 self._root_dir = self.address 122 if ignores: 123 self.ignores += ignores 124 125 def CleanupRoot(self): 126 """Override to prevent deletion.""" 127 pass 128 129 130 class P4Repo(Repo): 131 """Class for P4 repositories.""" 132 133 134 def __init__(self, address, mappings, revision=None): 135 Repo.__init__(self) 136 self.repo_type = 'p4' 137 self.address = address 138 self.mappings = mappings 139 self.revision = revision 140 141 def PullSources(self): 142 client_name = socket.gethostname() 143 client_name += tempfile.mkstemp()[1].replace('/', '-') 144 mappings = self.mappings 145 p4view = perforce.View('depot2', GetCanonicalMappings(mappings)) 146 p4client = perforce.CommandsFactory(self._root_dir, 147 p4view, 148 name=client_name) 149 command = p4client.SetupAndDo(p4client.Sync(self.revision)) 150 ret = self._ce.RunCommand(command) 151 assert ret == 0, 'Could not setup client.' 152 command = p4client.InCheckoutDir(p4client.SaveCurrentCLNumber()) 153 ret, o, _ = self._ce.RunCommandWOutput(command) 154 assert ret == 0, 'Could not get version from client.' 155 self.revision = re.search('^\d+$', o.strip(), re.MULTILINE).group(0) 156 command = p4client.InCheckoutDir(p4client.Remove()) 157 ret = self._ce.RunCommand(command) 158 assert ret == 0, 'Could not delete client.' 159 return 0 160 161 162 class SvnRepo(Repo): 163 """Class for svn repositories.""" 164 165 def __init__(self, address, mappings): 166 Repo.__init__(self) 167 self.repo_type = 'svn' 168 self.address = address 169 self.mappings = mappings 170 171 def PullSources(self): 172 with misc.WorkingDirectory(self._root_dir): 173 for mapping in self.mappings: 174 remote_path, local_path = SplitMapping(mapping) 175 command = 'svn co %s/%s %s' % (self.address, remote_path, local_path) 176 ret = self._ce.RunCommand(command) 177 if ret: 178 return ret 179 180 self.revision = '' 181 for mapping in self.mappings: 182 remote_path, local_path = SplitMapping(mapping) 183 command = 'cd %s && svnversion -c .' % (local_path) 184 ret, o, _ = self._ce.RunCommandWOutput(command) 185 self.revision += o.strip().split(':')[-1] 186 if ret: 187 return ret 188 return 0 189 190 191 class GitRepo(Repo): 192 """Class for git repositories.""" 193 194 def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None): 195 Repo.__init__(self) 196 self.repo_type = 'git' 197 self.address = address 198 self.branch = branch or 'master' 199 if ignores: 200 self.ignores += ignores 201 self.mappings = mappings 202 self.gerrit = gerrit 203 204 def _CloneSources(self): 205 with misc.WorkingDirectory(self._root_dir): 206 command = 'git clone %s .' % (self.address) 207 return self._ce.RunCommand(command) 208 209 def PullSources(self): 210 with misc.WorkingDirectory(self._root_dir): 211 ret = self._CloneSources() 212 if ret: 213 return ret 214 215 command = 'git checkout %s' % self.branch 216 ret = self._ce.RunCommand(command) 217 if ret: 218 return ret 219 220 command = 'git describe --always' 221 ret, o, _ = self._ce.RunCommandWOutput(command) 222 self.revision = o.strip() 223 return ret 224 225 def SetupForPush(self): 226 with misc.WorkingDirectory(self._root_dir): 227 ret = self._CloneSources() 228 logger.GetLogger().LogFatalIf(ret, 'Could not clone git repo %s.' % 229 self.address) 230 231 command = 'git branch -a | grep -wq %s' % self.branch 232 ret = self._ce.RunCommand(command) 233 234 if ret == 0: 235 if self.branch != 'master': 236 command = ('git branch --track %s remotes/origin/%s' % 237 (self.branch, self.branch)) 238 else: 239 command = 'pwd' 240 command += '&& git checkout %s' % self.branch 241 else: 242 command = 'git symbolic-ref HEAD refs/heads/%s' % self.branch 243 command += '&& rm -rf *' 244 ret = self._ce.RunCommand(command) 245 return ret 246 247 def CommitLocally(self, commit_message=None, message_file=None): 248 with misc.WorkingDirectory(self._root_dir): 249 command = 'pwd' 250 for ignore in self.ignores: 251 command += '&& echo \'%s\' >> .git/info/exclude' % ignore 252 command += '&& git add -Av .' 253 if message_file: 254 message_arg = '-F %s' % message_file 255 elif commit_message: 256 message_arg = '-m \'%s\'' % commit_message 257 else: 258 raise RuntimeError('No commit message given!') 259 command += '&& git commit -v %s' % message_arg 260 return self._ce.RunCommand(command) 261 262 def PushSources(self, commit_message=None, dry_run=False, message_file=None): 263 ret = self.CommitLocally(commit_message, message_file) 264 if ret: 265 return ret 266 push_args = '' 267 if dry_run: 268 push_args += ' -n ' 269 with misc.WorkingDirectory(self._root_dir): 270 if self.gerrit: 271 label = 'somelabel' 272 command = 'git remote add %s %s' % (label, self.address) 273 command += ('&& git push %s %s HEAD:refs/for/master' % 274 (push_args, label)) 275 else: 276 command = 'git push -v %s origin %s:%s' % (push_args, self.branch, 277 self.branch) 278 ret = self._ce.RunCommand(command) 279 return ret 280 281 def MapSources(self, root_dir): 282 if not self.mappings: 283 self._RsyncExcludingRepoDirs(self._root_dir, root_dir) 284 return 285 with misc.WorkingDirectory(self._root_dir): 286 for mapping in self.mappings: 287 remote_path, local_path = SplitMapping(mapping) 288 remote_path.rstrip('...') 289 local_path.rstrip('...') 290 full_local_path = os.path.join(root_dir, local_path) 291 ret = self._RsyncExcludingRepoDirs(remote_path, full_local_path) 292 if ret: 293 return ret 294 return 0 295 296 297 class RepoReader(object): 298 """Class for reading repositories.""" 299 300 def __init__(self, filename): 301 self.filename = filename 302 self.main_dict = {} 303 self.input_repos = [] 304 self.output_repos = [] 305 306 def ParseFile(self): 307 with open(self.filename) as f: 308 self.main_dict = json.load(f) 309 self.CreateReposFromDict(self.main_dict) 310 return [self.input_repos, self.output_repos] 311 312 def CreateReposFromDict(self, main_dict): 313 for key, repo_list in main_dict.items(): 314 for repo_dict in repo_list: 315 repo = self.CreateRepoFromDict(repo_dict) 316 if key == 'input': 317 self.input_repos.append(repo) 318 elif key == 'output': 319 self.output_repos.append(repo) 320 else: 321 logger.GetLogger().LogFatal('Unknown key: %s found' % key) 322 323 def CreateRepoFromDict(self, repo_dict): 324 repo_type = repo_dict.get('type', None) 325 repo_address = repo_dict.get('address', None) 326 repo_mappings = repo_dict.get('mappings', None) 327 repo_ignores = repo_dict.get('ignores', None) 328 repo_branch = repo_dict.get('branch', None) 329 gerrit = repo_dict.get('gerrit', None) 330 revision = repo_dict.get('revision', None) 331 332 if repo_type == 'p4': 333 repo = P4Repo(repo_address, repo_mappings, revision=revision) 334 elif repo_type == 'svn': 335 repo = SvnRepo(repo_address, repo_mappings) 336 elif repo_type == 'git': 337 repo = GitRepo(repo_address, 338 repo_branch, 339 mappings=repo_mappings, 340 ignores=repo_ignores, 341 gerrit=gerrit) 342 elif repo_type == 'file': 343 repo = FileRepo(repo_address) 344 else: 345 logger.GetLogger().LogFatal('Unknown repo type: %s' % repo_type) 346 return repo 347 348 349 @logger.HandleUncaughtExceptions 350 def Main(argv): 351 parser = argparse.ArgumentParser() 352 parser.add_argument('-i', 353 '--input_file', 354 dest='input_file', 355 help='The input file that contains repo descriptions.') 356 357 parser.add_argument('-n', 358 '--dry_run', 359 dest='dry_run', 360 action='store_true', 361 default=False, 362 help='Do a dry run of the push.') 363 364 parser.add_argument('-F', 365 '--message_file', 366 dest='message_file', 367 default=None, 368 help=('Use contents of the log file as the commit ' 369 'message.')) 370 371 options = parser.parse_args(argv) 372 if not options.input_file: 373 parser.print_help() 374 return 1 375 rr = RepoReader(options.input_file) 376 [input_repos, output_repos] = rr.ParseFile() 377 378 # Make sure FileRepo is not used as output destination. 379 for output_repo in output_repos: 380 if output_repo.repo_type == 'file': 381 logger.GetLogger().LogFatal( 382 'FileRepo is only supported as an input repo.') 383 384 for output_repo in output_repos: 385 ret = output_repo.SetupForPush() 386 if ret: 387 return ret 388 389 input_revisions = [] 390 for input_repo in input_repos: 391 ret = input_repo.PullSources() 392 if ret: 393 return ret 394 input_revisions.append(input_repo.revision) 395 396 for input_repo in input_repos: 397 for output_repo in output_repos: 398 ret = input_repo.MapSources(output_repo.GetRoot()) 399 if ret: 400 return ret 401 402 commit_message = 'Synced repos to: %s' % ','.join(input_revisions) 403 for output_repo in output_repos: 404 ret = output_repo.PushSources(commit_message=commit_message, 405 dry_run=options.dry_run, 406 message_file=options.message_file) 407 if ret: 408 return ret 409 410 if not options.dry_run: 411 for output_repo in output_repos: 412 output_repo.CleanupRoot() 413 for input_repo in input_repos: 414 input_repo.CleanupRoot() 415 416 return ret 417 418 419 if __name__ == '__main__': 420 retval = Main(sys.argv[1:]) 421 sys.exit(retval) 422
