1 #!/usr/bin/env python2 2 # 3 # Copyright 2010 Google Inc. All Rights Reserved. 4 """Module for transferring files between various types of repositories.""" 5 6 from __future__ import print_function 7 8 __author__ = 'asharif (at] google.com (Ahmad Sharif)' 9 10 import argparse 11 import datetime 12 import json 13 import os 14 import re 15 import socket 16 import sys 17 import tempfile 18 19 from automation.clients.helper import perforce 20 from cros_utils import command_executer 21 from cros_utils import logger 22 from cros_utils import misc 23 24 # pylint: disable=anomalous-backslash-in-string 25 26 27 def GetCanonicalMappings(mappings): 28 canonical_mappings = [] 29 for mapping in mappings: 30 remote_path, local_path = mapping.split() 31 if local_path.endswith('/') and not remote_path.endswith('/'): 32 local_path = os.path.join(local_path, os.path.basename(remote_path)) 33 remote_path = remote_path.lstrip('/').split('/', 1)[1] 34 canonical_mappings.append(perforce.PathMapping(remote_path, local_path)) 35 return canonical_mappings 36 37 38 def SplitMapping(mapping): 39 parts = mapping.split() 40 assert len(parts) <= 2, 'Mapping %s invalid' % mapping 41 remote_path = parts[0] 42 if len(parts) == 2: 43 local_path = parts[1] 44 else: 45 local_path = '.' 46 return remote_path, local_path 47 48 49 class Repo(object): 50 """Basic repository base class.""" 51 52 def __init__(self, no_create_tmp_dir=False): 53 self.repo_type = None 54 self.address = None 55 self.mappings = None 56 self.revision = None 57 self.ignores = ['.gitignore', '.p4config', 'README.google'] 58 if no_create_tmp_dir: 59 self._root_dir = None 60 else: 61 self._root_dir = tempfile.mkdtemp() 62 self._ce = command_executer.GetCommandExecuter() 63 self._logger = logger.GetLogger() 64 65 def PullSources(self): 66 """Pull all sources into an internal dir.""" 67 pass 68 69 def SetupForPush(self): 70 """Setup a repository for pushing later.""" 71 pass 72 73 def PushSources(self, commit_message=None, dry_run=False, message_file=None): 74 """Push to the external repo with the commit message.""" 75 pass 76 77 def _RsyncExcludingRepoDirs(self, source_dir, dest_dir): 78 for f in os.listdir(source_dir): 79 if f in ['.git', '.svn', '.p4config']: 80 continue 81 dest_file = os.path.join(dest_dir, f) 82 source_file = os.path.join(source_dir, f) 83 if os.path.exists(dest_file): 84 command = 'rm -rf %s' % dest_file 85 self._ce.RunCommand(command) 86 command = 'rsync -a %s %s' % (source_file, dest_dir) 87 self._ce.RunCommand(command) 88 return 0 89 90 def MapSources(self, dest_dir): 91 """Copy sources from the internal dir to root_dir.""" 92 return self._RsyncExcludingRepoDirs(self._root_dir, dest_dir) 93 94 def GetRoot(self): 95 return self._root_dir 96 97 def SetRoot(self, directory): 98 self._root_dir = directory 99 100 def CleanupRoot(self): 101 command = 'rm -rf %s' % self._root_dir 102 return self._ce.RunCommand(command) 103 104 def __str__(self): 105 return '\n'.join( 106 str(s) for s in [self.repo_type, self.address, self.mappings]) 107 108 109 # Note - this type of repo is used only for "readonly", in other words, this 110 # only serves as a incoming repo. 111 class FileRepo(Repo): 112 """Class for file repositories.""" 113 114 def __init__(self, address, ignores=None): 115 Repo.__init__(self, no_create_tmp_dir=True) 116 self.repo_type = 'file' 117 self.address = address 118 self.mappings = None 119 self.branch = None 120 self.revision = '{0} (as of "{1}")'.format(address, datetime.datetime.now()) 121 self.gerrit = None 122 self._root_dir = self.address 123 if ignores: 124 self.ignores += ignores 125 126 def CleanupRoot(self): 127 """Override to prevent deletion.""" 128 pass 129 130 131 class P4Repo(Repo): 132 """Class for P4 repositories.""" 133 134 def __init__(self, address, mappings, revision=None): 135 Repo.__init__(self) 136 self.repo_type = 'p4' 137 self.address = address 138 self.mappings = mappings 139 self.revision = revision 140 141 def PullSources(self): 142 client_name = socket.gethostname() 143 client_name += tempfile.mkstemp()[1].replace('/', '-') 144 mappings = self.mappings 145 p4view = perforce.View('depot2', GetCanonicalMappings(mappings)) 146 p4client = perforce.CommandsFactory( 147 self._root_dir, p4view, name=client_name) 148 command = p4client.SetupAndDo(p4client.Sync(self.revision)) 149 ret = self._ce.RunCommand(command) 150 assert ret == 0, 'Could not setup client.' 151 command = p4client.InCheckoutDir(p4client.SaveCurrentCLNumber()) 152 ret, o, _ = self._ce.RunCommandWOutput(command) 153 assert ret == 0, 'Could not get version from client.' 154 self.revision = re.search('^\d+$', o.strip(), re.MULTILINE).group(0) 155 command = p4client.InCheckoutDir(p4client.Remove()) 156 ret = self._ce.RunCommand(command) 157 assert ret == 0, 'Could not delete client.' 158 return 0 159 160 161 class SvnRepo(Repo): 162 """Class for svn repositories.""" 163 164 def __init__(self, address, mappings): 165 Repo.__init__(self) 166 self.repo_type = 'svn' 167 self.address = address 168 self.mappings = mappings 169 170 def PullSources(self): 171 with misc.WorkingDirectory(self._root_dir): 172 for mapping in self.mappings: 173 remote_path, local_path = SplitMapping(mapping) 174 command = 'svn co %s/%s %s' % (self.address, remote_path, local_path) 175 ret = self._ce.RunCommand(command) 176 if ret: 177 return ret 178 179 self.revision = '' 180 for mapping in self.mappings: 181 remote_path, local_path = SplitMapping(mapping) 182 command = 'cd %s && svnversion -c .' % (local_path) 183 ret, o, _ = self._ce.RunCommandWOutput(command) 184 self.revision += o.strip().split(':')[-1] 185 if ret: 186 return ret 187 return 0 188 189 190 class GitRepo(Repo): 191 """Class for git repositories.""" 192 193 def __init__(self, address, branch, mappings=None, ignores=None, gerrit=None): 194 Repo.__init__(self) 195 self.repo_type = 'git' 196 self.address = address 197 self.branch = branch or 'master' 198 if ignores: 199 self.ignores += ignores 200 self.mappings = mappings 201 self.gerrit = gerrit 202 203 def _CloneSources(self): 204 with misc.WorkingDirectory(self._root_dir): 205 command = 'git clone %s .' % (self.address) 206 return self._ce.RunCommand(command) 207 208 def PullSources(self): 209 with misc.WorkingDirectory(self._root_dir): 210 ret = self._CloneSources() 211 if ret: 212 return ret 213 214 command = 'git checkout %s' % self.branch 215 ret = self._ce.RunCommand(command) 216 if ret: 217 return ret 218 219 command = 'git describe --always' 220 ret, o, _ = self._ce.RunCommandWOutput(command) 221 self.revision = o.strip() 222 return ret 223 224 def SetupForPush(self): 225 with misc.WorkingDirectory(self._root_dir): 226 ret = self._CloneSources() 227 logger.GetLogger().LogFatalIf( 228 ret, 'Could not clone git repo %s.' % self.address) 229 230 command = 'git branch -a | grep -wq %s' % self.branch 231 ret = self._ce.RunCommand(command) 232 233 if ret == 0: 234 if self.branch != 'master': 235 command = ('git branch --track %s remotes/origin/%s' % (self.branch, 236 self.branch)) 237 else: 238 command = 'pwd' 239 command += '&& git checkout %s' % self.branch 240 else: 241 command = 'git symbolic-ref HEAD refs/heads/%s' % self.branch 242 command += '&& rm -rf *' 243 ret = self._ce.RunCommand(command) 244 return ret 245 246 def CommitLocally(self, commit_message=None, message_file=None): 247 with misc.WorkingDirectory(self._root_dir): 248 command = 'pwd' 249 for ignore in self.ignores: 250 command += '&& echo \'%s\' >> .git/info/exclude' % ignore 251 command += '&& git add -Av .' 252 if message_file: 253 message_arg = '-F %s' % message_file 254 elif commit_message: 255 message_arg = '-m \'%s\'' % commit_message 256 else: 257 raise RuntimeError('No commit message given!') 258 command += '&& git commit -v %s' % message_arg 259 return self._ce.RunCommand(command) 260 261 def PushSources(self, commit_message=None, dry_run=False, message_file=None): 262 ret = self.CommitLocally(commit_message, message_file) 263 if ret: 264 return ret 265 push_args = '' 266 if dry_run: 267 push_args += ' -n ' 268 with misc.WorkingDirectory(self._root_dir): 269 if self.gerrit: 270 label = 'somelabel' 271 command = 'git remote add %s %s' % (label, self.address) 272 command += ('&& git push %s %s HEAD:refs/for/master' % (push_args, 273 label)) 274 else: 275 command = 'git push -v %s origin %s:%s' % (push_args, self.branch, 276 self.branch) 277 ret = self._ce.RunCommand(command) 278 return ret 279 280 def MapSources(self, root_dir): 281 if not self.mappings: 282 self._RsyncExcludingRepoDirs(self._root_dir, root_dir) 283 return 284 with misc.WorkingDirectory(self._root_dir): 285 for mapping in self.mappings: 286 remote_path, local_path = SplitMapping(mapping) 287 remote_path.rstrip('...') 288 local_path.rstrip('...') 289 full_local_path = os.path.join(root_dir, local_path) 290 ret = self._RsyncExcludingRepoDirs(remote_path, full_local_path) 291 if ret: 292 return ret 293 return 0 294 295 296 class RepoReader(object): 297 """Class for reading repositories.""" 298 299 def __init__(self, filename): 300 self.filename = filename 301 self.main_dict = {} 302 self.input_repos = [] 303 self.output_repos = [] 304 305 def ParseFile(self): 306 with open(self.filename) as f: 307 self.main_dict = json.load(f) 308 self.CreateReposFromDict(self.main_dict) 309 return [self.input_repos, self.output_repos] 310 311 def CreateReposFromDict(self, main_dict): 312 for key, repo_list in main_dict.items(): 313 for repo_dict in repo_list: 314 repo = self.CreateRepoFromDict(repo_dict) 315 if key == 'input': 316 self.input_repos.append(repo) 317 elif key == 'output': 318 self.output_repos.append(repo) 319 else: 320 logger.GetLogger().LogFatal('Unknown key: %s found' % key) 321 322 def CreateRepoFromDict(self, repo_dict): 323 repo_type = repo_dict.get('type', None) 324 repo_address = repo_dict.get('address', None) 325 repo_mappings = repo_dict.get('mappings', None) 326 repo_ignores = repo_dict.get('ignores', None) 327 repo_branch = repo_dict.get('branch', None) 328 gerrit = repo_dict.get('gerrit', None) 329 revision = repo_dict.get('revision', None) 330 331 if repo_type == 'p4': 332 repo = P4Repo(repo_address, repo_mappings, revision=revision) 333 elif repo_type == 'svn': 334 repo = SvnRepo(repo_address, repo_mappings) 335 elif repo_type == 'git': 336 repo = GitRepo( 337 repo_address, 338 repo_branch, 339 mappings=repo_mappings, 340 ignores=repo_ignores, 341 gerrit=gerrit) 342 elif repo_type == 'file': 343 repo = FileRepo(repo_address) 344 else: 345 logger.GetLogger().LogFatal('Unknown repo type: %s' % repo_type) 346 return repo 347 348 349 @logger.HandleUncaughtExceptions 350 def Main(argv): 351 parser = argparse.ArgumentParser() 352 parser.add_argument( 353 '-i', 354 '--input_file', 355 dest='input_file', 356 help='The input file that contains repo descriptions.') 357 358 parser.add_argument( 359 '-n', 360 '--dry_run', 361 dest='dry_run', 362 action='store_true', 363 default=False, 364 help='Do a dry run of the push.') 365 366 parser.add_argument( 367 '-F', 368 '--message_file', 369 dest='message_file', 370 default=None, 371 help=('Use contents of the log file as the commit ' 372 'message.')) 373 374 options = parser.parse_args(argv) 375 if not options.input_file: 376 parser.print_help() 377 return 1 378 rr = RepoReader(options.input_file) 379 [input_repos, output_repos] = rr.ParseFile() 380 381 # Make sure FileRepo is not used as output destination. 382 for output_repo in output_repos: 383 if output_repo.repo_type == 'file': 384 logger.GetLogger().LogFatal( 385 'FileRepo is only supported as an input repo.') 386 387 for output_repo in output_repos: 388 ret = output_repo.SetupForPush() 389 if ret: 390 return ret 391 392 input_revisions = [] 393 for input_repo in input_repos: 394 ret = input_repo.PullSources() 395 if ret: 396 return ret 397 input_revisions.append(input_repo.revision) 398 399 for input_repo in input_repos: 400 for output_repo in output_repos: 401 ret = input_repo.MapSources(output_repo.GetRoot()) 402 if ret: 403 return ret 404 405 commit_message = 'Synced repos to: %s' % ','.join(input_revisions) 406 for output_repo in output_repos: 407 ret = output_repo.PushSources( 408 commit_message=commit_message, 409 dry_run=options.dry_run, 410 message_file=options.message_file) 411 if ret: 412 return ret 413 414 if not options.dry_run: 415 for output_repo in output_repos: 416 output_repo.CleanupRoot() 417 for input_repo in input_repos: 418 input_repo.CleanupRoot() 419 420 return ret 421 422 423 if __name__ == '__main__': 424 retval = Main(sys.argv[1:]) 425 sys.exit(retval) 426
