1 #!/usr/bin/python2 2 3 # Copyright 2014 Google Inc. 4 # 5 # Use of this source code is governed by a BSD-style license that can be 6 # found in the LICENSE file. 7 8 """Skia's Chromium DEPS roll script. 9 10 This script: 11 - searches through the last N Skia git commits to find out the hash that is 12 associated with the SVN revision number. 13 - creates a new branch in the Chromium tree, modifies the DEPS file to 14 point at the given Skia commit, commits, uploads to Rietveld, and 15 deletes the local copy of the branch. 16 - creates a whitespace-only commit and uploads that to to Rietveld. 17 - returns the Chromium tree to its previous state. 18 19 To specify the location of the git executable, set the GIT_EXECUTABLE 20 environment variable. 21 22 Usage: 23 %prog -c CHROMIUM_PATH -r REVISION [OPTIONAL_OPTIONS] 24 """ 25 26 27 import optparse 28 import os 29 import re 30 import shutil 31 import subprocess 32 import sys 33 import tempfile 34 35 import git_utils 36 import misc_utils 37 38 39 DEFAULT_BOTS_LIST = [ 40 'android_clang_dbg', 41 'android_dbg', 42 'android_rel', 43 'cros_daisy', 44 'linux', 45 'linux_asan', 46 'linux_chromeos', 47 'linux_chromeos_asan', 48 'linux_chromium_gn_dbg', 49 'linux_gpu', 50 'linux_layout', 51 'linux_layout_rel', 52 'mac', 53 'mac_asan', 54 'mac_gpu', 55 'mac_layout', 56 'mac_layout_rel', 57 'win', 58 'win_gpu', 59 'win_layout', 60 'win_layout_rel', 61 ] 62 63 64 class DepsRollConfig(object): 65 """Contains configuration options for this module. 66 67 Attributes: 68 git: (string) The git executable. 69 chromium_path: (string) path to a local chromium git repository. 70 save_branches: (boolean) iff false, delete temporary branches. 71 verbose: (boolean) iff false, suppress the output from git-cl. 72 search_depth: (int) how far back to look for the revision. 73 skia_url: (string) Skia's git repository. 74 self.skip_cl_upload: (boolean) 75 self.cl_bot_list: (list of strings) 76 """ 77 78 # pylint: disable=I0011,R0903,R0902 79 def __init__(self, options=None): 80 self.skia_url = 'https://skia.googlesource.com/skia.git' 81 self.revision_format = ( 82 'git-svn-id: http://skia.googlecode.com/svn/trunk@%d ') 83 84 self.git = git_utils.git_executable() 85 86 if not options: 87 options = DepsRollConfig.GetOptionParser() 88 # pylint: disable=I0011,E1103 89 self.verbose = options.verbose 90 self.vsp = misc_utils.VerboseSubprocess(self.verbose) 91 self.save_branches = not options.delete_branches 92 self.search_depth = options.search_depth 93 self.chromium_path = options.chromium_path 94 self.skip_cl_upload = options.skip_cl_upload 95 # Split and remove empty strigns from the bot list. 96 self.cl_bot_list = [bot for bot in options.bots.split(',') if bot] 97 self.skia_git_checkout_path = options.skia_git_path 98 self.default_branch_name = 'autogenerated_deps_roll_branch' 99 self.reviewers_list = ','.join([ 100 # 'rmistry (at] google.com', 101 # 'reed (at] google.com', 102 # 'bsalomon (at] google.com', 103 # 'robertphillips (at] google.com', 104 ]) 105 self.cc_list = ','.join([ 106 # 'skia-team (at] google.com', 107 ]) 108 109 @staticmethod 110 def GetOptionParser(): 111 # pylint: disable=I0011,C0103 112 """Returns an optparse.OptionParser object. 113 114 Returns: 115 An optparse.OptionParser object. 116 117 Called by the main() function. 118 """ 119 option_parser = optparse.OptionParser(usage=__doc__) 120 # Anyone using this script on a regular basis should set the 121 # CHROMIUM_CHECKOUT_PATH environment variable. 122 option_parser.add_option( 123 '-c', '--chromium_path', help='Path to local Chromium Git' 124 ' repository checkout, defaults to CHROMIUM_CHECKOUT_PATH' 125 ' if that environment variable is set.', 126 default=os.environ.get('CHROMIUM_CHECKOUT_PATH')) 127 option_parser.add_option( 128 '-r', '--revision', type='int', default=None, 129 help='The Skia SVN revision number, defaults to top of tree.') 130 option_parser.add_option( 131 '-g', '--git_hash', default=None, 132 help='A partial Skia Git hash. Do not set this and revision.') 133 134 # Anyone using this script on a regular basis should set the 135 # SKIA_GIT_CHECKOUT_PATH environment variable. 136 option_parser.add_option( 137 '', '--skia_git_path', 138 help='Path of a pure-git Skia repository checkout. If empty,' 139 ' a temporary will be cloned. Defaults to SKIA_GIT_CHECKOUT' 140 '_PATH, if that environment variable is set.', 141 default=os.environ.get('SKIA_GIT_CHECKOUT_PATH')) 142 option_parser.add_option( 143 '', '--search_depth', type='int', default=100, 144 help='How far back to look for the revision.') 145 option_parser.add_option( 146 '', '--delete_branches', help='Delete the temporary branches', 147 action='store_true', dest='delete_branches', default=False) 148 option_parser.add_option( 149 '', '--verbose', help='Do not suppress the output from `git cl`.', 150 action='store_true', dest='verbose', default=False) 151 option_parser.add_option( 152 '', '--skip_cl_upload', help='Skip the cl upload step; useful' 153 ' for testing.', 154 action='store_true', default=False) 155 156 default_bots_help = ( 157 'Comma-separated list of bots, defaults to a list of %d bots.' 158 ' To skip `git cl try`, set this to an empty string.' 159 % len(DEFAULT_BOTS_LIST)) 160 default_bots = ','.join(DEFAULT_BOTS_LIST) 161 option_parser.add_option( 162 '', '--bots', help=default_bots_help, default=default_bots) 163 164 return option_parser 165 166 167 class DepsRollError(Exception): 168 """Exceptions specific to this module.""" 169 pass 170 171 172 def get_svn_revision(config, commit): 173 """Works in both git and git-svn. returns a string.""" 174 svn_format = ( 175 '(git-svn-id: [^@ ]+@|SVN changes up to revision |' 176 'LKGR w/ DEPS up to revision )(?P<return>[0-9]+)') 177 svn_revision = misc_utils.ReSearch.search_within_output( 178 config.verbose, svn_format, None, 179 [config.git, 'log', '-n', '1', '--format=format:%B', commit]) 180 if not svn_revision: 181 raise DepsRollError( 182 'Revision number missing from Chromium origin/master.') 183 return int(svn_revision) 184 185 186 class SkiaGitCheckout(object): 187 """Class to create a temporary skia git checkout, if necessary. 188 """ 189 # pylint: disable=I0011,R0903 190 191 def __init__(self, config, depth): 192 self._config = config 193 self._depth = depth 194 self._use_temp = None 195 self._original_cwd = None 196 197 def __enter__(self): 198 config = self._config 199 git = config.git 200 skia_dir = None 201 self._original_cwd = os.getcwd() 202 if config.skia_git_checkout_path: 203 if config.skia_git_checkout_path != os.curdir: 204 skia_dir = config.skia_git_checkout_path 205 ## Update origin/master if needed. 206 if self._config.verbose: 207 print '~~$', 'cd', skia_dir 208 os.chdir(skia_dir) 209 config.vsp.check_call([git, 'fetch', '-q', 'origin']) 210 self._use_temp = None 211 else: 212 skia_dir = tempfile.mkdtemp(prefix='git_skia_tmp_') 213 self._use_temp = skia_dir 214 try: 215 os.chdir(skia_dir) 216 config.vsp.check_call( 217 [git, 'clone', '-q', '--depth=%d' % self._depth, 218 '--single-branch', config.skia_url, '.']) 219 except (OSError, subprocess.CalledProcessError) as error: 220 shutil.rmtree(skia_dir) 221 raise error 222 223 def __exit__(self, etype, value, traceback): 224 if self._config.skia_git_checkout_path != os.curdir: 225 if self._config.verbose: 226 print '~~$', 'cd', self._original_cwd 227 os.chdir(self._original_cwd) 228 if self._use_temp: 229 shutil.rmtree(self._use_temp) 230 231 232 def revision_and_hash(config): 233 """Finds revision number and git hash of origin/master in the Skia tree. 234 235 Args: 236 config: (roll_deps.DepsRollConfig) object containing options. 237 238 Returns: 239 A tuple (revision, hash) 240 revision: (int) SVN revision number. 241 git_hash: (string) full Git commit hash. 242 243 Raises: 244 roll_deps.DepsRollError: if the revision can't be found. 245 OSError: failed to execute git or git-cl. 246 subprocess.CalledProcessError: git returned unexpected status. 247 """ 248 with SkiaGitCheckout(config, 1): 249 revision = get_svn_revision(config, 'origin/master') 250 git_hash = config.vsp.strip_output( 251 [config.git, 'show-ref', 'origin/master', '--hash']) 252 if not git_hash: 253 raise DepsRollError('Git hash can not be found.') 254 return revision, git_hash 255 256 257 def revision_and_hash_from_revision(config, revision): 258 """Finds revision number and git hash of a commit in the Skia tree. 259 260 Args: 261 config: (roll_deps.DepsRollConfig) object containing options. 262 revision: (int) SVN revision number. 263 264 Returns: 265 A tuple (revision, hash) 266 revision: (int) SVN revision number. 267 git_hash: (string) full Git commit hash. 268 269 Raises: 270 roll_deps.DepsRollError: if the revision can't be found. 271 OSError: failed to execute git or git-cl. 272 subprocess.CalledProcessError: git returned unexpected status. 273 """ 274 with SkiaGitCheckout(config, config.search_depth): 275 revision_regex = config.revision_format % revision 276 git_hash = config.vsp.strip_output( 277 [config.git, 'log', '--grep', revision_regex, 278 '--format=format:%H', 'origin/master']) 279 if not git_hash: 280 raise DepsRollError('Git hash can not be found.') 281 return revision, git_hash 282 283 284 def revision_and_hash_from_partial(config, partial_hash): 285 """Returns the SVN revision number and full git hash. 286 287 Args: 288 config: (roll_deps.DepsRollConfig) object containing options. 289 partial_hash: (string) Partial git commit hash. 290 291 Returns: 292 A tuple (revision, hash) 293 revision: (int) SVN revision number. 294 git_hash: (string) full Git commit hash. 295 296 Raises: 297 roll_deps.DepsRollError: if the revision can't be found. 298 OSError: failed to execute git or git-cl. 299 subprocess.CalledProcessError: git returned unexpected status. 300 """ 301 with SkiaGitCheckout(config, config.search_depth): 302 git_hash = config.vsp.strip_output( 303 ['git', 'log', '-n', '1', '--format=format:%H', partial_hash]) 304 if not git_hash: 305 raise DepsRollError('Partial Git hash can not be found.') 306 revision = get_svn_revision(config, git_hash) 307 return revision, git_hash 308 309 310 def change_skia_deps(revision, git_hash, depspath): 311 """Update the DEPS file. 312 313 Modify the skia_revision and skia_hash entries in the given DEPS file. 314 315 Args: 316 revision: (int) Skia SVN revision. 317 git_hash: (string) Skia Git hash. 318 depspath: (string) path to DEPS file. 319 """ 320 temp_file = tempfile.NamedTemporaryFile(delete=False, 321 prefix='skia_DEPS_ROLL_tmp_') 322 try: 323 deps_regex_rev = re.compile('"skia_revision": "[0-9]*",') 324 deps_regex_hash = re.compile('"skia_hash": "[0-9a-f]*",') 325 326 deps_regex_rev_repl = '"skia_revision": "%d",' % revision 327 deps_regex_hash_repl = '"skia_hash": "%s",' % git_hash 328 329 with open(depspath, 'r') as input_stream: 330 for line in input_stream: 331 line = deps_regex_rev.sub(deps_regex_rev_repl, line) 332 line = deps_regex_hash.sub(deps_regex_hash_repl, line) 333 temp_file.write(line) 334 finally: 335 temp_file.close() 336 shutil.move(temp_file.name, depspath) 337 338 339 def git_cl_uploader(config, message, file_list): 340 """Create a commit in the current git branch; upload via git-cl. 341 342 Assumes that you are already on the branch you want to be on. 343 344 Args: 345 config: (roll_deps.DepsRollConfig) object containing options. 346 message: (string) the commit message, can be multiline. 347 file_list: (list of strings) list of filenames to pass to `git add`. 348 349 Returns: 350 The output of `git cl issue`, if not config.skip_cl_upload, else ''. 351 """ 352 353 git, vsp = config.git, config.vsp 354 svn_info = str(get_svn_revision(config, 'HEAD')) 355 356 for filename in file_list: 357 assert os.path.exists(filename) 358 vsp.check_call([git, 'add', filename]) 359 360 vsp.check_call([git, 'commit', '-q', '-m', message]) 361 362 git_cl = [git, 'cl', 'upload', '-f', 363 '--bypass-hooks', '--bypass-watchlists'] 364 if config.cc_list: 365 git_cl.append('--cc=%s' % config.cc_list) 366 if config.reviewers_list: 367 git_cl.append('--reviewers=%s' % config.reviewers_list) 368 369 git_try = [ 370 git, 'cl', 'try', '-m', 'tryserver.chromium', '--revision', svn_info] 371 git_try.extend([arg for bot in config.cl_bot_list for arg in ('-b', bot)]) 372 373 branch_name = git_utils.git_branch_name(vsp.verbose) 374 375 if config.skip_cl_upload: 376 space = ' ' 377 print 'You should call:' 378 print '%scd %s' % (space, os.getcwd()) 379 misc_utils.print_subprocess_args(space, [git, 'checkout', branch_name]) 380 misc_utils.print_subprocess_args(space, git_cl) 381 if config.cl_bot_list: 382 misc_utils.print_subprocess_args(space, git_try) 383 print 384 return '' 385 else: 386 vsp.check_call(git_cl) 387 issue = vsp.strip_output([git, 'cl', 'issue']) 388 if config.cl_bot_list: 389 vsp.check_call(git_try) 390 return issue 391 392 393 def roll_deps(config, revision, git_hash): 394 """Upload changed DEPS and a whitespace change. 395 396 Given the correct git_hash, create two Reitveld issues. 397 398 Args: 399 config: (roll_deps.DepsRollConfig) object containing options. 400 revision: (int) Skia SVN revision. 401 git_hash: (string) Skia Git hash. 402 403 Returns: 404 a tuple containing textual description of the two issues. 405 406 Raises: 407 OSError: failed to execute git or git-cl. 408 subprocess.CalledProcessError: git returned unexpected status. 409 """ 410 411 git = config.git 412 with misc_utils.ChangeDir(config.chromium_path, config.verbose): 413 config.vsp.check_call([git, 'fetch', '-q', 'origin']) 414 415 old_revision = misc_utils.ReSearch.search_within_output( 416 config.verbose, '"skia_revision": "(?P<return>[0-9]+)",', None, 417 [git, 'show', 'origin/master:DEPS']) 418 assert old_revision 419 if revision == int(old_revision): 420 print 'DEPS is up to date!' 421 return (None, None) 422 423 master_hash = config.vsp.strip_output( 424 [git, 'show-ref', 'origin/master', '--hash']) 425 master_revision = get_svn_revision(config, 'origin/master') 426 427 # master_hash[8] gives each whitespace CL a unique name. 428 if config.save_branches: 429 branch = 'control_%s' % master_hash[:8] 430 else: 431 branch = None 432 message = ('whitespace change %s\n\n' 433 'Chromium base revision: %d / %s\n\n' 434 'This CL was created by Skia\'s roll_deps.py script.\n' 435 ) % (master_hash[:8], master_revision, master_hash[:8]) 436 with git_utils.ChangeGitBranch(branch, 'origin/master', 437 config.verbose): 438 branch = git_utils.git_branch_name(config.vsp.verbose) 439 440 with open('build/whitespace_file.txt', 'a') as output_stream: 441 output_stream.write('\nCONTROL\n') 442 443 whitespace_cl = git_cl_uploader( 444 config, message, ['build/whitespace_file.txt']) 445 446 control_url = misc_utils.ReSearch.search_within_string( 447 whitespace_cl, '(?P<return>https?://[^) ]+)', '?') 448 if config.save_branches: 449 whitespace_cl = '%s\n branch: %s' % (whitespace_cl, branch) 450 451 if config.save_branches: 452 branch = 'roll_%d_%s' % (revision, master_hash[:8]) 453 else: 454 branch = None 455 message = ( 456 'roll skia DEPS to %d\n\n' 457 'Chromium base revision: %d / %s\n' 458 'Old Skia revision: %s\n' 459 'New Skia revision: %d\n' 460 'Control CL: %s\n\n' 461 'This CL was created by Skia\'s roll_deps.py script.\n\n' 462 'Bypassing commit queue trybots:\n' 463 'NOTRY=true\n' 464 % (revision, master_revision, master_hash[:8], 465 old_revision, revision, control_url)) 466 with git_utils.ChangeGitBranch(branch, 'origin/master', 467 config.verbose): 468 branch = git_utils.git_branch_name(config.vsp.verbose) 469 470 change_skia_deps(revision, git_hash, 'DEPS') 471 deps_cl = git_cl_uploader(config, message, ['DEPS']) 472 if config.save_branches: 473 deps_cl = '%s\n branch: %s' % (deps_cl, branch) 474 475 return deps_cl, whitespace_cl 476 477 478 def find_hash_and_roll_deps(config, revision=None, partial_hash=None): 479 """Call find_hash_from_revision() and roll_deps(). 480 481 The calls to git will be verbose on standard output. After a 482 successful upload of both issues, print links to the new 483 codereview issues. 484 485 Args: 486 config: (roll_deps.DepsRollConfig) object containing options. 487 revision: (int or None) the Skia SVN revision number or None 488 to use the tip of the tree. 489 partial_hash: (string or None) a partial pure-git Skia commit 490 hash. Don't pass both partial_hash and revision. 491 492 Raises: 493 roll_deps.DepsRollError: if the revision can't be found. 494 OSError: failed to execute git or git-cl. 495 subprocess.CalledProcessError: git returned unexpected status. 496 """ 497 498 if revision and partial_hash: 499 raise DepsRollError('Pass revision or partial_hash, not both.') 500 501 if partial_hash: 502 revision, git_hash = revision_and_hash_from_partial( 503 config, partial_hash) 504 elif revision: 505 revision, git_hash = revision_and_hash_from_revision(config, revision) 506 else: 507 revision, git_hash = revision_and_hash(config) 508 509 print 'revision=%r\nhash=%r\n' % (revision, git_hash) 510 511 deps_issue, whitespace_issue = roll_deps(config, revision, git_hash) 512 513 if deps_issue and whitespace_issue: 514 print 'DEPS roll:\n %s\n' % deps_issue 515 print 'Whitespace change:\n %s\n' % whitespace_issue 516 else: 517 print >> sys.stderr, 'No issues created.' 518 519 520 def main(args): 521 """main function; see module-level docstring and GetOptionParser help. 522 523 Args: 524 args: sys.argv[1:]-type argument list. 525 """ 526 option_parser = DepsRollConfig.GetOptionParser() 527 options = option_parser.parse_args(args)[0] 528 529 if not options.chromium_path: 530 option_parser.error('Must specify chromium_path.') 531 if not os.path.isdir(options.chromium_path): 532 option_parser.error('chromium_path must be a directory.') 533 534 if not git_utils.git_executable(): 535 option_parser.error('Invalid git executable.') 536 537 config = DepsRollConfig(options) 538 find_hash_and_roll_deps(config, options.revision, options.git_hash) 539 540 541 if __name__ == '__main__': 542 main(sys.argv[1:]) 543 544
