1 #!/usr/bin/env python 2 # Copyright 2017 The LibYuv Project Authors. All rights reserved. 3 # 4 # Use of this source code is governed by a BSD-style license 5 # that can be found in the LICENSE file in the root of the source 6 # tree. An additional intellectual property rights grant can be found 7 # in the file PATENTS. All contributing project authors may 8 # be found in the AUTHORS file in the root of the source tree. 9 10 # This is a modified copy of the script in 11 # https://chromium.googlesource.com/external/webrtc/+/master/tools-webrtc/autoroller/roll_deps.py 12 # customized for libyuv. 13 14 15 """Script to automatically roll dependencies in the libyuv DEPS file.""" 16 17 import argparse 18 import base64 19 import collections 20 import logging 21 import os 22 import re 23 import subprocess 24 import sys 25 import urllib 26 27 28 # Skip these dependencies (list without solution name prefix). 29 DONT_AUTOROLL_THESE = [ 30 'src/third_party/gflags/src', 31 ] 32 33 LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv' 34 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' 35 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' 36 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' 37 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' 38 39 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') 40 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$') 41 ROLL_BRANCH_NAME = 'roll_chromium_revision' 42 43 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 44 CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, 45 os.pardir)) 46 CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) 47 48 sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) 49 import find_depot_tools 50 find_depot_tools.add_depot_tools_to_path() 51 from gclient import GClientKeywords 52 53 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' 54 CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', 55 'clang', 'scripts', 'update.py') 56 57 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') 58 ChangedDep = collections.namedtuple('ChangedDep', 59 'path url current_rev new_rev') 60 61 class RollError(Exception): 62 pass 63 64 65 def ParseDepsDict(deps_content): 66 local_scope = {} 67 var = GClientKeywords.VarImpl({}, local_scope) 68 global_scope = { 69 'From': GClientKeywords.FromImpl, 70 'Var': var.Lookup, 71 'deps_os': {}, 72 } 73 exec(deps_content, global_scope, local_scope) 74 return local_scope 75 76 77 def ParseLocalDepsFile(filename): 78 with open(filename, 'rb') as f: 79 deps_content = f.read() 80 return ParseDepsDict(deps_content) 81 82 83 def ParseRemoteCrDepsFile(revision): 84 deps_content = ReadRemoteCrFile('DEPS', revision) 85 return ParseDepsDict(deps_content) 86 87 88 def ParseCommitPosition(commit_message): 89 for line in reversed(commit_message.splitlines()): 90 m = COMMIT_POSITION_RE.match(line.strip()) 91 if m: 92 return m.group(1) 93 logging.error('Failed to parse commit position id from:\n%s\n', 94 commit_message) 95 sys.exit(-1) 96 97 98 def _RunCommand(command, working_dir=None, ignore_exit_code=False, 99 extra_env=None): 100 """Runs a command and returns the output from that command. 101 102 If the command fails (exit code != 0), the function will exit the process. 103 104 Returns: 105 A tuple containing the stdout and stderr outputs as strings. 106 """ 107 working_dir = working_dir or CHECKOUT_SRC_DIR 108 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) 109 env = os.environ.copy() 110 if extra_env: 111 assert all(type(value) == str for value in extra_env.values()) 112 logging.debug('extra env: %s', extra_env) 113 env.update(extra_env) 114 p = subprocess.Popen(command, stdout=subprocess.PIPE, 115 stderr=subprocess.PIPE, env=env, 116 cwd=working_dir, universal_newlines=True) 117 std_output = p.stdout.read() 118 err_output = p.stderr.read() 119 p.wait() 120 p.stdout.close() 121 p.stderr.close() 122 if not ignore_exit_code and p.returncode != 0: 123 logging.error('Command failed: %s\n' 124 'stdout:\n%s\n' 125 'stderr:\n%s\n', ' '.join(command), std_output, err_output) 126 sys.exit(p.returncode) 127 return std_output, err_output 128 129 130 def _GetBranches(): 131 """Returns a tuple of active,branches. 132 133 The 'active' is the name of the currently active branch and 'branches' is a 134 list of all branches. 135 """ 136 lines = _RunCommand(['git', 'branch'])[0].split('\n') 137 branches = [] 138 active = '' 139 for line in lines: 140 if '*' in line: 141 # The assumption is that the first char will always be the '*'. 142 active = line[1:].strip() 143 branches.append(active) 144 else: 145 branch = line.strip() 146 if branch: 147 branches.append(branch) 148 return active, branches 149 150 151 def _ReadGitilesContent(url): 152 # Download and decode BASE64 content until 153 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. 154 base64_content = ReadUrlContent(url + '?format=TEXT') 155 return base64.b64decode(base64_content[0]) 156 157 158 def ReadRemoteCrFile(path_below_src, revision): 159 """Reads a remote Chromium file of a specific revision. Returns a string.""" 160 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, 161 path_below_src)) 162 163 164 def ReadRemoteCrCommit(revision): 165 """Reads a remote Chromium commit message. Returns a string.""" 166 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) 167 168 169 def ReadUrlContent(url): 170 """Connect to a remote host and read the contents. Returns a list of lines.""" 171 conn = urllib.urlopen(url) 172 try: 173 return conn.readlines() 174 except IOError as e: 175 logging.exception('Error connecting to %s. Error: %s', url, e) 176 raise 177 finally: 178 conn.close() 179 180 181 def GetMatchingDepsEntries(depsentry_dict, dir_path): 182 """Gets all deps entries matching the provided path. 183 184 This list may contain more than one DepsEntry object. 185 Example: dir_path='src/testing' would give results containing both 186 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. 187 Example 2: dir_path='src/build' should return 'src/build' but not 188 'src/buildtools'. 189 190 Returns: 191 A list of DepsEntry objects. 192 """ 193 result = [] 194 for path, depsentry in depsentry_dict.iteritems(): 195 if path == dir_path: 196 result.append(depsentry) 197 else: 198 parts = path.split('/') 199 if all(part == parts[i] 200 for i, part in enumerate(dir_path.split('/'))): 201 result.append(depsentry) 202 return result 203 204 205 def BuildDepsentryDict(deps_dict): 206 """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict.""" 207 result = {} 208 def AddDepsEntries(deps_subdict): 209 for path, deps_url in deps_subdict.iteritems(): 210 if not result.has_key(path): 211 url, revision = deps_url.split('@') if deps_url else (None, None) 212 result[path] = DepsEntry(path, url, revision) 213 214 AddDepsEntries(deps_dict['deps']) 215 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: 216 AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) 217 return result 218 219 220 def CalculateChangedDeps(libyuv_deps, new_cr_deps): 221 """ 222 Calculate changed deps entries based on entries defined in the libyuv DEPS 223 file: 224 - If a shared dependency with the Chromium DEPS file: roll it to the same 225 revision as Chromium (i.e. entry in the new_cr_deps dict) 226 - If it's a Chromium sub-directory, roll it to the HEAD revision (notice 227 this means it may be ahead of the chromium_revision, but generally these 228 should be close). 229 - If it's another DEPS entry (not shared with Chromium), roll it to HEAD 230 unless it's configured to be skipped. 231 232 Returns: 233 A list of ChangedDep objects representing the changed deps. 234 """ 235 result = [] 236 libyuv_entries = BuildDepsentryDict(libyuv_deps) 237 new_cr_entries = BuildDepsentryDict(new_cr_deps) 238 for path, libyuv_deps_entry in libyuv_entries.iteritems(): 239 if path in DONT_AUTOROLL_THESE: 240 continue 241 cr_deps_entry = new_cr_entries.get(path) 242 if cr_deps_entry: 243 # Use the revision from Chromium's DEPS file. 244 new_rev = cr_deps_entry.revision 245 assert libyuv_deps_entry.url == cr_deps_entry.url, ( 246 'Libyuv DEPS entry %s has a different URL (%s) than Chromium (%s).' % 247 (path, libyuv_deps_entry.url, cr_deps_entry.url)) 248 else: 249 # Use the HEAD of the deps repo. 250 stdout, _ = _RunCommand(['git', 'ls-remote', libyuv_deps_entry.url, 251 'HEAD']) 252 new_rev = stdout.strip().split('\t')[0] 253 254 # Check if an update is necessary. 255 if libyuv_deps_entry.revision != new_rev: 256 logging.debug('Roll dependency %s to %s', path, new_rev) 257 result.append(ChangedDep(path, libyuv_deps_entry.url, 258 libyuv_deps_entry.revision, new_rev)) 259 return sorted(result) 260 261 262 def CalculateChangedClang(new_cr_rev): 263 def GetClangRev(lines): 264 for line in lines: 265 match = CLANG_REVISION_RE.match(line) 266 if match: 267 return match.group(1) 268 raise RollError('Could not parse Clang revision!') 269 270 with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f: 271 current_lines = f.readlines() 272 current_rev = GetClangRev(current_lines) 273 274 new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, 275 new_cr_rev).splitlines() 276 new_rev = GetClangRev(new_clang_update_py) 277 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) 278 279 280 def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, 281 new_commit_pos, changed_deps_list, clang_change): 282 current_cr_rev = current_cr_rev[0:10] 283 new_cr_rev = new_cr_rev[0:10] 284 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) 285 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) 286 287 commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, 288 git_number_interval)] 289 commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) 290 commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % 291 rev_interval)) 292 # TBR field will be empty unless in some custom cases, where some engineers 293 # are added. 294 tbr_authors = '' 295 if changed_deps_list: 296 commit_msg.append('Changed dependencies:') 297 298 for c in changed_deps_list: 299 commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url, 300 c.current_rev[0:10], 301 c.new_rev[0:10])) 302 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') 303 commit_msg.append('DEPS diff: %s\n' % change_url) 304 else: 305 commit_msg.append('No dependencies changed.') 306 307 if clang_change.current_rev != clang_change.new_rev: 308 commit_msg.append('Clang version changed %s:%s' % 309 (clang_change.current_rev, clang_change.new_rev)) 310 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 311 CLANG_UPDATE_SCRIPT_URL_PATH) 312 commit_msg.append('Details: %s\n' % change_url) 313 else: 314 commit_msg.append('No update to Clang.\n') 315 316 commit_msg.append('TBR=%s' % tbr_authors) 317 commit_msg.append('BUG=None') 318 return '\n'.join(commit_msg) 319 320 321 def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision, 322 changed_deps): 323 """Update the DEPS file with the new revision.""" 324 325 # Update the chromium_revision variable. 326 with open(deps_filename, 'rb') as deps_file: 327 deps_content = deps_file.read() 328 deps_content = deps_content.replace(old_cr_revision, new_cr_revision) 329 with open(deps_filename, 'wb') as deps_file: 330 deps_file.write(deps_content) 331 332 # Update each individual DEPS entry. 333 for dep in changed_deps: 334 local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) 335 if not os.path.isdir(local_dep_dir): 336 raise RollError( 337 'Cannot find local directory %s. Either run\n' 338 'gclient sync --deps=all\n' 339 'or make sure the .gclient file for your solution contains all ' 340 'platforms in the target_os list, i.e.\n' 341 'target_os = ["android", "unix", "mac", "ios", "win"];\n' 342 'Then run "gclient sync" again.' % local_dep_dir) 343 _, stderr = _RunCommand( 344 ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev], 345 working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True) 346 if stderr: 347 logging.warning('roll-dep-svn: %s', stderr) 348 349 350 def _IsTreeClean(): 351 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) 352 if len(stdout) == 0: 353 return True 354 355 logging.error('Dirty/unversioned files:\n%s', stdout) 356 return False 357 358 359 def _EnsureUpdatedMasterBranch(dry_run): 360 current_branch = _RunCommand( 361 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] 362 if current_branch != 'master': 363 logging.error('Please checkout the master branch and re-run this script.') 364 if not dry_run: 365 sys.exit(-1) 366 367 logging.info('Updating master branch...') 368 _RunCommand(['git', 'pull']) 369 370 371 def _CreateRollBranch(dry_run): 372 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) 373 if not dry_run: 374 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) 375 376 377 def _RemovePreviousRollBranch(dry_run): 378 active_branch, branches = _GetBranches() 379 if active_branch == ROLL_BRANCH_NAME: 380 active_branch = 'master' 381 if ROLL_BRANCH_NAME in branches: 382 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) 383 if not dry_run: 384 _RunCommand(['git', 'checkout', active_branch]) 385 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) 386 387 388 def _LocalCommit(commit_msg, dry_run): 389 logging.info('Committing changes locally.') 390 if not dry_run: 391 _RunCommand(['git', 'add', '--update', '.']) 392 _RunCommand(['git', 'commit', '-m', commit_msg]) 393 394 395 def _UploadCL(dry_run, rietveld_email=None): 396 logging.info('Uploading CL...') 397 if not dry_run: 398 cmd = ['git', 'cl', 'upload', '-f'] 399 if rietveld_email: 400 cmd.append('--email=%s' % rietveld_email) 401 _RunCommand(cmd, extra_env={'EDITOR': 'true'}) 402 403 404 def _SendToCQ(dry_run, skip_cq): 405 logging.info('Sending the CL to the CQ...') 406 if not dry_run and not skip_cq: 407 _RunCommand(['git', 'cl', 'set_commit']) 408 logging.info('Sent the CL to the CQ.') 409 410 411 def main(): 412 p = argparse.ArgumentParser() 413 p.add_argument('--clean', action='store_true', default=False, 414 help='Removes any previous local roll branch.') 415 p.add_argument('-r', '--revision', 416 help=('Chromium Git revision to roll to. Defaults to the ' 417 'Chromium HEAD revision if omitted.')) 418 p.add_argument('-u', '--rietveld-email', 419 help=('E-mail address to use for creating the CL at Rietveld' 420 'If omitted a previously cached one will be used or an ' 421 'error will be thrown during upload.')) 422 p.add_argument('--dry-run', action='store_true', default=False, 423 help=('Calculate changes and modify DEPS, but don\'t create ' 424 'any local branch, commit, upload CL or send any ' 425 'tryjobs.')) 426 p.add_argument('-i', '--ignore-unclean-workdir', action='store_true', 427 default=False, 428 help=('Ignore if the current branch is not master or if there ' 429 'are uncommitted changes (default: %(default)s).')) 430 p.add_argument('--skip-cq', action='store_true', default=False, 431 help='Skip sending the CL to the CQ (default: %(default)s)') 432 p.add_argument('-v', '--verbose', action='store_true', default=False, 433 help='Be extra verbose in printing of log messages.') 434 opts = p.parse_args() 435 436 if opts.verbose: 437 logging.basicConfig(level=logging.DEBUG) 438 else: 439 logging.basicConfig(level=logging.INFO) 440 441 if not opts.ignore_unclean_workdir and not _IsTreeClean(): 442 logging.error('Please clean your local checkout first.') 443 return 1 444 445 if opts.clean: 446 _RemovePreviousRollBranch(opts.dry_run) 447 448 if not opts.ignore_unclean_workdir: 449 _EnsureUpdatedMasterBranch(opts.dry_run) 450 451 new_cr_rev = opts.revision 452 if not new_cr_rev: 453 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) 454 head_rev = stdout.strip().split('\t')[0] 455 logging.info('No revision specified. Using HEAD: %s', head_rev) 456 new_cr_rev = head_rev 457 458 deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') 459 libyuv_deps = ParseLocalDepsFile(deps_filename) 460 current_cr_rev = libyuv_deps['vars']['chromium_revision'] 461 462 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) 463 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) 464 465 new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) 466 changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) 467 clang_change = CalculateChangedClang(new_cr_rev) 468 commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, 469 current_commit_pos, new_commit_pos, 470 changed_deps, clang_change) 471 logging.debug('Commit message:\n%s', commit_msg) 472 473 _CreateRollBranch(opts.dry_run) 474 UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps) 475 _LocalCommit(commit_msg, opts.dry_run) 476 _UploadCL(opts.dry_run, opts.rietveld_email) 477 _SendToCQ(opts.dry_run, opts.skip_cq) 478 return 0 479 480 481 if __name__ == '__main__': 482 sys.exit(main()) 483
