1 #!/usr/bin/env python 2 # Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 # 4 # Use of this source code is governed by a BSD-style license 5 # that can be found in the LICENSE file in the root of the source 6 # tree. An additional intellectual property rights grant can be found 7 # in the file PATENTS. All contributing project authors may 8 # be found in the AUTHORS file in the root of the source tree. 9 10 """Script to roll chromium_revision in the WebRTC DEPS file.""" 11 12 import argparse 13 import base64 14 import collections 15 import logging 16 import os 17 import re 18 import subprocess 19 import sys 20 import urllib 21 22 23 CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' 24 CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' 25 CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' 26 CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' 27 28 COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') 29 CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION=(\d+)$') 30 ROLL_BRANCH_NAME = 'roll_chromium_revision' 31 32 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) 33 CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir, 34 os.pardir)) 35 sys.path.append(CHECKOUT_ROOT_DIR) 36 import setup_links 37 38 sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build')) 39 import find_depot_tools 40 find_depot_tools.add_depot_tools_to_path() 41 from gclient import GClientKeywords 42 43 CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' 44 CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts', 45 'update.py') 46 47 DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') 48 ChangedDep = collections.namedtuple('ChangedDep', 49 'path url current_rev new_rev') 50 51 52 def ParseDepsDict(deps_content): 53 local_scope = {} 54 var = GClientKeywords.VarImpl({}, local_scope) 55 global_scope = { 56 'File': GClientKeywords.FileImpl, 57 'From': GClientKeywords.FromImpl, 58 'Var': var.Lookup, 59 'deps_os': {}, 60 } 61 exec(deps_content, global_scope, local_scope) 62 return local_scope 63 64 65 def ParseLocalDepsFile(filename): 66 with open(filename, 'rb') as f: 67 deps_content = f.read() 68 return ParseDepsDict(deps_content) 69 70 71 def ParseRemoteCrDepsFile(revision): 72 deps_content = ReadRemoteCrFile('DEPS', revision) 73 return ParseDepsDict(deps_content) 74 75 76 def ParseCommitPosition(commit_message): 77 for line in reversed(commit_message.splitlines()): 78 m = COMMIT_POSITION_RE.match(line.strip()) 79 if m: 80 return m.group(1) 81 logging.error('Failed to parse commit position id from:\n%s\n', 82 commit_message) 83 sys.exit(-1) 84 85 86 def _RunCommand(command, working_dir=None, ignore_exit_code=False, 87 extra_env=None): 88 """Runs a command and returns the output from that command. 89 90 If the command fails (exit code != 0), the function will exit the process. 91 92 Returns: 93 A tuple containing the stdout and stderr outputs as strings. 94 """ 95 working_dir = working_dir or CHECKOUT_ROOT_DIR 96 logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) 97 env = os.environ.copy() 98 if extra_env: 99 assert all(type(value) == str for value in extra_env.values()) 100 logging.debug('extra env: %s', extra_env) 101 env.update(extra_env) 102 p = subprocess.Popen(command, stdout=subprocess.PIPE, 103 stderr=subprocess.PIPE, env=env, 104 cwd=working_dir, universal_newlines=True) 105 std_output = p.stdout.read() 106 err_output = p.stderr.read() 107 p.wait() 108 p.stdout.close() 109 p.stderr.close() 110 if not ignore_exit_code and p.returncode != 0: 111 logging.error('Command failed: %s\n' 112 'stdout:\n%s\n' 113 'stderr:\n%s\n', ' '.join(command), std_output, err_output) 114 sys.exit(p.returncode) 115 return std_output, err_output 116 117 118 def _GetBranches(): 119 """Returns a tuple of active,branches. 120 121 The 'active' is the name of the currently active branch and 'branches' is a 122 list of all branches. 123 """ 124 lines = _RunCommand(['git', 'branch'])[0].split('\n') 125 branches = [] 126 active = '' 127 for line in lines: 128 if '*' in line: 129 # The assumption is that the first char will always be the '*'. 130 active = line[1:].strip() 131 branches.append(active) 132 else: 133 branch = line.strip() 134 if branch: 135 branches.append(branch) 136 return active, branches 137 138 139 def _ReadGitilesContent(url): 140 # Download and decode BASE64 content until 141 # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. 142 base64_content = ReadUrlContent(url + '?format=TEXT') 143 return base64.b64decode(base64_content[0]) 144 145 146 def ReadRemoteCrFile(path_below_src, revision): 147 """Reads a remote Chromium file of a specific revision. Returns a string.""" 148 return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision, 149 path_below_src)) 150 151 152 def ReadRemoteCrCommit(revision): 153 """Reads a remote Chromium commit message. Returns a string.""" 154 return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) 155 156 157 def ReadUrlContent(url): 158 """Connect to a remote host and read the contents. Returns a list of lines.""" 159 conn = urllib.urlopen(url) 160 try: 161 return conn.readlines() 162 except IOError as e: 163 logging.exception('Error connecting to %s. Error: %s', url, e) 164 raise 165 finally: 166 conn.close() 167 168 169 def GetMatchingDepsEntries(depsentry_dict, dir_path): 170 """Gets all deps entries matching the provided path. 171 172 This list may contain more than one DepsEntry object. 173 Example: dir_path='src/testing' would give results containing both 174 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS. 175 Example 2: dir_path='src/build' should return 'src/build' but not 176 'src/buildtools'. 177 178 Returns: 179 A list of DepsEntry objects. 180 """ 181 result = [] 182 for path, depsentry in depsentry_dict.iteritems(): 183 if path == dir_path: 184 result.append(depsentry) 185 else: 186 parts = path.split('/') 187 if all(part == parts[i] 188 for i, part in enumerate(dir_path.split('/'))): 189 result.append(depsentry) 190 return result 191 192 193 def BuildDepsentryDict(deps_dict): 194 """Builds a dict of DepsEntry object from a raw parsed deps dict.""" 195 result = {} 196 def AddDepsEntries(deps_subdict): 197 for path, deps_url in deps_subdict.iteritems(): 198 if not result.has_key(path): 199 url, revision = deps_url.split('@') if deps_url else (None, None) 200 result[path] = DepsEntry(path, url, revision) 201 202 AddDepsEntries(deps_dict['deps']) 203 for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: 204 AddDepsEntries(deps_dict['deps_os'].get(deps_os, {})) 205 return result 206 207 208 def CalculateChangedDeps(current_deps, new_deps): 209 result = [] 210 current_entries = BuildDepsentryDict(current_deps) 211 new_entries = BuildDepsentryDict(new_deps) 212 213 all_deps_dirs = setup_links.DIRECTORIES 214 for deps_dir in all_deps_dirs: 215 # All deps have 'src' prepended to the path in the Chromium DEPS file. 216 dir_path = 'src/%s' % deps_dir 217 218 for entry in GetMatchingDepsEntries(current_entries, dir_path): 219 new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path) 220 assert len(new_matching_entries) <= 1, ( 221 'Should never find more than one entry matching %s in %s, found %d' % 222 (entry.path, new_entries, len(new_matching_entries))) 223 if not new_matching_entries: 224 result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None')) 225 elif entry != new_matching_entries[0]: 226 result.append(ChangedDep(entry.path, entry.url, entry.revision, 227 new_matching_entries[0].revision)) 228 return result 229 230 231 def CalculateChangedClang(new_cr_rev): 232 def GetClangRev(lines): 233 for line in lines: 234 match = CLANG_REVISION_RE.match(line) 235 if match: 236 return match.group(1) 237 return None 238 239 chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src', 240 CLANG_UPDATE_SCRIPT_LOCAL_PATH) 241 with open(chromium_src_path, 'rb') as f: 242 current_lines = f.readlines() 243 current_rev = GetClangRev(current_lines) 244 245 new_clang_update_sh = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, 246 new_cr_rev).splitlines() 247 new_rev = GetClangRev(new_clang_update_sh) 248 return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) 249 250 251 def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos, 252 new_commit_pos, changed_deps_list, clang_change): 253 current_cr_rev = current_cr_rev[0:7] 254 new_cr_rev = new_cr_rev[0:7] 255 rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) 256 git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) 257 258 commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval, 259 git_number_interval)] 260 commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval)) 261 commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % 262 rev_interval)) 263 # TBR field will be empty unless in some custom cases, where some engineers 264 # are added. 265 tbr_authors = '' 266 if changed_deps_list: 267 commit_msg.append('Changed dependencies:') 268 269 for c in changed_deps_list: 270 commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url, 271 c.current_rev[0:7], 272 c.new_rev[0:7])) 273 if 'libvpx' in c.path: 274 tbr_authors += 'marpan (at] webrtc.org, stefan (at] webrtc.org, ' 275 276 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') 277 commit_msg.append('DEPS diff: %s\n' % change_url) 278 else: 279 commit_msg.append('No dependencies changed.') 280 281 if clang_change.current_rev != clang_change.new_rev: 282 commit_msg.append('Clang version changed %s:%s' % 283 (clang_change.current_rev, clang_change.new_rev)) 284 change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 285 CLANG_UPDATE_SCRIPT_URL_PATH) 286 commit_msg.append('Details: %s\n' % change_url) 287 tbr_authors += 'pbos (at] webrtc.org' 288 else: 289 commit_msg.append('No update to Clang.\n') 290 291 commit_msg.append('TBR=%s' % tbr_authors) 292 return '\n'.join(commit_msg) 293 294 295 def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision): 296 """Update the DEPS file with the new revision.""" 297 with open(deps_filename, 'rb') as deps_file: 298 deps_content = deps_file.read() 299 deps_content = deps_content.replace(old_cr_revision, new_cr_revision) 300 with open(deps_filename, 'wb') as deps_file: 301 deps_file.write(deps_content) 302 303 def _IsTreeClean(): 304 stdout, _ = _RunCommand(['git', 'status', '--porcelain']) 305 if len(stdout) == 0: 306 return True 307 308 logging.error('Dirty/unversioned files:\n%s', stdout) 309 return False 310 311 312 def _EnsureUpdatedMasterBranch(dry_run): 313 current_branch = _RunCommand( 314 ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0] 315 if current_branch != 'master': 316 logging.error('Please checkout the master branch and re-run this script.') 317 if not dry_run: 318 sys.exit(-1) 319 320 logging.info('Updating master branch...') 321 _RunCommand(['git', 'pull']) 322 323 324 def _CreateRollBranch(dry_run): 325 logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) 326 if not dry_run: 327 _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) 328 329 330 def _RemovePreviousRollBranch(dry_run): 331 active_branch, branches = _GetBranches() 332 if active_branch == ROLL_BRANCH_NAME: 333 active_branch = 'master' 334 if ROLL_BRANCH_NAME in branches: 335 logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) 336 if not dry_run: 337 _RunCommand(['git', 'checkout', active_branch]) 338 _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) 339 340 341 def _LocalCommit(commit_msg, dry_run): 342 logging.info('Committing changes locally.') 343 if not dry_run: 344 _RunCommand(['git', 'add', '--update', '.']) 345 _RunCommand(['git', 'commit', '-m', commit_msg]) 346 347 348 def _UploadCL(dry_run, rietveld_email=None): 349 logging.info('Uploading CL...') 350 if not dry_run: 351 cmd = ['git', 'cl', 'upload', '-f'] 352 if rietveld_email: 353 cmd.append('--email=%s' % rietveld_email) 354 _RunCommand(cmd, extra_env={'EDITOR': 'true'}) 355 356 357 def _LaunchTrybots(dry_run, skip_try): 358 logging.info('Sending tryjobs...') 359 if not dry_run and not skip_try: 360 _RunCommand(['git', 'cl', 'try']) 361 362 363 def _SendToCQ(dry_run, skip_cq): 364 logging.info('Sending the CL to the CQ...') 365 if not dry_run and not skip_cq: 366 _RunCommand(['git', 'cl', 'set_commit']) 367 logging.info('Sent the CL to the CQ.') 368 369 370 def main(): 371 p = argparse.ArgumentParser() 372 p.add_argument('--clean', action='store_true', default=False, 373 help='Removes any previous local roll branch.') 374 p.add_argument('-r', '--revision', 375 help=('Chromium Git revision to roll to. Defaults to the ' 376 'Chromium HEAD revision if omitted.')) 377 p.add_argument('-u', '--rietveld-email', 378 help=('E-mail address to use for creating the CL at Rietveld' 379 'If omitted a previously cached one will be used or an ' 380 'error will be thrown during upload.')) 381 p.add_argument('--dry-run', action='store_true', default=False, 382 help=('Calculate changes and modify DEPS, but don\'t create ' 383 'any local branch, commit, upload CL or send any ' 384 'tryjobs.')) 385 p.add_argument('--allow-reverse', action='store_true', default=False, 386 help=('Allow rolling back in time (disabled by default but ' 387 'may be useful to be able do to manually).')) 388 p.add_argument('-s', '--skip-try', action='store_true', default=False, 389 help='Skip sending tryjobs (default: %(default)s)') 390 p.add_argument('--skip-cq', action='store_true', default=False, 391 help='Skip sending the CL to the CQ (default: %(default)s)') 392 p.add_argument('-v', '--verbose', action='store_true', default=False, 393 help='Be extra verbose in printing of log messages.') 394 opts = p.parse_args() 395 396 if opts.verbose: 397 logging.basicConfig(level=logging.DEBUG) 398 else: 399 logging.basicConfig(level=logging.INFO) 400 401 if not _IsTreeClean(): 402 logging.error('Please clean your local checkout first.') 403 return 1 404 405 if opts.clean: 406 _RemovePreviousRollBranch(opts.dry_run) 407 408 _EnsureUpdatedMasterBranch(opts.dry_run) 409 410 new_cr_rev = opts.revision 411 if not new_cr_rev: 412 stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) 413 head_rev = stdout.strip().split('\t')[0] 414 logging.info('No revision specified. Using HEAD: %s', head_rev) 415 new_cr_rev = head_rev 416 417 deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS') 418 local_deps = ParseLocalDepsFile(deps_filename) 419 current_cr_rev = local_deps['vars']['chromium_revision'] 420 421 current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev)) 422 new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev)) 423 424 current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev) 425 new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev) 426 427 if new_commit_pos > current_commit_pos or opts.allow_reverse: 428 changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps)) 429 clang_change = CalculateChangedClang(new_cr_rev) 430 commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev, 431 current_commit_pos, new_commit_pos, 432 changed_deps, clang_change) 433 logging.debug('Commit message:\n%s', commit_msg) 434 else: 435 logging.info('Currently pinned chromium_revision: %s (#%s) is newer than ' 436 '%s (#%s). To roll to older revisions, you must pass the ' 437 '--allow-reverse flag.\n' 438 'Aborting without action.', current_cr_rev, current_commit_pos, 439 new_cr_rev, new_commit_pos) 440 return 0 441 442 _CreateRollBranch(opts.dry_run) 443 UpdateDeps(deps_filename, current_cr_rev, new_cr_rev) 444 _LocalCommit(commit_msg, opts.dry_run) 445 _UploadCL(opts.dry_run, opts.rietveld_email) 446 _LaunchTrybots(opts.dry_run, opts.skip_try) 447 _SendToCQ(opts.dry_run, opts.skip_cq) 448 return 0 449 450 451 if __name__ == '__main__': 452 sys.exit(main()) 453
