1 #!/usr/bin/python 2 # 3 # Copyright (C) 2012 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Merge master-chromium to master within the Android tree.""" 18 19 import logging 20 import optparse 21 import os 22 import re 23 import shutil 24 import subprocess 25 import sys 26 27 import merge_common 28 29 30 AUTOGEN_MESSAGE = 'This commit was generated by merge_to_master.py.' 31 WEBVIEW_PROJECT = 'frameworks/webview' 32 33 34 def _GetAbsPath(project): 35 """Returns the full path to a given project (either Chromium or Android).""" 36 if project in merge_common.ALL_PROJECTS: 37 abs_path = os.path.join(merge_common.REPOSITORY_ROOT, project) 38 else: 39 abs_path = os.path.join(os.environ['ANDROID_BUILD_TOP'], project) 40 if not os.path.exists(abs_path): 41 raise merge_common.MergeError('Cannot find path ' + abs_path) 42 return abs_path 43 44 45 def _CheckoutSingleProject(project, target_branch): 46 """Checks out the tip of the target_branch into a local branch (merge-to-XXX). 47 48 Args: 49 project: a Chromium project (., third_party/foo) or frameworks/webview. 50 target_branch: name of the target branch (in the goog remote). 51 """ 52 dest_dir = _GetAbsPath(project) 53 tracking_branch = 'goog/' + target_branch 54 logging.debug('Check out %-45s at %-16s', project, tracking_branch) 55 merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'], 56 cwd=dest_dir) 57 merge_common.GetCommandStdout(['git', 'checkout', 58 '-b', 'merge-to-' + target_branch, 59 '-t', tracking_branch], cwd=dest_dir) 60 61 62 def _FetchSingleProject(project, remote, remote_ref): 63 """Fetches a remote ref for the given project and returns the fetched SHA. 64 65 Args: 66 project: a Chromium project (., third_party/foo) or frameworks/webview. 67 remote: Git remote name (goog for most projects, history for squashed ones). 68 remote_ref: the remote ref to fetch (e.g., refs/archive/chromium-XXX). 69 70 Returns: 71 The SHA1 of the FETCH_HEAD. 72 """ 73 dest_dir = _GetAbsPath(project) 74 logging.debug('Fetch %-45s %s:%s', project, remote, remote_ref) 75 merge_common.GetCommandStdout(['git', 'fetch', remote, remote_ref], 76 cwd=dest_dir) 77 return merge_common.GetCommandStdout(['git', 'rev-parse', 'FETCH_HEAD'], 78 cwd=dest_dir).strip() 79 80 81 def _MergeSingleProject(project, merge_sha, revision, target_branch, flatten): 82 """Merges a single project at a given SHA. 83 84 Args: 85 project: a Chromium project (., third_party/foo) or frameworks/webview. 86 merge_sha: the SHA to merge. 87 revision: Abbrev. commitish in the main Chromium repository. 88 target_branch: name of the target branch. 89 flatten: True: squash history while merging; False: perform a normal merge. 90 """ 91 dest_dir = _GetAbsPath(project) 92 if flatten: 93 # Make the previous merges into grafts so we can do a correct merge. 94 old_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], 95 cwd=dest_dir).strip() 96 merge_log = os.path.join(dest_dir, '.merged-revisions') 97 if os.path.exists(merge_log): 98 shutil.copyfile(merge_log, 99 os.path.join(dest_dir, '.git', 'info', 'grafts')) 100 101 # Early out if there is nothing to merge. 102 if not merge_common.GetCommandStdout(['git', 'rev-list', '-1', 103 'HEAD..' + merge_sha], cwd=dest_dir): 104 logging.debug('No new commits to merge in project %s', project) 105 return 106 107 logging.debug('Merging project %s (flatten: %s)...', project, flatten) 108 merge_cmd = ['git', 'merge', '--no-commit'] 109 merge_cmd += ['--squash'] if flatten else ['--no-ff'] 110 merge_cmd += [merge_sha] 111 # Merge conflicts cause 'git merge' to return 1, so ignore errors 112 merge_common.GetCommandStdout(merge_cmd, cwd=dest_dir, ignore_errors=True) 113 114 if flatten: 115 dirs_to_prune = merge_common.PRUNE_WHEN_FLATTENING.get(project, []) 116 if dirs_to_prune: 117 merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', '-rf'] + 118 dirs_to_prune, cwd=dest_dir) 119 120 if project in merge_common.ALL_PROJECTS: 121 commit_msg = 'Merge from Chromium at DEPS revision %s' % revision 122 else: 123 commit_msg = 'Merge master-chromium into %s at %s' % (target_branch, 124 revision) 125 commit_msg += '\n\n' + AUTOGEN_MESSAGE 126 merge_common.CheckNoConflictsAndCommitMerge(commit_msg, cwd=dest_dir) 127 128 if flatten: 129 # Generate the new grafts file and commit it on top of the merge. 130 new_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'], 131 cwd=dest_dir).strip() 132 with open(merge_log, 'a+') as f: 133 f.write('%s %s %s\n' % (new_sha, old_sha, merge_sha)) 134 merge_common.GetCommandStdout(['git', 'add', '.merged-revisions'], 135 cwd=dest_dir) 136 merge_common.GetCommandStdout( 137 ['git', 'commit', '-m', 138 'Record Chromium merge at DEPS revision %s\n\n%s' % 139 (revision, AUTOGEN_MESSAGE)], cwd=dest_dir) 140 141 142 def _IsAncestor(ref1, ref2, cwd): 143 """Checks whether ref1 is a ancestor of ref2 in the given Git repo.""" 144 cmd = ['git', 'merge-base', '--is-ancestor', ref1, ref2] 145 ret = subprocess.call(cmd, cwd=cwd) 146 if ret == 0: 147 return True 148 elif ret == 1: 149 return False 150 else: 151 raise merge_common.CommandError(ret, ' '.join(cmd), cwd, 'N/A', 'N/A') 152 153 154 def _MergeChromiumProjects(revision, target_branch, repo_shas=None, 155 force=False): 156 """Merges the Chromium projects from master-chromium to target_branch. 157 158 The larger projects' histories are flattened in the process. 159 When repo_shas != None, it checks that the SHAs of the projects in the 160 archive match exactly the SHAs of the projects in repo.prop. 161 162 Args: 163 revision: Abbrev. commitish in the main Chromium repository. 164 target_branch: target branch name to merge and push to. 165 repo_shas: optional dict. of expected revisions (only for --repo-prop). 166 force: True: merge anyways using the SHAs from repo.prop; False: bail out if 167 projects mismatch (archive vs repo.prop). 168 """ 169 # Sync and checkout ToT for all projects (creating the merge-to-XXX branch) 170 # and fetch the archive snapshot. 171 fetched_shas = {} 172 remote_ref = 'refs/archive/chromium-%s' % revision 173 for project in merge_common.PROJECTS_WITH_FLAT_HISTORY: 174 _CheckoutSingleProject(project, target_branch) 175 fetched_shas[project] = _FetchSingleProject(project, 'history', remote_ref) 176 for project in merge_common.PROJECTS_WITH_FULL_HISTORY: 177 _CheckoutSingleProject(project, target_branch) 178 fetched_shas[project] = _FetchSingleProject(project, 'goog', remote_ref) 179 180 if repo_shas: 181 project_shas_mismatch = False 182 for project, merge_sha in fetched_shas.items(): # the dict can be modified. 183 expected_sha = repo_shas.get(project) 184 if expected_sha != merge_sha: 185 logging.warn('The SHA for project %s specified in the repo.prop (%s) ' 186 'and the one in the archive (%s) differ.', 187 project, expected_sha, merge_sha) 188 dest_dir = _GetAbsPath(project) 189 if expected_sha is None: 190 reason = 'cannot find a SHA in the repo.pro for %s' % project 191 elif _IsAncestor(merge_sha, expected_sha, cwd=dest_dir): 192 reason = 'the SHA in repo.prop is ahead of the SHA in the archive. ' 193 log_cmd = ['git', 'log', '--oneline', '--graph', '--max-count=10', 194 '%s..%s' % (merge_sha, expected_sha)] 195 log_cmd_output = merge_common.GetCommandStdout(log_cmd, cwd=dest_dir) 196 reason += 'showing partial log (%s): \n %s' % (' '.join(log_cmd), 197 log_cmd_output) 198 elif _IsAncestor(expected_sha, merge_sha, cwd=dest_dir): 199 reason = 'The SHA is already merged in the archive' 200 else: 201 reason = 'The project history diverged. Consult your Git historian.' 202 203 project_shas_mismatch = True 204 if force: 205 logging.debug('Merging the SHA in repo.prop anyways (due to --force)') 206 fetched_shas[project] = expected_sha 207 else: 208 logging.debug('Reason: %s', reason) 209 if not force and project_shas_mismatch: 210 raise merge_common.MergeError( 211 'The revision of some projects in the archive is different from the ' 212 'one provided in build.prop. See the log for more details. Re-run ' 213 'with --force to continue.') 214 215 for project in merge_common.PROJECTS_WITH_FLAT_HISTORY: 216 _MergeSingleProject(project, fetched_shas[project], revision, target_branch, 217 flatten=True) 218 for project in merge_common.PROJECTS_WITH_FULL_HISTORY: 219 _MergeSingleProject(project, fetched_shas[project], revision, target_branch, 220 flatten=False) 221 222 223 def _GetNearestUpstreamAbbrevSHA(reference='history/master-chromium'): 224 """Returns the abbrev. upstream SHA which closest to the given reference.""" 225 logging.debug('Getting upstream SHA for %s...', reference) 226 merge_common.GetCommandStdout(['git', 'remote', 'update', 'history']) 227 upstream_commit = merge_common.Abbrev(merge_common.GetCommandStdout([ 228 'git', 'merge-base', 'history/upstream-master', reference])) 229 230 # Pedantic check: look for the existence of a merge commit which contains the 231 # |upstream_commit| in its message and is its children. 232 merge_parents = merge_common.GetCommandStdout([ 233 'git', 'rev-list', reference, '--grep', upstream_commit, '--merges', 234 '--parents', '-1']) 235 if upstream_commit not in merge_parents: 236 raise merge_common.MergeError( 237 'Found upstream commit %s, but the merge child (%s) could not be found ' 238 'or is not a parent of the upstream SHA') 239 logging.debug('Found nearest Chromium revision %s', upstream_commit) 240 return upstream_commit 241 242 243 def _MergeWithRepoProp(repo_prop_file, target_branch, force): 244 """Performs a merge using a repo.prop file (from Android build waterfall). 245 246 This does NOT merge (unless forced with force=True) the pinned 247 revisions in repo.prop, as a repo.prop can snapshot an intermediate state 248 (between two automerger cycles). Instead, this looks up the archived snapshot 249 (generated by the chromium->master-chromium auto-merger) which is closest to 250 the given repo.prop (following the main Chromium project) and merges that one. 251 If the projects revisions don't match, it fails with detailed error messages. 252 253 Args: 254 repo_prop_file: Path to a downloaded repo.prop file. 255 target_branch: name of the target branch to merget to. 256 force: ignores the aforementioned check and merged anyways. 257 """ 258 chromium_sha = None 259 webview_sha = None 260 repo_shas = {} # 'project/path' -> 'sha' 261 with open(repo_prop_file) as prop: 262 for line in prop: 263 repo, sha = line.split() 264 # Translate the Android repo paths into the relative project paths used in 265 # merge_common (e.g., platform/external/chromium_org/foo -> foo). 266 m = ( 267 re.match(r'^platform/(frameworks/.+)$', repo) or 268 re.match(r'^platform/external/chromium_org/?(.*?)(-history)?$', repo)) 269 if m: 270 project = m.group(1) if m.group(1) else '.' # '.' = Main project. 271 repo_shas[project] = sha 272 273 chromium_sha = repo_shas.get('.') 274 webview_sha = repo_shas.get(WEBVIEW_PROJECT) 275 if not chromium_sha or not webview_sha: 276 raise merge_common.MergeError('SHAs for projects not found; ' 277 'invalid build.prop?') 278 279 # Check that the revisions in repo.prop and the on in the archive match. 280 archived_chromium_revision = _GetNearestUpstreamAbbrevSHA(chromium_sha) 281 logging.info('Merging Chromium at %s and WebView at %s', 282 archived_chromium_revision, webview_sha) 283 _MergeChromiumProjects(archived_chromium_revision, target_branch, repo_shas, 284 force) 285 286 _CheckoutSingleProject(WEBVIEW_PROJECT, target_branch) 287 _MergeSingleProject(WEBVIEW_PROJECT, webview_sha, 288 archived_chromium_revision, target_branch, flatten=False) 289 290 291 def Push(target_branch): 292 """Push the finished snapshot to the Android repository. 293 294 Creates first a CL for frameworks/webview (if the merge-to-XXX branch exists) 295 then wait for user confirmation and pushes the Chromium merges. This is to 296 give an opportunity to get a +2 for frameworks/webview and then push both 297 frameworks/webview and the Chromium projects atomically(ish). 298 299 Args: 300 target_branch: name of the target branch (in the goog remote). 301 """ 302 merge_branch = 'merge-to-%s' % target_branch 303 304 # Create a Gerrit CL for the frameworks/webview project (if needed). 305 dest_dir = _GetAbsPath(WEBVIEW_PROJECT) 306 did_upload_webview_cl = False 307 if merge_common.GetCommandStdout(['git', 'branch', '--list', merge_branch], 308 cwd=dest_dir): 309 # Check that there was actually something to merge. 310 merge_range = 'goog/%s..%s' % (target_branch, merge_branch) 311 if merge_common.GetCommandStdout(['git', 'rev-list', '-1', merge_range], 312 cwd=dest_dir): 313 logging.info('Uploading a merge CL for %s...', WEBVIEW_PROJECT) 314 refspec = '%s:refs/for/%s' % (merge_branch, target_branch) 315 upload = merge_common.GetCommandStdout(['git', 'push', 'goog', refspec], 316 cwd=dest_dir) 317 logging.info(upload) 318 did_upload_webview_cl = True 319 320 prompt_msg = 'About push the Chromium projects merge. ' 321 if not did_upload_webview_cl: 322 logging.info('No merge CL needed for %s.', WEBVIEW_PROJECT) 323 else: 324 prompt_msg += ('At this point you should have the CL +2-ed and merge it ' 325 'together with this push.') 326 prompt_msg += '\nPress "y" to continue: ' 327 if raw_input(prompt_msg) != 'y': 328 logging.warn('Push aborted by the user!') 329 return 330 331 logging.debug('Pushing Chromium projects to %s ...', target_branch) 332 refspec = '%s:%s' % (merge_branch, target_branch) 333 for path in merge_common.ALL_PROJECTS: 334 logging.debug('Pushing %s', path) 335 dest_dir = _GetAbsPath(path) 336 # Delete the graft before pushing otherwise git will attempt to push all the 337 # grafted-in objects to the server as well as the ones we want. 338 graftfile = os.path.join(dest_dir, '.git', 'info', 'grafts') 339 if os.path.exists(graftfile): 340 os.remove(graftfile) 341 merge_common.GetCommandStdout(['git', 'push', 'goog', refspec], 342 cwd=dest_dir) 343 344 345 def main(): 346 parser = optparse.OptionParser(usage='%prog [options]') 347 parser.epilog = ('Takes the current master-chromium branch of the Chromium ' 348 'projects in Android and merges them into master to publish ' 349 'them.') 350 parser.add_option( 351 '', '--revision', 352 default=None, 353 help=('Merge to the specified archived master-chromium revision (abbrev. ' 354 'SHA or release version) rather than using HEAD. e.g., ' 355 '--revision=a1b2c3d4e5f6 or --revision=38.0.2125.24')) 356 parser.add_option( 357 '', '--repo-prop', 358 default=None, metavar='FILE', 359 help=('Merge to the revisions specified in this repo.prop file.')) 360 parser.add_option( 361 '', '--force', 362 default=False, action='store_true', 363 help=('Skip history checks and merged anyways (only for --repo-prop).')) 364 parser.add_option( 365 '', '--push', 366 default=False, action='store_true', 367 help=('Push the result of a previous merge to the server.')) 368 parser.add_option( 369 '', '--target', 370 default='master', metavar='BRANCH', 371 help=('Target branch to push to. Defaults to master.')) 372 (options, args) = parser.parse_args() 373 if args: 374 parser.print_help() 375 return 1 376 377 logging.basicConfig(format='%(message)s', level=logging.DEBUG, 378 stream=sys.stdout) 379 380 if options.push: 381 Push(options.target) 382 elif options.repo_prop: 383 _MergeWithRepoProp(os.path.expanduser(options.repo_prop), 384 options.target, options.force) 385 elif options.revision: 386 _MergeChromiumProjects(options.revision, options.target) 387 else: 388 first_upstream_sha = _GetNearestUpstreamAbbrevSHA() 389 _MergeChromiumProjects(first_upstream_sha, options.target) 390 391 return 0 392 393 if __name__ == '__main__': 394 sys.exit(main()) 395
