1 #!/usr/bin/python 2 # 3 # Copyright (C) 2012 The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 """Merge Chromium into the Android tree.""" 18 19 import logging 20 import optparse 21 import os 22 import re 23 import sys 24 25 import merge_common 26 27 28 # We need to import this *after* merging from upstream to get the latest 29 # version. Set it to none here to catch uses before it's imported. 30 webview_licenses = None 31 32 33 AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.' 34 SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master' 35 36 37 def _ReadGitFile(sha1, path, git_url=None, git_branch=None): 38 """Reads a file from a (possibly remote) git project at a specific revision. 39 40 Args: 41 sha1: The SHA1 at which to read. 42 path: The relative path of the file to read. 43 git_url: The URL of the git server, if reading a remote project. 44 git_branch: The branch to fetch, if reading a remote project. 45 Returns: 46 The contents of the specified file. 47 """ 48 if git_url: 49 merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, git_branch]) 50 return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)]) 51 52 53 def _ParseDEPS(deps_content): 54 """Parses the DEPS file from Chromium and returns its contents. 55 56 Args: 57 deps_content: The contents of the DEPS file as text. 58 Returns: 59 A dictionary of the contents of DEPS at the specified revision 60 """ 61 62 class FromImpl(object): 63 """Used to implement the From syntax.""" 64 65 def __init__(self, module_name): 66 self.module_name = module_name 67 68 def __str__(self): 69 return 'From("%s")' % self.module_name 70 71 class _VarImpl(object): 72 73 def __init__(self, custom_vars, local_scope): 74 self._custom_vars = custom_vars 75 self._local_scope = local_scope 76 77 def Lookup(self, var_name): 78 """Implements the Var syntax.""" 79 if var_name in self._custom_vars: 80 return self._custom_vars[var_name] 81 elif var_name in self._local_scope.get('vars', {}): 82 return self._local_scope['vars'][var_name] 83 raise Exception('Var is not defined: %s' % var_name) 84 85 tmp_locals = {} 86 var = _VarImpl({}, tmp_locals) 87 tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}} 88 exec(deps_content) in tmp_globals, tmp_locals # pylint: disable=W0122 89 return tmp_locals 90 91 92 def _GetProjectMergeInfo(projects, deps_vars): 93 """Gets the git URL and SHA1 for each project based on DEPS. 94 95 Args: 96 projects: The list of projects to consider. 97 deps_vars: The dictionary of dependencies from DEPS. 98 Returns: 99 A dictionary from project to git URL and SHA1 - 'path: (url, sha1)' 100 Raises: 101 TemporaryMergeError: if a project to be merged is not found in DEPS. 102 """ 103 deps_fallback_order = [ 104 deps_vars['deps'], 105 deps_vars['deps_os']['unix'], 106 deps_vars['deps_os']['android'], 107 ] 108 result = {} 109 for path in projects: 110 for deps in deps_fallback_order: 111 if path: 112 upstream_path = os.path.join('src', path) 113 else: 114 upstream_path = 'src' 115 url_plus_sha1 = deps.get(upstream_path) 116 if url_plus_sha1: 117 break 118 else: 119 raise merge_common.TemporaryMergeError( 120 'Could not find DEPS entry for project %s. This probably ' 121 'means that the project list in merge_from_chromium.py needs to be ' 122 'updated.' % path) 123 match = re.match('(.*?)@(.*)', url_plus_sha1) 124 url = match.group(1) 125 sha1 = match.group(2) 126 logging.debug(' Got URL %s and SHA1 %s for project %s', url, sha1, path) 127 result[path] = {'url': url, 'sha1': sha1} 128 return result 129 130 131 def _MergeProjects(version, root_sha1, target, unattended, buildspec_url): 132 """Merges each required Chromium project into the Android repository. 133 134 DEPS is consulted to determine which revision each project must be merged 135 at. Only a whitelist of required projects are merged. 136 137 Args: 138 version: The version to mention in generated commit messages. 139 root_sha1: The git hash to merge in the root repository. 140 target: The target branch to merge to. 141 unattended: Run in unattended mode. 142 buildspec_url: URL for buildspec repository, when merging a branch. 143 Returns: 144 The abbrev sha1 merged. It will be either |root_sha1| itself (when merging 145 chromium trunk) or the upstream sha1 of the release. 146 Raises: 147 TemporaryMergeError: If incompatibly licensed code is left after pruning. 148 """ 149 # The logic for this step lives here, in the Android tree, as it makes no 150 # sense for a Chromium tree to know about this merge. 151 152 if unattended: 153 branch_create_flag = '-B' 154 else: 155 branch_create_flag = '-b' 156 branch_name = 'merge-from-chromium-%s' % version 157 158 logging.debug('Parsing DEPS ...') 159 if root_sha1: 160 deps_content = _ReadGitFile(root_sha1, 'DEPS') 161 else: 162 # TODO(primiano): At some point the release branches will use DEPS as well, 163 # instead of .DEPS.git. Rename below when that day will come. 164 deps_content = _ReadGitFile('FETCH_HEAD', 165 'releases/' + version + '/.DEPS.git', 166 buildspec_url, 167 'master') 168 169 deps_vars = _ParseDEPS(deps_content) 170 171 merge_info = _GetProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS, 172 deps_vars) 173 174 for path in merge_info: 175 # webkit needs special handling as we have a local mirror 176 local_mirrored = path == 'third_party/WebKit' 177 url = merge_info[path]['url'] 178 sha1 = merge_info[path]['sha1'] 179 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 180 if local_mirrored: 181 remote = 'history' 182 else: 183 remote = 'goog' 184 merge_common.GetCommandStdout(['git', 'checkout', 185 branch_create_flag, branch_name, 186 '-t', remote + '/' + target], 187 cwd=dest_dir) 188 if not local_mirrored or not root_sha1: 189 logging.debug('Fetching project %s at %s ...', path, sha1) 190 fetch_args = ['git', 'fetch', url, sha1] 191 merge_common.GetCommandStdout(fetch_args, cwd=dest_dir) 192 if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1], 193 cwd=dest_dir): 194 logging.debug('Merging project %s at %s ...', path, sha1) 195 # Merge conflicts make git merge return 1, so ignore errors 196 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1], 197 cwd=dest_dir, ignore_errors=True) 198 merge_common.CheckNoConflictsAndCommitMerge( 199 'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE), 200 cwd=dest_dir, unattended=unattended) 201 else: 202 logging.debug('No new commits to merge in project %s', path) 203 204 # Handle root repository separately. 205 merge_common.GetCommandStdout(['git', 'checkout', 206 branch_create_flag, branch_name, 207 '-t', 'history/' + target]) 208 if not root_sha1: 209 merge_info = _GetProjectMergeInfo([''], deps_vars) 210 url = merge_info['']['url'] 211 merged_sha1 = merge_info['']['sha1'] 212 merge_common.GetCommandStdout(['git', 'fetch', url, merged_sha1]) 213 merged_sha1 = merge_common.Abbrev(merged_sha1) 214 merge_msg_version = '%s (%s)' % (version, merged_sha1) 215 else: 216 merge_msg_version = root_sha1 217 merged_sha1 = root_sha1 218 219 logging.debug('Merging Chromium at %s ...', merged_sha1) 220 # Merge conflicts make git merge return 1, so ignore errors 221 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', merged_sha1], 222 ignore_errors=True) 223 merge_common.CheckNoConflictsAndCommitMerge( 224 'Merge Chromium at %s\n\n%s' 225 % (merge_msg_version, AUTOGEN_MESSAGE), unattended=unattended) 226 227 logging.debug('Getting directories to exclude ...') 228 229 # We import this now that we have merged the latest version. 230 # It imports to a global in order that it can be used to generate NOTICE 231 # later. We also disable writing bytecode to keep the source tree clean. 232 sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview', 233 'tools')) 234 sys.dont_write_bytecode = True 235 global webview_licenses # pylint: disable=W0602 236 import webview_licenses # pylint: disable=W0621,W0612,C6204 237 import known_issues # pylint: disable=C6204 238 239 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): 240 logging.debug(' %s', '\n '.join(os.path.join(path, x) for x in 241 exclude_list)) 242 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 243 merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] + 244 exclude_list, cwd=dest_dir) 245 if _ModifiedFilesInIndex(dest_dir): 246 merge_common.GetCommandStdout(['git', 'commit', '-m', 247 'Exclude unwanted directories'], 248 cwd=dest_dir) 249 assert(root_sha1 is None or root_sha1 == merged_sha1) 250 return merged_sha1 251 252 253 def _CheckLicenses(): 254 """Check that no incompatibly licensed directories exist.""" 255 directories_left_over = webview_licenses.GetIncompatibleDirectories() 256 if directories_left_over: 257 raise merge_common.TemporaryMergeError( 258 'Incompatibly licensed directories remain: ' + 259 '\n'.join(directories_left_over)) 260 261 262 def _GenerateMakefiles(version, unattended): 263 """Run gyp to generate the Android build system makefiles. 264 265 Args: 266 version: The version to mention in generated commit messages. 267 unattended: Run in unattended mode. 268 """ 269 logging.debug('Generating makefiles ...') 270 271 # TODO(torne): come up with a way to deal with hooks from DEPS properly 272 273 # TODO(torne): The .tmp files are generated by 274 # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source 275 # tree. We should avoid this, or at least use a more specific name to avoid 276 # accidentally removing or adding other files. 277 for path in merge_common.ALL_PROJECTS: 278 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 279 merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', 280 'GypAndroid.*.mk', '*.target.*.mk', 281 '*.host.*.mk', '*.tmp'], cwd=dest_dir) 282 283 try: 284 merge_common.GetCommandStdout(['android_webview/tools/gyp_webview', 'all']) 285 except merge_common.MergeError as e: 286 if not unattended: 287 raise 288 else: 289 for path in merge_common.ALL_PROJECTS: 290 merge_common.GetCommandStdout( 291 ['git', 'reset', '--hard'], 292 cwd=os.path.join(merge_common.REPOSITORY_ROOT, path)) 293 raise merge_common.TemporaryMergeError('Makefile generation failed: ' + 294 str(e)) 295 296 for path in merge_common.ALL_PROJECTS: 297 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 298 # git add doesn't have an --ignore-unmatch so we have to do this instead: 299 merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'], 300 ignore_errors=True, cwd=dest_dir) 301 merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'], 302 ignore_errors=True, cwd=dest_dir) 303 merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'], 304 ignore_errors=True, cwd=dest_dir) 305 merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'], 306 ignore_errors=True, cwd=dest_dir) 307 # Only try to commit the makefiles if something has actually changed. 308 if _ModifiedFilesInIndex(dest_dir): 309 merge_common.GetCommandStdout( 310 ['git', 'commit', '-m', 311 'Update makefiles after merge of Chromium at %s\n\n%s' % 312 (version, AUTOGEN_MESSAGE)], cwd=dest_dir) 313 314 315 def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT): 316 """Returns true if git's index contains any changes.""" 317 status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'], 318 cwd=cwd) 319 return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None 320 321 322 def _GenerateNoticeFile(version): 323 """Generates and commits a NOTICE file containing code licenses. 324 325 This covers all third-party code (from Android's perspective) that lives in 326 the Chromium tree. 327 328 Args: 329 version: The version to mention in generated commit messages. 330 """ 331 logging.debug('Regenerating NOTICE file ...') 332 333 contents = webview_licenses.GenerateNoticeFile() 334 335 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f: 336 f.write(contents) 337 merge_common.GetCommandStdout(['git', 'add', 'NOTICE']) 338 # Only try to commit the NOTICE update if the file has actually changed. 339 if _ModifiedFilesInIndex(): 340 merge_common.GetCommandStdout([ 341 'git', 'commit', '-m', 342 'Update NOTICE file after merge of Chromium at %s\n\n%s' 343 % (version, AUTOGEN_MESSAGE)]) 344 345 346 def _GenerateLastChange(version, root_sha1): 347 """Write a build/util/LASTCHANGE file containing the current revision. 348 349 The revision number is compiled into the binary at build time from this file. 350 351 Args: 352 version: The version to mention in generated commit messages. 353 root_sha1: The SHA1 of the main project (before the merge). 354 """ 355 logging.debug('Updating LASTCHANGE ...') 356 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'), 357 'w') as f: 358 f.write('LASTCHANGE=%s\n' % merge_common.Abbrev(root_sha1)) 359 merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE']) 360 logging.debug('Updating LASTCHANGE.blink ...') 361 with open(os.path.join(merge_common.REPOSITORY_ROOT, 362 'build/util/LASTCHANGE.blink'), 'w') as f: 363 f.write('LASTCHANGE=%s\n' % _GetBlinkRevision()) 364 merge_common.GetCommandStdout(['git', 'add', '-f', 365 'build/util/LASTCHANGE.blink']) 366 if _ModifiedFilesInIndex(): 367 merge_common.GetCommandStdout([ 368 'git', 'commit', '-m', 369 'Update LASTCHANGE file after merge of Chromium at %s\n\n%s' 370 % (version, AUTOGEN_MESSAGE)]) 371 372 373 def GetHEAD(): 374 """Fetch the latest HEAD revision from the Chromium Git mirror. 375 376 Returns: 377 The latest HEAD revision (A Git abbrev SHA1). 378 """ 379 return _GetGitAbbrevSHA1(SRC_GIT_BRANCH, 'HEAD') 380 381 382 def _ParseSvnRevisionFromGitCommitMessage(commit_message): 383 return re.search(r'^git-svn-id: .*@([0-9]+)', commit_message, 384 flags=re.MULTILINE).group(1) 385 386 387 def _GetGitAbbrevSHA1(git_branch, revision): 388 """Returns an abbrev. SHA for the given revision (or branch, if HEAD).""" 389 assert revision 390 logging.debug('Getting Git revision for %s ...', revision) 391 392 upstream = git_branch if revision == 'HEAD' else revision 393 394 # Make sure the remote and the branch exist locally. 395 try: 396 merge_common.GetCommandStdout([ 397 'git', 'show-ref', '--verify', '--quiet', git_branch]) 398 except merge_common.CommandError: 399 raise merge_common.TemporaryMergeError( 400 'Cannot find the branch %s. Have you sync\'d master-chromium in this ' 401 'checkout?' % git_branch) 402 403 # Make sure the |upstream| Git object has been mirrored. 404 try: 405 merge_common.GetCommandStdout([ 406 'git', 'merge-base', '--is-ancestor', upstream, git_branch]) 407 except merge_common.CommandError: 408 raise merge_common.TemporaryMergeError( 409 'Upstream object (%s) not reachable from %s' % (upstream, git_branch)) 410 411 abbrev_sha = merge_common.Abbrev(merge_common.GetCommandStdout( 412 ['git', 'rev-list', '--max-count=1', upstream]).split()[0]) 413 return abbrev_sha 414 415 416 def _GetBlinkRevision(): 417 # TODO(primiano): Switch to Git as soon as Blink gets migrated as well. 418 commit = merge_common.GetCommandStdout( 419 ['git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b'], 420 cwd=os.path.join(merge_common.REPOSITORY_ROOT, 'third_party', 'WebKit')) 421 return _ParseSvnRevisionFromGitCommitMessage(commit) 422 423 424 def Snapshot(root_sha1, release, target, unattended, buildspec_url): 425 """Takes a snapshot of the Chromium tree and merges it into Android. 426 427 Android makefiles and a top-level NOTICE file are generated and committed 428 after the merge. 429 430 Args: 431 root_sha1: The abbrev sha1 in the Chromium git mirror to merge from. 432 release: The Chromium release version to merge from (e.g. "30.0.1599.20"). 433 Only one of root_sha1 and release should be specified. 434 target: The target branch to merge to. 435 unattended: Run in unattended mode. 436 buildspec_url: URL for buildspec repository, used when merging a release. 437 438 Returns: 439 True if new commits were merged; False if no new commits were present. 440 """ 441 if release: 442 root_sha1 = None 443 version = release 444 else: 445 root_sha1 = _GetGitAbbrevSHA1(SRC_GIT_BRANCH, root_sha1) 446 version = root_sha1 447 448 assert (root_sha1 is not None and len(root_sha1) > 6) or version == release 449 450 if root_sha1 and not merge_common.GetCommandStdout( 451 ['git', 'rev-list', '-1', 'HEAD..' + root_sha1]): 452 logging.info('No new commits to merge at %s (%s)', version, root_sha1) 453 return False 454 455 logging.info('Snapshotting Chromium at %s (%s)', version, root_sha1) 456 457 # 1. Merge, accounting for excluded directories 458 merged_sha1 = _MergeProjects(version, root_sha1, target, unattended, 459 buildspec_url) 460 461 # 2. Generate Android makefiles 462 _GenerateMakefiles(version, unattended) 463 464 # 3. Check for incompatible licenses 465 _CheckLicenses() 466 467 # 4. Generate Android NOTICE file 468 _GenerateNoticeFile(version) 469 470 # 5. Generate LASTCHANGE file 471 _GenerateLastChange(version, merged_sha1) 472 473 return True 474 475 476 def Push(version, target): 477 """Push the finished snapshot to the Android repository.""" 478 src = 'merge-from-chromium-%s' % version 479 # Use forced pushes ('+' prefix) for the temporary and archive branches in 480 # case they already got updated by a previous (possibly failed?) merge, but 481 # do not force push to the real master-chromium branch as this could erase 482 # downstream changes. 483 refspecs = ['%s:%s' % (src, target), 484 '+%s:refs/archive/chromium-%s' % (src, version)] 485 if target == 'master-chromium': 486 refspecs.insert(0, '+%s:master-chromium-merge' % src) 487 for refspec in refspecs: 488 logging.debug('Pushing to server (%s) ...', refspec) 489 for path in merge_common.ALL_PROJECTS: 490 if path in merge_common.PROJECTS_WITH_FLAT_HISTORY: 491 remote = 'history' 492 else: 493 remote = 'goog' 494 logging.debug('Pushing %s', path) 495 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 496 merge_common.GetCommandStdout(['git', 'push', remote, refspec], 497 cwd=dest_dir) 498 499 500 def main(): 501 parser = optparse.OptionParser(usage='%prog [options]') 502 parser.epilog = ('Takes a snapshot of the Chromium tree at the specified ' 503 'Chromium Git revision and merges it into this repository. ' 504 'Paths marked as excluded for license reasons are removed ' 505 'as part of the merge. Also generates Android makefiles and ' 506 'generates a top-level NOTICE file suitable for use in the ' 507 'Android build.') 508 parser.add_option( 509 '', '--sha1', 510 default='HEAD', 511 help=('Merge to the specified chromium sha1 revision from ' + 512 SRC_GIT_BRANCH + ' branch. Default is HEAD, to merge from ToT.')) 513 parser.add_option( 514 '', '--release', 515 default=None, 516 help=('Merge to the specified chromium release buildspec (e.g., "30.0.' 517 '1599.20"). Only one of --sha1 and --release should be specified')) 518 parser.add_option( 519 '', '--buildspec_url', 520 default=None, 521 help=('Git URL for buildspec repository.')) 522 parser.add_option( 523 '', '--target', 524 default='master-chromium', metavar='BRANCH', 525 help=('Target branch to push to. Defaults to master-chromium.')) 526 parser.add_option( 527 '', '--push', 528 default=False, action='store_true', 529 help=('Push the result of a previous merge to the server. Note ' 530 '--sha1 must be given.')) 531 parser.add_option( 532 '', '--get_head', 533 default=False, action='store_true', 534 help=('Just print the current HEAD revision on stdout and exit.')) 535 parser.add_option( 536 '', '--unattended', 537 default=False, action='store_true', 538 help=('Run in unattended mode.')) 539 parser.add_option( 540 '', '--no_changes_exit', 541 default=0, type='int', 542 help=('Exit code to use if there are no changes to merge, for scripts.')) 543 (options, args) = parser.parse_args() 544 if args: 545 parser.print_help() 546 return 1 547 548 if 'ANDROID_BUILD_TOP' not in os.environ: 549 print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.' 550 return 1 551 552 if os.environ.get('GYP_DEFINES'): 553 print >>sys.stderr, ( 554 'The environment is defining GYP_DEFINES (=%s). It will affect the ' 555 ' generated makefiles.' % os.environ['GYP_DEFINES']) 556 if not options.unattended and raw_input('Continue? [y/N]') != 'y': 557 return 1 558 559 logging.basicConfig(format='%(message)s', level=logging.DEBUG, 560 stream=sys.stdout) 561 562 if options.get_head: 563 logging.disable(logging.CRITICAL) # Prevent log messages 564 print GetHEAD() 565 elif options.push: 566 if options.release: 567 Push(options.release, options.target) 568 elif options.sha1: 569 Push(options.sha1, options.target) 570 else: 571 print >>sys.stderr, 'You need to pass the version to push.' 572 return 1 573 else: 574 if not Snapshot(options.sha1, options.release, options.target, 575 options.unattended, options.buildspec_url): 576 return options.no_changes_exit 577 578 return 0 579 580 if __name__ == '__main__': 581 sys.exit(main()) 582
