1 #!/usr/bin/env python 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Script that reads omahaproxy and gsutil to determine version of SDK to put 7 in manifest. 8 """ 9 10 # pylint is convinced the email module is missing attributes 11 # pylint: disable=E1101 12 13 import buildbot_common 14 import csv 15 import cStringIO 16 import difflib 17 import email 18 import json 19 import logging 20 import logging.handlers 21 import manifest_util 22 import optparse 23 import os 24 import posixpath 25 import re 26 import smtplib 27 import subprocess 28 import sys 29 import time 30 import traceback 31 import urllib2 32 33 MANIFEST_BASENAME = 'naclsdk_manifest2.json' 34 SCRIPT_DIR = os.path.dirname(__file__) 35 REPO_MANIFEST = os.path.join(SCRIPT_DIR, 'json', MANIFEST_BASENAME) 36 GS_BUCKET_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/' 37 GS_SDK_MANIFEST = GS_BUCKET_PATH + MANIFEST_BASENAME 38 GS_SDK_MANIFEST_LOG = GS_BUCKET_PATH + MANIFEST_BASENAME + '.log' 39 GS_MANIFEST_BACKUP_DIR = GS_BUCKET_PATH + 'manifest_backups/' 40 41 CANARY_BUNDLE_NAME = 'pepper_canary' 42 CANARY = 'canary' 43 NACLPORTS_ARCHIVE_NAME = 'naclports.tar.bz2' 44 45 46 logger = logging.getLogger(__name__) 47 48 49 def SplitVersion(version_string): 50 """Split a version string (e.g. "18.0.1025.163") into its components. 51 52 Note that this function doesn't handle versions in the form "trunk.###". 53 """ 54 return tuple(map(int, version_string.split('.'))) 55 56 57 def JoinVersion(version_tuple): 58 """Create a string from a version tuple. 59 60 The tuple should be of the form (18, 0, 1025, 163). 61 """ 62 return '.'.join(map(str, version_tuple)) 63 64 65 def GetTimestampManifestName(): 66 """Create a manifest name with a timestamp. 67 68 Returns: 69 A manifest name with an embedded date. This should make it easier to roll 70 back if necessary. 71 """ 72 return time.strftime('naclsdk_manifest2.%Y_%m_%d_%H_%M_%S.json', 73 time.gmtime()) 74 75 76 def GetPlatformArchiveName(platform): 77 """Get the basename of an archive given a platform string. 78 79 Args: 80 platform: One of ('win', 'mac', 'linux'). 81 82 Returns: 83 The basename of the sdk archive for that platform. 84 """ 85 return 'naclsdk_%s.tar.bz2' % platform 86 87 88 def GetCanonicalArchiveName(url): 89 """Get the canonical name of an archive given its URL. 90 91 This will convert "naclsdk_linux.bz2" -> "naclsdk_linux.tar.bz2", and also 92 remove everything but the filename of the URL. 93 94 This is used below to determine if an expected bundle is found in an version 95 directory; the archives all have the same name, but may not exist for a given 96 version. 97 98 Args: 99 url: The url to parse. 100 101 Returns: 102 The canonical name as described above. 103 """ 104 name = posixpath.basename(url) 105 match = re.match(r'naclsdk_(.*?)(?:\.tar)?\.bz2', name) 106 if match: 107 return 'naclsdk_%s.tar.bz2' % match.group(1) 108 109 return name 110 111 112 class Delegate(object): 113 """Delegate all external access; reading/writing to filesystem, gsutil etc.""" 114 115 def GetRepoManifest(self): 116 """Read the manifest file from the NaCl SDK repository. 117 118 This manifest is used as a template for the auto updater; only pepper 119 bundles with no archives are considered for auto updating. 120 121 Returns: 122 A manifest_util.SDKManifest object read from the NaCl SDK repo.""" 123 raise NotImplementedError() 124 125 def GetHistory(self): 126 """Read Chrome release history from omahaproxy.appspot.com 127 128 Here is an example of data from this URL: 129 cros,stable,18.0.1025.168,2012-05-01 17:04:05.962578\n 130 win,canary,20.0.1123.0,2012-05-01 13:59:31.703020\n 131 mac,canary,20.0.1123.0,2012-05-01 11:54:13.041875\n 132 win,stable,18.0.1025.168,2012-04-30 20:34:56.078490\n 133 mac,stable,18.0.1025.168,2012-04-30 20:34:55.231141\n 134 ... 135 Where each line has comma separated values in the following format: 136 platform, channel, version, date/time\n 137 138 Returns: 139 A list where each element is a line from the document, represented as a 140 tuple.""" 141 raise NotImplementedError() 142 143 def GetTrunkRevision(self, version): 144 """Given a Chrome version, get its trunk revision. 145 146 Args: 147 version: A version string of the form '18.0.1025.64' 148 Returns: 149 The revision number for that version, as a string.""" 150 raise NotImplementedError() 151 152 def GsUtil_ls(self, url): 153 """Runs gsutil ls |url| 154 155 Args: 156 url: The commondatastorage url to list. 157 Returns: 158 A list of URLs, all with the gs:// schema.""" 159 raise NotImplementedError() 160 161 def GsUtil_cat(self, url): 162 """Runs gsutil cat |url| 163 164 Args: 165 url: The commondatastorage url to read from. 166 Returns: 167 A string with the contents of the file at |url|.""" 168 raise NotImplementedError() 169 170 def GsUtil_cp(self, src, dest, stdin=None): 171 """Runs gsutil cp |src| |dest| 172 173 Args: 174 src: The file path or url to copy from. 175 dest: The file path or url to copy to. 176 stdin: If src is '-', this is used as the stdin to give to gsutil. The 177 effect is that text in stdin is copied to |dest|.""" 178 raise NotImplementedError() 179 180 def SendMail(self, subject, text): 181 """Send an email. 182 183 Args: 184 subject: The subject of the email. 185 text: The text of the email. 186 """ 187 raise NotImplementedError() 188 189 190 class RealDelegate(Delegate): 191 def __init__(self, dryrun=False, gsutil=None, mailfrom=None, mailto=None): 192 super(RealDelegate, self).__init__() 193 self.dryrun = dryrun 194 self.mailfrom = mailfrom 195 self.mailto = mailto 196 if gsutil: 197 self.gsutil = gsutil 198 else: 199 self.gsutil = buildbot_common.GetGsutil() 200 201 def GetRepoManifest(self): 202 """See Delegate.GetRepoManifest""" 203 with open(REPO_MANIFEST, 'r') as sdk_stream: 204 sdk_json_string = sdk_stream.read() 205 206 manifest = manifest_util.SDKManifest() 207 manifest.LoadDataFromString(sdk_json_string, add_missing_info=True) 208 return manifest 209 210 def GetHistory(self): 211 """See Delegate.GetHistory""" 212 url_stream = urllib2.urlopen('https://omahaproxy.appspot.com/history') 213 return [(platform, channel, version, date) 214 for platform, channel, version, date in csv.reader(url_stream)] 215 216 def GetTrunkRevision(self, version): 217 """See Delegate.GetTrunkRevision""" 218 url = 'http://omahaproxy.appspot.com/revision.json?version=%s' % (version,) 219 data = json.loads(urllib2.urlopen(url).read()) 220 return 'trunk.%s' % int(data['chromium_revision']) 221 222 def GsUtil_ls(self, url): 223 """See Delegate.GsUtil_ls""" 224 try: 225 stdout = self._RunGsUtil(None, False, 'ls', url) 226 except subprocess.CalledProcessError: 227 return [] 228 229 # filter out empty lines 230 return filter(None, stdout.split('\n')) 231 232 def GsUtil_cat(self, url): 233 """See Delegate.GsUtil_cat""" 234 return self._RunGsUtil(None, True, 'cat', url) 235 236 def GsUtil_cp(self, src, dest, stdin=None): 237 """See Delegate.GsUtil_cp""" 238 if self.dryrun: 239 logger.info("Skipping upload: %s -> %s" % (src, dest)) 240 if src == '-': 241 logger.info(' contents = """%s"""' % stdin) 242 return 243 244 return self._RunGsUtil(stdin, True, 'cp', '-a', 'public-read', src, dest) 245 246 def SendMail(self, subject, text): 247 """See Delegate.SendMail""" 248 if self.mailfrom and self.mailto: 249 msg = email.MIMEMultipart.MIMEMultipart() 250 msg['From'] = self.mailfrom 251 msg['To'] = ', '.join(self.mailto) 252 msg['Date'] = email.Utils.formatdate(localtime=True) 253 msg['Subject'] = subject 254 msg.attach(email.MIMEText.MIMEText(text)) 255 smtp_obj = smtplib.SMTP('localhost') 256 smtp_obj.sendmail(self.mailfrom, self.mailto, msg.as_string()) 257 smtp_obj.close() 258 259 def _RunGsUtil(self, stdin, log_errors, *args): 260 """Run gsutil as a subprocess. 261 262 Args: 263 stdin: If non-None, used as input to the process. 264 log_errors: If True, write errors to stderr. 265 *args: Arguments to pass to gsutil. The first argument should be an 266 operation such as ls, cp or cat. 267 Returns: 268 The stdout from the process.""" 269 cmd = [self.gsutil] + list(args) 270 logger.debug("Running: %s" % str(cmd)) 271 if stdin: 272 stdin_pipe = subprocess.PIPE 273 else: 274 stdin_pipe = None 275 276 try: 277 process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE, 278 stderr=subprocess.PIPE) 279 stdout, stderr = process.communicate(stdin) 280 except OSError as e: 281 raise manifest_util.Error("Unable to run '%s': %s" % (cmd[0], str(e))) 282 283 if process.returncode: 284 if log_errors: 285 logger.error(stderr) 286 raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd)) 287 return stdout 288 289 290 class GsutilLoggingHandler(logging.handlers.BufferingHandler): 291 def __init__(self, delegate): 292 logging.handlers.BufferingHandler.__init__(self, capacity=0) 293 self.delegate = delegate 294 295 def shouldFlush(self, record): 296 # BufferingHandler.shouldFlush automatically flushes if the length of the 297 # buffer is greater than self.capacity. We don't want that behavior, so 298 # return False here. 299 return False 300 301 def flush(self): 302 # Do nothing here. We want to be explicit about uploading the log. 303 pass 304 305 def upload(self): 306 output_list = [] 307 for record in self.buffer: 308 output_list.append(self.format(record)) 309 output = '\n'.join(output_list) 310 self.delegate.GsUtil_cp('-', GS_SDK_MANIFEST_LOG, stdin=output) 311 312 logging.handlers.BufferingHandler.flush(self) 313 314 315 class NoSharedVersionException(Exception): 316 pass 317 318 319 class VersionFinder(object): 320 """Finds a version of a pepper bundle that all desired platforms share. 321 322 Args: 323 delegate: See Delegate class above. 324 platforms: A sequence of platforms to consider, e.g. 325 ('mac', 'linux', 'win') 326 extra_archives: A sequence of tuples: (archive_basename, minimum_version), 327 e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')] 328 These archives must exist to consider a version for inclusion, as 329 long as that version is greater than the archive's minimum version. 330 """ 331 def __init__(self, delegate, platforms, extra_archives=None): 332 self.delegate = delegate 333 self.history = delegate.GetHistory() 334 self.platforms = platforms 335 self.extra_archives = extra_archives 336 337 def GetMostRecentSharedVersion(self, major_version): 338 """Returns the most recent version of a pepper bundle that exists on all 339 given platforms. 340 341 Specifically, the resulting version should be the most recently released 342 (meaning closest to the top of the listing on 343 omahaproxy.appspot.com/history) version that has a Chrome release on all 344 given platforms, and has a pepper bundle archive for each platform as well. 345 346 Args: 347 major_version: The major version of the pepper bundle, e.g. 19. 348 Returns: 349 A tuple (version, channel, archives). The version is a string such as 350 "19.0.1084.41". The channel is one of ('stable', 'beta', or 'dev'). 351 |archives| is a list of archive URLs.""" 352 def GetPlatformHistory(platform): 353 return self._GetPlatformMajorVersionHistory(major_version, platform) 354 355 shared_version_generator = self._FindNextSharedVersion(self.platforms, 356 GetPlatformHistory) 357 return self._DoGetMostRecentSharedVersion(shared_version_generator, 358 allow_trunk_revisions=False) 359 360 def GetMostRecentSharedCanary(self): 361 """Returns the most recent version of a canary pepper bundle that exists on 362 all given platforms. 363 364 Canary is special-cased because we don't care about its major version; we 365 always use the most recent canary, regardless of major version. 366 367 Returns: 368 A tuple (version, channel, archives). The version is a string such as 369 "19.0.1084.41". The channel is always 'canary'. |archives| is a list of 370 archive URLs.""" 371 # We don't ship canary on Linux, so it won't appear in self.history. 372 # Instead, we can use the matching Linux trunk build for that version. 373 shared_version_generator = self._FindNextSharedVersion( 374 set(self.platforms) - set(('linux',)), 375 self._GetPlatformCanaryHistory) 376 return self._DoGetMostRecentSharedVersion(shared_version_generator, 377 allow_trunk_revisions=True) 378 379 def GetAvailablePlatformArchivesFor(self, version, allow_trunk_revisions): 380 """Returns a sequence of archives that exist for a given version, on the 381 given platforms. 382 383 The second element of the returned tuple is a list of all platforms that do 384 not have an archive for the given version. 385 386 Args: 387 version: The version to find archives for. (e.g. "18.0.1025.164") 388 allow_trunk_revisions: If True, will search for archives using the 389 trunk revision that matches the branch version. 390 Returns: 391 A tuple (archives, missing_archives). |archives| is a list of archive 392 URLs, |missing_archives| is a list of archive names. 393 """ 394 archive_urls = self._GetAvailableArchivesFor(version) 395 platform_archives = set(GetPlatformArchiveName(p) for p in self.platforms) 396 expected_archives = platform_archives 397 if self.extra_archives: 398 for extra_archive, extra_archive_min_version in self.extra_archives: 399 if SplitVersion(version) >= SplitVersion(extra_archive_min_version): 400 expected_archives.add(extra_archive) 401 found_archives = set(GetCanonicalArchiveName(a) for a in archive_urls) 402 missing_archives = expected_archives - found_archives 403 if allow_trunk_revisions and missing_archives: 404 # Try to find trunk versions of any missing archives. 405 trunk_version = self.delegate.GetTrunkRevision(version) 406 trunk_archives = self._GetAvailableArchivesFor(trunk_version) 407 for trunk_archive_url in trunk_archives: 408 trunk_archive = GetCanonicalArchiveName(trunk_archive_url) 409 if trunk_archive in missing_archives: 410 archive_urls.append(trunk_archive_url) 411 missing_archives.discard(trunk_archive) 412 413 # Only return archives that are "expected". 414 def IsExpected(url): 415 return GetCanonicalArchiveName(url) in expected_archives 416 417 expected_archive_urls = [u for u in archive_urls if IsExpected(u)] 418 return expected_archive_urls, missing_archives 419 420 def _DoGetMostRecentSharedVersion(self, shared_version_generator, 421 allow_trunk_revisions): 422 """Returns the most recent version of a pepper bundle that exists on all 423 given platforms. 424 425 This function does the real work for the public GetMostRecentShared* above. 426 427 Args: 428 shared_version_generator: A generator that will yield (version, channel) 429 tuples in order of most recent to least recent. 430 allow_trunk_revisions: If True, will search for archives using the 431 trunk revision that matches the branch version. 432 Returns: 433 A tuple (version, channel, archives). The version is a string such as 434 "19.0.1084.41". The channel is one of ('stable', 'beta', 'dev', 435 'canary'). |archives| is a list of archive URLs.""" 436 version = None 437 skipped_versions = [] 438 channel = '' 439 while True: 440 try: 441 version, channel = shared_version_generator.next() 442 except StopIteration: 443 msg = 'No shared version for platforms: %s\n' % ( 444 ', '.join(self.platforms)) 445 msg += 'Last version checked = %s.\n' % (version,) 446 if skipped_versions: 447 msg += 'Versions skipped due to missing archives:\n' 448 for version, channel, missing_archives in skipped_versions: 449 archive_msg = '(missing %s)' % (', '.join(missing_archives)) 450 msg += ' %s (%s) %s\n' % (version, channel, archive_msg) 451 raise NoSharedVersionException(msg) 452 453 logger.info('Found shared version: %s, channel: %s' % ( 454 version, channel)) 455 456 archives, missing_archives = self.GetAvailablePlatformArchivesFor( 457 version, allow_trunk_revisions) 458 459 if not missing_archives: 460 return version, channel, archives 461 462 logger.info(' skipping. Missing archives: %s' % ( 463 ', '.join(missing_archives))) 464 465 skipped_versions.append((version, channel, missing_archives)) 466 467 def _GetPlatformMajorVersionHistory(self, with_major_version, with_platform): 468 """Yields Chrome history for a given platform and major version. 469 470 Args: 471 with_major_version: The major version to filter for. If 0, match all 472 versions. 473 with_platform: The name of the platform to filter for. 474 Returns: 475 A generator that yields a tuple (channel, version) for each version that 476 matches the platform and major version. The version returned is a tuple as 477 returned from SplitVersion. 478 """ 479 for platform, channel, version, _ in self.history: 480 version = SplitVersion(version) 481 if (with_platform == platform and 482 (with_major_version == 0 or with_major_version == version[0])): 483 yield channel, version 484 485 def _GetPlatformCanaryHistory(self, with_platform): 486 """Yields Chrome history for a given platform, but only for canary 487 versions. 488 489 Args: 490 with_platform: The name of the platform to filter for. 491 Returns: 492 A generator that yields a tuple (channel, version) for each version that 493 matches the platform and uses the canary channel. The version returned is 494 a tuple as returned from SplitVersion. 495 """ 496 for platform, channel, version, _ in self.history: 497 version = SplitVersion(version) 498 if with_platform == platform and channel == CANARY: 499 yield channel, version 500 501 502 def _FindNextSharedVersion(self, platforms, generator_func): 503 """Yields versions of Chrome that exist on all given platforms, in order of 504 newest to oldest. 505 506 Versions are compared in reverse order of release. That is, the most 507 recently updated version will be tested first. 508 509 Args: 510 platforms: A sequence of platforms to consider, e.g. 511 ('mac', 'linux', 'win') 512 generator_func: A function which takes a platform and returns a 513 generator that yields (channel, version) tuples. 514 Returns: 515 A generator that yields a tuple (version, channel) for each version that 516 matches all platforms and the major version. The version returned is a 517 string (e.g. "18.0.1025.164"). 518 """ 519 platform_generators = [] 520 for platform in platforms: 521 platform_generators.append(generator_func(platform)) 522 523 shared_version = None 524 platform_versions = [] 525 for platform_gen in platform_generators: 526 platform_versions.append(platform_gen.next()) 527 528 while True: 529 if logger.isEnabledFor(logging.INFO): 530 msg_info = [] 531 for i, platform in enumerate(platforms): 532 msg_info.append('%s: %s' % ( 533 platform, JoinVersion(platform_versions[i][1]))) 534 logger.info('Checking versions: %s' % ', '.join(msg_info)) 535 536 shared_version = min(v for c, v in platform_versions) 537 538 if all(v == shared_version for c, v in platform_versions): 539 # grab the channel from an arbitrary platform 540 first_platform = platform_versions[0] 541 channel = first_platform[0] 542 yield JoinVersion(shared_version), channel 543 544 # force increment to next version for all platforms 545 shared_version = None 546 547 # Find the next version for any platform that isn't at the shared version. 548 try: 549 for i, platform_gen in enumerate(platform_generators): 550 if platform_versions[i][1] != shared_version: 551 platform_versions[i] = platform_gen.next() 552 except StopIteration: 553 return 554 555 556 def _GetAvailableArchivesFor(self, version_string): 557 """Downloads a list of all available archives for a given version. 558 559 Args: 560 version_string: The version to find archives for. (e.g. "18.0.1025.164") 561 Returns: 562 A list of strings, each of which is a platform-specific archive URL. (e.g. 563 "gs://nativeclient_mirror/nacl/nacl_sdk/18.0.1025.164/" 564 "naclsdk_linux.tar.bz2"). 565 566 All returned URLs will use the gs:// schema.""" 567 files = self.delegate.GsUtil_ls(GS_BUCKET_PATH + version_string) 568 569 assert all(file.startswith('gs://') for file in files) 570 571 archives = [f for f in files if not f.endswith('.json')] 572 manifests = [f for f in files if f.endswith('.json')] 573 574 # don't include any archives that don't have an associated manifest. 575 return filter(lambda a: a + '.json' in manifests, archives) 576 577 578 class UnknownLockedBundleException(Exception): 579 pass 580 581 582 class Updater(object): 583 def __init__(self, delegate): 584 self.delegate = delegate 585 self.versions_to_update = [] 586 self.locked_bundles = [] 587 self.online_manifest = manifest_util.SDKManifest() 588 self._FetchOnlineManifest() 589 590 def AddVersionToUpdate(self, bundle_name, version, channel, archives): 591 """Add a pepper version to update in the uploaded manifest. 592 593 Args: 594 bundle_name: The name of the pepper bundle, e.g. 'pepper_18' 595 version: The version of the pepper bundle, e.g. '18.0.1025.64' 596 channel: The stability of the pepper bundle, e.g. 'beta' 597 archives: A sequence of archive URLs for this bundle.""" 598 self.versions_to_update.append((bundle_name, version, channel, archives)) 599 600 def AddLockedBundle(self, bundle_name): 601 """Add a "locked" bundle to the updater. 602 603 A locked bundle is a bundle that wasn't found in the history. When this 604 happens, the bundle is now "locked" to whatever was last found. We want to 605 ensure that the online manifest has this bundle. 606 607 Args: 608 bundle_name: The name of the locked bundle. 609 """ 610 self.locked_bundles.append(bundle_name) 611 612 def Update(self, manifest): 613 """Update a manifest and upload it. 614 615 Note that bundles will not be updated if the current version is newer. 616 That is, the updater will never automatically update to an older version of 617 a bundle. 618 619 Args: 620 manifest: The manifest used as a template for updating. Only pepper 621 bundles that contain no archives will be considered for auto-updating.""" 622 # Make sure there is only one stable branch: the one with the max version. 623 # All others are post-stable. 624 stable_major_versions = [SplitVersion(version)[0] for _, version, channel, _ 625 in self.versions_to_update if channel == 'stable'] 626 # Add 0 in case there are no stable versions. 627 max_stable_version = max([0] + stable_major_versions) 628 629 # Ensure that all locked bundles exist in the online manifest. 630 for bundle_name in self.locked_bundles: 631 online_bundle = self.online_manifest.GetBundle(bundle_name) 632 if online_bundle: 633 manifest.SetBundle(online_bundle) 634 else: 635 msg = ('Attempted to update bundle "%s", but no shared versions were ' 636 'found, and there is no online bundle with that name.') 637 raise UnknownLockedBundleException(msg % bundle_name) 638 639 if self.locked_bundles: 640 # Send a nagging email that we shouldn't be wasting time looking for 641 # bundles that are no longer in the history. 642 scriptname = os.path.basename(sys.argv[0]) 643 subject = '[%s] Reminder: remove bundles from %s' % (scriptname, 644 MANIFEST_BASENAME) 645 text = 'These bundles are not in the omahaproxy history anymore: ' + \ 646 ', '.join(self.locked_bundles) 647 self.delegate.SendMail(subject, text) 648 649 650 # Update all versions. 651 logger.info('>>> Updating bundles...') 652 for bundle_name, version, channel, archives in self.versions_to_update: 653 logger.info('Updating %s to %s...' % (bundle_name, version)) 654 bundle = manifest.GetBundle(bundle_name) 655 for archive in archives: 656 platform_bundle = self._GetPlatformArchiveBundle(archive) 657 # Normally the manifest snippet's bundle name matches our bundle name. 658 # pepper_canary, however is called "pepper_###" in the manifest 659 # snippet. 660 platform_bundle.name = bundle_name 661 bundle.MergeWithBundle(platform_bundle) 662 663 # Fix the stability and recommended values 664 major_version = SplitVersion(version)[0] 665 if major_version < max_stable_version: 666 bundle.stability = 'post_stable' 667 else: 668 bundle.stability = channel 669 # We always recommend the stable version. 670 if bundle.stability == 'stable': 671 bundle.recommended = 'yes' 672 else: 673 bundle.recommended = 'no' 674 675 # Check to ensure this bundle is newer than the online bundle. 676 online_bundle = self.online_manifest.GetBundle(bundle_name) 677 if online_bundle: 678 # This test used to be online_bundle.revision >= bundle.revision. 679 # That doesn't do quite what we want: sometimes the metadata changes 680 # but the revision stays the same -- we still want to push those 681 # changes. 682 if online_bundle.revision > bundle.revision or online_bundle == bundle: 683 logger.info( 684 ' Revision %s is not newer than than online revision %s. ' 685 'Skipping.' % (bundle.revision, online_bundle.revision)) 686 687 manifest.SetBundle(online_bundle) 688 continue 689 self._UploadManifest(manifest) 690 logger.info('Done.') 691 692 def _GetPlatformArchiveBundle(self, archive): 693 """Downloads the manifest "snippet" for an archive, and reads it as a 694 Bundle. 695 696 Args: 697 archive: A full URL of a platform-specific archive, using the gs schema. 698 Returns: 699 An object of type manifest_util.Bundle, read from a JSON file storing 700 metadata for this archive. 701 """ 702 stdout = self.delegate.GsUtil_cat(archive + '.json') 703 bundle = manifest_util.Bundle('') 704 bundle.LoadDataFromString(stdout) 705 # Some snippets were uploaded with revisions and versions as strings. Fix 706 # those here. 707 bundle.revision = int(bundle.revision) 708 bundle.version = int(bundle.version) 709 710 # HACK. The naclports archive specifies host_os as linux. Change it to all. 711 for archive in bundle.GetArchives(): 712 if NACLPORTS_ARCHIVE_NAME in archive.url: 713 archive.host_os = 'all' 714 return bundle 715 716 def _UploadManifest(self, manifest): 717 """Upload a serialized manifest_util.SDKManifest object. 718 719 Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to 720 gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json. 721 722 Args: 723 manifest: The new manifest to upload. 724 """ 725 new_manifest_string = manifest.GetDataAsString() 726 online_manifest_string = self.online_manifest.GetDataAsString() 727 728 if self.delegate.dryrun: 729 logger.info(''.join(list(difflib.unified_diff( 730 online_manifest_string.splitlines(1), 731 new_manifest_string.splitlines(1))))) 732 return 733 else: 734 online_manifest = manifest_util.SDKManifest() 735 online_manifest.LoadDataFromString(online_manifest_string) 736 737 if online_manifest == manifest: 738 logger.info('New manifest doesn\'t differ from online manifest.' 739 'Skipping upload.') 740 return 741 742 timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \ 743 GetTimestampManifestName() 744 self.delegate.GsUtil_cp('-', timestamp_manifest_path, 745 stdin=manifest.GetDataAsString()) 746 747 # copy from timestampped copy over the official manifest. 748 self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST) 749 750 def _FetchOnlineManifest(self): 751 try: 752 online_manifest_string = self.delegate.GsUtil_cat(GS_SDK_MANIFEST) 753 except subprocess.CalledProcessError: 754 # It is not a failure if the online manifest doesn't exist. 755 online_manifest_string = '' 756 757 if online_manifest_string: 758 self.online_manifest.LoadDataFromString(online_manifest_string) 759 760 761 def Run(delegate, platforms, extra_archives, fixed_bundle_versions=None): 762 """Entry point for the auto-updater. 763 764 Args: 765 delegate: The Delegate object to use for reading Urls, files, etc. 766 platforms: A sequence of platforms to consider, e.g. 767 ('mac', 'linux', 'win') 768 extra_archives: A sequence of tuples: (archive_basename, minimum_version), 769 e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')] 770 These archives must exist to consider a version for inclusion, as 771 long as that version is greater than the archive's minimum version. 772 fixed_bundle_versions: A sequence of tuples (bundle_name, version_string). 773 e.g. ('pepper_21', '21.0.1145.0') 774 """ 775 if fixed_bundle_versions: 776 fixed_bundle_versions = dict(fixed_bundle_versions) 777 else: 778 fixed_bundle_versions = {} 779 780 manifest = delegate.GetRepoManifest() 781 auto_update_bundles = [] 782 for bundle in manifest.GetBundles(): 783 if not bundle.name.startswith('pepper_'): 784 continue 785 archives = bundle.GetArchives() 786 if not archives: 787 auto_update_bundles.append(bundle) 788 789 if not auto_update_bundles: 790 logger.info('No versions need auto-updating.') 791 return 792 793 version_finder = VersionFinder(delegate, platforms, extra_archives) 794 updater = Updater(delegate) 795 796 for bundle in auto_update_bundles: 797 try: 798 if bundle.name == CANARY_BUNDLE_NAME: 799 logger.info('>>> Looking for most recent pepper_canary...') 800 version, channel, archives = version_finder.GetMostRecentSharedCanary() 801 else: 802 logger.info('>>> Looking for most recent pepper_%s...' % 803 bundle.version) 804 version, channel, archives = version_finder.GetMostRecentSharedVersion( 805 bundle.version) 806 except NoSharedVersionException: 807 # If we can't find a shared version, make sure that there is an uploaded 808 # bundle with that name already. 809 updater.AddLockedBundle(bundle.name) 810 continue 811 812 if bundle.name in fixed_bundle_versions: 813 # Ensure this version is valid for all platforms. 814 # If it is, use the channel found above (because the channel for this 815 # version may not be in the history.) 816 version = fixed_bundle_versions[bundle.name] 817 logger.info('Fixed bundle version: %s, %s' % (bundle.name, version)) 818 allow_trunk_revisions = bundle.name == CANARY_BUNDLE_NAME 819 archives, missing = version_finder.GetAvailablePlatformArchivesFor( 820 version, allow_trunk_revisions) 821 if missing: 822 logger.warn( 823 'Some archives for version %s of bundle %s don\'t exist: ' 824 'Missing %s' % (version, bundle.name, ', '.join(missing))) 825 return 826 827 updater.AddVersionToUpdate(bundle.name, version, channel, archives) 828 829 updater.Update(manifest) 830 831 832 class CapturedFile(object): 833 """A file-like object that captures text written to it, but also passes it 834 through to an underlying file-like object.""" 835 def __init__(self, passthrough): 836 self.passthrough = passthrough 837 self.written = cStringIO.StringIO() 838 839 def write(self, s): 840 self.written.write(s) 841 if self.passthrough: 842 self.passthrough.write(s) 843 844 def getvalue(self): 845 return self.written.getvalue() 846 847 848 def main(args): 849 parser = optparse.OptionParser() 850 parser.add_option('--gsutil', help='path to gsutil.') 851 parser.add_option('-d', '--debug', help='run in debug mode.', 852 action='store_true') 853 parser.add_option('--mailfrom', help='email address of sender.') 854 parser.add_option('--mailto', help='send error mails to...', action='append') 855 parser.add_option('-n', '--dryrun', help="don't upload the manifest.", 856 action='store_true') 857 parser.add_option('-v', '--verbose', help='print more diagnotic messages. ' 858 'Use more than once for more info.', 859 action='count') 860 parser.add_option('--log-file', metavar='FILE', help='log to FILE') 861 parser.add_option('--upload-log', help='Upload log alongside the manifest.', 862 action='store_true') 863 parser.add_option('--bundle-version', 864 help='Manually set a bundle version. This can be passed more than once. ' 865 'format: --bundle-version pepper_24=24.0.1312.25', action='append') 866 options, args = parser.parse_args(args[1:]) 867 868 if (options.mailfrom is None) != (not options.mailto): 869 options.mailfrom = None 870 options.mailto = None 871 logger.warning('Disabling email, one of --mailto or --mailfrom ' 872 'was missing.\n') 873 874 if options.verbose >= 2: 875 logging.basicConfig(level=logging.DEBUG, filename=options.log_file) 876 elif options.verbose: 877 logging.basicConfig(level=logging.INFO, filename=options.log_file) 878 else: 879 logging.basicConfig(level=logging.WARNING, filename=options.log_file) 880 881 # Parse bundle versions. 882 fixed_bundle_versions = {} 883 if options.bundle_version: 884 for bundle_version_string in options.bundle_version: 885 bundle_name, version = bundle_version_string.split('=') 886 fixed_bundle_versions[bundle_name] = version 887 888 if options.mailfrom and options.mailto: 889 # Capture stderr so it can be emailed, if necessary. 890 sys.stderr = CapturedFile(sys.stderr) 891 892 try: 893 try: 894 delegate = RealDelegate(options.dryrun, options.gsutil, 895 options.mailfrom, options.mailto) 896 897 if options.upload_log: 898 gsutil_logging_handler = GsutilLoggingHandler(delegate) 899 logger.addHandler(gsutil_logging_handler) 900 901 # Only look for naclports archives >= 27. The old ports bundles don't 902 # include license information. 903 extra_archives = [('naclports.tar.bz2', '27.0.0.0')] 904 Run(delegate, ('mac', 'win', 'linux'), extra_archives, 905 fixed_bundle_versions) 906 return 0 907 except Exception: 908 if options.mailfrom and options.mailto: 909 traceback.print_exc() 910 scriptname = os.path.basename(sys.argv[0]) 911 subject = '[%s] Failed to update manifest' % (scriptname,) 912 text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname, 913 sys.stderr.getvalue()) 914 delegate.SendMail(subject, text) 915 return 1 916 else: 917 raise 918 finally: 919 if options.upload_log: 920 gsutil_logging_handler.upload() 921 except manifest_util.Error as e: 922 if options.debug: 923 raise 924 sys.stderr.write(str(e) + '\n') 925 return 1 926 927 928 if __name__ == '__main__': 929 sys.exit(main(sys.argv)) 930