1 #!/usr/bin/env python 2 # 3 # Copyright 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Updates the Chrome reference builds. 8 9 Usage: 10 $ /path/to/update_reference_build.py 11 $ git commit -a 12 $ git cl upload 13 """ 14 15 import collections 16 import logging 17 import os 18 import shutil 19 import subprocess 20 import sys 21 import urllib2 22 import zipfile 23 24 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils')) 25 26 from py_utils import cloud_storage 27 from dependency_manager import base_config 28 29 30 def BuildNotFoundError(error_string): 31 raise ValueError(error_string) 32 33 34 _CHROME_BINARIES_CONFIG = os.path.join( 35 os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common', 36 'py_utils', 'py_utils', 'chrome_binaries.json') 37 38 CHROME_GS_BUCKET = 'chrome-unsigned' 39 40 41 # Remove a platform name from this list to disable updating it. 42 # Add one to enable updating it. (Must also update _PLATFORM_MAP.) 43 _PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64', 44 'android_k_armeabi-v7a', 'android_l_arm64-v8a', 45 'android_l_armeabi-v7a'] 46 47 # Remove a channal name from this list to disable updating it. 48 # Add one to enable updating it. 49 _CHANNELS_TO_UPDATE = ['stable'] # 'canary', 'dev' 50 51 52 # Omaha is Chrome's autoupdate server. It reports the current versions used 53 # by each platform on each channel. 54 _OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'], 55 'dev': ['linux'], 'canary': ['mac', 'win']} 56 57 58 # All of the information we need to update each platform. 59 # omaha: name omaha uses for the plaftorms. 60 # zip_name: name of the zip file to be retrieved from cloud storage. 61 # gs_build: name of the Chrome build platform used in cloud storage. 62 # destination: Name of the folder to download the reference build to. 63 UpdateInfo = collections.namedtuple('UpdateInfo', 64 'omaha, gs_folder, gs_build, zip_name') 65 _PLATFORM_MAP = { 'mac_x86_64': UpdateInfo(omaha='mac', 66 gs_folder='desktop-*', 67 gs_build='mac64', 68 zip_name='chrome-mac.zip'), 69 'win_x86': UpdateInfo(omaha='win', 70 gs_folder='desktop-*', 71 gs_build='win', 72 zip_name='chrome-win.zip'), 73 'win_AMD64': UpdateInfo(omaha='win', 74 gs_folder='desktop-*', 75 gs_build='win64', 76 zip_name='chrome-win64.zip'), 77 'linux_x86_64': UpdateInfo(omaha='linux', 78 gs_folder='desktop-*', 79 gs_build='linux64', 80 zip_name='chrome-linux64.zip'), 81 'android_k_armeabi-v7a': UpdateInfo(omaha='android', 82 gs_folder='android-*', 83 gs_build='arm', 84 zip_name='Chrome.apk'), 85 'android_l_arm64-v8a': UpdateInfo(omaha='android', 86 gs_folder='android-*', 87 gs_build='arm_64', 88 zip_name='ChromeModern.apk'), 89 'android_l_armeabi-v7a': UpdateInfo(omaha='android', 90 gs_folder='android-*', 91 gs_build='arm', 92 zip_name='Chrome.apk'), 93 } 94 95 96 def _ChannelVersionsMap(channel): 97 rows = _OmahaReportVersionInfo(channel) 98 omaha_versions_map = _OmahaVersionsMap(rows, channel) 99 channel_versions_map = {} 100 for platform in _PLATFORMS_TO_UPDATE: 101 omaha_platform = _PLATFORM_MAP[platform].omaha 102 if omaha_platform in omaha_versions_map: 103 channel_versions_map[platform] = omaha_versions_map[omaha_platform] 104 return channel_versions_map 105 106 107 def _OmahaReportVersionInfo(channel): 108 url ='https://omahaproxy.appspot.com/all?channel=%s' % channel 109 lines = urllib2.urlopen(url).readlines() 110 return [l.split(',') for l in lines] 111 112 113 def _OmahaVersionsMap(rows, channel): 114 platforms = _OMAHA_PLATFORMS.get(channel, []) 115 if (len(rows) < 1 or 116 not rows[0][0:3] == ['os', 'channel', 'current_version']): 117 raise ValueError( 118 'Omaha report is not in the expected form: %s.' % rows) 119 versions_map = {} 120 for row in rows[1:]: 121 if row[1] != channel: 122 raise ValueError( 123 'Omaha report contains a line with the channel %s' % row[1]) 124 if row[0] in platforms: 125 versions_map[row[0]] = row[2] 126 logging.warn('versions map: %s' % versions_map) 127 if not all(platform in versions_map for platform in platforms): 128 raise ValueError( 129 'Omaha report did not contain all desired platforms for channel %s' % channel) 130 return versions_map 131 132 133 def _QueuePlatformUpdate(platform, version, config, channel): 134 """ platform: the name of the platform for the browser to 135 be downloaded & updated from cloud storage. """ 136 platform_info = _PLATFORM_MAP[platform] 137 filename = platform_info.zip_name 138 # remote_path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip 139 remote_path = '%s/%s/%s/%s' % ( 140 platform_info.gs_folder, version, platform_info.gs_build, filename) 141 if not cloud_storage.Exists(CHROME_GS_BUCKET, remote_path): 142 raise BuildNotFoundError( 143 'Failed to find %s build for version %s at path %s.' % (platform, version, remote_path)) 144 reference_builds_folder = os.path.join( 145 os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build', 146 'reference_builds', channel) 147 if not os.path.exists(reference_builds_folder): 148 os.makedirs(reference_builds_folder) 149 local_dest_path = os.path.join(reference_builds_folder, filename) 150 cloud_storage.Get(CHROME_GS_BUCKET, remote_path, local_dest_path) 151 config.AddCloudStorageDependencyUpdateJob( 152 'chrome_%s' % channel, platform, local_dest_path, version=version, 153 execute_job=False) 154 155 156 def UpdateBuilds(): 157 config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True) 158 for channel in _CHANNELS_TO_UPDATE: 159 channel_versions_map = _ChannelVersionsMap(channel) 160 for platform in channel_versions_map: 161 print 'Downloading Chrome (%s channel) on %s' % (channel, platform) 162 current_version = config.GetVersion('chrome_%s' % channel, platform) 163 channel_version = channel_versions_map.get(platform) 164 print 'current: %s, channel: %s' % (current_version, channel_version) 165 if current_version and current_version == channel_version: 166 continue 167 _QueuePlatformUpdate(platform, channel_version, config, channel) 168 # TODO: move execute update jobs here, and add committing/uploading the cl. 169 170 print 'Updating chrome builds with downloaded binaries' 171 config.ExecuteUpdateJobs(force=True) 172 173 174 def main(): 175 logging.getLogger().setLevel(logging.DEBUG) 176 #TODO(aiolos): alert sheriffs via email when an error is seen. 177 #This should be added when alerts are added when updating the build. 178 UpdateBuilds() 179 # TODO(aiolos): Add --commit flag. crbug.com/547229 180 181 if __name__ == '__main__': 182 main() 183
