1 #!/usr/bin/env python 2 # 3 # Copyright 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Updates the Chrome reference builds. 8 9 Usage: 10 $ /path/to/update_reference_build.py 11 $ git commit -a 12 $ git cl upload 13 """ 14 15 import collections 16 import logging 17 import os 18 import shutil 19 import subprocess 20 import sys 21 import urllib2 22 import zipfile 23 24 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils')) 25 26 from py_utils import cloud_storage 27 from dependency_manager import base_config 28 29 30 def BuildNotFoundError(error_string): 31 raise ValueError(error_string) 32 33 34 _CHROME_BINARIES_CONFIG = os.path.join( 35 os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common', 36 'py_utils', 'py_utils', 'chrome_binaries.json') 37 38 CHROME_GS_BUCKET = 'chrome-unsigned' 39 40 41 # Remove a platform name from this list to disable updating it. 42 # Add one to enable updating it. (Must also update _PLATFORM_MAP.) 43 _PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64', 44 'android_k_armeabi-v7a', 'android_l_arm64-v8a', 45 'android_l_armeabi-v7a', 'android_n_armeabi-v7a'] 46 47 # Remove a channal name from this list to disable updating it. 48 # Add one to enable updating it. 49 _CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev'] 50 51 52 # Omaha is Chrome's autoupdate server. It reports the current versions used 53 # by each platform on each channel. 54 _OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'], 55 'dev': ['linux'], 'canary': ['mac', 'win']} 56 57 58 # All of the information we need to update each platform. 59 # omaha: name omaha uses for the plaftorms. 60 # zip_name: name of the zip file to be retrieved from cloud storage. 61 # gs_build: name of the Chrome build platform used in cloud storage. 62 # destination: Name of the folder to download the reference build to. 63 UpdateInfo = collections.namedtuple('UpdateInfo', 64 'omaha, gs_folder, gs_build, zip_name') 65 _PLATFORM_MAP = {'mac_x86_64': UpdateInfo(omaha='mac', 66 gs_folder='desktop-*', 67 gs_build='mac64', 68 zip_name='chrome-mac.zip'), 69 'win_x86': UpdateInfo(omaha='win', 70 gs_folder='desktop-*', 71 gs_build='win-clang', 72 zip_name='chrome-win-clang.zip'), 73 'win_AMD64': UpdateInfo(omaha='win', 74 gs_folder='desktop-*', 75 gs_build='win64-clang', 76 zip_name='chrome-win64-clang.zip'), 77 'linux_x86_64': UpdateInfo(omaha='linux', 78 gs_folder='desktop-*', 79 gs_build='linux64', 80 zip_name='chrome-linux64.zip'), 81 'android_k_armeabi-v7a': UpdateInfo(omaha='android', 82 gs_folder='android-*', 83 gs_build='arm', 84 zip_name='Chrome.apk'), 85 'android_l_arm64-v8a': UpdateInfo(omaha='android', 86 gs_folder='android-*', 87 gs_build='arm_64', 88 zip_name='ChromeModern.apk'), 89 'android_l_armeabi-v7a': UpdateInfo(omaha='android', 90 gs_folder='android-*', 91 gs_build='arm', 92 zip_name='Chrome.apk'), 93 'android_n_armeabi-v7a': UpdateInfo(omaha='android', 94 gs_folder='android-*', 95 gs_build='arm', 96 zip_name='Monochrome.apk'), 97 98 } 99 100 101 def _ChannelVersionsMap(channel): 102 rows = _OmahaReportVersionInfo(channel) 103 omaha_versions_map = _OmahaVersionsMap(rows, channel) 104 channel_versions_map = {} 105 for platform in _PLATFORMS_TO_UPDATE: 106 omaha_platform = _PLATFORM_MAP[platform].omaha 107 if omaha_platform in omaha_versions_map: 108 channel_versions_map[platform] = omaha_versions_map[omaha_platform] 109 return channel_versions_map 110 111 112 def _OmahaReportVersionInfo(channel): 113 url ='https://omahaproxy.appspot.com/all?channel=%s' % channel 114 lines = urllib2.urlopen(url).readlines() 115 return [l.split(',') for l in lines] 116 117 118 def _OmahaVersionsMap(rows, channel): 119 platforms = _OMAHA_PLATFORMS.get(channel, []) 120 if (len(rows) < 1 or 121 not rows[0][0:3] == ['os', 'channel', 'current_version']): 122 raise ValueError( 123 'Omaha report is not in the expected form: %s.' % rows) 124 versions_map = {} 125 for row in rows[1:]: 126 if row[1] != channel: 127 raise ValueError( 128 'Omaha report contains a line with the channel %s' % row[1]) 129 if row[0] in platforms: 130 versions_map[row[0]] = row[2] 131 logging.warn('versions map: %s' % versions_map) 132 if not all(platform in versions_map for platform in platforms): 133 raise ValueError( 134 'Omaha report did not contain all desired platforms for channel %s' % channel) 135 return versions_map 136 137 138 def _QueuePlatformUpdate(platform, version, config, channel): 139 """ platform: the name of the platform for the browser to 140 be downloaded & updated from cloud storage. """ 141 platform_info = _PLATFORM_MAP[platform] 142 filename = platform_info.zip_name 143 # remote_path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip 144 remote_path = '%s/%s/%s/%s' % ( 145 platform_info.gs_folder, version, platform_info.gs_build, filename) 146 if not cloud_storage.Exists(CHROME_GS_BUCKET, remote_path): 147 cloud_storage_path = 'gs://%s/%s' % (CHROME_GS_BUCKET, remote_path) 148 raise BuildNotFoundError( 149 'Failed to find %s build for version %s at path %s.' % ( 150 platform, version, cloud_storage_path)) 151 reference_builds_folder = os.path.join( 152 os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build', 153 'reference_builds', channel) 154 if not os.path.exists(reference_builds_folder): 155 os.makedirs(reference_builds_folder) 156 local_dest_path = os.path.join(reference_builds_folder, filename) 157 cloud_storage.Get(CHROME_GS_BUCKET, remote_path, local_dest_path) 158 config.AddCloudStorageDependencyUpdateJob( 159 'chrome_%s' % channel, platform, local_dest_path, version=version, 160 execute_job=False) 161 162 163 def UpdateBuilds(): 164 config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True) 165 for channel in _CHANNELS_TO_UPDATE: 166 channel_versions_map = _ChannelVersionsMap(channel) 167 for platform in channel_versions_map: 168 print 'Downloading Chrome (%s channel) on %s' % (channel, platform) 169 current_version = config.GetVersion('chrome_%s' % channel, platform) 170 channel_version = channel_versions_map.get(platform) 171 print 'current: %s, channel: %s' % (current_version, channel_version) 172 if current_version and current_version == channel_version: 173 continue 174 _QueuePlatformUpdate(platform, channel_version, config, channel) 175 # TODO: move execute update jobs here, and add committing/uploading the cl. 176 177 print 'Updating chrome builds with downloaded binaries' 178 config.ExecuteUpdateJobs(force=True) 179 180 181 def main(): 182 logging.getLogger().setLevel(logging.DEBUG) 183 #TODO(aiolos): alert sheriffs via email when an error is seen. 184 #This should be added when alerts are added when updating the build. 185 UpdateBuilds() 186 # TODO(aiolos): Add --commit flag. crbug.com/547229 187 188 if __name__ == '__main__': 189 main() 190
