1 #!/usr/bin/env python 2 # 3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Updates the Chrome reference builds. 8 9 Use -r option to update a Chromium reference build, or -b option for Chrome 10 official builds. 11 12 Usage: 13 $ cd /tmp 14 $ /path/to/update_reference_build.py -r <revision> 15 $ cd reference_builds/reference_builds 16 $ gcl change 17 $ gcl upload <change> 18 $ gcl commit <change> 19 """ 20 21 import logging 22 import optparse 23 import os 24 import shutil 25 import subprocess 26 import sys 27 import time 28 import urllib 29 import urllib2 30 import zipfile 31 32 # Example chromium build location: 33 # gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip 34 CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/' 35 'chromium-browser-snapshots/%s/%s/%s') 36 37 # Chrome official build storage 38 # https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds 39 40 # Internal Google archive of official Chrome builds, example: 41 # https://goto.google.com/chrome_official_builds/ 42 # 32.0.1677.0/precise32bit/chrome-precise32bit.zip 43 CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/' 44 'official_builds/%s/%s/%s') 45 46 # Google storage location (no public web URL's), example: 47 # gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip 48 CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s') 49 50 51 class BuildUpdater(object): 52 _PLATFORM_FILES_MAP = { 53 'Win': [ 54 'chrome-win32.zip', 55 'chrome-win32-syms.zip', 56 ], 57 'Mac': [ 58 'chrome-mac.zip', 59 ], 60 'Linux': [ 61 'chrome-linux.zip', 62 ], 63 'Linux_x64': [ 64 'chrome-linux.zip', 65 ], 66 } 67 68 _CHROME_PLATFORM_FILES_MAP = { 69 'Win': [ 70 'chrome-win32.zip', 71 'chrome-win32-syms.zip', 72 ], 73 'Mac': [ 74 'chrome-mac.zip', 75 ], 76 'Linux': [ 77 'chrome-precise32bit.zip', 78 ], 79 'Linux_x64': [ 80 'chrome-precise64bit.zip', 81 ], 82 } 83 84 # Map of platform names to gs:// Chrome build names. 85 _BUILD_PLATFORM_MAP = { 86 'Linux': 'precise32bit', 87 'Linux_x64': 'precise64bit', 88 'Win': 'win', 89 'Mac': 'mac', 90 } 91 92 _PLATFORM_DEST_MAP = { 93 'Linux': 'chrome_linux', 94 'Linux_x64': 'chrome_linux64', 95 'Win': 'chrome_win', 96 'Mac': 'chrome_mac', 97 } 98 99 def __init__(self, options): 100 self._platforms = options.platforms.split(',') 101 self._revision = options.build_number or int(options.revision) 102 self._use_build_number = bool(options.build_number) 103 self._use_gs = options.use_gs 104 105 @staticmethod 106 def _GetCmdStatusAndOutput(args, cwd=None, shell=False): 107 """Executes a subprocess and returns its exit code and output. 108 109 Args: 110 args: A string or a sequence of program arguments. 111 cwd: If not None, the subprocess's current directory will be changed to 112 |cwd| before it's executed. 113 shell: Whether to execute args as a shell command. 114 115 Returns: 116 The tuple (exit code, output). 117 """ 118 logging.info(str(args) + ' ' + (cwd or '')) 119 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, 120 stderr=subprocess.PIPE, shell=shell) 121 stdout, stderr = p.communicate() 122 exit_code = p.returncode 123 if stderr: 124 logging.critical(stderr) 125 logging.info(stdout) 126 return (exit_code, stdout) 127 128 def _GetBuildUrl(self, platform, revision, filename): 129 if self._use_build_number: 130 # Chrome Google storage bucket. 131 if self._use_gs: 132 release = revision[:revision.find('.')] 133 return (CHROME_GS_URL_FMT % ( 134 release, 135 revision, 136 self._BUILD_PLATFORM_MAP[platform], 137 filename)) 138 # Chrome internal archive. 139 return (CHROME_INTERNAL_URL_FMT % ( 140 revision, 141 self._BUILD_PLATFORM_MAP[platform], 142 filename)) 143 # Chromium archive. 144 return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename) 145 146 def _FindBuildRevision(self, platform, revision, filename): 147 # TODO(shadi): Iterate over build numbers to find a valid one. 148 if self._use_build_number: 149 return (revision 150 if self._DoesBuildExist(platform, revision, filename) else None) 151 152 MAX_REVISIONS_PER_BUILD = 100 153 for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD): 154 if self._DoesBuildExist(platform, revision_guess, filename): 155 return revision_guess 156 else: 157 time.sleep(.1) 158 return None 159 160 def _DoesBuildExist(self, platform, build_number, filename): 161 url = self._GetBuildUrl(platform, build_number, filename) 162 if self._use_gs: 163 return self._DoesGSFileExist(url) 164 165 r = urllib2.Request(url) 166 r.get_method = lambda: 'HEAD' 167 try: 168 urllib2.urlopen(r) 169 return True 170 except urllib2.HTTPError, err: 171 if err.code == 404: 172 return False 173 174 def _DoesGSFileExist(self, gs_file_name): 175 exit_code = BuildUpdater._GetCmdStatusAndOutput( 176 ['gsutil', 'ls', gs_file_name])[0] 177 return not exit_code 178 179 def _GetPlatformFiles(self, platform): 180 if self._use_build_number: 181 return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform] 182 return BuildUpdater._PLATFORM_FILES_MAP[platform] 183 184 def _DownloadBuilds(self): 185 for platform in self._platforms: 186 for f in self._GetPlatformFiles(platform): 187 output = os.path.join('dl', platform, 188 '%s_%s_%s' % (platform, self._revision, f)) 189 if os.path.exists(output): 190 logging.info('%s alread exists, skipping download', output) 191 continue 192 build_revision = self._FindBuildRevision(platform, self._revision, f) 193 if not build_revision: 194 logging.critical('Failed to find %s build for r%s\n', platform, 195 self._revision) 196 sys.exit(1) 197 dirname = os.path.dirname(output) 198 if dirname and not os.path.exists(dirname): 199 os.makedirs(dirname) 200 url = self._GetBuildUrl(platform, build_revision, f) 201 self._DownloadFile(url, output) 202 203 def _DownloadFile(self, url, output): 204 logging.info('Downloading %s, saving to %s', url, output) 205 if self._use_build_number and self._use_gs: 206 BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output]) 207 else: 208 r = urllib2.urlopen(url) 209 with file(output, 'wb') as f: 210 f.write(r.read()) 211 212 def _FetchSvnRepos(self): 213 if not os.path.exists('reference_builds'): 214 os.makedirs('reference_builds') 215 BuildUpdater._GetCmdStatusAndOutput( 216 ['gclient', 'config', 217 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], 218 'reference_builds') 219 BuildUpdater._GetCmdStatusAndOutput( 220 ['gclient', 'sync'], 'reference_builds') 221 222 def _UnzipFile(self, dl_file, dest_dir): 223 if not zipfile.is_zipfile(dl_file): 224 return False 225 logging.info('Opening %s', dl_file) 226 with zipfile.ZipFile(dl_file, 'r') as z: 227 for content in z.namelist(): 228 dest = os.path.join(dest_dir, content[content.find('/')+1:]) 229 # Create dest parent dir if it does not exist. 230 if not os.path.isdir(os.path.dirname(dest)): 231 os.makedirs(os.path.dirname(dest)) 232 # If dest is just a dir listing, do nothing. 233 if not os.path.basename(dest): 234 continue 235 if not os.path.isdir(os.path.dirname(dest)): 236 os.makedirs(os.path.dirname(dest)) 237 with z.open(content) as unzipped_content: 238 logging.info('Extracting %s to %s (%s)', content, dest, dl_file) 239 with file(dest, 'wb') as dest_file: 240 dest_file.write(unzipped_content.read()) 241 permissions = z.getinfo(content).external_attr >> 16 242 if permissions: 243 os.chmod(dest, permissions) 244 return True 245 246 def _ClearDir(self, dir): 247 """Clears all files in |dir| except for hidden files and folders.""" 248 for root, dirs, files in os.walk(dir): 249 # Skip hidden files and folders (like .svn and .git). 250 files = [f for f in files if f[0] != '.'] 251 dirs[:] = [d for d in dirs if d[0] != '.'] 252 253 for f in files: 254 os.remove(os.path.join(root, f)) 255 256 def _ExtractBuilds(self): 257 for platform in self._platforms: 258 if os.path.exists('tmp_unzip'): 259 os.path.unlink('tmp_unzip') 260 dest_dir = os.path.join('reference_builds', 'reference_builds', 261 BuildUpdater._PLATFORM_DEST_MAP[platform]) 262 self._ClearDir(dest_dir) 263 for root, _, dl_files in os.walk(os.path.join('dl', platform)): 264 for dl_file in dl_files: 265 dl_file = os.path.join(root, dl_file) 266 if not self._UnzipFile(dl_file, dest_dir): 267 logging.info('Copying %s to %s', dl_file, dest_dir) 268 shutil.copy(dl_file, dest_dir) 269 270 def _SvnAddAndRemove(self): 271 svn_dir = os.path.join('reference_builds', 'reference_builds') 272 # List all changes without ignoring any files. 273 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'], 274 svn_dir)[1] 275 for line in stat.splitlines(): 276 action, filename = line.split(None, 1) 277 # Add new and ignored files. 278 if action == '?' or action == 'I': 279 BuildUpdater._GetCmdStatusAndOutput( 280 ['svn', 'add', filename], svn_dir) 281 elif action == '!': 282 BuildUpdater._GetCmdStatusAndOutput( 283 ['svn', 'delete', filename], svn_dir) 284 filepath = os.path.join(svn_dir, filename) 285 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): 286 BuildUpdater._GetCmdStatusAndOutput( 287 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) 288 289 def DownloadAndUpdateBuilds(self): 290 self._DownloadBuilds() 291 self._FetchSvnRepos() 292 self._ExtractBuilds() 293 self._SvnAddAndRemove() 294 295 296 def ParseOptions(argv): 297 parser = optparse.OptionParser() 298 usage = 'usage: %prog <options>' 299 parser.set_usage(usage) 300 parser.add_option('-b', dest='build_number', 301 help='Chrome official build number to pick up.') 302 parser.add_option('--gs', dest='use_gs', action='store_true', default=False, 303 help='Use Google storage for official builds. Used with -b ' 304 'option. Default is false (i.e. use internal storage.') 305 parser.add_option('-p', dest='platforms', 306 default='Win,Mac,Linux,Linux_x64', 307 help='Comma separated list of platforms to download ' 308 '(as defined by the chromium builders).') 309 parser.add_option('-r', dest='revision', 310 help='Revision to pick up.') 311 312 (options, _) = parser.parse_args(argv) 313 if not options.revision and not options.build_number: 314 logging.critical('Must specify either -r or -b.\n') 315 sys.exit(1) 316 if options.revision and options.build_number: 317 logging.critical('Must specify either -r or -b but not both.\n') 318 sys.exit(1) 319 if options.use_gs and not options.build_number: 320 logging.critical('Can only use --gs with -b option.\n') 321 sys.exit(1) 322 323 return options 324 325 326 def main(argv): 327 logging.getLogger().setLevel(logging.DEBUG) 328 options = ParseOptions(argv) 329 b = BuildUpdater(options) 330 b.DownloadAndUpdateBuilds() 331 logging.info('Successfully updated reference builds. Move to ' 332 'reference_builds/reference_builds and make a change with gcl.') 333 334 if __name__ == '__main__': 335 sys.exit(main(sys.argv)) 336
