1 #!/usr/bin/env python 2 # 3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Updates the Chrome reference builds. 8 9 To update Chromium (unofficial) reference build, use the -r option and give a 10 Chromium SVN revision number, like 228977. To update a Chrome (official) build, 11 use the -v option and give a Chrome version number like 30.0.1595.0. 12 13 If you're getting a Chrome build, you can give the flag --gs to fetch from 14 Google Storage. Otherwise, it will be fetched from go/chrome_official_builds. 15 16 Before running this script, you should first verify that you are authenticated 17 for SVN. You can do this by running: 18 $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds 19 You may need to get your SVN password from https://chromium-access.appspot.com/. 20 21 Usage: 22 $ cd /tmp 23 $ /path/to/update_reference_build.py --gs -v <version> 24 $ cd reference_builds/reference_builds 25 $ gcl change 26 $ gcl upload <change> 27 $ gcl commit <change> 28 """ 29 30 import logging 31 import optparse 32 import os 33 import shutil 34 import subprocess 35 import sys 36 import time 37 import urllib 38 import urllib2 39 import zipfile 40 41 # Example chromium build location: 42 # gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip 43 CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/' 44 'chromium-browser-snapshots/%s/%s/%s') 45 46 # Chrome official build storage 47 # https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds 48 49 # Internal Google archive of official Chrome builds, example: 50 # https://goto.google.com/chrome_official_builds/ 51 # 32.0.1677.0/precise32bit/chrome-precise32bit.zip 52 CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/' 53 'official_builds/%s/%s/%s') 54 55 # Google storage location (no public web URL's), example: 56 # gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip 57 CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s') 58 59 60 class BuildUpdater(object): 61 _PLATFORM_FILES_MAP = { 62 'Win': [ 63 'chrome-win32.zip', 64 'chrome-win32-syms.zip', 65 ], 66 'Mac': [ 67 'chrome-mac.zip', 68 ], 69 'Linux': [ 70 'chrome-linux.zip', 71 ], 72 'Linux_x64': [ 73 'chrome-linux.zip', 74 ], 75 } 76 77 _CHROME_PLATFORM_FILES_MAP = { 78 'Win': [ 79 'chrome-win32.zip', 80 'chrome-win32-syms.zip', 81 ], 82 'Mac': [ 83 'chrome-mac.zip', 84 ], 85 'Linux': [ 86 'chrome-precise32bit.zip', 87 ], 88 'Linux_x64': [ 89 'chrome-precise64bit.zip', 90 ], 91 } 92 93 # Map of platform names to gs:// Chrome build names. 94 _BUILD_PLATFORM_MAP = { 95 'Linux': 'precise32bit', 96 'Linux_x64': 'precise64bit', 97 'Win': 'win', 98 'Mac': 'mac', 99 } 100 101 _PLATFORM_DEST_MAP = { 102 'Linux': 'chrome_linux', 103 'Linux_x64': 'chrome_linux64', 104 'Win': 'chrome_win', 105 'Mac': 'chrome_mac', 106 } 107 108 def __init__(self, options): 109 self._platforms = options.platforms.split(',') 110 self._version_or_revision = options.version or int(options.revision) 111 self._use_official_version = bool(options.version) 112 self._use_gs = options.use_gs 113 114 @staticmethod 115 def _GetCmdStatusAndOutput(args, cwd=None, shell=False): 116 """Executes a subprocess and returns its exit code and output. 117 118 Args: 119 args: A string or a sequence of program arguments. 120 cwd: If not None, the subprocess's current directory will be changed to 121 |cwd| before it's executed. 122 shell: Whether to execute args as a shell command. 123 124 Returns: 125 The tuple (exit code, output). 126 """ 127 logging.info(str(args) + ' ' + (cwd or '')) 128 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, 129 stderr=subprocess.PIPE, shell=shell) 130 stdout, stderr = p.communicate() 131 exit_code = p.returncode 132 if stderr: 133 logging.critical(stderr) 134 logging.info(stdout) 135 return (exit_code, stdout) 136 137 def _GetBuildUrl(self, platform, version_or_revision, filename): 138 """Returns the URL for fetching one file. 139 140 Args: 141 platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|. 142 version_or_revision: Either an SVN revision, e.g. 234567, or a Chrome 143 version number, e.g. 30.0.1600.1. 144 filename: Name of the file to fetch. 145 146 Returns: 147 The URL for fetching a file. This may be a GS or HTTP URL. 148 """ 149 if self._use_official_version: 150 # Chrome Google storage bucket. 151 version = version_or_revision 152 if self._use_gs: 153 release = version[:version.find('.')] 154 return (CHROME_GS_URL_FMT % ( 155 release, 156 version, 157 self._BUILD_PLATFORM_MAP[platform], 158 filename)) 159 # Chrome internal archive. 160 return (CHROME_INTERNAL_URL_FMT % ( 161 version, 162 self._BUILD_PLATFORM_MAP[platform], 163 filename)) 164 # Chromium archive. 165 revision = version_or_revision 166 return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename) 167 168 def _FindBuildVersionOrRevision( 169 self, platform, version_or_revision, filename): 170 """Searches for a version or revision where a filename can be found. 171 172 Args: 173 platform: Platform name. 174 version_or_revision: Either Chrome version or Chromium revision. 175 filename: Filename to look for. 176 177 Returns: 178 A version or revision where the file could be found, or None. 179 """ 180 # TODO(shadi): Iterate over official versions to find a valid one. 181 if self._use_official_version: 182 version = version_or_revision 183 return (version 184 if self._DoesBuildExist(platform, version, filename) else None) 185 186 revision = version_or_revision 187 MAX_REVISIONS_PER_BUILD = 100 188 for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD): 189 if self._DoesBuildExist(platform, revision_guess, filename): 190 return revision_guess 191 else: 192 time.sleep(.1) 193 return None 194 195 def _DoesBuildExist(self, platform, version, filename): 196 """Checks whether a file can be found for the given Chrome version. 197 198 Args: 199 platform: Platform name. 200 version: Chrome version number, e.g. 30.0.1600.1. 201 filename: Filename to look for. 202 203 Returns: 204 True if the file could be found, False otherwise. 205 """ 206 url = self._GetBuildUrl(platform, version, filename) 207 if self._use_gs: 208 return self._DoesGSFileExist(url) 209 210 request = urllib2.Request(url) 211 request.get_method = lambda: 'HEAD' 212 try: 213 urllib2.urlopen(request) 214 return True 215 except urllib2.HTTPError, err: 216 if err.code == 404: 217 return False 218 219 def _DoesGSFileExist(self, gs_file_name): 220 """Returns True if the GS file can be found, False otherwise.""" 221 exit_code = BuildUpdater._GetCmdStatusAndOutput( 222 ['gsutil', 'ls', gs_file_name])[0] 223 return not exit_code 224 225 def _GetPlatformFiles(self, platform): 226 """Returns a list of filenames to fetch for the given platform.""" 227 if self._use_official_version: 228 return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform] 229 return BuildUpdater._PLATFORM_FILES_MAP[platform] 230 231 def _DownloadBuilds(self): 232 for platform in self._platforms: 233 for filename in self._GetPlatformFiles(platform): 234 output = os.path.join('dl', platform, 235 '%s_%s_%s' % (platform, 236 self._version_or_revision, 237 filename)) 238 if os.path.exists(output): 239 logging.info('%s alread exists, skipping download', output) 240 continue 241 version_or_revision = self._FindBuildVersionOrRevision( 242 platform, self._version_or_revision, filename) 243 if not version_or_revision: 244 logging.critical('Failed to find %s build for r%s\n', platform, 245 self._version_or_revision) 246 sys.exit(1) 247 dirname = os.path.dirname(output) 248 if dirname and not os.path.exists(dirname): 249 os.makedirs(dirname) 250 url = self._GetBuildUrl(platform, version_or_revision, filename) 251 self._DownloadFile(url, output) 252 253 def _DownloadFile(self, url, output): 254 logging.info('Downloading %s, saving to %s', url, output) 255 if self._use_official_version and self._use_gs: 256 BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output]) 257 else: 258 response = urllib2.urlopen(url) 259 with file(output, 'wb') as f: 260 f.write(response.read()) 261 262 def _FetchSvnRepos(self): 263 if not os.path.exists('reference_builds'): 264 os.makedirs('reference_builds') 265 BuildUpdater._GetCmdStatusAndOutput( 266 ['gclient', 'config', 267 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], 268 'reference_builds') 269 BuildUpdater._GetCmdStatusAndOutput( 270 ['gclient', 'sync'], 'reference_builds') 271 272 def _UnzipFile(self, dl_file, dest_dir): 273 """Unzips a file if it is a zip file. 274 275 Args: 276 dl_file: The downloaded file to unzip. 277 dest_dir: The destination directory to unzip to. 278 279 Returns: 280 True if the file was unzipped. False if it wasn't a zip file. 281 """ 282 if not zipfile.is_zipfile(dl_file): 283 return False 284 logging.info('Opening %s', dl_file) 285 with zipfile.ZipFile(dl_file, 'r') as z: 286 for content in z.namelist(): 287 dest = os.path.join(dest_dir, content[content.find('/')+1:]) 288 # Create dest parent dir if it does not exist. 289 if not os.path.isdir(os.path.dirname(dest)): 290 os.makedirs(os.path.dirname(dest)) 291 # If dest is just a dir listing, do nothing. 292 if not os.path.basename(dest): 293 continue 294 if not os.path.isdir(os.path.dirname(dest)): 295 os.makedirs(os.path.dirname(dest)) 296 with z.open(content) as unzipped_content: 297 logging.info('Extracting %s to %s (%s)', content, dest, dl_file) 298 with file(dest, 'wb') as dest_file: 299 dest_file.write(unzipped_content.read()) 300 permissions = z.getinfo(content).external_attr >> 16 301 if permissions: 302 os.chmod(dest, permissions) 303 return True 304 305 def _ClearDir(self, dir): 306 """Clears all files in |dir| except for hidden files and folders.""" 307 for root, dirs, files in os.walk(dir): 308 # Skip hidden files and folders (like .svn and .git). 309 files = [f for f in files if f[0] != '.'] 310 dirs[:] = [d for d in dirs if d[0] != '.'] 311 312 for f in files: 313 os.remove(os.path.join(root, f)) 314 315 def _ExtractBuilds(self): 316 for platform in self._platforms: 317 if os.path.exists('tmp_unzip'): 318 os.path.unlink('tmp_unzip') 319 dest_dir = os.path.join('reference_builds', 'reference_builds', 320 BuildUpdater._PLATFORM_DEST_MAP[platform]) 321 self._ClearDir(dest_dir) 322 for root, _, dl_files in os.walk(os.path.join('dl', platform)): 323 for dl_file in dl_files: 324 dl_file = os.path.join(root, dl_file) 325 if not self._UnzipFile(dl_file, dest_dir): 326 logging.info('Copying %s to %s', dl_file, dest_dir) 327 shutil.copy(dl_file, dest_dir) 328 329 def _SvnAddAndRemove(self): 330 svn_dir = os.path.join('reference_builds', 'reference_builds') 331 # List all changes without ignoring any files. 332 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'], 333 svn_dir)[1] 334 for line in stat.splitlines(): 335 action, filename = line.split(None, 1) 336 # Add new and ignored files. 337 if action == '?' or action == 'I': 338 BuildUpdater._GetCmdStatusAndOutput( 339 ['svn', 'add', filename], svn_dir) 340 elif action == '!': 341 BuildUpdater._GetCmdStatusAndOutput( 342 ['svn', 'delete', filename], svn_dir) 343 filepath = os.path.join(svn_dir, filename) 344 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): 345 BuildUpdater._GetCmdStatusAndOutput( 346 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) 347 348 def DownloadAndUpdateBuilds(self): 349 self._DownloadBuilds() 350 self._FetchSvnRepos() 351 self._ExtractBuilds() 352 self._SvnAddAndRemove() 353 354 355 def ParseOptions(argv): 356 parser = optparse.OptionParser() 357 usage = 'usage: %prog <options>' 358 parser.set_usage(usage) 359 parser.add_option('-v', dest='version', 360 help='Chrome official version to pick up ' 361 '(e.g. 30.0.1600.1).') 362 parser.add_option('--gs', dest='use_gs', action='store_true', default=False, 363 help='Use Google storage for official builds. Used with -b ' 364 'option. Default is false (i.e. use internal storage.') 365 parser.add_option('-p', dest='platforms', 366 default='Win,Mac,Linux,Linux_x64', 367 help='Comma separated list of platforms to download ' 368 '(as defined by the chromium builders).') 369 parser.add_option('-r', dest='revision', 370 help='Chromium revision to pick up (e.g. 234567).') 371 372 (options, _) = parser.parse_args(argv) 373 if not options.revision and not options.version: 374 logging.critical('Must specify either -r or -v.\n') 375 sys.exit(1) 376 if options.revision and options.version: 377 logging.critical('Must specify either -r or -v but not both.\n') 378 sys.exit(1) 379 if options.use_gs and not options.version: 380 logging.critical('Can only use --gs with -v option.\n') 381 sys.exit(1) 382 383 return options 384 385 386 def main(argv): 387 logging.getLogger().setLevel(logging.DEBUG) 388 options = ParseOptions(argv) 389 b = BuildUpdater(options) 390 b.DownloadAndUpdateBuilds() 391 logging.info('Successfully updated reference builds. Move to ' 392 'reference_builds/reference_builds and make a change with gcl.') 393 394 if __name__ == '__main__': 395 sys.exit(main(sys.argv)) 396
