1 #!/usr/bin/env python 2 # 3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """Updates the Chrome reference builds. 8 9 Usage: 10 $ cd /tmp 11 $ /path/to/update_reference_build.py -r <revision> 12 $ cd reference_builds/reference_builds 13 $ gcl change 14 $ gcl upload <change> 15 $ gcl commit <change> 16 """ 17 18 import errno 19 import logging 20 import optparse 21 import os 22 import shutil 23 import subprocess 24 import sys 25 import time 26 import urllib 27 import urllib2 28 import zipfile 29 30 31 class BuildUpdater(object): 32 _PLATFORM_FILES_MAP = { 33 'Win': [ 34 'chrome-win32.zip', 35 'chrome-win32-syms.zip', 36 'chrome-win32.test/_pyautolib.pyd', 37 'chrome-win32.test/pyautolib.py', 38 ], 39 'Mac': [ 40 'chrome-mac.zip', 41 'chrome-mac.test/_pyautolib.so', 42 'chrome-mac.test/pyautolib.py', 43 ], 44 'Linux': [ 45 'chrome-linux.zip', 46 ], 47 'Linux_x64': [ 48 'chrome-linux.zip', 49 ], 50 } 51 52 _PLATFORM_DEST_MAP = { 53 'Linux': 'chrome_linux', 54 'Linux_x64': 'chrome_linux64', 55 'Win': 'chrome_win', 56 'Mac': 'chrome_mac', 57 } 58 59 def __init__(self, options): 60 self._platforms = options.platforms.split(',') 61 self._revision = int(options.revision) 62 63 @staticmethod 64 def _GetCmdStatusAndOutput(args, cwd=None, shell=False): 65 """Executes a subprocess and returns its exit code and output. 66 67 Args: 68 args: A string or a sequence of program arguments. 69 cwd: If not None, the subprocess's current directory will be changed to 70 |cwd| before it's executed. 71 shell: Whether to execute args as a shell command. 72 73 Returns: 74 The tuple (exit code, output). 75 """ 76 logging.info(str(args) + ' ' + (cwd or '')) 77 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, 78 stderr=subprocess.PIPE, shell=shell) 79 stdout, stderr = p.communicate() 80 exit_code = p.returncode 81 if stderr: 82 logging.critical(stderr) 83 logging.info(stdout) 84 return (exit_code, stdout) 85 86 def _GetBuildUrl(self, platform, revision, filename): 87 URL_FMT = ('http://commondatastorage.googleapis.com/' 88 'chromium-browser-snapshots/%s/%s/%s') 89 return URL_FMT % (urllib.quote_plus(platform), revision, filename) 90 91 def _FindBuildRevision(self, platform, revision, filename): 92 MAX_REVISIONS_PER_BUILD = 100 93 for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD): 94 r = urllib2.Request(self._GetBuildUrl(platform, revision_guess, filename)) 95 r.get_method = lambda: 'HEAD' 96 try: 97 response = urllib2.urlopen(r) 98 return revision_guess 99 except urllib2.HTTPError, err: 100 if err.code == 404: 101 time.sleep(.1) 102 continue 103 return None 104 105 def _DownloadBuilds(self): 106 for platform in self._platforms: 107 for f in BuildUpdater._PLATFORM_FILES_MAP[platform]: 108 output = os.path.join('dl', platform, 109 '%s_%s_%s' % (platform, self._revision, f)) 110 if os.path.exists(output): 111 logging.info('%s alread exists, skipping download' % output) 112 continue 113 build_revision = self._FindBuildRevision(platform, self._revision, f) 114 if not build_revision: 115 logging.critical('Failed to find %s build for r%s\n' % ( 116 platform, self._revision)) 117 sys.exit(1) 118 dirname = os.path.dirname(output) 119 if dirname and not os.path.exists(dirname): 120 os.makedirs(dirname) 121 url = self._GetBuildUrl(platform, build_revision, f) 122 logging.info('Downloading %s, saving to %s' % (url, output)) 123 r = urllib2.urlopen(url) 124 with file(output, 'wb') as f: 125 f.write(r.read()) 126 127 def _FetchSvnRepos(self): 128 if not os.path.exists('reference_builds'): 129 os.makedirs('reference_builds') 130 BuildUpdater._GetCmdStatusAndOutput( 131 ['gclient', 'config', 132 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], 133 'reference_builds') 134 BuildUpdater._GetCmdStatusAndOutput( 135 ['gclient', 'sync'], 'reference_builds') 136 137 def _UnzipFile(self, dl_file, dest_dir): 138 if not zipfile.is_zipfile(dl_file): 139 return False 140 logging.info('Opening %s' % dl_file) 141 with zipfile.ZipFile(dl_file, 'r') as z: 142 for content in z.namelist(): 143 dest = os.path.join(dest_dir, content[content.find('/')+1:]) 144 if not os.path.basename(dest): 145 if not os.path.isdir(dest): 146 os.makedirs(dest) 147 continue 148 with z.open(content) as unzipped_content: 149 logging.info('Extracting %s to %s (%s)' % (content, dest, dl_file)) 150 with file(dest, 'wb') as dest_file: 151 dest_file.write(unzipped_content.read()) 152 permissions = z.getinfo(content).external_attr >> 16 153 if permissions: 154 os.chmod(dest, permissions) 155 return True 156 157 def _ClearDir(self, dir): 158 """Clears all files in |dir| except for hidden files and folders.""" 159 for root, dirs, files in os.walk(dir): 160 # Skip hidden files and folders (like .svn and .git). 161 files = [f for f in files if f[0] != '.'] 162 dirs[:] = [d for d in dirs if d[0] != '.'] 163 164 for f in files: 165 os.remove(os.path.join(root, f)) 166 167 def _ExtractBuilds(self): 168 for platform in self._platforms: 169 if os.path.exists('tmp_unzip'): 170 os.path.unlink('tmp_unzip') 171 dest_dir = os.path.join('reference_builds', 'reference_builds', 172 BuildUpdater._PLATFORM_DEST_MAP[platform]) 173 self._ClearDir(dest_dir) 174 for root, _, dl_files in os.walk(os.path.join('dl', platform)): 175 for dl_file in dl_files: 176 dl_file = os.path.join(root, dl_file) 177 if not self._UnzipFile(dl_file, dest_dir): 178 logging.info('Copying %s to %s' % (dl_file, dest_dir)) 179 shutil.copy(dl_file, dest_dir) 180 181 def _SvnAddAndRemove(self): 182 svn_dir = os.path.join('reference_builds', 'reference_builds') 183 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat'], svn_dir)[1] 184 for line in stat.splitlines(): 185 action, filename = line.split(None, 1) 186 if action == '?': 187 BuildUpdater._GetCmdStatusAndOutput( 188 ['svn', 'add', filename], svn_dir) 189 elif action == '!': 190 BuildUpdater._GetCmdStatusAndOutput( 191 ['svn', 'delete', filename], svn_dir) 192 filepath = os.path.join(svn_dir, filename) 193 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): 194 BuildUpdater._GetCmdStatusAndOutput( 195 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) 196 197 def DownloadAndUpdateBuilds(self): 198 self._DownloadBuilds() 199 self._FetchSvnRepos() 200 self._ExtractBuilds() 201 self._SvnAddAndRemove() 202 203 204 def ParseOptions(argv): 205 parser = optparse.OptionParser() 206 usage = 'usage: %prog <options>' 207 parser.set_usage(usage) 208 parser.add_option('-r', dest='revision', 209 help='Revision to pickup') 210 parser.add_option('-p', dest='platforms', 211 default='Win,Mac,Linux,Linux_x64', 212 help='Comma separated list of platforms to download ' 213 '(as defined by the chromium builders).') 214 (options, _) = parser.parse_args(argv) 215 if not options.revision: 216 logging.critical('Must specify -r\n') 217 sys.exit(1) 218 219 return options 220 221 222 def main(argv): 223 logging.getLogger().setLevel(logging.DEBUG) 224 options = ParseOptions(argv) 225 b = BuildUpdater(options) 226 b.DownloadAndUpdateBuilds() 227 logging.info('Successfully updated reference builds. Move to ' 228 'reference_builds/reference_builds and make a change with gcl.') 229 230 if __name__ == '__main__': 231 sys.exit(main(sys.argv)) 232
