1 #!/usr/bin/env python 2 # Copyright 2014 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # Modified from go/bootstrap.py in Chromium infrastructure's repository to patch 7 # out everything but the core toolchain. 8 # 9 # https://chromium.googlesource.com/infra/infra/ 10 11 """Prepares a local hermetic Go installation. 12 13 - Downloads and unpacks the Go toolset in ../golang. 14 """ 15 16 import contextlib 17 import logging 18 import os 19 import platform 20 import shutil 21 import stat 22 import subprocess 23 import sys 24 import tarfile 25 import tempfile 26 import urllib 27 import zipfile 28 29 # TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to 30 # git completely. 31 32 LOGGER = logging.getLogger(__name__) 33 34 35 # /path/to/util/bot 36 ROOT = os.path.dirname(os.path.abspath(__file__)) 37 38 # Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go. 39 TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang') 40 41 # Default workspace with infra go code. 42 WORKSPACE = os.path.join(ROOT, 'go') 43 44 # Platform depended suffix for executable files. 45 EXE_SFX = '.exe' if sys.platform == 'win32' else '' 46 47 # Pinned version of Go toolset to download. 48 TOOLSET_VERSION = 'go1.4' 49 50 # Platform dependent portion of a download URL. See http://golang.org/dl/. 51 TOOLSET_VARIANTS = { 52 ('darwin', 'x86-32'): 'darwin-386-osx10.8.tar.gz', 53 ('darwin', 'x86-64'): 'darwin-amd64-osx10.8.tar.gz', 54 ('linux2', 'x86-32'): 'linux-386.tar.gz', 55 ('linux2', 'x86-64'): 'linux-amd64.tar.gz', 56 ('win32', 'x86-32'): 'windows-386.zip', 57 ('win32', 'x86-64'): 'windows-amd64.zip', 58 } 59 60 # Download URL root. 61 DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang' 62 63 64 class Failure(Exception): 65 """Bootstrap failed.""" 66 67 68 def get_toolset_url(): 69 """URL of a platform specific Go toolset archive.""" 70 # TODO(vadimsh): Support toolset for cross-compilation. 71 arch = { 72 'amd64': 'x86-64', 73 'x86_64': 'x86-64', 74 'i386': 'x86-32', 75 'x86': 'x86-32', 76 }.get(platform.machine().lower()) 77 variant = TOOLSET_VARIANTS.get((sys.platform, arch)) 78 if not variant: 79 # TODO(vadimsh): Compile go lang from source. 80 raise Failure('Unrecognized platform') 81 return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant) 82 83 84 def read_file(path): 85 """Returns contents of a given file or None if not readable.""" 86 assert isinstance(path, (list, tuple)) 87 try: 88 with open(os.path.join(*path), 'r') as f: 89 return f.read() 90 except IOError: 91 return None 92 93 94 def write_file(path, data): 95 """Writes |data| to a file.""" 96 assert isinstance(path, (list, tuple)) 97 with open(os.path.join(*path), 'w') as f: 98 f.write(data) 99 100 101 def remove_directory(path): 102 """Recursively removes a directory.""" 103 assert isinstance(path, (list, tuple)) 104 p = os.path.join(*path) 105 if not os.path.exists(p): 106 return 107 LOGGER.info('Removing %s', p) 108 # Crutch to remove read-only file (.git/* in particular) on Windows. 109 def onerror(func, path, _exc_info): 110 if not os.access(path, os.W_OK): 111 os.chmod(path, stat.S_IWUSR) 112 func(path) 113 else: 114 raise 115 shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None) 116 117 118 def install_toolset(toolset_root, url): 119 """Downloads and installs Go toolset. 120 121 GOROOT would be <toolset_root>/go/. 122 """ 123 if not os.path.exists(toolset_root): 124 os.makedirs(toolset_root) 125 pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:]) 126 127 LOGGER.info('Downloading %s...', url) 128 download_file(url, pkg_path) 129 130 LOGGER.info('Extracting...') 131 if pkg_path.endswith('.zip'): 132 with zipfile.ZipFile(pkg_path, 'r') as f: 133 f.extractall(toolset_root) 134 elif pkg_path.endswith('.tar.gz'): 135 with tarfile.open(pkg_path, 'r:gz') as f: 136 f.extractall(toolset_root) 137 else: 138 raise Failure('Unrecognized archive format') 139 140 LOGGER.info('Validating...') 141 if not check_hello_world(toolset_root): 142 raise Failure('Something is not right, test program doesn\'t work') 143 144 145 def download_file(url, path): 146 """Fetches |url| to |path|.""" 147 last_progress = [0] 148 def report(a, b, c): 149 progress = int(a * b * 100.0 / c) 150 if progress != last_progress[0]: 151 print >> sys.stderr, 'Downloading... %d%%' % progress 152 last_progress[0] = progress 153 # TODO(vadimsh): Use something less crippled, something that validates SSL. 154 urllib.urlretrieve(url, path, reporthook=report) 155 156 157 @contextlib.contextmanager 158 def temp_dir(path): 159 """Creates a temporary directory, then deletes it.""" 160 tmp = tempfile.mkdtemp(dir=path) 161 try: 162 yield tmp 163 finally: 164 remove_directory([tmp]) 165 166 167 def check_hello_world(toolset_root): 168 """Compiles and runs 'hello world' program to verify that toolset works.""" 169 with temp_dir(toolset_root) as tmp: 170 path = os.path.join(tmp, 'hello.go') 171 write_file([path], r""" 172 package main 173 func main() { println("hello, world\n") } 174 """) 175 out = subprocess.check_output( 176 [get_go_exe(toolset_root), 'run', path], 177 env=get_go_environ(toolset_root, tmp), 178 stderr=subprocess.STDOUT) 179 if out.strip() != 'hello, world': 180 LOGGER.error('Failed to run sample program:\n%s', out) 181 return False 182 return True 183 184 185 def ensure_toolset_installed(toolset_root): 186 """Installs or updates Go toolset if necessary. 187 188 Returns True if new toolset was installed. 189 """ 190 installed = read_file([toolset_root, 'INSTALLED_TOOLSET']) 191 available = get_toolset_url() 192 if installed == available: 193 LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION) 194 return False 195 196 LOGGER.info('Installing Go toolset.') 197 LOGGER.info(' Old toolset is %s', installed) 198 LOGGER.info(' New toolset is %s', available) 199 remove_directory([toolset_root]) 200 install_toolset(toolset_root, available) 201 LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION) 202 write_file([toolset_root, 'INSTALLED_TOOLSET'], available) 203 return True 204 205 206 def get_go_environ( 207 toolset_root, 208 workspace=None): 209 """Returns a copy of os.environ with added GO* environment variables. 210 211 Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent. 212 213 Args: 214 toolset_root: GOROOT would be <toolset_root>/go. 215 workspace: main workspace directory or None if compiling in GOROOT. 216 """ 217 env = os.environ.copy() 218 env['GOROOT'] = os.path.join(toolset_root, 'go') 219 if workspace: 220 env['GOBIN'] = os.path.join(workspace, 'bin') 221 else: 222 env.pop('GOBIN', None) 223 224 all_go_paths = [] 225 if workspace: 226 all_go_paths.append(workspace) 227 env['GOPATH'] = os.pathsep.join(all_go_paths) 228 229 # New PATH entries. 230 paths_to_add = [ 231 os.path.join(env['GOROOT'], 'bin'), 232 env.get('GOBIN'), 233 ] 234 235 # Make sure not to add duplicates entries to PATH over and over again when 236 # get_go_environ is invoked multiple times. 237 path = env['PATH'].split(os.pathsep) 238 paths_to_add = [p for p in paths_to_add if p and p not in path] 239 env['PATH'] = os.pathsep.join(paths_to_add + path) 240 241 return env 242 243 244 def get_go_exe(toolset_root): 245 """Returns path to go executable.""" 246 return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX) 247 248 249 def bootstrap(logging_level): 250 """Installs all dependencies in default locations. 251 252 Supposed to be called at the beginning of some script (it modifies logger). 253 254 Args: 255 logging_level: logging level of bootstrap process. 256 """ 257 logging.basicConfig() 258 LOGGER.setLevel(logging_level) 259 ensure_toolset_installed(TOOLSET_ROOT) 260 261 262 def prepare_go_environ(): 263 """Returns dict with environment variables to set to use Go toolset. 264 265 Installs or updates the toolset if necessary. 266 """ 267 bootstrap(logging.INFO) 268 return get_go_environ(TOOLSET_ROOT, WORKSPACE) 269 270 271 def find_executable(name, workspaces): 272 """Returns full path to an executable in some bin/ (in GOROOT or GOBIN).""" 273 basename = name 274 if EXE_SFX and basename.endswith(EXE_SFX): 275 basename = basename[:-len(EXE_SFX)] 276 roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')] 277 for path in workspaces: 278 roots.extend([ 279 os.path.join(path, 'bin'), 280 ]) 281 for root in roots: 282 full_path = os.path.join(root, basename + EXE_SFX) 283 if os.path.exists(full_path): 284 return full_path 285 return name 286 287 288 def main(args): 289 if args: 290 print >> sys.stderr, sys.modules[__name__].__doc__, 291 return 2 292 bootstrap(logging.DEBUG) 293 return 0 294 295 296 if __name__ == '__main__': 297 sys.exit(main(sys.argv[1:])) 298
