1 #!/usr/bin/env python 2 # Copyright 2014 The Chromium Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 # Modified from go/bootstrap.py in Chromium infrastructure's repository to patch 7 # out everything but the core toolchain. 8 # 9 # https://chromium.googlesource.com/infra/infra/ 10 11 """Prepares a local hermetic Go installation. 12 13 - Downloads and unpacks the Go toolset in ../golang. 14 """ 15 16 import contextlib 17 import logging 18 import os 19 import platform 20 import shutil 21 import stat 22 import subprocess 23 import sys 24 import tarfile 25 import tempfile 26 import urllib 27 import zipfile 28 29 # TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to 30 # git completely. 31 32 LOGGER = logging.getLogger(__name__) 33 34 35 # /path/to/util/bot 36 ROOT = os.path.dirname(os.path.abspath(__file__)) 37 38 # Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go. 39 TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang') 40 41 # Default workspace with infra go code. 42 WORKSPACE = os.path.join(ROOT, 'go') 43 44 # Platform depended suffix for executable files. 45 EXE_SFX = '.exe' if sys.platform == 'win32' else '' 46 47 # Pinned version of Go toolset to download. 48 TOOLSET_VERSION = 'go1.8.3' 49 50 # Platform dependent portion of a download URL. See http://golang.org/dl/. 51 TOOLSET_VARIANTS = { 52 ('darwin', 'x86-64'): 'darwin-amd64.tar.gz', 53 ('linux2', 'x86-32'): 'linux-386.tar.gz', 54 ('linux2', 'x86-64'): 'linux-amd64.tar.gz', 55 ('win32', 'x86-32'): 'windows-386.zip', 56 ('win32', 'x86-64'): 'windows-amd64.zip', 57 } 58 59 # Download URL root. 60 DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang' 61 62 63 class Failure(Exception): 64 """Bootstrap failed.""" 65 66 67 def get_toolset_url(): 68 """URL of a platform specific Go toolset archive.""" 69 # TODO(vadimsh): Support toolset for cross-compilation. 70 arch = { 71 'amd64': 'x86-64', 72 'x86_64': 'x86-64', 73 'i386': 'x86-32', 74 'x86': 'x86-32', 75 }.get(platform.machine().lower()) 76 variant = TOOLSET_VARIANTS.get((sys.platform, arch)) 77 if not variant: 78 # TODO(vadimsh): Compile go lang from source. 79 raise Failure('Unrecognized platform') 80 return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant) 81 82 83 def read_file(path): 84 """Returns contents of a given file or None if not readable.""" 85 assert isinstance(path, (list, tuple)) 86 try: 87 with open(os.path.join(*path), 'r') as f: 88 return f.read() 89 except IOError: 90 return None 91 92 93 def write_file(path, data): 94 """Writes |data| to a file.""" 95 assert isinstance(path, (list, tuple)) 96 with open(os.path.join(*path), 'w') as f: 97 f.write(data) 98 99 100 def remove_directory(path): 101 """Recursively removes a directory.""" 102 assert isinstance(path, (list, tuple)) 103 p = os.path.join(*path) 104 if not os.path.exists(p): 105 return 106 LOGGER.info('Removing %s', p) 107 # Crutch to remove read-only file (.git/* in particular) on Windows. 108 def onerror(func, path, _exc_info): 109 if not os.access(path, os.W_OK): 110 os.chmod(path, stat.S_IWUSR) 111 func(path) 112 else: 113 raise 114 shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None) 115 116 117 def install_toolset(toolset_root, url): 118 """Downloads and installs Go toolset. 119 120 GOROOT would be <toolset_root>/go/. 121 """ 122 if not os.path.exists(toolset_root): 123 os.makedirs(toolset_root) 124 pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:]) 125 126 LOGGER.info('Downloading %s...', url) 127 download_file(url, pkg_path) 128 129 LOGGER.info('Extracting...') 130 if pkg_path.endswith('.zip'): 131 with zipfile.ZipFile(pkg_path, 'r') as f: 132 f.extractall(toolset_root) 133 elif pkg_path.endswith('.tar.gz'): 134 with tarfile.open(pkg_path, 'r:gz') as f: 135 f.extractall(toolset_root) 136 else: 137 raise Failure('Unrecognized archive format') 138 139 LOGGER.info('Validating...') 140 if not check_hello_world(toolset_root): 141 raise Failure('Something is not right, test program doesn\'t work') 142 143 144 def download_file(url, path): 145 """Fetches |url| to |path|.""" 146 last_progress = [0] 147 def report(a, b, c): 148 progress = int(a * b * 100.0 / c) 149 if progress != last_progress[0]: 150 print >> sys.stderr, 'Downloading... %d%%' % progress 151 last_progress[0] = progress 152 # TODO(vadimsh): Use something less crippled, something that validates SSL. 153 urllib.urlretrieve(url, path, reporthook=report) 154 155 156 @contextlib.contextmanager 157 def temp_dir(path): 158 """Creates a temporary directory, then deletes it.""" 159 tmp = tempfile.mkdtemp(dir=path) 160 try: 161 yield tmp 162 finally: 163 remove_directory([tmp]) 164 165 166 def check_hello_world(toolset_root): 167 """Compiles and runs 'hello world' program to verify that toolset works.""" 168 with temp_dir(toolset_root) as tmp: 169 path = os.path.join(tmp, 'hello.go') 170 write_file([path], r""" 171 package main 172 func main() { println("hello, world\n") } 173 """) 174 out = subprocess.check_output( 175 [get_go_exe(toolset_root), 'run', path], 176 env=get_go_environ(toolset_root, tmp), 177 stderr=subprocess.STDOUT) 178 if out.strip() != 'hello, world': 179 LOGGER.error('Failed to run sample program:\n%s', out) 180 return False 181 return True 182 183 184 def ensure_toolset_installed(toolset_root): 185 """Installs or updates Go toolset if necessary. 186 187 Returns True if new toolset was installed. 188 """ 189 installed = read_file([toolset_root, 'INSTALLED_TOOLSET']) 190 available = get_toolset_url() 191 if installed == available: 192 LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION) 193 return False 194 195 LOGGER.info('Installing Go toolset.') 196 LOGGER.info(' Old toolset is %s', installed) 197 LOGGER.info(' New toolset is %s', available) 198 remove_directory([toolset_root]) 199 install_toolset(toolset_root, available) 200 LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION) 201 write_file([toolset_root, 'INSTALLED_TOOLSET'], available) 202 return True 203 204 205 def get_go_environ( 206 toolset_root, 207 workspace=None): 208 """Returns a copy of os.environ with added GO* environment variables. 209 210 Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent. 211 212 Args: 213 toolset_root: GOROOT would be <toolset_root>/go. 214 workspace: main workspace directory or None if compiling in GOROOT. 215 """ 216 env = os.environ.copy() 217 env['GOROOT'] = os.path.join(toolset_root, 'go') 218 if workspace: 219 env['GOBIN'] = os.path.join(workspace, 'bin') 220 else: 221 env.pop('GOBIN', None) 222 223 all_go_paths = [] 224 if workspace: 225 all_go_paths.append(workspace) 226 env['GOPATH'] = os.pathsep.join(all_go_paths) 227 228 # New PATH entries. 229 paths_to_add = [ 230 os.path.join(env['GOROOT'], 'bin'), 231 env.get('GOBIN'), 232 ] 233 234 # Make sure not to add duplicates entries to PATH over and over again when 235 # get_go_environ is invoked multiple times. 236 path = env['PATH'].split(os.pathsep) 237 paths_to_add = [p for p in paths_to_add if p and p not in path] 238 env['PATH'] = os.pathsep.join(paths_to_add + path) 239 240 return env 241 242 243 def get_go_exe(toolset_root): 244 """Returns path to go executable.""" 245 return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX) 246 247 248 def bootstrap(logging_level): 249 """Installs all dependencies in default locations. 250 251 Supposed to be called at the beginning of some script (it modifies logger). 252 253 Args: 254 logging_level: logging level of bootstrap process. 255 """ 256 logging.basicConfig() 257 LOGGER.setLevel(logging_level) 258 ensure_toolset_installed(TOOLSET_ROOT) 259 260 261 def prepare_go_environ(): 262 """Returns dict with environment variables to set to use Go toolset. 263 264 Installs or updates the toolset if necessary. 265 """ 266 bootstrap(logging.INFO) 267 return get_go_environ(TOOLSET_ROOT, WORKSPACE) 268 269 270 def find_executable(name, workspaces): 271 """Returns full path to an executable in some bin/ (in GOROOT or GOBIN).""" 272 basename = name 273 if EXE_SFX and basename.endswith(EXE_SFX): 274 basename = basename[:-len(EXE_SFX)] 275 roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')] 276 for path in workspaces: 277 roots.extend([ 278 os.path.join(path, 'bin'), 279 ]) 280 for root in roots: 281 full_path = os.path.join(root, basename + EXE_SFX) 282 if os.path.exists(full_path): 283 return full_path 284 return name 285 286 287 def main(args): 288 if args: 289 print >> sys.stderr, sys.modules[__name__].__doc__, 290 return 2 291 bootstrap(logging.DEBUG) 292 return 0 293 294 295 if __name__ == '__main__': 296 sys.exit(main(sys.argv[1:])) 297
