Home | History | Annotate | Download | only in go
      1 #!/usr/bin/env python
      2 # Copyright 2014 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 # Modified from go/bootstrap.py in Chromium infrastructure's repository to patch
      7 # out everything but the core toolchain.
      8 #
      9 # https://chromium.googlesource.com/infra/infra/
     10 
     11 """Prepares a local hermetic Go installation.
     12 
     13 - Downloads and unpacks the Go toolset in ../golang.
     14 """
     15 
     16 import contextlib
     17 import logging
     18 import os
     19 import platform
     20 import shutil
     21 import stat
     22 import subprocess
     23 import sys
     24 import tarfile
     25 import tempfile
     26 import urllib
     27 import zipfile
     28 
     29 # TODO(vadimsh): Migrate to new golang.org/x/ paths once Golang moves to
     30 # git completely.
     31 
     32 LOGGER = logging.getLogger(__name__)
     33 
     34 
     35 # /path/to/util/bot
     36 ROOT = os.path.dirname(os.path.abspath(__file__))
     37 
     38 # Where to install Go toolset to. GOROOT would be <TOOLSET_ROOT>/go.
     39 TOOLSET_ROOT = os.path.join(os.path.dirname(ROOT), 'golang')
     40 
     41 # Default workspace with infra go code.
     42 WORKSPACE = os.path.join(ROOT, 'go')
     43 
     44 # Platform depended suffix for executable files.
     45 EXE_SFX = '.exe' if sys.platform == 'win32' else ''
     46 
     47 # Pinned version of Go toolset to download.
     48 TOOLSET_VERSION = 'go1.8.3'
     49 
     50 # Platform dependent portion of a download URL. See http://golang.org/dl/.
     51 TOOLSET_VARIANTS = {
     52   ('darwin', 'x86-64'): 'darwin-amd64.tar.gz',
     53   ('linux2', 'x86-32'): 'linux-386.tar.gz',
     54   ('linux2', 'x86-64'): 'linux-amd64.tar.gz',
     55   ('win32', 'x86-32'): 'windows-386.zip',
     56   ('win32', 'x86-64'): 'windows-amd64.zip',
     57 }
     58 
     59 # Download URL root.
     60 DOWNLOAD_URL_PREFIX = 'https://storage.googleapis.com/golang'
     61 
     62 
     63 class Failure(Exception):
     64   """Bootstrap failed."""
     65 
     66 
     67 def get_toolset_url():
     68   """URL of a platform specific Go toolset archive."""
     69   # TODO(vadimsh): Support toolset for cross-compilation.
     70   arch = {
     71     'amd64': 'x86-64',
     72     'x86_64': 'x86-64',
     73     'i386': 'x86-32',
     74     'x86': 'x86-32',
     75   }.get(platform.machine().lower())
     76   variant = TOOLSET_VARIANTS.get((sys.platform, arch))
     77   if not variant:
     78     # TODO(vadimsh): Compile go lang from source.
     79     raise Failure('Unrecognized platform')
     80   return '%s/%s.%s' % (DOWNLOAD_URL_PREFIX, TOOLSET_VERSION, variant)
     81 
     82 
     83 def read_file(path):
     84   """Returns contents of a given file or None if not readable."""
     85   assert isinstance(path, (list, tuple))
     86   try:
     87     with open(os.path.join(*path), 'r') as f:
     88       return f.read()
     89   except IOError:
     90     return None
     91 
     92 
     93 def write_file(path, data):
     94   """Writes |data| to a file."""
     95   assert isinstance(path, (list, tuple))
     96   with open(os.path.join(*path), 'w') as f:
     97     f.write(data)
     98 
     99 
    100 def remove_directory(path):
    101   """Recursively removes a directory."""
    102   assert isinstance(path, (list, tuple))
    103   p = os.path.join(*path)
    104   if not os.path.exists(p):
    105     return
    106   LOGGER.info('Removing %s', p)
    107   # Crutch to remove read-only file (.git/* in particular) on Windows.
    108   def onerror(func, path, _exc_info):
    109     if not os.access(path, os.W_OK):
    110       os.chmod(path, stat.S_IWUSR)
    111       func(path)
    112     else:
    113       raise
    114   shutil.rmtree(p, onerror=onerror if sys.platform == 'win32' else None)
    115 
    116 
    117 def install_toolset(toolset_root, url):
    118   """Downloads and installs Go toolset.
    119 
    120   GOROOT would be <toolset_root>/go/.
    121   """
    122   if not os.path.exists(toolset_root):
    123     os.makedirs(toolset_root)
    124   pkg_path = os.path.join(toolset_root, url[url.rfind('/')+1:])
    125 
    126   LOGGER.info('Downloading %s...', url)
    127   download_file(url, pkg_path)
    128 
    129   LOGGER.info('Extracting...')
    130   if pkg_path.endswith('.zip'):
    131     with zipfile.ZipFile(pkg_path, 'r') as f:
    132       f.extractall(toolset_root)
    133   elif pkg_path.endswith('.tar.gz'):
    134     with tarfile.open(pkg_path, 'r:gz') as f:
    135       f.extractall(toolset_root)
    136   else:
    137     raise Failure('Unrecognized archive format')
    138 
    139   LOGGER.info('Validating...')
    140   if not check_hello_world(toolset_root):
    141     raise Failure('Something is not right, test program doesn\'t work')
    142 
    143 
    144 def download_file(url, path):
    145   """Fetches |url| to |path|."""
    146   last_progress = [0]
    147   def report(a, b, c):
    148     progress = int(a * b * 100.0 / c)
    149     if progress != last_progress[0]:
    150       print >> sys.stderr, 'Downloading... %d%%' % progress
    151       last_progress[0] = progress
    152   # TODO(vadimsh): Use something less crippled, something that validates SSL.
    153   urllib.urlretrieve(url, path, reporthook=report)
    154 
    155 
    156 @contextlib.contextmanager
    157 def temp_dir(path):
    158   """Creates a temporary directory, then deletes it."""
    159   tmp = tempfile.mkdtemp(dir=path)
    160   try:
    161     yield tmp
    162   finally:
    163     remove_directory([tmp])
    164 
    165 
    166 def check_hello_world(toolset_root):
    167   """Compiles and runs 'hello world' program to verify that toolset works."""
    168   with temp_dir(toolset_root) as tmp:
    169     path = os.path.join(tmp, 'hello.go')
    170     write_file([path], r"""
    171         package main
    172         func main() { println("hello, world\n") }
    173     """)
    174     out = subprocess.check_output(
    175         [get_go_exe(toolset_root), 'run', path],
    176         env=get_go_environ(toolset_root, tmp),
    177         stderr=subprocess.STDOUT)
    178     if out.strip() != 'hello, world':
    179       LOGGER.error('Failed to run sample program:\n%s', out)
    180       return False
    181     return True
    182 
    183 
    184 def ensure_toolset_installed(toolset_root):
    185   """Installs or updates Go toolset if necessary.
    186 
    187   Returns True if new toolset was installed.
    188   """
    189   installed = read_file([toolset_root, 'INSTALLED_TOOLSET'])
    190   available = get_toolset_url()
    191   if installed == available:
    192     LOGGER.debug('Go toolset is up-to-date: %s', TOOLSET_VERSION)
    193     return False
    194 
    195   LOGGER.info('Installing Go toolset.')
    196   LOGGER.info('  Old toolset is %s', installed)
    197   LOGGER.info('  New toolset is %s', available)
    198   remove_directory([toolset_root])
    199   install_toolset(toolset_root, available)
    200   LOGGER.info('Go toolset installed: %s', TOOLSET_VERSION)
    201   write_file([toolset_root, 'INSTALLED_TOOLSET'], available)
    202   return True
    203 
    204 
    205 def get_go_environ(
    206     toolset_root,
    207     workspace=None):
    208   """Returns a copy of os.environ with added GO* environment variables.
    209 
    210   Overrides GOROOT, GOPATH and GOBIN. Keeps everything else. Idempotent.
    211 
    212   Args:
    213     toolset_root: GOROOT would be <toolset_root>/go.
    214     workspace: main workspace directory or None if compiling in GOROOT.
    215   """
    216   env = os.environ.copy()
    217   env['GOROOT'] = os.path.join(toolset_root, 'go')
    218   if workspace:
    219     env['GOBIN'] = os.path.join(workspace, 'bin')
    220   else:
    221     env.pop('GOBIN', None)
    222 
    223   all_go_paths = []
    224   if workspace:
    225     all_go_paths.append(workspace)
    226   env['GOPATH'] = os.pathsep.join(all_go_paths)
    227 
    228   # New PATH entries.
    229   paths_to_add = [
    230     os.path.join(env['GOROOT'], 'bin'),
    231     env.get('GOBIN'),
    232   ]
    233 
    234   # Make sure not to add duplicates entries to PATH over and over again when
    235   # get_go_environ is invoked multiple times.
    236   path = env['PATH'].split(os.pathsep)
    237   paths_to_add = [p for p in paths_to_add if p and p not in path]
    238   env['PATH'] = os.pathsep.join(paths_to_add + path)
    239 
    240   return env
    241 
    242 
    243 def get_go_exe(toolset_root):
    244   """Returns path to go executable."""
    245   return os.path.join(toolset_root, 'go', 'bin', 'go' + EXE_SFX)
    246 
    247 
    248 def bootstrap(logging_level):
    249   """Installs all dependencies in default locations.
    250 
    251   Supposed to be called at the beginning of some script (it modifies logger).
    252 
    253   Args:
    254     logging_level: logging level of bootstrap process.
    255   """
    256   logging.basicConfig()
    257   LOGGER.setLevel(logging_level)
    258   ensure_toolset_installed(TOOLSET_ROOT)
    259 
    260 
    261 def prepare_go_environ():
    262   """Returns dict with environment variables to set to use Go toolset.
    263 
    264   Installs or updates the toolset if necessary.
    265   """
    266   bootstrap(logging.INFO)
    267   return get_go_environ(TOOLSET_ROOT, WORKSPACE)
    268 
    269 
    270 def find_executable(name, workspaces):
    271   """Returns full path to an executable in some bin/ (in GOROOT or GOBIN)."""
    272   basename = name
    273   if EXE_SFX and basename.endswith(EXE_SFX):
    274     basename = basename[:-len(EXE_SFX)]
    275   roots = [os.path.join(TOOLSET_ROOT, 'go', 'bin')]
    276   for path in workspaces:
    277     roots.extend([
    278       os.path.join(path, 'bin'),
    279     ])
    280   for root in roots:
    281     full_path = os.path.join(root, basename + EXE_SFX)
    282     if os.path.exists(full_path):
    283       return full_path
    284   return name
    285 
    286 
    287 def main(args):
    288   if args:
    289     print >> sys.stderr, sys.modules[__name__].__doc__,
    290     return 2
    291   bootstrap(logging.DEBUG)
    292   return 0
    293 
    294 
    295 if __name__ == '__main__':
    296   sys.exit(main(sys.argv[1:]))
    297