1 #!/usr/bin/env python 2 3 # Copyright 2017 The LUCI Authors. All rights reserved. 4 # Use of this source code is governed under the Apache License, Version 2.0 5 # that can be found in the LICENSE file. 6 7 """Bootstrap script to clone and forward to the recipe engine tool. 8 9 ******************* 10 ** DO NOT MODIFY ** 11 ******************* 12 13 This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/recipes.py. 14 To fix bugs, fix in the googlesource repo then run the autoroller. 15 """ 16 17 import argparse 18 import json 19 import logging 20 import os 21 import random 22 import subprocess 23 import sys 24 import time 25 import urlparse 26 27 from collections import namedtuple 28 29 from cStringIO import StringIO 30 31 # The dependency entry for the recipe_engine in the client repo's recipes.cfg 32 # 33 # url (str) - the url to the engine repo we want to use. 34 # revision (str) - the git revision for the engine to get. 35 # branch (str) - the branch to fetch for the engine as an absolute ref (e.g. 36 # refs/heads/master) 37 EngineDep = namedtuple('EngineDep', 38 'url revision branch') 39 40 41 class MalformedRecipesCfg(Exception): 42 def __init__(self, msg, path): 43 super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r' 44 % (msg, path)) 45 46 47 def parse(repo_root, recipes_cfg_path): 48 """Parse is a lightweight a recipes.cfg file parser. 49 50 Args: 51 repo_root (str) - native path to the root of the repo we're trying to run 52 recipes for. 53 recipes_cfg_path (str) - native path to the recipes.cfg file to process. 54 55 Returns (as tuple): 56 engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the 57 current repo IS the recipe_engine. 58 recipes_path (str) - native path to where the recipes live inside of the 59 current repo (i.e. the folder containing `recipes/` and/or 60 `recipe_modules`) 61 """ 62 with open(recipes_cfg_path, 'rU') as fh: 63 pb = json.load(fh) 64 65 try: 66 if pb['api_version'] != 2: 67 raise MalformedRecipesCfg('unknown version %d' % pb['api_version'], 68 recipes_cfg_path) 69 70 # If we're running ./recipes.py from the recipe_engine repo itself, then 71 # return None to signal that there's no EngineDep. 72 if pb['project_id'] == 'recipe_engine': 73 return None, pb.get('recipes_path', '') 74 75 engine = pb['deps']['recipe_engine'] 76 77 if 'url' not in engine: 78 raise MalformedRecipesCfg( 79 'Required field "url" in dependency "recipe_engine" not found', 80 recipes_cfg_path) 81 82 engine.setdefault('revision', '') 83 engine.setdefault('branch', 'refs/heads/master') 84 recipes_path = pb.get('recipes_path', '') 85 86 # TODO(iannucci): only support absolute refs 87 if not engine['branch'].startswith('refs/'): 88 engine['branch'] = 'refs/heads/' + engine['branch'] 89 90 recipes_path = os.path.join( 91 repo_root, recipes_path.replace('/', os.path.sep)) 92 return EngineDep(**engine), recipes_path 93 except KeyError as ex: 94 raise MalformedRecipesCfg(ex.message, recipes_cfg_path) 95 96 97 _BAT = '.bat' if sys.platform.startswith(('win', 'cygwin')) else '' 98 GIT = 'git' + _BAT 99 VPYTHON = 'vpython' + _BAT 100 101 102 def _subprocess_call(argv, **kwargs): 103 logging.info('Running %r', argv) 104 return subprocess.call(argv, **kwargs) 105 106 107 def _git_check_call(argv, **kwargs): 108 argv = [GIT]+argv 109 logging.info('Running %r', argv) 110 subprocess.check_call(argv, **kwargs) 111 112 113 def _git_output(argv, **kwargs): 114 argv = [GIT]+argv 115 logging.info('Running %r', argv) 116 return subprocess.check_output(argv, **kwargs) 117 118 119 def parse_args(argv): 120 """This extracts a subset of the arguments that this bootstrap script cares 121 about. Currently this consists of: 122 * an override for the recipe engine in the form of `-O recipe_engine=/path` 123 * the --package option. 124 """ 125 PREFIX = 'recipe_engine=' 126 127 p = argparse.ArgumentParser(add_help=False) 128 p.add_argument('-O', '--project-override', action='append') 129 p.add_argument('--package', type=os.path.abspath) 130 args, _ = p.parse_known_args(argv) 131 for override in args.project_override or (): 132 if override.startswith(PREFIX): 133 return override[len(PREFIX):], args.package 134 return None, args.package 135 136 137 def checkout_engine(engine_path, repo_root, recipes_cfg_path): 138 dep, recipes_path = parse(repo_root, recipes_cfg_path) 139 if dep is None: 140 # we're running from the engine repo already! 141 return os.path.join(repo_root, recipes_path) 142 143 url = dep.url 144 145 if not engine_path and url.startswith('file://'): 146 engine_path = urlparse.urlparse(url).path 147 148 if not engine_path: 149 revision = dep.revision 150 branch = dep.branch 151 152 # Ensure that we have the recipe engine cloned. 153 engine_path = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine') 154 155 with open(os.devnull, 'w') as NUL: 156 # Note: this logic mirrors the logic in recipe_engine/fetch.py 157 _git_check_call(['init', engine_path], stdout=NUL) 158 159 try: 160 _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision], 161 cwd=engine_path, stdout=NUL, stderr=NUL) 162 except subprocess.CalledProcessError: 163 _git_check_call(['fetch', url, branch], cwd=engine_path, stdout=NUL, 164 stderr=NUL) 165 166 try: 167 _git_check_call(['diff', '--quiet', revision], cwd=engine_path) 168 except subprocess.CalledProcessError: 169 _git_check_call(['reset', '-q', '--hard', revision], cwd=engine_path) 170 171 return engine_path 172 173 174 def main(): 175 if '--verbose' in sys.argv: 176 logging.getLogger().setLevel(logging.INFO) 177 178 args = sys.argv[1:] 179 engine_override, recipes_cfg_path = parse_args(args) 180 181 if recipes_cfg_path: 182 # calculate repo_root from recipes_cfg_path 183 repo_root = os.path.dirname( 184 os.path.dirname( 185 os.path.dirname(recipes_cfg_path))) 186 else: 187 # find repo_root with git and calculate recipes_cfg_path 188 repo_root = (_git_output( 189 ['rev-parse', '--show-toplevel'], 190 cwd=os.path.abspath(os.path.dirname(__file__))).strip()) 191 repo_root = os.path.abspath(repo_root) 192 recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') 193 args = ['--package', recipes_cfg_path] + args 194 195 engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path) 196 197 return _subprocess_call([ 198 VPYTHON, '-u', 199 os.path.join(engine_path, 'recipe_engine', 'main.py')] + args) 200 201 202 if __name__ == '__main__': 203 sys.exit(main()) 204
