1 #!/usr/bin/env python 2 3 # Copyright 2017 The LUCI Authors. All rights reserved. 4 # Use of this source code is governed under the Apache License, Version 2.0 5 # that can be found in the LICENSE file. 6 7 """Bootstrap script to clone and forward to the recipe engine tool. 8 9 ******************* 10 ** DO NOT MODIFY ** 11 ******************* 12 13 This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/doc/recipes.py. 14 To fix bugs, fix in the googlesource repo then run the autoroller. 15 """ 16 17 import argparse 18 import json 19 import logging 20 import os 21 import random 22 import subprocess 23 import sys 24 import time 25 import urlparse 26 27 from collections import namedtuple 28 29 from cStringIO import StringIO 30 31 # The dependency entry for the recipe_engine in the client repo's recipes.cfg 32 # 33 # url (str) - the url to the engine repo we want to use. 34 # revision (str) - the git revision for the engine to get. 35 # path_override (str) - the subdirectory in the engine repo we should use to 36 # find it's recipes.py entrypoint. This is here for completeness, but will 37 # essentially always be empty. It would be used if the recipes-py repo was 38 # merged as a subdirectory of some other repo and you depended on that 39 # subdirectory. 40 # branch (str) - the branch to fetch for the engine as an absolute ref (e.g. 41 # refs/heads/master) 42 # repo_type ("GIT"|"GITILES") - An ignored enum which will be removed soon. 43 EngineDep = namedtuple('EngineDep', 44 'url revision path_override branch repo_type') 45 46 47 class MalformedRecipesCfg(Exception): 48 def __init__(self, msg, path): 49 super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r' 50 % (msg, path)) 51 52 53 def parse(repo_root, recipes_cfg_path): 54 """Parse is a lightweight a recipes.cfg file parser. 55 56 Args: 57 repo_root (str) - native path to the root of the repo we're trying to run 58 recipes for. 59 recipes_cfg_path (str) - native path to the recipes.cfg file to process. 60 61 Returns (as tuple): 62 engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the 63 current repo IS the recipe_engine. 64 recipes_path (str) - native path to where the recipes live inside of the 65 current repo (i.e. the folder containing `recipes/` and/or 66 `recipe_modules`) 67 """ 68 with open(recipes_cfg_path, 'rU') as fh: 69 pb = json.load(fh) 70 71 try: 72 if pb['api_version'] != 2: 73 raise MalformedRecipesCfg('unknown version %d' % pb['api_version'], 74 recipes_cfg_path) 75 76 # If we're running ./doc/recipes.py from the recipe_engine repo itself, then 77 # return None to signal that there's no EngineDep. 78 if pb['project_id'] == 'recipe_engine': 79 return None, pb.get('recipes_path', '') 80 81 engine = pb['deps']['recipe_engine'] 82 83 if 'url' not in engine: 84 raise MalformedRecipesCfg( 85 'Required field "url" in dependency "recipe_engine" not found', 86 recipes_cfg_path) 87 88 engine.setdefault('revision', '') 89 engine.setdefault('path_override', '') 90 engine.setdefault('branch', 'refs/heads/master') 91 recipes_path = pb.get('recipes_path', '') 92 93 # TODO(iannucci): only support absolute refs 94 if not engine['branch'].startswith('refs/'): 95 engine['branch'] = 'refs/heads/' + engine['branch'] 96 97 engine.setdefault('repo_type', 'GIT') 98 if engine['repo_type'] not in ('GIT', 'GITILES'): 99 raise MalformedRecipesCfg( 100 'Unsupported "repo_type" value in dependency "recipe_engine"', 101 recipes_cfg_path) 102 103 recipes_path = os.path.join( 104 repo_root, recipes_path.replace('/', os.path.sep)) 105 return EngineDep(**engine), recipes_path 106 except KeyError as ex: 107 raise MalformedRecipesCfg(ex.message, recipes_cfg_path) 108 109 110 GIT = 'git.bat' if sys.platform.startswith(('win', 'cygwin')) else 'git' 111 112 113 def _subprocess_call(argv, **kwargs): 114 logging.info('Running %r', argv) 115 return subprocess.call(argv, **kwargs) 116 117 118 def _git_check_call(argv, **kwargs): 119 argv = [GIT]+argv 120 logging.info('Running %r', argv) 121 subprocess.check_call(argv, **kwargs) 122 123 124 def _git_output(argv, **kwargs): 125 argv = [GIT]+argv 126 logging.info('Running %r', argv) 127 return subprocess.check_output(argv, **kwargs) 128 129 130 def parse_args(argv): 131 """This extracts a subset of the arguments that this bootstrap script cares 132 about. Currently this consists of: 133 * an override for the recipe engine in the form of `-O recipe_engin=/path` 134 * the --package option. 135 """ 136 PREFIX = 'recipe_engine=' 137 138 p = argparse.ArgumentParser(add_help=False) 139 p.add_argument('-O', '--project-override', action='append') 140 p.add_argument('--package', type=os.path.abspath) 141 args, _ = p.parse_known_args(argv) 142 for override in args.project_override or (): 143 if override.startswith(PREFIX): 144 return override[len(PREFIX):], args.package 145 return None, args.package 146 147 148 def checkout_engine(engine_path, repo_root, recipes_cfg_path): 149 dep, recipes_path = parse(repo_root, recipes_cfg_path) 150 if dep is None: 151 # we're running from the engine repo already! 152 return os.path.join(repo_root, recipes_path) 153 154 url = dep.url 155 156 if not engine_path and url.startswith('file://'): 157 engine_path = urlparse.urlparse(url).path 158 159 if not engine_path: 160 revision = dep.revision 161 subpath = dep.path_override 162 branch = dep.branch 163 164 # Ensure that we have the recipe engine cloned. 165 engine = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine') 166 engine_path = os.path.join(engine, subpath) 167 168 with open(os.devnull, 'w') as NUL: 169 # Note: this logic mirrors the logic in recipe_engine/fetch.py 170 _git_check_call(['init', engine], stdout=NUL) 171 172 try: 173 _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision], 174 cwd=engine, stdout=NUL, stderr=NUL) 175 except subprocess.CalledProcessError: 176 _git_check_call(['fetch', url, branch], cwd=engine, stdout=NUL, 177 stderr=NUL) 178 179 try: 180 _git_check_call(['diff', '--quiet', revision], cwd=engine) 181 except subprocess.CalledProcessError: 182 _git_check_call(['reset', '-q', '--hard', revision], cwd=engine) 183 184 return engine_path 185 186 187 def main(): 188 if '--verbose' in sys.argv: 189 logging.getLogger().setLevel(logging.INFO) 190 191 args = sys.argv[1:] 192 engine_override, recipes_cfg_path = parse_args(args) 193 194 if recipes_cfg_path: 195 # calculate repo_root from recipes_cfg_path 196 repo_root = os.path.dirname( 197 os.path.dirname( 198 os.path.dirname(recipes_cfg_path))) 199 else: 200 # find repo_root with git and calculate recipes_cfg_path 201 repo_root = (_git_output( 202 ['rev-parse', '--show-toplevel'], 203 cwd=os.path.abspath(os.path.dirname(__file__))).strip()) 204 repo_root = os.path.abspath(repo_root) 205 recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') 206 args = ['--package', recipes_cfg_path] + args 207 208 engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path) 209 210 return _subprocess_call([ 211 sys.executable, '-u', 212 os.path.join(engine_path, 'recipes.py')] + args) 213 214 215 if __name__ == '__main__': 216 sys.exit(main()) 217
