1 #!/usr/bin/env python 2 3 # Copyright 2017 The LUCI Authors. All rights reserved. 4 # Use of this source code is governed under the Apache License, Version 2.0 5 # that can be found in the LICENSE file. 6 7 """Bootstrap script to clone and forward to the recipe engine tool. 8 9 ******************* 10 ** DO NOT MODIFY ** 11 ******************* 12 13 This is a copy of https://chromium.googlesource.com/infra/luci/recipes-py/+/master/recipes.py. 14 To fix bugs, fix in the googlesource repo then run the autoroller. 15 """ 16 17 import argparse 18 import json 19 import logging 20 import os 21 import random 22 import subprocess 23 import sys 24 import time 25 import urlparse 26 27 from collections import namedtuple 28 29 from cStringIO import StringIO 30 31 # The dependency entry for the recipe_engine in the client repo's recipes.cfg 32 # 33 # url (str) - the url to the engine repo we want to use. 34 # revision (str) - the git revision for the engine to get. 35 # branch (str) - the branch to fetch for the engine as an absolute ref (e.g. 36 # refs/heads/master) 37 EngineDep = namedtuple('EngineDep', 38 'url revision branch') 39 40 41 class MalformedRecipesCfg(Exception): 42 def __init__(self, msg, path): 43 super(MalformedRecipesCfg, self).__init__('malformed recipes.cfg: %s: %r' 44 % (msg, path)) 45 46 47 def parse(repo_root, recipes_cfg_path): 48 """Parse is a lightweight a recipes.cfg file parser. 49 50 Args: 51 repo_root (str) - native path to the root of the repo we're trying to run 52 recipes for. 53 recipes_cfg_path (str) - native path to the recipes.cfg file to process. 54 55 Returns (as tuple): 56 engine_dep (EngineDep|None): The recipe_engine dependency, or None, if the 57 current repo IS the recipe_engine. 58 recipes_path (str) - native path to where the recipes live inside of the 59 current repo (i.e. the folder containing `recipes/` and/or 60 `recipe_modules`) 61 """ 62 with open(recipes_cfg_path, 'rU') as fh: 63 pb = json.load(fh) 64 65 try: 66 if pb['api_version'] != 2: 67 raise MalformedRecipesCfg('unknown version %d' % pb['api_version'], 68 recipes_cfg_path) 69 70 # If we're running ./recipes.py from the recipe_engine repo itself, then 71 # return None to signal that there's no EngineDep. 72 repo_name = pb.get('repo_name') 73 if not repo_name: 74 repo_name = pb['project_id'] 75 if repo_name == 'recipe_engine': 76 return None, pb.get('recipes_path', '') 77 78 engine = pb['deps']['recipe_engine'] 79 80 if 'url' not in engine: 81 raise MalformedRecipesCfg( 82 'Required field "url" in dependency "recipe_engine" not found', 83 recipes_cfg_path) 84 85 engine.setdefault('revision', '') 86 engine.setdefault('branch', 'refs/heads/master') 87 recipes_path = pb.get('recipes_path', '') 88 89 # TODO(iannucci): only support absolute refs 90 if not engine['branch'].startswith('refs/'): 91 engine['branch'] = 'refs/heads/' + engine['branch'] 92 93 recipes_path = os.path.join( 94 repo_root, recipes_path.replace('/', os.path.sep)) 95 return EngineDep(**engine), recipes_path 96 except KeyError as ex: 97 raise MalformedRecipesCfg(ex.message, recipes_cfg_path) 98 99 100 _BAT = '.bat' if sys.platform.startswith(('win', 'cygwin')) else '' 101 GIT = 'git' + _BAT 102 VPYTHON = 'vpython' + _BAT 103 104 105 def _subprocess_call(argv, **kwargs): 106 logging.info('Running %r', argv) 107 return subprocess.call(argv, **kwargs) 108 109 110 def _git_check_call(argv, **kwargs): 111 argv = [GIT]+argv 112 logging.info('Running %r', argv) 113 subprocess.check_call(argv, **kwargs) 114 115 116 def _git_output(argv, **kwargs): 117 argv = [GIT]+argv 118 logging.info('Running %r', argv) 119 return subprocess.check_output(argv, **kwargs) 120 121 122 def parse_args(argv): 123 """This extracts a subset of the arguments that this bootstrap script cares 124 about. Currently this consists of: 125 * an override for the recipe engine in the form of `-O recipe_engine=/path` 126 * the --package option. 127 """ 128 PREFIX = 'recipe_engine=' 129 130 p = argparse.ArgumentParser(add_help=False) 131 p.add_argument('-O', '--project-override', action='append') 132 p.add_argument('--package', type=os.path.abspath) 133 args, _ = p.parse_known_args(argv) 134 for override in args.project_override or (): 135 if override.startswith(PREFIX): 136 return override[len(PREFIX):], args.package 137 return None, args.package 138 139 140 def checkout_engine(engine_path, repo_root, recipes_cfg_path): 141 dep, recipes_path = parse(repo_root, recipes_cfg_path) 142 if dep is None: 143 # we're running from the engine repo already! 144 return os.path.join(repo_root, recipes_path) 145 146 url = dep.url 147 148 if not engine_path and url.startswith('file://'): 149 engine_path = urlparse.urlparse(url).path 150 151 if not engine_path: 152 revision = dep.revision 153 branch = dep.branch 154 155 # Ensure that we have the recipe engine cloned. 156 engine_path = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine') 157 158 with open(os.devnull, 'w') as NUL: 159 # Note: this logic mirrors the logic in recipe_engine/fetch.py 160 _git_check_call(['init', engine_path], stdout=NUL) 161 162 try: 163 _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision], 164 cwd=engine_path, stdout=NUL, stderr=NUL) 165 except subprocess.CalledProcessError: 166 _git_check_call(['fetch', url, branch], cwd=engine_path, stdout=NUL, 167 stderr=NUL) 168 169 try: 170 _git_check_call(['diff', '--quiet', revision], cwd=engine_path) 171 except subprocess.CalledProcessError: 172 _git_check_call(['reset', '-q', '--hard', revision], cwd=engine_path) 173 174 return engine_path 175 176 177 def main(): 178 if '--verbose' in sys.argv: 179 logging.getLogger().setLevel(logging.INFO) 180 181 args = sys.argv[1:] 182 engine_override, recipes_cfg_path = parse_args(args) 183 184 if recipes_cfg_path: 185 # calculate repo_root from recipes_cfg_path 186 repo_root = os.path.dirname( 187 os.path.dirname( 188 os.path.dirname(recipes_cfg_path))) 189 else: 190 # find repo_root with git and calculate recipes_cfg_path 191 repo_root = (_git_output( 192 ['rev-parse', '--show-toplevel'], 193 cwd=os.path.abspath(os.path.dirname(__file__))).strip()) 194 repo_root = os.path.abspath(repo_root) 195 recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') 196 args = ['--package', recipes_cfg_path] + args 197 198 engine_path = checkout_engine(engine_override, repo_root, recipes_cfg_path) 199 200 return _subprocess_call([ 201 VPYTHON, '-u', 202 os.path.join(engine_path, 'recipe_engine', 'main.py')] + args) 203 204 205 if __name__ == '__main__': 206 sys.exit(main()) 207
