1 #!/usr/bin/python 2 3 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 8 """ 9 submit_try: Submit a try request. 10 11 This is a thin wrapper around the try request utilities in depot_tools which 12 adds some validation and supports both git and svn. 13 """ 14 15 16 import httplib 17 import json 18 import os 19 import re 20 import shutil 21 import subprocess 22 import svn 23 import sys 24 import tempfile 25 26 import retrieve_from_googlesource 27 28 29 # Alias which can be used to run a try on every builder. 30 ALL_BUILDERS = 'all' 31 # Alias which can be used to run a try on all compile builders. 32 COMPILE_BUILDERS = 'compile' 33 # Alias which can be used to run a try on all builders that are run in the CQ. 34 CQ_BUILDERS = 'cq' 35 # Alias which can be used to specify a regex to choose builders. 36 REGEX = 'regex' 37 38 ALL_ALIASES = [ALL_BUILDERS, COMPILE_BUILDERS, REGEX, CQ_BUILDERS] 39 40 LARGE_NUMBER_OF_BOTS = 5 41 42 GIT = 'git.bat' if os.name == 'nt' else 'git' 43 44 # URL of the slaves.cfg file in the Skia buildbot sources. 45 SKIA_REPO = 'https://skia.googlesource.com/buildbot' 46 SLAVES_CFG_PATH = 'master/slaves.cfg' 47 48 # All try builders have this suffix. 49 TRYBOT_SUFFIX = '-Trybot' 50 51 # String for matching the svn url of the try server inside codereview.settings. 52 TRYSERVER_SVN_URL = 'TRYSERVER_SVN_URL: ' 53 54 # Strings used for matching svn config properties. 55 URL_STR = 'URL' 56 REPO_ROOT_STR = 'Repository Root' 57 58 59 def FindDepotTools(): 60 """ Find depot_tools on the local machine and return its location. """ 61 which_cmd = 'where' if os.name == 'nt' else 'which' 62 cmd = [which_cmd, 'gcl'] 63 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 64 if proc.wait() != 0: 65 raise Exception('Couldn\'t find depot_tools in PATH!') 66 gcl = proc.communicate()[0].split('\n')[0].rstrip() 67 depot_tools_dir = os.path.dirname(gcl) 68 return depot_tools_dir 69 70 71 def GetCheckoutRoot(): 72 """ Determine where the local checkout is rooted.""" 73 cmd = ['git', 'rev-parse', '--show-toplevel'] 74 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, 75 stderr=subprocess.STDOUT) 76 if proc.wait() != 0: 77 raise Exception('Couldn\'t find checkout root!') 78 return os.path.basename(proc.communicate()[0]) 79 80 81 def GetTryRepo(): 82 """Determine the TRYSERVER_SVN_URL from the codereview.settings file.""" 83 codereview_settings_file = os.path.join(os.path.dirname(__file__), os.pardir, 84 'codereview.settings') 85 with open(codereview_settings_file) as f: 86 for line in f: 87 if line.startswith(TRYSERVER_SVN_URL): 88 return line[len(TRYSERVER_SVN_URL):].rstrip() 89 raise Exception('Couldn\'t determine the TRYSERVER_SVN_URL. Make sure it is ' 90 'defined in the %s file.' % codereview_settings_file) 91 92 93 def RetrieveTrybotList(): 94 """Retrieve the list of known trybots from the checked-in buildbot 95 configuration.""" 96 # Retrieve the slaves.cfg file from the repository. 97 slaves_cfg_text = retrieve_from_googlesource.get(SKIA_REPO, SLAVES_CFG_PATH) 98 99 # Execute the slaves.cfg file to obtain the list of slaves. 100 vars = {} 101 exec(slaves_cfg_text, vars) 102 slaves_cfg = vars['slaves'] 103 104 # Pull the list of known builders from the slaves list. 105 trybots = set() 106 for slave in slaves_cfg: 107 for builder in slave['builder']: 108 if not builder.endswith(TRYBOT_SUFFIX): 109 trybots.add(builder) 110 111 return list(trybots), vars['cq_trybots'] 112 113 114 def ValidateArgs(argv, trybots, cq_trybots, is_svn=True): 115 """ Parse and validate command-line arguments. If the arguments are valid, 116 returns a tuple of (<changelist name>, <list of trybots>). 117 118 trybots: list of strings; A list of the known try builders. 119 cq_trybots: list of strings; Trybots who get run by the commit queue. 120 is_svn: bool; whether or not we're in an svn checkout. 121 """ 122 123 class CollectedArgs(object): 124 def __init__(self, bots, changelist, revision): 125 self._bots = bots 126 self._changelist = changelist 127 self._revision = revision 128 129 @property 130 def bots(self): 131 for bot in self._bots: 132 yield bot 133 134 @property 135 def changelist(self): 136 return self._changelist 137 138 @property 139 def revision(self): 140 return self._revision 141 142 usage = ( 143 """submit_try: Submit a try request. 144 submit_try %s--bot <buildername> [<buildername> ...] 145 146 -b, --bot Builder(s) or Alias on which to run the try. Required. 147 Allowed aliases: %s 148 -h, --help Show this message. 149 -r <revision#> Revision from which to run the try. 150 -l, --list_bots List the available try builders and aliases and exit. 151 """ % ('<changelist> ' if is_svn else '', ALL_ALIASES)) 152 153 def Error(msg=None): 154 if msg: 155 print msg 156 print usage 157 sys.exit(1) 158 159 using_bots = None 160 changelist = None 161 revision = None 162 163 while argv: 164 arg = argv.pop(0) 165 if arg == '-h' or arg == '--help': 166 Error() 167 elif arg == '-l' or arg == '--list_bots': 168 format_args = ['\n '.join(sorted(trybots))] + \ 169 ALL_ALIASES + \ 170 ['\n '.join(sorted(cq_trybots))] 171 print ( 172 """ 173 submit_try: Available builders:\n %s 174 175 Can also use the following aliases to run on groups of builders- 176 %s: Will run against all trybots. 177 %s: Will run against all compile trybots. 178 %s: You will be prompted to enter a regex to select builders with. 179 %s: Will run against the same trybots as the commit queue:\n %s 180 181 """ % tuple(format_args)) 182 sys.exit(0) 183 elif arg == '-b' or arg == '--bot': 184 if using_bots: 185 Error('--bot specified multiple times.') 186 if len(argv) < 1: 187 Error('You must specify a builder with "--bot".') 188 using_bots = [] 189 while argv and not argv[0].startswith('-'): 190 for bot in argv.pop(0).split(','): 191 if bot in ALL_ALIASES: 192 if using_bots: 193 Error('Cannot specify "%s" with additional builder names or ' 194 'aliases.' % bot) 195 elif bot == COMPILE_BUILDERS: 196 using_bots = [t for t in trybots if t.startswith('Build')] 197 elif bot == CQ_BUILDERS: 198 using_bots = cq_trybots 199 elif bot == REGEX: 200 while True: 201 regex = raw_input("Enter your trybot regex: ") 202 p = re.compile(regex) 203 using_bots = [t for t in trybots if p.match(t)] 204 print '\n\nTrybots that match your regex:\n%s\n\n' % '\n'.join( 205 using_bots) 206 if raw_input('Re-enter regex? [y,n]: ') == 'n': 207 break 208 break 209 else: 210 if not bot in trybots: 211 Error('Unrecognized builder: %s' % bot) 212 using_bots.append(bot) 213 elif arg == '-r': 214 if len(argv) < 1: 215 Error('You must specify a revision with "-r".') 216 revision = argv.pop(0) 217 else: 218 if changelist or not is_svn: 219 Error('Unknown argument: %s' % arg) 220 changelist = arg 221 if is_svn and not changelist: 222 Error('You must specify a changelist name.') 223 if not using_bots: 224 Error('You must specify one or more builders using --bot.') 225 if len(using_bots) > LARGE_NUMBER_OF_BOTS: 226 are_you_sure = raw_input('Running a try on a large number of bots is very ' 227 'expensive. You may be able to get enough ' 228 'information by running on a smaller set of bots. ' 229 'Are you sure you want to do this? [y,n]: ') 230 if are_you_sure != 'y': 231 Error() 232 return CollectedArgs(bots=using_bots, changelist=changelist, 233 revision=revision) 234 235 236 def SubmitTryRequest(trybots, revision=None): 237 """ Submits a try request on the given list of trybots. 238 239 Args: 240 trybots: list of strings; the names of the try builders to run. 241 revision: optional string; the revision from which to run the try. 242 """ 243 botlist = ','.join(['%s%s' % (bot, TRYBOT_SUFFIX) for bot in trybots]) 244 # Find depot_tools. This is needed to import git_cl and trychange. 245 sys.path.append(FindDepotTools()) 246 import git_cl 247 import trychange 248 249 cmd = [GIT, 'diff', git_cl.Changelist().GetUpstreamBranch(), 250 '--no-ext-diff'] 251 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 252 git_data = proc.communicate() 253 if git_data[0] is None: 254 raise Exception('Failed to capture git diff!') 255 256 temp_dir = tempfile.mkdtemp() 257 try: 258 diff_file = os.path.join(temp_dir, 'patch.diff') 259 with open(diff_file, 'wb') as f: 260 f.write(git_data[0]) 261 f.close() 262 263 try_args = ['--use_svn', 264 '--svn_repo', GetTryRepo(), 265 '--root', GetCheckoutRoot(), 266 '--bot', botlist, 267 '--diff', diff_file, 268 ] 269 if revision: 270 try_args.extend(['-r', revision]) 271 272 # Submit the try request. 273 trychange.TryChange(try_args, None, False) 274 finally: 275 shutil.rmtree(temp_dir) 276 277 278 def main(): 279 # Retrieve the list of active try builders from the build master. 280 trybots, cq_trybots = RetrieveTrybotList() 281 282 # Determine if we're in an SVN checkout. 283 is_svn = os.path.isdir('.svn') 284 285 # Parse and validate the command-line arguments. 286 args = ValidateArgs(sys.argv[1:], trybots=trybots, cq_trybots=cq_trybots, 287 is_svn=is_svn) 288 289 # Submit the try request. 290 SubmitTryRequest(args.bots, args.revision) 291 292 293 if __name__ == '__main__': 294 sys.exit(main()) 295
