1 # Copyright 2017 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Utilities for launching and accessing ChromeOS buildbots.""" 5 6 from __future__ import print_function 7 8 import ast 9 import json 10 import os 11 import re 12 import time 13 14 from cros_utils import command_executer 15 from cros_utils import logger 16 17 INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot. 18 SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot. 19 20 # Some of our slower builders (llmv-next) are taking more 21 # than 8 hours. So, increase this TIME_OUT to 9 hours. 22 TIME_OUT = 32400 # Decide the build is dead or will never finish 23 24 25 class BuildbotTimeout(Exception): 26 """Exception to throw when a buildbot operation timesout.""" 27 pass 28 29 30 def RunCommandInPath(path, cmd): 31 ce = command_executer.GetCommandExecuter() 32 cwd = os.getcwd() 33 os.chdir(path) 34 status, stdout, stderr = ce.RunCommandWOutput(cmd, print_to_console=False) 35 os.chdir(cwd) 36 return status, stdout, stderr 37 38 39 def PeekTrybotImage(chromeos_root, buildbucket_id): 40 """Get the artifact URL of a given tryjob. 41 42 Args: 43 buildbucket_id: buildbucket-id 44 chromeos_root: root dir of chrome os checkout 45 46 Returns: 47 (status, url) where status can be 'pass', 'fail', 'running', 48 and url looks like: 49 gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 50 """ 51 command = ( 52 'cros buildresult --report json --buildbucket-id %s' % buildbucket_id) 53 rc, out, _ = RunCommandInPath(chromeos_root, command) 54 55 # Current implementation of cros buildresult returns fail when a job is still 56 # running. 57 if rc != 0: 58 return ('running', None) 59 60 results = json.loads(out)[buildbucket_id] 61 62 return (results['status'], results['artifacts_url'].rstrip('/')) 63 64 65 def ParseTryjobBuildbucketId(msg): 66 """Find the buildbucket-id in the messages from `cros tryjob`. 67 68 Args: 69 msg: messages from `cros tryjob` 70 71 Returns: 72 buildbucket-id, which will be passed to `cros buildresult` 73 """ 74 output_list = ast.literal_eval(msg) 75 output_dict = output_list[0] 76 if 'buildbucket_id' in output_dict: 77 return output_dict['buildbucket_id'] 78 return None 79 80 81 def SubmitTryjob(chromeos_root, 82 buildbot_name, 83 patch_list, 84 tryjob_flags=None, 85 build_toolchain=False): 86 """Calls `cros tryjob ...` 87 88 Args: 89 chromeos_root: the path to the ChromeOS root, needed for finding chromite 90 and launching the buildbot. 91 buildbot_name: the name of the buildbot queue, such as lumpy-release or 92 daisy-paladin. 93 patch_list: a python list of the patches, if any, for the buildbot to use. 94 tryjob_flags: See cros tryjob --help for available options. 95 build_toolchain: builds and uses the latest toolchain, rather than the 96 prebuilt one in SDK. 97 98 Returns: 99 buildbucket id 100 """ 101 patch_arg = '' 102 if patch_list: 103 for p in patch_list: 104 patch_arg = patch_arg + ' -g ' + repr(p) 105 if not tryjob_flags: 106 tryjob_flags = [] 107 if build_toolchain: 108 tryjob_flags.append('--latest-toolchain') 109 tryjob_flags = ' '.join(tryjob_flags) 110 111 # Launch buildbot with appropriate flags. 112 build = buildbot_name 113 command = ('cros tryjob --yes --json --nochromesdk %s %s %s' % 114 (tryjob_flags, patch_arg, build)) 115 print('CMD: %s' % command) 116 _, out, _ = RunCommandInPath(chromeos_root, command) 117 buildbucket_id = ParseTryjobBuildbucketId(out) 118 print('buildbucket_id: %s' % repr(buildbucket_id)) 119 if not buildbucket_id: 120 logger.GetLogger().LogFatal('Error occurred while launching trybot job: ' 121 '%s' % command) 122 return buildbucket_id 123 124 125 def GetTrybotImage(chromeos_root, 126 buildbot_name, 127 patch_list, 128 tryjob_flags=None, 129 build_toolchain=False, 130 async=False): 131 """Launch buildbot and get resulting trybot artifact name. 132 133 This function launches a buildbot with the appropriate flags to 134 build the test ChromeOS image, with the current ToT mobile compiler. It 135 checks every 10 minutes to see if the trybot has finished. When the trybot 136 has finished, it parses the resulting report logs to find the trybot 137 artifact (if one was created), and returns that artifact name. 138 139 Args: 140 chromeos_root: the path to the ChromeOS root, needed for finding chromite 141 and launching the buildbot. 142 buildbot_name: the name of the buildbot queue, such as lumpy-release or 143 daisy-paladin. 144 patch_list: a python list of the patches, if any, for the buildbot to use. 145 tryjob_flags: See cros tryjob --help for available options. 146 build_toolchain: builds and uses the latest toolchain, rather than the 147 prebuilt one in SDK. 148 async: don't wait for artifacts; just return the buildbucket id 149 150 Returns: 151 (buildbucket id, partial image url) e.g. 152 (8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596) 153 """ 154 buildbucket_id = SubmitTryjob(chromeos_root, buildbot_name, patch_list, 155 tryjob_flags, build_toolchain) 156 if async: 157 return buildbucket_id, ' ' 158 159 # The trybot generally takes more than 2 hours to finish. 160 # Wait two hours before polling the status. 161 time.sleep(INITIAL_SLEEP_TIME) 162 elapsed = INITIAL_SLEEP_TIME 163 status = 'running' 164 image = '' 165 while True: 166 status, image = PeekTrybotImage(chromeos_root, buildbucket_id) 167 if status == 'running': 168 if elapsed > TIME_OUT: 169 logger.GetLogger().LogFatal( 170 'Unable to get build result for target %s.' % buildbot_name) 171 else: 172 wait_msg = 'Unable to find build result; job may be running.' 173 logger.GetLogger().LogOutput(wait_msg) 174 logger.GetLogger().LogOutput('{0} minutes elapsed.'.format(elapsed / 60)) 175 logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME)) 176 time.sleep(SLEEP_TIME) 177 elapsed += SLEEP_TIME 178 else: 179 break 180 181 if not buildbot_name.endswith('-toolchain') and status == 'fail': 182 # For rotating testers, we don't care about their status 183 # result, because if any HWTest failed it will be non-zero. 184 # 185 # The nightly performance tests do not run HWTests, so if 186 # their status is non-zero, we do care. In this case 187 # non-zero means the image itself probably did not build. 188 image = '' 189 190 if not image: 191 logger.GetLogger().LogError( 192 'Trybot job (buildbucket id: %s) failed with' 193 'status %s; no trybot image generated. ' % (buildbucket_id, status)) 194 else: 195 # Convert full gs path to what crosperf expects. For example, convert 196 # gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789 197 # to 198 # trybot-elm-release-tryjob/R67-10468.0.0-b20789 199 image = '/'.join(image.split('/')[-2:]) 200 201 logger.GetLogger().LogOutput("image is '%s'" % image) 202 logger.GetLogger().LogOutput('status is %s' % status) 203 return buildbucket_id, image 204 205 206 def DoesImageExist(chromeos_root, build): 207 """Check if the image for the given build exists.""" 208 209 ce = command_executer.GetCommandExecuter() 210 command = ('gsutil ls gs://chromeos-image-archive/%s' 211 '/chromiumos_test_image.tar.xz' % (build)) 212 ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False) 213 return not ret 214 215 216 def WaitForImage(chromeos_root, build): 217 """Wait for an image to be ready.""" 218 219 elapsed_time = 0 220 while elapsed_time < TIME_OUT: 221 if DoesImageExist(chromeos_root, build): 222 return 223 logger.GetLogger().LogOutput( 224 'Image %s not ready, waiting for 10 minutes' % build) 225 time.sleep(SLEEP_TIME) 226 elapsed_time += SLEEP_TIME 227 228 logger.GetLogger().LogOutput( 229 'Image %s not found, waited for %d hours' % (build, (TIME_OUT / 3600))) 230 raise BuildbotTimeout('Timeout while waiting for image %s' % build) 231 232 233 def GetLatestImage(chromeos_root, path): 234 """Get latest image""" 235 236 fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)') 237 238 ce = command_executer.GetCommandExecuter() 239 command = ('gsutil ls gs://chromeos-image-archive/%s' % path) 240 _, out, _ = ce.ChrootRunCommandWOutput( 241 chromeos_root, command, print_to_console=False) 242 candidates = [l.split('/')[-2] for l in out.split()] 243 candidates = map(fmt.match, candidates) 244 candidates = [[int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m] 245 candidates.sort(reverse=True) 246 for c in candidates: 247 build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3]) 248 if DoesImageExist(chromeos_root, build): 249 return build 250
