1 # Copyright 2016 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Utilities for launching and accessing ChromeOS buildbots.""" 5 6 from __future__ import print_function 7 8 import base64 9 import json 10 import os 11 import time 12 import urllib2 13 14 # pylint: disable=no-name-in-module 15 from oauth2client.service_account import ServiceAccountCredentials 16 17 from cros_utils import command_executer 18 from cros_utils import logger 19 from cros_utils import buildbot_json 20 21 INITIAL_SLEEP_TIME = 7200 # 2 hours; wait time before polling buildbot. 22 SLEEP_TIME = 600 # 10 minutes; time between polling of buildbot. 23 TIME_OUT = 28800 # Decide the build is dead or will never finish 24 # after this time (8 hours). 25 OK_STATUS = [ # List of result status values that are 'ok'. 26 # This was obtained from: 27 # https://chromium.googlesource.com/chromium/tools/build/+/ 28 # master/third_party/buildbot_8_4p1/buildbot/status/results.py 29 0, # "success" 30 1, # "warnings" 31 6, # "retry" 32 ] 33 34 35 class BuildbotTimeout(Exception): 36 """Exception to throw when a buildbot operation timesout.""" 37 pass 38 39 40 def ParseReportLog(url, build): 41 """Scrape the trybot image name off the Reports log page. 42 43 This takes the URL for a trybot Reports Stage web page, 44 and a trybot build type, such as 'daisy-release'. It 45 opens the web page and parses it looking for the trybot 46 artifact name (e.g. something like 47 'trybot-daisy-release/R40-6394.0.0-b1389'). It returns the 48 artifact name, if found. 49 """ 50 trybot_image = '' 51 url += '/text' 52 newurl = url.replace('uberchromegw', 'chromegw') 53 webpage = urllib2.urlopen(newurl) 54 data = webpage.read() 55 lines = data.split('\n') 56 for l in lines: 57 if l.find('Artifacts') > 0 and l.find('trybot') > 0: 58 trybot_name = 'trybot-%s' % build 59 start_pos = l.find(trybot_name) 60 end_pos = l.find('@https://storage') 61 trybot_image = l[start_pos:end_pos] 62 63 return trybot_image 64 65 66 def GetBuildData(buildbot_queue, build_id): 67 """Find the Reports stage web page for a trybot build. 68 69 This takes the name of a buildbot_queue, such as 'daisy-release' 70 and a build id (the build number), and uses the json buildbot api to 71 find the Reports stage web page for that build, if it exists. 72 """ 73 builder = buildbot_json.Buildbot( 74 'http://chromegw/p/tryserver.chromiumos/').builders[buildbot_queue] 75 build_data = builder.builds[build_id].data 76 logs = build_data['logs'] 77 for l in logs: 78 fname = l[1] 79 if 'steps/Report/' in fname: 80 return fname 81 82 return '' 83 84 85 def FindBuildRecordFromLog(description, build_info): 86 """Find the right build record in the build logs. 87 88 Get the first build record from build log with a reason field 89 that matches 'description'. ('description' is a special tag we 90 created when we launched the buildbot, so we could find it at this 91 point.) 92 """ 93 for build_log in build_info: 94 if description in build_log['reason']: 95 return build_log 96 return {} 97 98 99 def GetBuildInfo(file_dir, waterfall_builder): 100 """Get all the build records for the trybot builds.""" 101 102 builder = '' 103 if waterfall_builder.endswith('-release'): 104 builder = 'release' 105 elif waterfall_builder.endswith('-gcc-toolchain'): 106 builder = 'gcc_toolchain' 107 elif waterfall_builder.endswith('-llvm-toolchain'): 108 builder = 'llvm_toolchain' 109 elif waterfall_builder.endswith('-llvm-next-toolchain'): 110 builder = 'llvm_next_toolchain' 111 112 sa_file = os.path.expanduser( 113 os.path.join(file_dir, 'cros_utils', 114 'chromeos-toolchain-credentials.json')) 115 scopes = ['https://www.googleapis.com/auth/userinfo.email'] 116 117 credentials = ServiceAccountCredentials.from_json_keyfile_name( 118 sa_file, scopes=scopes) 119 url = ( 120 'https://luci-milo.appspot.com/prpc/milo.Buildbot/GetBuildbotBuildsJSON') 121 122 # NOTE: If we want to get build logs for the main waterfall builders, the 123 # 'master' field below should be 'chromeos' instead of 'chromiumos.tryserver'. 124 # Builder would be 'amd64-gcc-toolchain' or 'arm-llvm-toolchain', etc. 125 126 body = json.dumps({ 127 'master': 'chromiumos.tryserver', 128 'builder': builder, 129 'include_current': True, 130 'limit': 100 131 }) 132 access_token = credentials.get_access_token() 133 headers = { 134 'Accept': 'application/json', 135 'Content-Type': 'application/json', 136 'Authorization': 'Bearer %s' % access_token.access_token 137 } 138 r = urllib2.Request(url, body, headers) 139 u = urllib2.urlopen(r, timeout=60) 140 u.read(4) 141 o = json.load(u) 142 data = [base64.b64decode(item['data']) for item in o['builds']] 143 result = [] 144 for d in data: 145 tmp = json.loads(d) 146 result.append(tmp) 147 return result 148 149 150 def FindArchiveImage(chromeos_root, build, build_id): 151 """Returns name of the trybot artifact for board/build_id.""" 152 ce = command_executer.GetCommandExecuter() 153 command = ('gsutil ls gs://chromeos-image-archive/trybot-%s/*b%s' 154 '/chromiumos_test_image.tar.xz' % (build, build_id)) 155 _, out, _ = ce.ChrootRunCommandWOutput( 156 chromeos_root, command, print_to_console=False) 157 # 158 # If build_id is not unique, there may be multiple archive images 159 # to choose from; sort them & pick the first (newest). 160 # 161 # If there are multiple archive images found, out will look something 162 # like this: 163 # 164 # 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz 165 # gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' 166 # 167 out = out.rstrip('\n') 168 tmp_list = out.split('\n') 169 # After stripping the final '\n' and splitting on any other '\n', we get 170 # something like this: 171 # tmp_list = [ 'gs://.../R35-5692.0.0-b105/chromiumos_test_image.tar.xz' , 172 # 'gs://.../R46-7339.0.0-b105/chromiumos_test_image.tar.xz' ] 173 # 174 # If we sort this in descending order, we should end up with the most 175 # recent test image first, so that's what we do here. 176 # 177 if len(tmp_list) > 1: 178 tmp_list = sorted(tmp_list, reverse=True) 179 out = tmp_list[0] 180 181 trybot_image = '' 182 trybot_name = 'trybot-%s' % build 183 if out and out.find(trybot_name) > 0: 184 start_pos = out.find(trybot_name) 185 end_pos = out.find('/chromiumos_test_image') 186 trybot_image = out[start_pos:end_pos] 187 188 return trybot_image 189 190 191 def GetTrybotImage(chromeos_root, 192 buildbot_name, 193 patch_list, 194 build_tag, 195 other_flags=None, 196 build_toolchain=False, 197 async=False): 198 """Launch buildbot and get resulting trybot artifact name. 199 200 This function launches a buildbot with the appropriate flags to 201 build the test ChromeOS image, with the current ToT mobile compiler. It 202 checks every 10 minutes to see if the trybot has finished. When the trybot 203 has finished, it parses the resulting report logs to find the trybot 204 artifact (if one was created), and returns that artifact name. 205 206 chromeos_root is the path to the ChromeOS root, needed for finding chromite 207 and launching the buildbot. 208 209 buildbot_name is the name of the buildbot queue, such as lumpy-release or 210 daisy-paladin. 211 212 patch_list a python list of the patches, if any, for the buildbot to use. 213 214 build_tag is a (unique) string to be used to look up the buildbot results 215 from among all the build records. 216 """ 217 ce = command_executer.GetCommandExecuter() 218 cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot') 219 base_dir = os.getcwd() 220 patch_arg = '' 221 if patch_list: 222 for p in patch_list: 223 patch_arg = patch_arg + ' -g ' + repr(p) 224 toolchain_flags = '' 225 if build_toolchain: 226 toolchain_flags += '--latest-toolchain' 227 os.chdir(cbuildbot_path) 228 if other_flags: 229 optional_flags = ' '.join(other_flags) 230 else: 231 optional_flags = '' 232 233 # Launch buildbot with appropriate flags. 234 build = buildbot_name 235 description = build_tag 236 command_prefix = '' 237 if not patch_arg: 238 command_prefix = 'yes | ' 239 command = ('%s ./cbuildbot --remote --nochromesdk %s' 240 ' --remote-description=%s %s %s %s' % (command_prefix, 241 optional_flags, description, 242 toolchain_flags, patch_arg, 243 build)) 244 _, out, _ = ce.RunCommandWOutput(command) 245 if 'Tryjob submitted!' not in out: 246 logger.GetLogger().LogFatal('Error occurred while launching trybot job: ' 247 '%s' % command) 248 249 os.chdir(base_dir) 250 251 build_id = 0 252 build_status = None 253 # Wait for buildbot to finish running (check every 10 minutes). Wait 254 # 10 minutes before the first check to give the buildbot time to launch 255 # (so we don't start looking for build data before it's out there). 256 time.sleep(SLEEP_TIME) 257 done = False 258 pending = True 259 # pending_time is the time between when we submit the job and when the 260 # buildbot actually launches the build. running_time is the time between 261 # when the buildbot job launches and when it finishes. The job is 262 # considered 'pending' until we can find an entry for it in the buildbot 263 # logs. 264 pending_time = SLEEP_TIME 265 running_time = 0 266 long_slept = False 267 while not done: 268 done = True 269 build_info = GetBuildInfo(base_dir, build) 270 if not build_info: 271 if pending_time > TIME_OUT: 272 logger.GetLogger().LogFatal('Unable to get build logs for target %s.' % 273 build) 274 else: 275 pending_message = 'Unable to find build log; job may be pending.' 276 done = False 277 278 if done: 279 data_dict = FindBuildRecordFromLog(description, build_info) 280 if not data_dict: 281 # Trybot job may be pending (not actually launched yet). 282 if pending_time > TIME_OUT: 283 logger.GetLogger().LogFatal('Unable to find build record for trybot' 284 ' %s.' % description) 285 else: 286 pending_message = 'Unable to find build record; job may be pending.' 287 done = False 288 289 else: 290 # Now that we have actually found the entry for the build 291 # job in the build log, we know the job is actually 292 # runnning, not pending, so we flip the 'pending' flag. We 293 # still have to wait for the buildbot job to finish running 294 # however. 295 pending = False 296 build_id = data_dict['number'] 297 298 if async: 299 # Do not wait for trybot job to finish; return immediately 300 return build_id, ' ' 301 302 if not long_slept: 303 # The trybot generally takes more than 2 hours to finish. 304 # Wait two hours before polling the status. 305 long_slept = True 306 time.sleep(INITIAL_SLEEP_TIME) 307 pending_time += INITIAL_SLEEP_TIME 308 if True == data_dict['finished']: 309 build_status = data_dict['results'] 310 else: 311 done = False 312 313 if not done: 314 if pending: 315 logger.GetLogger().LogOutput(pending_message) 316 logger.GetLogger().LogOutput('Current pending time: %d minutes.' % 317 (pending_time / 60)) 318 pending_time += SLEEP_TIME 319 else: 320 logger.GetLogger().LogOutput('{0} minutes passed.'.format(running_time / 321 60)) 322 logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME)) 323 running_time += SLEEP_TIME 324 325 time.sleep(SLEEP_TIME) 326 if running_time > TIME_OUT: 327 done = True 328 329 trybot_image = '' 330 331 if build.endswith('-toolchain'): 332 # For rotating testers, we don't care about their build_status 333 # result, because if any HWTest failed it will be non-zero. 334 trybot_image = FindArchiveImage(chromeos_root, build, build_id) 335 else: 336 # The nightly performance tests do not run HWTests, so if 337 # their build_status is non-zero, we do care. In this case 338 # non-zero means the image itself probably did not build. 339 if build_status in OK_STATUS: 340 trybot_image = FindArchiveImage(chromeos_root, build, build_id) 341 if not trybot_image: 342 logger.GetLogger().LogError('Trybot job %s failed with status %d;' 343 ' no trybot image generated.' % 344 (description, build_status)) 345 346 logger.GetLogger().LogOutput("trybot_image is '%s'" % trybot_image) 347 logger.GetLogger().LogOutput('build_status is %d' % build_status) 348 return build_id, trybot_image 349 350 351 def GetGSContent(chromeos_root, path): 352 """gsutil cat path""" 353 354 ce = command_executer.GetCommandExecuter() 355 command = ('gsutil cat gs://chromeos-image-archive/%s' % path) 356 _, out, _ = ce.ChrootRunCommandWOutput( 357 chromeos_root, command, print_to_console=False) 358 return out 359 360 361 def DoesImageExist(chromeos_root, build): 362 """Check if the image for the given build exists.""" 363 364 ce = command_executer.GetCommandExecuter() 365 command = ('gsutil ls gs://chromeos-image-archive/%s' 366 '/chromiumos_test_image.tar.xz' % (build)) 367 ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False) 368 return not ret 369 370 371 def WaitForImage(chromeos_root, build): 372 """Wait for an image to be ready.""" 373 374 elapsed_time = 0 375 while elapsed_time < TIME_OUT: 376 if DoesImageExist(chromeos_root, build): 377 return 378 logger.GetLogger().LogOutput('Image %s not ready, waiting for 10 minutes' % 379 build) 380 time.sleep(SLEEP_TIME) 381 elapsed_time += SLEEP_TIME 382 383 logger.GetLogger().LogOutput('Image %s not found, waited for %d hours' % 384 (build, (TIME_OUT / 3600))) 385 raise BuildbotTimeout('Timeout while waiting for image %s' % build) 386
