1 #!/usr/bin/env python2 2 # 3 # Copyright 2016 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 """Script for running nightly compiler tests on ChromeOS. 7 8 This script launches a buildbot to build ChromeOS with the latest compiler on 9 a particular board; then it finds and downloads the trybot image and the 10 corresponding official image, and runs crosperf performance tests comparing 11 the two. It then generates a report, emails it to the c-compiler-chrome, as 12 well as copying the images into the seven-day reports directory. 13 """ 14 15 # Script to test different toolchains against ChromeOS benchmarks. 16 17 from __future__ import print_function 18 19 import argparse 20 import datetime 21 import os 22 import re 23 import sys 24 import time 25 26 from cros_utils import command_executer 27 from cros_utils import logger 28 29 from cros_utils import buildbot_utils 30 31 # CL that uses LLVM-Next to build the images (includes chrome). 32 USE_LLVM_NEXT_PATCH = '513590' 33 34 CROSTC_ROOT = '/usr/local/google/crostc' 35 ROLE_ACCOUNT = 'mobiletc-prebuild' 36 TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__)) 37 MAIL_PROGRAM = '~/var/bin/mail-sheriff' 38 PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') 39 NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') 40 41 IMAGE_DIR = '{board}-{image_type}' 42 IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}' 43 IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR 44 TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}' 45 PFQ_IMAGE_FS = IMAGE_FS + '-rc1' 46 IMAGE_RE_GROUPS = { 47 'board': r'(?P<board>\S+)', 48 'image_type': r'(?P<image_type>\S+)', 49 'chrome_version': r'(?P<chrome_version>R\d+)', 50 'tip': r'(?P<tip>\d+)', 51 'branch': r'(?P<branch>\d+)', 52 'branch_branch': r'(?P<branch_branch>\d+)', 53 'build_id': r'(?P<build_id>b\d+)' 54 } 55 TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS) 56 57 58 class ToolchainComparator(object): 59 """Class for doing the nightly tests work.""" 60 61 def __init__(self, 62 board, 63 remotes, 64 chromeos_root, 65 weekday, 66 patches, 67 noschedv2=False): 68 self._board = board 69 self._remotes = remotes 70 self._chromeos_root = chromeos_root 71 self._base_dir = os.getcwd() 72 self._ce = command_executer.GetCommandExecuter() 73 self._l = logger.GetLogger() 74 self._build = '%s-release' % board 75 self._patches = patches.split(',') if patches else [] 76 self._patches_string = '_'.join(str(p) for p in self._patches) 77 self._noschedv2 = noschedv2 78 79 if not weekday: 80 self._weekday = time.strftime('%a') 81 else: 82 self._weekday = weekday 83 timestamp = datetime.datetime.strftime(datetime.datetime.now(), 84 '%Y-%m-%d_%H:%M:%S') 85 self._reports_dir = os.path.join( 86 NIGHTLY_TESTS_DIR, 87 '%s.%s' % (timestamp, board),) 88 89 def _GetVanillaImageName(self, trybot_image): 90 """Given a trybot artifact name, get latest vanilla image name. 91 92 Args: 93 trybot_image: artifact name such as 94 'trybot-daisy-release/R40-6394.0.0-b1389' 95 96 Returns: 97 Latest official image name, e.g. 'daisy-release/R57-9089.0.0'. 98 """ 99 mo = re.search(TRYBOT_IMAGE_RE, trybot_image) 100 assert mo 101 dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict()) 102 return buildbot_utils.GetLatestImage(self._chromeos_root, dirname) 103 104 def _GetNonAFDOImageName(self, trybot_image): 105 """Given a trybot artifact name, get corresponding non-AFDO image name. 106 107 We get the non-AFDO image from the PFQ builders. This image 108 is not generated for all the boards and, the closest PFQ image 109 was the one build for the previous ChromeOS version (the chrome 110 used in the current version is the one validated in the previous 111 version). 112 The previous ChromeOS does not always exist either. So, we try 113 a couple of versions before. 114 115 Args: 116 trybot_image: artifact name such as 117 'trybot-daisy-release/R40-6394.0.0-b1389' 118 119 Returns: 120 Corresponding chrome PFQ image name, e.g. 121 'daisy-chrome-pfq/R40-6393.0.0-rc1'. 122 """ 123 mo = re.search(TRYBOT_IMAGE_RE, trybot_image) 124 assert mo 125 image_dict = mo.groupdict() 126 image_dict['image_type'] = 'chrome-pfq' 127 for _ in xrange(2): 128 image_dict['tip'] = str(int(image_dict['tip']) - 1) 129 nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict) 130 if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image): 131 return nonafdo_image 132 return '' 133 134 def _FinishSetup(self): 135 """Make sure testing_rsa file is properly set up.""" 136 # Fix protections on ssh key 137 command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' 138 '/chrome-src-internal/src/third_party/chromite/ssh_keys' 139 '/testing_rsa') 140 ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command) 141 if ret_val != 0: 142 raise RuntimeError('chmod for testing_rsa failed') 143 144 def _TestImages(self, trybot_image, vanilla_image, nonafdo_image): 145 """Create crosperf experiment file. 146 147 Given the names of the trybot, vanilla and non-AFDO images, create the 148 appropriate crosperf experiment file and launch crosperf on it. 149 """ 150 experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday) 151 experiment_file_name = '%s_toolchain_experiment.txt' % self._board 152 153 compiler_string = 'llvm' 154 if USE_LLVM_NEXT_PATCH in self._patches_string: 155 experiment_file_name = '%s_llvm_next_experiment.txt' % self._board 156 compiler_string = 'llvm_next' 157 158 experiment_file = os.path.join(experiment_file_dir, experiment_file_name) 159 experiment_header = """ 160 board: %s 161 remote: %s 162 retries: 1 163 """ % (self._board, self._remotes) 164 experiment_tests = """ 165 benchmark: all_toolchain_perf { 166 suite: telemetry_Crosperf 167 iterations: 0 168 } 169 170 benchmark: page_cycler_v2.typical_25 { 171 suite: telemetry_Crosperf 172 iterations: 0 173 run_local: False 174 retries: 0 175 } 176 """ 177 178 with open(experiment_file, 'w') as f: 179 f.write(experiment_header) 180 f.write(experiment_tests) 181 182 # Now add vanilla to test file. 183 official_image = """ 184 vanilla_image { 185 chromeos_root: %s 186 build: %s 187 compiler: llvm 188 } 189 """ % (self._chromeos_root, vanilla_image) 190 f.write(official_image) 191 192 # Now add non-AFDO image to test file. 193 if nonafdo_image: 194 official_nonafdo_image = """ 195 nonafdo_image { 196 chromeos_root: %s 197 build: %s 198 compiler: llvm 199 } 200 """ % (self._chromeos_root, nonafdo_image) 201 f.write(official_nonafdo_image) 202 203 label_string = '%s_trybot_image' % compiler_string 204 205 # Reuse autotest files from vanilla image for trybot images 206 autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files') 207 experiment_image = """ 208 %s { 209 chromeos_root: %s 210 build: %s 211 autotest_path: %s 212 compiler: %s 213 } 214 """ % (label_string, self._chromeos_root, trybot_image, 215 autotest_files, compiler_string) 216 f.write(experiment_image) 217 218 crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf') 219 noschedv2_opts = '--noschedv2' if self._noschedv2 else '' 220 command = ('{crosperf} --no_email=True --results_dir={r_dir} ' 221 '--json_report=True {noschedv2_opts} {exp_file}').format( 222 crosperf=crosperf, 223 r_dir=self._reports_dir, 224 noschedv2_opts=noschedv2_opts, 225 exp_file=experiment_file) 226 227 ret = self._ce.RunCommand(command) 228 if ret != 0: 229 raise RuntimeError('Crosperf execution error!') 230 else: 231 # Copy json report to pending archives directory. 232 command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) 233 ret = self._ce.RunCommand(command) 234 return 235 236 def _SendEmail(self): 237 """Find email message generated by crosperf and send it.""" 238 filename = os.path.join(self._reports_dir, 'msg_body.html') 239 if (os.path.exists(filename) and 240 os.path.exists(os.path.expanduser(MAIL_PROGRAM))): 241 email_title = 'buildbot llvm test results' 242 if USE_LLVM_NEXT_PATCH in self._patches_string: 243 email_title = 'buildbot llvm_next test results' 244 command = ('cat %s | %s -s "%s, %s" -team -html' % 245 (filename, MAIL_PROGRAM, email_title, self._board)) 246 self._ce.RunCommand(command) 247 248 def DoAll(self): 249 """Main function inside ToolchainComparator class. 250 251 Launch trybot, get image names, create crosperf experiment file, run 252 crosperf, and copy images into seven-day report directories. 253 """ 254 date_str = datetime.date.today() 255 description = 'master_%s_%s_%s' % (self._patches_string, self._build, 256 date_str) 257 build_id, trybot_image = buildbot_utils.GetTrybotImage( 258 self._chromeos_root, 259 self._build, 260 self._patches, 261 description, 262 other_flags=['--notests'], 263 build_toolchain=True) 264 265 print('trybot_url: \ 266 https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/release/builds/%s' 267 % build_id) 268 if len(trybot_image) == 0: 269 self._l.LogError('Unable to find trybot_image for %s!' % description) 270 return 1 271 272 vanilla_image = self._GetVanillaImageName(trybot_image) 273 nonafdo_image = self._GetNonAFDOImageName(trybot_image) 274 275 print('trybot_image: %s' % trybot_image) 276 print('vanilla_image: %s' % vanilla_image) 277 print('nonafdo_image: %s' % nonafdo_image) 278 279 if os.getlogin() == ROLE_ACCOUNT: 280 self._FinishSetup() 281 282 self._TestImages(trybot_image, vanilla_image, nonafdo_image) 283 self._SendEmail() 284 return 0 285 286 287 def Main(argv): 288 """The main function.""" 289 290 # Common initializations 291 command_executer.InitCommandExecuter() 292 parser = argparse.ArgumentParser() 293 parser.add_argument( 294 '--remote', dest='remote', help='Remote machines to run tests on.') 295 parser.add_argument( 296 '--board', dest='board', default='x86-zgb', help='The target board.') 297 parser.add_argument( 298 '--chromeos_root', 299 dest='chromeos_root', 300 help='The chromeos root from which to run tests.') 301 parser.add_argument( 302 '--weekday', 303 default='', 304 dest='weekday', 305 help='The day of the week for which to run tests.') 306 parser.add_argument( 307 '--patch', 308 dest='patches', 309 help='The patches to use for the testing, ' 310 "seprate the patch numbers with ',' " 311 'for more than one patches.') 312 parser.add_argument( 313 '--noschedv2', 314 dest='noschedv2', 315 action='store_true', 316 default=False, 317 help='Pass --noschedv2 to crosperf.') 318 319 options = parser.parse_args(argv[1:]) 320 if not options.board: 321 print('Please give a board.') 322 return 1 323 if not options.remote: 324 print('Please give at least one remote machine.') 325 return 1 326 if not options.chromeos_root: 327 print('Please specify the ChromeOS root directory.') 328 return 1 329 330 fc = ToolchainComparator(options.board, options.remote, options.chromeos_root, 331 options.weekday, options.patches, options.noschedv2) 332 return fc.DoAll() 333 334 335 if __name__ == '__main__': 336 retval = Main(sys.argv) 337 sys.exit(retval) 338
