1 #!/usr/bin/python2 2 3 # Script to test different toolchains against ChromeOS benchmarks. 4 """Toolchain team nightly performance test script (local builds).""" 5 6 from __future__ import print_function 7 8 import argparse 9 import datetime 10 import os 11 import sys 12 import build_chromeos 13 import setup_chromeos 14 import time 15 from cros_utils import command_executer 16 from cros_utils import misc 17 from cros_utils import logger 18 19 CROSTC_ROOT = '/usr/local/google/crostc' 20 MAIL_PROGRAM = '~/var/bin/mail-sheriff' 21 PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives') 22 NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports') 23 24 25 class GCCConfig(object): 26 """GCC configuration class.""" 27 28 def __init__(self, githash): 29 self.githash = githash 30 31 32 class ToolchainConfig(object): 33 """Toolchain configuration class.""" 34 35 def __init__(self, gcc_config=None): 36 self.gcc_config = gcc_config 37 38 39 class ChromeOSCheckout(object): 40 """Main class for checking out, building and testing ChromeOS.""" 41 42 def __init__(self, board, chromeos_root): 43 self._board = board 44 self._chromeos_root = chromeos_root 45 self._ce = command_executer.GetCommandExecuter() 46 self._l = logger.GetLogger() 47 self._build_num = None 48 49 def _DeleteChroot(self): 50 command = 'cd %s; cros_sdk --delete' % self._chromeos_root 51 return self._ce.RunCommand(command) 52 53 def _DeleteCcahe(self): 54 # crosbug.com/34956 55 command = 'sudo rm -rf %s' % os.path.join(self._chromeos_root, '.cache') 56 return self._ce.RunCommand(command) 57 58 def _GetBuildNumber(self): 59 """Get the build number of the ChromeOS image from the chroot. 60 61 This function assumes a ChromeOS image has been built in the chroot. 62 It translates the 'latest' symlink in the 63 <chroot>/src/build/images/<board> directory, to find the actual 64 ChromeOS build number for the image that was built. For example, if 65 src/build/image/lumpy/latest -> R37-5982.0.2014_06_23_0454-a1, then 66 This function would parse it out and assign 'R37-5982' to self._build_num. 67 This is used to determine the official, vanilla build to use for 68 comparison tests. 69 """ 70 # Get the path to 'latest' 71 sym_path = os.path.join( 72 misc.GetImageDir(self._chromeos_root, self._board), 'latest') 73 # Translate the symlink to its 'real' path. 74 real_path = os.path.realpath(sym_path) 75 # Break up the path and get the last piece 76 # (e.g. 'R37-5982.0.2014_06_23_0454-a1" 77 path_pieces = real_path.split('/') 78 last_piece = path_pieces[-1] 79 # Break this piece into the image number + other pieces, and get the 80 # image number [ 'R37-5982', '0', '2014_06_23_0454-a1'] 81 image_parts = last_piece.split('.') 82 self._build_num = image_parts[0] 83 84 def _BuildLabelName(self, config): 85 pieces = config.split('/') 86 compiler_version = pieces[-1] 87 label = compiler_version + '_tot_afdo' 88 return label 89 90 def _BuildAndImage(self, label=''): 91 if (not label or 92 not misc.DoesLabelExist(self._chromeos_root, self._board, label)): 93 build_chromeos_args = [ 94 build_chromeos.__file__, '--chromeos_root=%s' % self._chromeos_root, 95 '--board=%s' % self._board, '--rebuild' 96 ] 97 if self._public: 98 build_chromeos_args.append('--env=USE=-chrome_internal') 99 100 ret = build_chromeos.Main(build_chromeos_args) 101 if ret != 0: 102 raise RuntimeError("Couldn't build ChromeOS!") 103 104 if not self._build_num: 105 self._GetBuildNumber() 106 # Check to see if we need to create the symbolic link for the vanilla 107 # image, and do so if appropriate. 108 if not misc.DoesLabelExist(self._chromeos_root, self._board, 'vanilla'): 109 build_name = '%s-release/%s.0.0' % (self._board, self._build_num) 110 full_vanilla_path = os.path.join(os.getcwd(), self._chromeos_root, 111 'chroot/tmp', build_name) 112 misc.LabelLatestImage(self._chromeos_root, self._board, label, 113 full_vanilla_path) 114 else: 115 misc.LabelLatestImage(self._chromeos_root, self._board, label) 116 return label 117 118 def _SetupBoard(self, env_dict, usepkg_flag, clobber_flag): 119 env_string = misc.GetEnvStringFromDict(env_dict) 120 command = ('%s %s' % (env_string, misc.GetSetupBoardCommand( 121 self._board, usepkg=usepkg_flag, force=clobber_flag))) 122 ret = self._ce.ChrootRunCommand(self._chromeos_root, command) 123 error_str = "Could not setup board: '%s'" % command 124 assert ret == 0, error_str 125 126 def _UnInstallToolchain(self): 127 command = ('sudo CLEAN_DELAY=0 emerge -C cross-%s/gcc' % 128 misc.GetCtargetFromBoard(self._board, self._chromeos_root)) 129 ret = self._ce.ChrootRunCommand(self._chromeos_root, command) 130 if ret != 0: 131 raise RuntimeError("Couldn't uninstall the toolchain!") 132 133 def _CheckoutChromeOS(self): 134 # TODO(asharif): Setup a fixed ChromeOS version (quarterly snapshot). 135 if not os.path.exists(self._chromeos_root): 136 setup_chromeos_args = ['--dir=%s' % self._chromeos_root] 137 if self._public: 138 setup_chromeos_args.append('--public') 139 ret = setup_chromeos.Main(setup_chromeos_args) 140 if ret != 0: 141 raise RuntimeError("Couldn't run setup_chromeos!") 142 143 def _BuildToolchain(self, config): 144 # Call setup_board for basic, vanilla setup. 145 self._SetupBoard({}, usepkg_flag=True, clobber_flag=False) 146 # Now uninstall the vanilla compiler and setup/build our custom 147 # compiler. 148 self._UnInstallToolchain() 149 envdict = { 150 'USE': 'git_gcc', 151 'GCC_GITHASH': config.gcc_config.githash, 152 'EMERGE_DEFAULT_OPTS': '--exclude=gcc' 153 } 154 self._SetupBoard(envdict, usepkg_flag=False, clobber_flag=False) 155 156 157 class ToolchainComparator(ChromeOSCheckout): 158 """Main class for running tests and generating reports.""" 159 160 def __init__(self, 161 board, 162 remotes, 163 configs, 164 clean, 165 public, 166 force_mismatch, 167 noschedv2=False): 168 self._board = board 169 self._remotes = remotes 170 self._chromeos_root = 'chromeos' 171 self._configs = configs 172 self._clean = clean 173 self._public = public 174 self._force_mismatch = force_mismatch 175 self._ce = command_executer.GetCommandExecuter() 176 self._l = logger.GetLogger() 177 timestamp = datetime.datetime.strftime(datetime.datetime.now(), 178 '%Y-%m-%d_%H:%M:%S') 179 self._reports_dir = os.path.join( 180 NIGHTLY_TESTS_DIR, 181 '%s.%s' % (timestamp, board),) 182 self._noschedv2 = noschedv2 183 ChromeOSCheckout.__init__(self, board, self._chromeos_root) 184 185 def _FinishSetup(self): 186 # Get correct .boto file 187 current_dir = os.getcwd() 188 src = '/usr/local/google/home/mobiletc-prebuild/.boto' 189 dest = os.path.join(current_dir, self._chromeos_root, 190 'src/private-overlays/chromeos-overlay/' 191 'googlestorage_account.boto') 192 # Copy the file to the correct place 193 copy_cmd = 'cp %s %s' % (src, dest) 194 retv = self._ce.RunCommand(copy_cmd) 195 if retv != 0: 196 raise RuntimeError("Couldn't copy .boto file for google storage.") 197 198 # Fix protections on ssh key 199 command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target' 200 '/chrome-src-internal/src/third_party/chromite/ssh_keys' 201 '/testing_rsa') 202 retv = self._ce.ChrootRunCommand(self._chromeos_root, command) 203 if retv != 0: 204 raise RuntimeError('chmod for testing_rsa failed') 205 206 def _TestLabels(self, labels): 207 experiment_file = 'toolchain_experiment.txt' 208 image_args = '' 209 if self._force_mismatch: 210 image_args = '--force-mismatch' 211 experiment_header = """ 212 board: %s 213 remote: %s 214 retries: 1 215 """ % (self._board, self._remotes) 216 experiment_tests = """ 217 benchmark: all_toolchain_perf { 218 suite: telemetry_Crosperf 219 iterations: 3 220 } 221 """ 222 223 with open(experiment_file, 'w') as f: 224 f.write(experiment_header) 225 f.write(experiment_tests) 226 for label in labels: 227 # TODO(asharif): Fix crosperf so it accepts labels with symbols 228 crosperf_label = label 229 crosperf_label = crosperf_label.replace('-', '_') 230 crosperf_label = crosperf_label.replace('+', '_') 231 crosperf_label = crosperf_label.replace('.', '') 232 233 # Use the official build instead of building vanilla ourselves. 234 if label == 'vanilla': 235 build_name = '%s-release/%s.0.0' % (self._board, self._build_num) 236 237 # Now add 'official build' to test file. 238 official_image = """ 239 official_image { 240 chromeos_root: %s 241 build: %s 242 } 243 """ % (self._chromeos_root, build_name) 244 f.write(official_image) 245 246 else: 247 experiment_image = """ 248 %s { 249 chromeos_image: %s 250 image_args: %s 251 } 252 """ % (crosperf_label, os.path.join( 253 misc.GetImageDir(self._chromeos_root, self._board), label, 254 'chromiumos_test_image.bin'), image_args) 255 f.write(experiment_image) 256 257 crosperf = os.path.join(os.path.dirname(__file__), 'crosperf', 'crosperf') 258 noschedv2_opts = '--noschedv2' if self._noschedv2 else '' 259 command = ('{crosperf} --no_email=True --results_dir={r_dir} ' 260 '--json_report=True {noschedv2_opts} {exp_file}').format( 261 crosperf=crosperf, 262 r_dir=self._reports_dir, 263 noschedv2_opts=noschedv2_opts, 264 exp_file=experiment_file) 265 266 ret = self._ce.RunCommand(command) 267 if ret != 0: 268 raise RuntimeError('Crosperf execution error!') 269 else: 270 # Copy json report to pending archives directory. 271 command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR) 272 ret = self._ce.RunCommand(command) 273 return 274 275 def _SendEmail(self): 276 """Find email msesage generated by crosperf and send it.""" 277 filename = os.path.join(self._reports_dir, 'msg_body.html') 278 if (os.path.exists(filename) and 279 os.path.exists(os.path.expanduser(MAIL_PROGRAM))): 280 command = ('cat %s | %s -s "Nightly test results, %s" -team -html' % 281 (filename, MAIL_PROGRAM, self._board)) 282 self._ce.RunCommand(command) 283 284 def DoAll(self): 285 self._CheckoutChromeOS() 286 labels = [] 287 labels.append('vanilla') 288 for config in self._configs: 289 label = self._BuildLabelName(config.gcc_config.githash) 290 if not misc.DoesLabelExist(self._chromeos_root, self._board, label): 291 self._BuildToolchain(config) 292 label = self._BuildAndImage(label) 293 labels.append(label) 294 self._FinishSetup() 295 self._TestLabels(labels) 296 self._SendEmail() 297 if self._clean: 298 ret = self._DeleteChroot() 299 if ret != 0: 300 return ret 301 ret = self._DeleteCcahe() 302 if ret != 0: 303 return ret 304 return 0 305 306 307 def Main(argv): 308 """The main function.""" 309 # Common initializations 310 ### command_executer.InitCommandExecuter(True) 311 command_executer.InitCommandExecuter() 312 parser = argparse.ArgumentParser() 313 parser.add_argument( 314 '--remote', dest='remote', help='Remote machines to run tests on.') 315 parser.add_argument( 316 '--board', dest='board', default='x86-alex', help='The target board.') 317 parser.add_argument( 318 '--githashes', 319 dest='githashes', 320 default='master', 321 help='The gcc githashes to test.') 322 parser.add_argument( 323 '--clean', 324 dest='clean', 325 default=False, 326 action='store_true', 327 help='Clean the chroot after testing.') 328 parser.add_argument( 329 '--public', 330 dest='public', 331 default=False, 332 action='store_true', 333 help='Use the public checkout/build.') 334 parser.add_argument( 335 '--force-mismatch', 336 dest='force_mismatch', 337 default='', 338 help='Force the image regardless of board mismatch') 339 parser.add_argument( 340 '--noschedv2', 341 dest='noschedv2', 342 action='store_true', 343 default=False, 344 help='Pass --noschedv2 to crosperf.') 345 options = parser.parse_args(argv) 346 if not options.board: 347 print('Please give a board.') 348 return 1 349 if not options.remote: 350 print('Please give at least one remote machine.') 351 return 1 352 toolchain_configs = [] 353 for githash in options.githashes.split(','): 354 gcc_config = GCCConfig(githash=githash) 355 toolchain_config = ToolchainConfig(gcc_config=gcc_config) 356 toolchain_configs.append(toolchain_config) 357 fc = ToolchainComparator(options.board, options.remote, toolchain_configs, 358 options.clean, options.public, 359 options.force_mismatch, options.noschedv2) 360 return fc.DoAll() 361 362 363 if __name__ == '__main__': 364 retval = Main(sys.argv[1:]) 365 sys.exit(retval) 366
