1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """The experiment runner module.""" 5 from __future__ import print_function 6 7 import getpass 8 import os 9 import shutil 10 import time 11 12 import afe_lock_machine 13 import test_flag 14 15 from cros_utils import command_executer 16 from cros_utils import logger 17 from cros_utils.email_sender import EmailSender 18 from cros_utils.file_utils import FileUtils 19 20 import config 21 from experiment_status import ExperimentStatus 22 from results_cache import CacheConditions 23 from results_cache import ResultsCache 24 from results_report import HTMLResultsReport 25 from results_report import TextResultsReport 26 from results_report import JSONResultsReport 27 from schedv2 import Schedv2 28 29 def _WriteJSONReportToFile(experiment, results_dir, json_report): 30 """Writes a JSON report to a file in results_dir.""" 31 has_llvm = any('llvm' in l.compiler for l in experiment.labels) 32 compiler_string = 'llvm' if has_llvm else 'gcc' 33 board = experiment.labels[0].board 34 filename = 'report_%s_%s_%s.%s.json' % ( 35 board, json_report.date, json_report.time.replace(':', '.'), 36 compiler_string) 37 fullname = os.path.join(results_dir, filename) 38 report_text = json_report.GetReport() 39 with open(fullname, 'w') as out_file: 40 out_file.write(report_text) 41 42 43 class ExperimentRunner(object): 44 """ExperimentRunner Class.""" 45 46 STATUS_TIME_DELAY = 30 47 THREAD_MONITOR_DELAY = 2 48 49 def __init__(self, 50 experiment, 51 json_report, 52 using_schedv2=False, 53 log=None, 54 cmd_exec=None): 55 self._experiment = experiment 56 self.l = log or logger.GetLogger(experiment.log_dir) 57 self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l) 58 self._terminated = False 59 self.json_report = json_report 60 self.locked_machines = [] 61 if experiment.log_level != 'verbose': 62 self.STATUS_TIME_DELAY = 10 63 64 # Setting this to True will use crosperf sched v2 (feature in progress). 65 self._using_schedv2 = using_schedv2 66 67 def _GetMachineList(self): 68 """Return a list of all requested machines. 69 70 Create a list of all the requested machines, both global requests and 71 label-specific requests, and return the list. 72 """ 73 machines = self._experiment.remote 74 # All Label.remote is a sublist of experiment.remote. 75 for l in self._experiment.labels: 76 for r in l.remote: 77 assert r in machines 78 return machines 79 80 def _UpdateMachineList(self, locked_machines): 81 """Update machines lists to contain only locked machines. 82 83 Go through all the lists of requested machines, both global and 84 label-specific requests, and remove any machine that we were not 85 able to lock. 86 87 Args: 88 locked_machines: A list of the machines we successfully locked. 89 """ 90 for m in self._experiment.remote: 91 if m not in locked_machines: 92 self._experiment.remote.remove(m) 93 94 for l in self._experiment.labels: 95 for m in l.remote: 96 if m not in locked_machines: 97 l.remote.remove(m) 98 99 def _LockAllMachines(self, experiment): 100 """Attempt to globally lock all of the machines requested for run. 101 102 This method will use the AFE server to globally lock all of the machines 103 requested for this crosperf run, to prevent any other crosperf runs from 104 being able to update/use the machines while this experiment is running. 105 """ 106 if test_flag.GetTestMode(): 107 self.locked_machines = self._GetMachineList() 108 self._experiment.locked_machines = self.locked_machines 109 else: 110 lock_mgr = afe_lock_machine.AFELockManager( 111 self._GetMachineList(), 112 '', 113 experiment.labels[0].chromeos_root, 114 None, 115 log=self.l,) 116 for m in lock_mgr.machines: 117 if not lock_mgr.MachineIsKnown(m): 118 lock_mgr.AddLocalMachine(m) 119 machine_states = lock_mgr.GetMachineStates('lock') 120 lock_mgr.CheckMachineLocks(machine_states, 'lock') 121 self.locked_machines = lock_mgr.UpdateMachines(True) 122 self._experiment.locked_machines = self.locked_machines 123 self._UpdateMachineList(self.locked_machines) 124 self._experiment.machine_manager.RemoveNonLockedMachines( 125 self.locked_machines) 126 if len(self.locked_machines) == 0: 127 raise RuntimeError('Unable to lock any machines.') 128 129 def _UnlockAllMachines(self, experiment): 130 """Attempt to globally unlock all of the machines requested for run. 131 132 The method will use the AFE server to globally unlock all of the machines 133 requested for this crosperf run. 134 """ 135 if not self.locked_machines or test_flag.GetTestMode(): 136 return 137 138 lock_mgr = afe_lock_machine.AFELockManager( 139 self.locked_machines, 140 '', 141 experiment.labels[0].chromeos_root, 142 None, 143 log=self.l,) 144 machine_states = lock_mgr.GetMachineStates('unlock') 145 lock_mgr.CheckMachineLocks(machine_states, 'unlock') 146 lock_mgr.UpdateMachines(False) 147 148 def _ClearCacheEntries(self, experiment): 149 for br in experiment.benchmark_runs: 150 cache = ResultsCache() 151 cache.Init(br.label.chromeos_image, br.label.chromeos_root, 152 br.benchmark.test_name, br.iteration, br.test_args, 153 br.profiler_args, br.machine_manager, br.machine, 154 br.label.board, br.cache_conditions, br._logger, br.log_level, 155 br.label, br.share_cache, br.benchmark.suite, 156 br.benchmark.show_all_results, br.benchmark.run_local) 157 cache_dir = cache.GetCacheDirForWrite() 158 if os.path.exists(cache_dir): 159 self.l.LogOutput('Removing cache dir: %s' % cache_dir) 160 shutil.rmtree(cache_dir) 161 162 def _Run(self, experiment): 163 try: 164 if not experiment.locks_dir: 165 self._LockAllMachines(experiment) 166 if self._using_schedv2: 167 schedv2 = Schedv2(experiment) 168 experiment.set_schedv2(schedv2) 169 if CacheConditions.FALSE in experiment.cache_conditions: 170 self._ClearCacheEntries(experiment) 171 status = ExperimentStatus(experiment) 172 experiment.Run() 173 last_status_time = 0 174 last_status_string = '' 175 try: 176 if experiment.log_level != 'verbose': 177 self.l.LogStartDots() 178 while not experiment.IsComplete(): 179 if last_status_time + self.STATUS_TIME_DELAY < time.time(): 180 last_status_time = time.time() 181 border = '==============================' 182 if experiment.log_level == 'verbose': 183 self.l.LogOutput(border) 184 self.l.LogOutput(status.GetProgressString()) 185 self.l.LogOutput(status.GetStatusString()) 186 self.l.LogOutput(border) 187 else: 188 current_status_string = status.GetStatusString() 189 if current_status_string != last_status_string: 190 self.l.LogEndDots() 191 self.l.LogOutput(border) 192 self.l.LogOutput(current_status_string) 193 self.l.LogOutput(border) 194 last_status_string = current_status_string 195 else: 196 self.l.LogAppendDot() 197 time.sleep(self.THREAD_MONITOR_DELAY) 198 except KeyboardInterrupt: 199 self._terminated = True 200 self.l.LogError('Ctrl-c pressed. Cleaning up...') 201 experiment.Terminate() 202 raise 203 except SystemExit: 204 self._terminated = True 205 self.l.LogError('Unexpected exit. Cleaning up...') 206 experiment.Terminate() 207 raise 208 finally: 209 if not experiment.locks_dir: 210 self._UnlockAllMachines(experiment) 211 212 def _PrintTable(self, experiment): 213 self.l.LogOutput(TextResultsReport.FromExperiment(experiment).GetReport()) 214 215 def _Email(self, experiment): 216 # Only email by default if a new run was completed. 217 send_mail = False 218 for benchmark_run in experiment.benchmark_runs: 219 if not benchmark_run.cache_hit: 220 send_mail = True 221 break 222 if (not send_mail and not experiment.email_to or 223 config.GetConfig('no_email')): 224 return 225 226 label_names = [] 227 for label in experiment.labels: 228 label_names.append(label.name) 229 subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names)) 230 231 text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() 232 text_report += ('\nResults are stored in %s.\n' % 233 experiment.results_directory) 234 text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report 235 html_report = HTMLResultsReport.FromExperiment(experiment).GetReport() 236 attachment = EmailSender.Attachment('report.html', html_report) 237 email_to = experiment.email_to or [] 238 email_to.append(getpass.getuser()) 239 EmailSender().SendEmail(email_to, 240 subject, 241 text_report, 242 attachments=[attachment], 243 msg_type='html') 244 245 def _StoreResults(self, experiment): 246 if self._terminated: 247 return 248 results_directory = experiment.results_directory 249 FileUtils().RmDir(results_directory) 250 FileUtils().MkDirP(results_directory) 251 self.l.LogOutput('Storing experiment file in %s.' % results_directory) 252 experiment_file_path = os.path.join(results_directory, 'experiment.exp') 253 FileUtils().WriteFile(experiment_file_path, experiment.experiment_file) 254 255 self.l.LogOutput('Storing results report in %s.' % results_directory) 256 results_table_path = os.path.join(results_directory, 'results.html') 257 report = HTMLResultsReport.FromExperiment(experiment).GetReport() 258 if self.json_report: 259 json_report = JSONResultsReport.FromExperiment(experiment, 260 json_args={'indent': 2}) 261 _WriteJSONReportToFile(experiment, results_directory, json_report) 262 263 FileUtils().WriteFile(results_table_path, report) 264 265 self.l.LogOutput('Storing email message body in %s.' % results_directory) 266 msg_file_path = os.path.join(results_directory, 'msg_body.html') 267 text_report = TextResultsReport.FromExperiment(experiment, True).GetReport() 268 text_report += ('\nResults are stored in %s.\n' % 269 experiment.results_directory) 270 msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report 271 FileUtils().WriteFile(msg_file_path, msg_body) 272 273 self.l.LogOutput('Storing results of each benchmark run.') 274 for benchmark_run in experiment.benchmark_runs: 275 if benchmark_run.result: 276 benchmark_run_name = filter(str.isalnum, benchmark_run.name) 277 benchmark_run_path = os.path.join(results_directory, benchmark_run_name) 278 benchmark_run.result.CopyResultsTo(benchmark_run_path) 279 benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp) 280 281 def Run(self): 282 try: 283 self._Run(self._experiment) 284 finally: 285 # Always print the report at the end of the run. 286 self._PrintTable(self._experiment) 287 if not self._terminated: 288 self._StoreResults(self._experiment) 289 self._Email(self._experiment) 290 291 292 class MockExperimentRunner(ExperimentRunner): 293 """Mocked ExperimentRunner for testing.""" 294 295 def __init__(self, experiment, json_report): 296 super(MockExperimentRunner, self).__init__(experiment, json_report) 297 298 def _Run(self, experiment): 299 self.l.LogOutput("Would run the following experiment: '%s'." % 300 experiment.name) 301 302 def _PrintTable(self, experiment): 303 self.l.LogOutput('Would print the experiment table.') 304 305 def _Email(self, experiment): 306 self.l.LogOutput('Would send result email.') 307 308 def _StoreResults(self, experiment): 309 self.l.LogOutput('Would store the results.') 310