Home | History | Annotate | Download | only in crosperf 
      1 # Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 """The experiment runner module."""
      5 from __future__ import print_function
      6 
      7 import getpass
      8 import os
      9 import shutil
     10 import time
     11 
     12 import afe_lock_machine
     13 import test_flag
     14 
     15 from cros_utils import command_executer
     16 from cros_utils import logger
     17 from cros_utils.email_sender import EmailSender
     18 from cros_utils.file_utils import FileUtils
     19 
     20 import config
     21 from experiment_status import ExperimentStatus
     22 from results_cache import CacheConditions
     23 from results_cache import ResultsCache
     24 from results_report import HTMLResultsReport
     25 from results_report import TextResultsReport
     26 from results_report import JSONResultsReport
     27 from schedv2 import Schedv2
     28 
     29 def _WriteJSONReportToFile(experiment, results_dir, json_report):
     30   """Writes a JSON report to a file in results_dir."""
     31   has_llvm = any('llvm' in l.compiler for l in experiment.labels)
     32   compiler_string = 'llvm' if has_llvm else 'gcc'
     33   board = experiment.labels[0].board
     34   filename = 'report_%s_%s_%s.%s.json' % (
     35       board, json_report.date, json_report.time.replace(':', '.'),
     36       compiler_string)
     37   fullname = os.path.join(results_dir, filename)
     38   report_text = json_report.GetReport()
     39   with open(fullname, 'w') as out_file:
     40     out_file.write(report_text)
     41 
     42 
     43 class ExperimentRunner(object):
     44   """ExperimentRunner Class."""
     45 
     46   STATUS_TIME_DELAY = 30
     47   THREAD_MONITOR_DELAY = 2
     48 
     49   def __init__(self,
     50                experiment,
     51                json_report,
     52                using_schedv2=False,
     53                log=None,
     54                cmd_exec=None):
     55     self._experiment = experiment
     56     self.l = log or logger.GetLogger(experiment.log_dir)
     57     self._ce = cmd_exec or command_executer.GetCommandExecuter(self.l)
     58     self._terminated = False
     59     self.json_report = json_report
     60     self.locked_machines = []
     61     if experiment.log_level != 'verbose':
     62       self.STATUS_TIME_DELAY = 10
     63 
     64     # Setting this to True will use crosperf sched v2 (feature in progress).
     65     self._using_schedv2 = using_schedv2
     66 
     67   def _GetMachineList(self):
     68     """Return a list of all requested machines.
     69 
     70     Create a list of all the requested machines, both global requests and
     71     label-specific requests, and return the list.
     72     """
     73     machines = self._experiment.remote
     74     # All Label.remote is a sublist of experiment.remote.
     75     for l in self._experiment.labels:
     76       for r in l.remote:
     77         assert r in machines
     78     return machines
     79 
     80   def _UpdateMachineList(self, locked_machines):
     81     """Update machines lists to contain only locked machines.
     82 
     83     Go through all the lists of requested machines, both global and
     84     label-specific requests, and remove any machine that we were not
     85     able to lock.
     86 
     87     Args:
     88       locked_machines: A list of the machines we successfully locked.
     89     """
     90     for m in self._experiment.remote:
     91       if m not in locked_machines:
     92         self._experiment.remote.remove(m)
     93 
     94     for l in self._experiment.labels:
     95       for m in l.remote:
     96         if m not in locked_machines:
     97           l.remote.remove(m)
     98 
     99   def _LockAllMachines(self, experiment):
    100     """Attempt to globally lock all of the machines requested for run.
    101 
    102     This method will use the AFE server to globally lock all of the machines
    103     requested for this crosperf run, to prevent any other crosperf runs from
    104     being able to update/use the machines while this experiment is running.
    105     """
    106     if test_flag.GetTestMode():
    107       self.locked_machines = self._GetMachineList()
    108       self._experiment.locked_machines = self.locked_machines
    109     else:
    110       lock_mgr = afe_lock_machine.AFELockManager(
    111           self._GetMachineList(),
    112           '',
    113           experiment.labels[0].chromeos_root,
    114           None,
    115           log=self.l,)
    116       for m in lock_mgr.machines:
    117         if not lock_mgr.MachineIsKnown(m):
    118           lock_mgr.AddLocalMachine(m)
    119       machine_states = lock_mgr.GetMachineStates('lock')
    120       lock_mgr.CheckMachineLocks(machine_states, 'lock')
    121       self.locked_machines = lock_mgr.UpdateMachines(True)
    122       self._experiment.locked_machines = self.locked_machines
    123       self._UpdateMachineList(self.locked_machines)
    124       self._experiment.machine_manager.RemoveNonLockedMachines(
    125           self.locked_machines)
    126       if len(self.locked_machines) == 0:
    127         raise RuntimeError('Unable to lock any machines.')
    128 
    129   def _UnlockAllMachines(self, experiment):
    130     """Attempt to globally unlock all of the machines requested for run.
    131 
    132     The method will use the AFE server to globally unlock all of the machines
    133     requested for this crosperf run.
    134     """
    135     if not self.locked_machines or test_flag.GetTestMode():
    136       return
    137 
    138     lock_mgr = afe_lock_machine.AFELockManager(
    139         self.locked_machines,
    140         '',
    141         experiment.labels[0].chromeos_root,
    142         None,
    143         log=self.l,)
    144     machine_states = lock_mgr.GetMachineStates('unlock')
    145     lock_mgr.CheckMachineLocks(machine_states, 'unlock')
    146     lock_mgr.UpdateMachines(False)
    147 
    148   def _ClearCacheEntries(self, experiment):
    149     for br in experiment.benchmark_runs:
    150       cache = ResultsCache()
    151       cache.Init(br.label.chromeos_image, br.label.chromeos_root,
    152                  br.benchmark.test_name, br.iteration, br.test_args,
    153                  br.profiler_args, br.machine_manager, br.machine,
    154                  br.label.board, br.cache_conditions, br._logger, br.log_level,
    155                  br.label, br.share_cache, br.benchmark.suite,
    156                  br.benchmark.show_all_results, br.benchmark.run_local)
    157       cache_dir = cache.GetCacheDirForWrite()
    158       if os.path.exists(cache_dir):
    159         self.l.LogOutput('Removing cache dir: %s' % cache_dir)
    160         shutil.rmtree(cache_dir)
    161 
    162   def _Run(self, experiment):
    163     try:
    164       if not experiment.locks_dir:
    165         self._LockAllMachines(experiment)
    166       if self._using_schedv2:
    167         schedv2 = Schedv2(experiment)
    168         experiment.set_schedv2(schedv2)
    169       if CacheConditions.FALSE in experiment.cache_conditions:
    170         self._ClearCacheEntries(experiment)
    171       status = ExperimentStatus(experiment)
    172       experiment.Run()
    173       last_status_time = 0
    174       last_status_string = ''
    175       try:
    176         if experiment.log_level != 'verbose':
    177           self.l.LogStartDots()
    178         while not experiment.IsComplete():
    179           if last_status_time + self.STATUS_TIME_DELAY < time.time():
    180             last_status_time = time.time()
    181             border = '=============================='
    182             if experiment.log_level == 'verbose':
    183               self.l.LogOutput(border)
    184               self.l.LogOutput(status.GetProgressString())
    185               self.l.LogOutput(status.GetStatusString())
    186               self.l.LogOutput(border)
    187             else:
    188               current_status_string = status.GetStatusString()
    189               if current_status_string != last_status_string:
    190                 self.l.LogEndDots()
    191                 self.l.LogOutput(border)
    192                 self.l.LogOutput(current_status_string)
    193                 self.l.LogOutput(border)
    194                 last_status_string = current_status_string
    195               else:
    196                 self.l.LogAppendDot()
    197           time.sleep(self.THREAD_MONITOR_DELAY)
    198       except KeyboardInterrupt:
    199         self._terminated = True
    200         self.l.LogError('Ctrl-c pressed. Cleaning up...')
    201         experiment.Terminate()
    202         raise
    203       except SystemExit:
    204         self._terminated = True
    205         self.l.LogError('Unexpected exit. Cleaning up...')
    206         experiment.Terminate()
    207         raise
    208     finally:
    209       if not experiment.locks_dir:
    210         self._UnlockAllMachines(experiment)
    211 
    212   def _PrintTable(self, experiment):
    213     self.l.LogOutput(TextResultsReport.FromExperiment(experiment).GetReport())
    214 
    215   def _Email(self, experiment):
    216     # Only email by default if a new run was completed.
    217     send_mail = False
    218     for benchmark_run in experiment.benchmark_runs:
    219       if not benchmark_run.cache_hit:
    220         send_mail = True
    221         break
    222     if (not send_mail and not experiment.email_to or
    223         config.GetConfig('no_email')):
    224       return
    225 
    226     label_names = []
    227     for label in experiment.labels:
    228       label_names.append(label.name)
    229     subject = '%s: %s' % (experiment.name, ' vs. '.join(label_names))
    230 
    231     text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
    232     text_report += ('\nResults are stored in %s.\n' %
    233                     experiment.results_directory)
    234     text_report = "<pre style='font-size: 13px'>%s</pre>" % text_report
    235     html_report = HTMLResultsReport.FromExperiment(experiment).GetReport()
    236     attachment = EmailSender.Attachment('report.html', html_report)
    237     email_to = experiment.email_to or []
    238     email_to.append(getpass.getuser())
    239     EmailSender().SendEmail(email_to,
    240                             subject,
    241                             text_report,
    242                             attachments=[attachment],
    243                             msg_type='html')
    244 
    245   def _StoreResults(self, experiment):
    246     if self._terminated:
    247       return
    248     results_directory = experiment.results_directory
    249     FileUtils().RmDir(results_directory)
    250     FileUtils().MkDirP(results_directory)
    251     self.l.LogOutput('Storing experiment file in %s.' % results_directory)
    252     experiment_file_path = os.path.join(results_directory, 'experiment.exp')
    253     FileUtils().WriteFile(experiment_file_path, experiment.experiment_file)
    254 
    255     self.l.LogOutput('Storing results report in %s.' % results_directory)
    256     results_table_path = os.path.join(results_directory, 'results.html')
    257     report = HTMLResultsReport.FromExperiment(experiment).GetReport()
    258     if self.json_report:
    259       json_report = JSONResultsReport.FromExperiment(experiment,
    260                                                      json_args={'indent': 2})
    261       _WriteJSONReportToFile(experiment, results_directory, json_report)
    262 
    263     FileUtils().WriteFile(results_table_path, report)
    264 
    265     self.l.LogOutput('Storing email message body in %s.' % results_directory)
    266     msg_file_path = os.path.join(results_directory, 'msg_body.html')
    267     text_report = TextResultsReport.FromExperiment(experiment, True).GetReport()
    268     text_report += ('\nResults are stored in %s.\n' %
    269                     experiment.results_directory)
    270     msg_body = "<pre style='font-size: 13px'>%s</pre>" % text_report
    271     FileUtils().WriteFile(msg_file_path, msg_body)
    272 
    273     self.l.LogOutput('Storing results of each benchmark run.')
    274     for benchmark_run in experiment.benchmark_runs:
    275       if benchmark_run.result:
    276         benchmark_run_name = filter(str.isalnum, benchmark_run.name)
    277         benchmark_run_path = os.path.join(results_directory, benchmark_run_name)
    278         benchmark_run.result.CopyResultsTo(benchmark_run_path)
    279         benchmark_run.result.CleanUp(benchmark_run.benchmark.rm_chroot_tmp)
    280 
    281   def Run(self):
    282     try:
    283       self._Run(self._experiment)
    284     finally:
    285       # Always print the report at the end of the run.
    286       self._PrintTable(self._experiment)
    287       if not self._terminated:
    288         self._StoreResults(self._experiment)
    289         self._Email(self._experiment)
    290 
    291 
    292 class MockExperimentRunner(ExperimentRunner):
    293   """Mocked ExperimentRunner for testing."""
    294 
    295   def __init__(self, experiment, json_report):
    296     super(MockExperimentRunner, self).__init__(experiment, json_report)
    297 
    298   def _Run(self, experiment):
    299     self.l.LogOutput("Would run the following experiment: '%s'." %
    300                      experiment.name)
    301 
    302   def _PrintTable(self, experiment):
    303     self.l.LogOutput('Would print the experiment table.')
    304 
    305   def _Email(self, experiment):
    306     self.l.LogOutput('Would send result email.')
    307 
    308   def _StoreResults(self, experiment):
    309     self.l.LogOutput('Would store the results.')
    310