Home | History | Annotate | Download | only in crb 
      1 import datetime
      2 import getpass
      3 import glob
      4 import os
      5 import pickle
      6 import re
      7 import threading
      8 import time
      9 import image_chromeos
     10 import machine_manager_singleton
     11 import table_formatter
     12 from cros_utils import command_executer
     13 from cros_utils import logger
     14 
     15 SCRATCH_DIR = '/home/%s/cros_scratch' % getpass.getuser()
     16 PICKLE_FILE = 'pickle.txt'
     17 VERSION = '1'
     18 
     19 
     20 def ConvertToFilename(text):
     21   ret = text
     22   ret = re.sub('/', '__', ret)
     23   ret = re.sub(' ', '_', ret)
     24   ret = re.sub('=', '', ret)
     25   ret = re.sub("\"", '', ret)
     26   return ret
     27 
     28 
     29 class AutotestRun(threading.Thread):
     30 
     31   def __init__(self,
     32                autotest,
     33                chromeos_root='',
     34                chromeos_image='',
     35                board='',
     36                remote='',
     37                iteration=0,
     38                image_checksum='',
     39                exact_remote=False,
     40                rerun=False,
     41                rerun_if_failed=False):
     42     self.autotest = autotest
     43     self.chromeos_root = chromeos_root
     44     self.chromeos_image = chromeos_image
     45     self.board = board
     46     self.remote = remote
     47     self.iteration = iteration
     48     l = logger.GetLogger()
     49     l.LogFatalIf(not image_checksum, "Checksum shouldn't be None")
     50     self.image_checksum = image_checksum
     51     self.results = {}
     52     threading.Thread.__init__(self)
     53     self.terminate = False
     54     self.retval = None
     55     self.status = 'PENDING'
     56     self.run_completed = False
     57     self.exact_remote = exact_remote
     58     self.rerun = rerun
     59     self.rerun_if_failed = rerun_if_failed
     60     self.results_dir = None
     61     self.full_name = None
     62 
     63   @staticmethod
     64   def MeanExcludingSlowest(array):
     65     mean = sum(array) / len(array)
     66     array2 = []
     67 
     68     for v in array:
     69       if mean != 0 and abs(v - mean) / mean < 0.2:
     70         array2.append(v)
     71 
     72     if array2:
     73       return sum(array2) / len(array2)
     74     else:
     75       return mean
     76 
     77   @staticmethod
     78   def AddComposite(results_dict):
     79     composite_keys = []
     80     composite_dict = {}
     81     for key in results_dict:
     82       mo = re.match('(.*){\d+}', key)
     83       if mo:
     84         composite_keys.append(mo.group(1))
     85     for key in results_dict:
     86       for composite_key in composite_keys:
     87         if (key.count(composite_key) != 0 and
     88             table_formatter.IsFloat(results_dict[key])):
     89           if composite_key not in composite_dict:
     90             composite_dict[composite_key] = []
     91           composite_dict[composite_key].append(float(results_dict[key]))
     92           break
     93 
     94     for composite_key in composite_dict:
     95       v = composite_dict[composite_key]
     96       results_dict['%s[c]' % composite_key] = sum(v) / len(v)
     97       mean_excluding_slowest = AutotestRun.MeanExcludingSlowest(v)
     98       results_dict['%s[ce]' % composite_key] = mean_excluding_slowest
     99 
    100     return results_dict
    101 
    102   def ParseOutput(self):
    103     p = re.compile('^-+.*?^-+', re.DOTALL | re.MULTILINE)
    104     matches = p.findall(self.out)
    105     for i in range(len(matches)):
    106       results = matches[i]
    107       results_dict = {}
    108       for line in results.splitlines()[1:-1]:
    109         mo = re.match('(.*\S)\s+\[\s+(PASSED|FAILED)\s+\]', line)
    110         if mo:
    111           results_dict[mo.group(1)] = mo.group(2)
    112           continue
    113         mo = re.match('(.*\S)\s+(.*)', line)
    114         if mo:
    115           results_dict[mo.group(1)] = mo.group(2)
    116 
    117       # Add a composite keyval for tests like startup.
    118       results_dict = AutotestRun.AddComposite(results_dict)
    119 
    120       self.results = results_dict
    121 
    122       # This causes it to not parse the table again
    123       # Autotest recently added a secondary table
    124       # That reports errors and screws up the final pretty output.
    125       break
    126     mo = re.search('Results placed in (\S+)', self.out)
    127     if mo:
    128       self.results_dir = mo.group(1)
    129       self.full_name = os.path.basename(self.results_dir)
    130 
    131   def GetCacheHashBase(self):
    132     ret = ('%s %s %s' %
    133            (self.image_checksum, self.autotest.name, self.iteration))
    134     if self.autotest.args:
    135       ret += ' %s' % self.autotest.args
    136     ret += '-%s' % VERSION
    137     return ret
    138 
    139   def GetLabel(self):
    140     ret = '%s %s remote:%s' % (self.chromeos_image, self.autotest.name,
    141                                self.remote)
    142     return ret
    143 
    144   def TryToLoadFromCache(self):
    145     base = self.GetCacheHashBase()
    146     if self.exact_remote:
    147       if not self.remote:
    148         return False
    149       cache_dir_glob = '%s_%s' % (ConvertToFilename(base), self.remote)
    150     else:
    151       cache_dir_glob = '%s*' % ConvertToFilename(base)
    152     cache_path_glob = os.path.join(SCRATCH_DIR, cache_dir_glob)
    153     matching_dirs = glob.glob(cache_path_glob)
    154     if matching_dirs:
    155       matching_dir = matching_dirs[0]
    156       cache_file = os.path.join(matching_dir, PICKLE_FILE)
    157       assert os.path.isfile(cache_file)
    158       self._logger.LogOutput('Trying to read from cache file: %s' % cache_file)
    159       return self.ReadFromCache(cache_file)
    160     self._logger.LogOutput('Cache miss. AM going to run: %s for: %s' %
    161                            (self.autotest.name, self.chromeos_image))
    162     return False
    163 
    164   def ReadFromCache(self, cache_file):
    165     with open(cache_file, 'rb') as f:
    166       self.retval = pickle.load(f)
    167       self.out = pickle.load(f)
    168       self.err = pickle.load(f)
    169       self._logger.LogOutput(self.out)
    170       return True
    171     return False
    172 
    173   def StoreToCache(self):
    174     base = self.GetCacheHashBase()
    175     self.cache_dir = os.path.join(SCRATCH_DIR,
    176                                   '%s_%s' % (ConvertToFilename(base),
    177                                              self.remote))
    178     cache_file = os.path.join(self.cache_dir, PICKLE_FILE)
    179     command = 'mkdir -p %s' % os.path.dirname(cache_file)
    180     ret = self._ce.RunCommand(command)
    181     assert ret == 0, "Couldn't create cache dir"
    182     with open(cache_file, 'wb') as f:
    183       pickle.dump(self.retval, f)
    184       pickle.dump(self.out, f)
    185       pickle.dump(self.err, f)
    186 
    187   def run(self):
    188     self._logger = logger.Logger(
    189         os.path.dirname(__file__), '%s.%s' % (os.path.basename(__file__),
    190                                               self.name), True)
    191     self._ce = command_executer.GetCommandExecuter(self._logger)
    192     self.RunCached()
    193 
    194   def RunCached(self):
    195     self.status = 'WAITING'
    196     cache_hit = False
    197     if not self.rerun:
    198       cache_hit = self.TryToLoadFromCache()
    199     else:
    200       self._logger.LogOutput('--rerun passed. Not using cached results.')
    201     if self.rerun_if_failed and self.retval:
    202       self._logger.LogOutput('--rerun_if_failed passed and existing test '
    203                              'failed. Rerunning...')
    204       cache_hit = False
    205     if not cache_hit:
    206       # Get machine
    207       while True:
    208         if self.terminate:
    209           return 1
    210         self.machine = (machine_manager_singleton.MachineManagerSingleton(
    211         ).AcquireMachine(self.image_checksum))
    212         if self.machine:
    213           self._logger.LogOutput('%s: Machine %s acquired at %s' %
    214                                  (self.name, self.machine.name,
    215                                   datetime.datetime.now()))
    216           break
    217         else:
    218           sleep_duration = 10
    219           time.sleep(sleep_duration)
    220       try:
    221         self.remote = self.machine.name
    222 
    223         if self.machine.checksum != self.image_checksum:
    224           self.retval = self.ImageTo(self.machine.name)
    225           if self.retval:
    226             return self.retval
    227           self.machine.checksum = self.image_checksum
    228           self.machine.image = self.chromeos_image
    229         self.status = 'RUNNING: %s' % self.autotest.name
    230         [self.retval, self.out, self.err] = self.RunTestOn(self.machine.name)
    231         self.run_completed = True
    232 
    233       finally:
    234         self._logger.LogOutput('Releasing machine: %s' % self.machine.name)
    235         machine_manager_singleton.MachineManagerSingleton().ReleaseMachine(
    236             self.machine)
    237         self._logger.LogOutput('Released machine: %s' % self.machine.name)
    238 
    239       self.StoreToCache()
    240 
    241     if not self.retval:
    242       self.status = 'SUCCEEDED'
    243     else:
    244       self.status = 'FAILED'
    245 
    246     self.ParseOutput()
    247     # Copy results directory to the scratch dir
    248     if (not cache_hit and not self.retval and self.autotest.args and
    249         '--profile' in self.autotest.args):
    250       results_dir = os.path.join(self.chromeos_root, 'chroot',
    251                                  self.results_dir.lstrip('/'))
    252       tarball = os.path.join(
    253           self.cache_dir, os.path.basename(os.path.dirname(self.results_dir)))
    254       command = ('cd %s && tar cjf %s.tbz2 .' % (results_dir, tarball))
    255       self._ce.RunCommand(command)
    256       perf_data_file = os.path.join(self.results_dir, self.full_name,
    257                                     'profiling/iteration.1/perf.data')
    258 
    259       # Attempt to build a perf report and keep it with the results.
    260       command = ('cd %s/src/scripts &&'
    261                  ' cros_sdk -- /usr/sbin/perf report --symfs=/build/%s'
    262                  ' -i %s --stdio' % (self.chromeos_root, self.board,
    263                                      perf_data_file))
    264       ret, out, err = self._ce.RunCommandWOutput(command)
    265       with open(os.path.join(self.cache_dir, 'perf.report'), 'wb') as f:
    266         f.write(out)
    267     return self.retval
    268 
    269   def ImageTo(self, machine_name):
    270     image_args = [image_chromeos.__file__, '--chromeos_root=%s' %
    271                   self.chromeos_root, '--image=%s' % self.chromeos_image,
    272                   '--remote=%s' % machine_name]
    273     if self.board:
    274       image_args.append('--board=%s' % self.board)
    275 
    276 ###    devserver_port = 8080
    277 ###    mo = re.search("\d+", self.name)
    278 ###    if mo:
    279 ###      to_add = int(mo.group(0))
    280 ###      assert to_add < 100, "Too many threads launched!"
    281 ###      devserver_port += to_add
    282 
    283 ###    # I tried --noupdate_stateful, but that still fails when run in parallel.
    284 ###    image_args.append("--image_to_live_args=\"--devserver_port=%s"
    285 ###                      " --noupdate_stateful\"" % devserver_port)
    286 ###    image_args.append("--image_to_live_args=--devserver_port=%s" %
    287 ###                      devserver_port)
    288 
    289 # Currently can't image two machines at once.
    290 # So have to serialized on this lock.
    291     self.status = 'WAITING ON IMAGE_LOCK'
    292     with machine_manager_singleton.MachineManagerSingleton().image_lock:
    293       self.status = 'IMAGING'
    294       retval = self._ce.RunCommand(' '.join(['python'] + image_args))
    295       machine_manager_singleton.MachineManagerSingleton().num_reimages += 1
    296       if retval:
    297         self.status = 'ABORTED DUE TO IMAGE FAILURE'
    298     return retval
    299 
    300   def DoPowerdHack(self):
    301     command = 'sudo initctl stop powerd'
    302     self._ce.CrosRunCommand(command,
    303                             machine=self.machine.name,
    304                             chromeos_root=self.chromeos_root)
    305 
    306   def RunTestOn(self, machine_name):
    307     command = 'cd %s/src/scripts' % self.chromeos_root
    308     options = ''
    309     if self.board:
    310       options += ' --board=%s' % self.board
    311     if self.autotest.args:
    312       options += " --args='%s'" % self.autotest.args
    313     if 'tegra2' in self.board:
    314       self.DoPowerdHack()
    315     command += ('&& cros_sdk -- /usr/bin/test_that %s %s %s' %
    316                 (options, machine_name, self.autotest.name))
    317     return self._ce.RunCommand(command, True)
    318