Home | History | Annotate | Download | only in lucifer 
      1 # Copyright 2018 The Chromium Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 """Library for frontend.afe.models.JobHandoff and job cleanup."""
      6 
      7 from __future__ import absolute_import
      8 from __future__ import division
      9 from __future__ import print_function
     10 
     11 import datetime
     12 import logging
     13 import socket
     14 
     15 from lucifer import autotest
     16 
     17 logger = logging.getLogger(__name__)
     18 
     19 
     20 _JOB_GRACE_SECS = 10
     21 
     22 
     23 def incomplete():
     24     """Return a QuerySet of incomplete JobHandoffs.
     25 
     26     JobHandoff created within a cutoff period are exempt to allow the
     27     job the chance to acquire its lease file; otherwise, incomplete jobs
     28     without an active lease are considered dead.
     29 
     30     @returns: Django QuerySet
     31     """
     32     models = autotest.load('frontend.afe.models')
     33     Q = autotest.deps_load('django.db.models').Q
     34     # Time ---*---------|---------*-------|--->
     35     #    incomplete   cutoff   newborn   now
     36     cutoff = (datetime.datetime.now()
     37               - datetime.timedelta(seconds=_JOB_GRACE_SECS))
     38     return (models.JobHandoff.objects
     39             .filter(completed=False, created__lt=cutoff)
     40             .filter(Q(drone=socket.gethostname()) | Q(drone=None)))
     41 
     42 
     43 def clean_up(job_ids):
     44     """Clean up failed jobs failed in database.
     45 
     46     This brings the database into a clean state, which includes marking
     47     the job, HQEs, and hosts.
     48     """
     49     if not job_ids:
     50         return
     51     models = autotest.load('frontend.afe.models')
     52     logger.info('Cleaning up failed jobs: %r', job_ids)
     53     hqes = models.HostQueueEntry.objects.filter(job_id__in=job_ids)
     54     logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
     55     _clean_up_hqes(hqes)
     56     host_ids = {id for id in hqes.values_list('host_id', flat=True)
     57                 if id is not None}
     58     logger.debug('Found Hosts associated with jobs: %r', host_ids)
     59     _clean_up_hosts(host_ids)
     60 
     61 
     62 def _clean_up_hqes(hqes):
     63     models = autotest.load('frontend.afe.models')
     64     logger.debug('Cleaning up HQEs: %r', hqes.values_list('id', flat=True))
     65     hqes.update(complete=True,
     66                 active=False,
     67                 status=models.HostQueueEntry.Status.FAILED)
     68     (hqes.exclude(started_on=None)
     69      .update(finished_on=datetime.datetime.now()))
     70 
     71 
     72 def _clean_up_hosts(host_ids):
     73     models = autotest.load('frontend.afe.models')
     74     transaction = autotest.deps_load('django.db.transaction')
     75     with transaction.commit_on_success():
     76         active_hosts = {
     77             id for id in (models.HostQueueEntry.objects
     78                           .filter(active=True, complete=False)
     79                           .values_list('host_id', flat=True))
     80             if id is not None}
     81         logger.debug('Found active Hosts: %r', active_hosts)
     82         (models.Host.objects
     83          .filter(id__in=host_ids)
     84          .exclude(id__in=active_hosts)
     85          .update(status=None))
     86 
     87 
     88 def mark_complete(job_ids):
     89     """Mark the corresponding JobHandoffs as completed."""
     90     if not job_ids:
     91         return
     92     models = autotest.load('frontend.afe.models')
     93     logger.info('Marking job handoffs complete: %r', job_ids)
     94     (models.JobHandoff.objects
     95      .filter(job_id__in=job_ids)
     96      .update(completed=True))
     97