Home | History | Annotate | Download | only in scheduler 
      1 
      2 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """Scheduler helper libraries.
      7 """
      8 import logging
      9 import os
     10 
     11 import common
     12 
     13 from autotest_lib.client.common_lib import global_config
     14 from autotest_lib.client.common_lib import logging_config
     15 from autotest_lib.client.common_lib import logging_manager
     16 from autotest_lib.client.common_lib import utils
     17 from autotest_lib.database import database_connection
     18 from autotest_lib.frontend import setup_django_environment
     19 from autotest_lib.frontend.afe import readonly_connection
     20 from autotest_lib.server import utils as server_utils
     21 
     22 
     23 DB_CONFIG_SECTION = 'AUTOTEST_WEB'
     24 
     25 # Translations necessary for scheduler queries to work with SQLite.
     26 # Though this is only used for testing it is included in this module to avoid
     27 # circular imports.
     28 _re_translator = database_connection.TranslatingDatabase.make_regexp_translator
     29 _DB_TRANSLATORS = (
     30         _re_translator(r'NOW\(\)', 'time("now")'),
     31         _re_translator(r'LAST_INSERT_ID\(\)', 'LAST_INSERT_ROWID()'),
     32         # older SQLite doesn't support group_concat, so just don't bother until
     33         # it arises in an important query
     34         _re_translator(r'GROUP_CONCAT\((.*?)\)', r'\1'),
     35         _re_translator(r'TRUNCATE TABLE', 'DELETE FROM'),
     36         _re_translator(r'ISNULL\(([a-z,_]+)\)',
     37                        r'ifnull(nullif(\1, NULL), \1) DESC'),
     38 )
     39 
     40 
     41 class SchedulerError(Exception):
     42     """General parent class for exceptions raised by scheduler code."""
     43 
     44 
     45 class MalformedRecordError(SchedulerError):
     46     """Exception raised when an individual job or record is malformed.
     47 
     48     Code that handles individual records (e.g. afe jobs, hqe entries, special
     49     tasks) should treat such an exception as a signal to skip or permanently
     50     discard this record."""
     51 
     52 
     53 class NoHostIdError(MalformedRecordError):
     54     """Raised by the scheduler when a non-hostless job's host is None."""
     55 
     56 
     57 class ConnectionManager(object):
     58     """Manager for the django database connections.
     59 
     60     The connection is used through scheduler_models and monitor_db.
     61     """
     62     __metaclass__ = server_utils.Singleton
     63 
     64     def __init__(self, readonly=True, autocommit=True):
     65         """Set global django database options for correct connection handling.
     66 
     67         @param readonly: Globally disable readonly connections.
     68         @param autocommit: Initialize django autocommit options.
     69         """
     70         self.db_connection = None
     71         # bypass the readonly connection
     72         readonly_connection.set_globally_disabled(readonly)
     73         if autocommit:
     74             # ensure Django connection is in autocommit
     75             setup_django_environment.enable_autocommit()
     76 
     77 
     78     @classmethod
     79     def open_connection(cls):
     80         """Open a new database connection.
     81 
     82         @return: An instance of the newly opened connection.
     83         """
     84         db = database_connection.DatabaseConnection(DB_CONFIG_SECTION)
     85         db.connect(db_type='django')
     86         return db
     87 
     88 
     89     def get_connection(self):
     90         """Get a connection.
     91 
     92         @return: A database connection.
     93         """
     94         if self.db_connection is None:
     95             self.db_connection = self.open_connection()
     96         return self.db_connection
     97 
     98 
     99     def disconnect(self):
    100         """Close the database connection."""
    101         try:
    102             self.db_connection.disconnect()
    103         except Exception as e:
    104             logging.debug('Could not close the db connection. %s', e)
    105 
    106 
    107     def __del__(self):
    108         self.disconnect()
    109 
    110 
    111 class SchedulerLoggingConfig(logging_config.LoggingConfig):
    112     """Configure timestamped logging for a scheduler."""
    113     GLOBAL_LEVEL = logging.INFO
    114 
    115     @classmethod
    116     def get_log_name(cls, timestamped_logfile_prefix):
    117         """Get the name of a logfile.
    118 
    119         @param timestamped_logfile_prefix: The prefix to apply to the
    120             a timestamped log. Eg: 'scheduler' will create a logfile named
    121             scheduler.log.2014-05-12-17.24.02.
    122 
    123         @return: The timestamped log name.
    124         """
    125         return cls.get_timestamped_log_name(timestamped_logfile_prefix)
    126 
    127 
    128     def configure_logging(self, log_dir=None, logfile_name=None,
    129                           timestamped_logfile_prefix='scheduler'):
    130         """Configure logging to a specified logfile.
    131 
    132         @param log_dir: The directory to log into.
    133         @param logfile_name: The name of the log file.
    134         @timestamped_logfile_prefix: The prefix to apply to the name of
    135             the logfile, if a log file name isn't specified.
    136         """
    137         super(SchedulerLoggingConfig, self).configure_logging(use_console=True)
    138 
    139         if log_dir is None:
    140             log_dir = self.get_server_log_dir()
    141         if not logfile_name:
    142             logfile_name = self.get_log_name(timestamped_logfile_prefix)
    143 
    144         self.add_file_handler(logfile_name, logging.DEBUG, log_dir=log_dir)
    145         symlink_path = os.path.join(
    146                 log_dir, '%s.latest' % timestamped_logfile_prefix)
    147         try:
    148             os.unlink(symlink_path)
    149         except OSError:
    150             pass
    151         os.symlink(os.path.join(log_dir, logfile_name), symlink_path)
    152 
    153 
    154 def setup_logging(log_dir, log_name, timestamped_logfile_prefix='scheduler'):
    155     """Setup logging to a given log directory and log file.
    156 
    157     @param log_dir: The directory to log into.
    158     @param log_name: Name of the log file.
    159     @param timestamped_logfile_prefix: The prefix to apply to the logfile.
    160     """
    161     logging_manager.configure_logging(
    162             SchedulerLoggingConfig(), log_dir=log_dir, logfile_name=log_name,
    163             timestamped_logfile_prefix=timestamped_logfile_prefix)
    164 
    165 
    166 def check_production_settings(scheduler_options):
    167     """Check the scheduler option's production settings.
    168 
    169     @param scheduler_options: Settings for scheduler.
    170 
    171     @raises SchedulerError: If a loclhost scheduler is started with
    172        production settings.
    173     """
    174     db_server = global_config.global_config.get_config_value('AUTOTEST_WEB',
    175                                                              'host')
    176     if (not scheduler_options.production and
    177         not utils.is_localhost(db_server)):
    178         raise SchedulerError('Scheduler is not running in production mode, you '
    179                              'should not set database to hosts other than '
    180                              'localhost. It\'s currently set to %s.\nAdd option'
    181                              ' --production if you want to skip this check.' %
    182                              db_server)
    183