1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """CrOS suite scheduler. Will schedule suites based on configured triggers. 8 9 The Scheduler understands two main primitives: Events and Tasks. Each stanza 10 in the config file specifies a Task that triggers on a given Event. 11 12 Events: 13 The scheduler supports two kinds of Events: timed events, and 14 build system events -- like a particular build artifact becoming available. 15 Every Event has a set of Tasks that get run whenever the event happens. 16 17 Tasks: 18 Basically, event handlers. A Task is specified in the config file like so: 19 [NightlyPower] 20 suite: power 21 run_on: nightly 22 pool: remote_power 23 branch_specs: >=R20,factory 24 25 This specifies a Task that gets run whenever the 'nightly' event occurs. 26 The Task schedules a suite of tests called 'power' on the pool of machines 27 called 'remote_power', for both the factory branch and all active release 28 branches from R20 on. 29 30 31 On startup, the scheduler reads in a config file that provides a few 32 parameters for certain supported Events (the time/day of the 'weekly' 33 and 'nightly' triggers, for example), and configures all the Tasks 34 that will be in play. 35 """ 36 37 import getpass, logging, logging.handlers, optparse, os, re, signal, sys 38 import traceback 39 import common 40 import board_enumerator, deduping_scheduler, driver, forgiving_config_parser 41 import manifest_versions, sanity 42 from autotest_lib.client.common_lib import global_config 43 from autotest_lib.client.common_lib import logging_config, logging_manager 44 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 45 try: 46 from autotest_lib.frontend import setup_django_environment 47 # server_manager_utils depend on django which 48 # may not be available when people run checks with --sanity 49 from autotest_lib.site_utils import server_manager_utils 50 except ImportError: 51 server_manager_utils = None 52 logging.debug('Could not load server_manager_utils module, expected ' 53 'if you are running sanity check or pre-submit hook') 54 55 56 CONFIG_SECTION = 'SCHEDULER' 57 58 CONFIG_SECTION_SERVER = 'SERVER' 59 60 61 def signal_handler(signal, frame): 62 """Singnal hanlder to exit gracefully. 63 64 @param signal: signum 65 @param frame: stack frame object 66 """ 67 logging.info('Signal %d received. Exiting gracefully...', signal) 68 sys.exit(0) 69 70 71 class SeverityFilter(logging.Filter): 72 """Filters out messages of anything other than self._level""" 73 def __init__(self, level): 74 self._level = level 75 76 77 def filter(self, record): 78 """Causes only messages of |self._level| severity to be logged.""" 79 return record.levelno == self._level 80 81 82 class SchedulerLoggingConfig(logging_config.LoggingConfig): 83 """Configure loggings for scheduler, e.g., email setup.""" 84 def __init__(self): 85 super(SchedulerLoggingConfig, self).__init__() 86 self._from_address = global_config.global_config.get_config_value( 87 CONFIG_SECTION, "notify_email_from", default=getpass.getuser()) 88 89 self._notify_address = global_config.global_config.get_config_value( 90 CONFIG_SECTION, "notify_email", 91 default='chromeos-lab-admins (at] google.com') 92 93 self._smtp_server = global_config.global_config.get_config_value( 94 CONFIG_SECTION_SERVER, "smtp_server", default='localhost') 95 96 self._smtp_port = global_config.global_config.get_config_value( 97 CONFIG_SECTION_SERVER, "smtp_port", default=None) 98 99 self._smtp_user = global_config.global_config.get_config_value( 100 CONFIG_SECTION_SERVER, "smtp_user", default='') 101 102 self._smtp_password = global_config.global_config.get_config_value( 103 CONFIG_SECTION_SERVER, "smtp_password", default='') 104 105 106 @classmethod 107 def get_log_name(cls): 108 """Get timestamped log name of suite_scheduler, e.g., 109 suite_scheduler.log.2013-2-1-02-05-06. 110 111 @param cls: class 112 """ 113 return cls.get_timestamped_log_name('suite_scheduler') 114 115 116 def add_smtp_handler(self, subject, level=logging.ERROR): 117 """Add smtp handler to logging handler to trigger email when logging 118 occurs. 119 120 @param subject: email subject. 121 @param level: level of logging to trigger smtp handler. 122 """ 123 if not self._smtp_user or not self._smtp_password: 124 creds = None 125 else: 126 creds = (self._smtp_user, self._smtp_password) 127 server = self._smtp_server 128 if self._smtp_port: 129 server = (server, self._smtp_port) 130 131 handler = logging.handlers.SMTPHandler(server, 132 self._from_address, 133 [self._notify_address], 134 subject, 135 creds) 136 handler.setLevel(level) 137 # We want to send mail for the given level, and only the given level. 138 # One can add more handlers to send messages for other levels. 139 handler.addFilter(SeverityFilter(level)) 140 handler.setFormatter( 141 logging.Formatter('%(asctime)s %(levelname)-5s %(message)s')) 142 self.logger.addHandler(handler) 143 return handler 144 145 146 def configure_logging(self, log_dir=None): 147 super(SchedulerLoggingConfig, self).configure_logging(use_console=True) 148 149 if not log_dir: 150 return 151 base = self.get_log_name() 152 153 self.add_file_handler(base + '.DEBUG', logging.DEBUG, log_dir=log_dir) 154 self.add_file_handler(base + '.INFO', logging.INFO, log_dir=log_dir) 155 self.add_smtp_handler('Suite scheduler ERROR', logging.ERROR) 156 self.add_smtp_handler('Suite scheduler WARNING', logging.WARN) 157 158 159 def parse_options(): 160 """Parse commandline options.""" 161 usage = "usage: %prog [options]" 162 parser = optparse.OptionParser(usage=usage) 163 parser.add_option('-f', '--config_file', dest='config_file', 164 metavar='/path/to/config', default='suite_scheduler.ini', 165 help='Scheduler config. Defaults to suite_scheduler.ini') 166 parser.add_option('-e', '--events', dest='events', 167 metavar='list,of,events', 168 help='Handle listed events once each, then exit. '\ 169 'Must also specify a build to test.') 170 parser.add_option('-i', '--build', dest='build', 171 help='If handling a list of events, the build to test.'\ 172 ' Ignored otherwise.') 173 parser.add_option('-d', '--log_dir', dest='log_dir', 174 help='Log to a file in the specified directory.') 175 parser.add_option('-l', '--list_events', dest='list', 176 action='store_true', default=False, 177 help='List supported events and exit.') 178 parser.add_option('-r', '--repo_dir', dest='tmp_repo_dir', default=None, 179 help=('Path to a tmpdir containing manifest versions. ' 180 'This option is only used for testing.')) 181 parser.add_option('-t', '--sanity', dest='sanity', action='store_true', 182 default=False, 183 help='Check the config file for any issues.') 184 parser.add_option('-b', '--file_bug', dest='file_bug', action='store_true', 185 default=False, 186 help='File bugs for known suite scheduling exceptions.') 187 188 189 options, args = parser.parse_args() 190 return parser, options, args 191 192 193 def main(): 194 """Entry point for suite_scheduler.py""" 195 signal.signal(signal.SIGINT, signal_handler) 196 signal.signal(signal.SIGHUP, signal_handler) 197 signal.signal(signal.SIGTERM, signal_handler) 198 199 parser, options, args = parse_options() 200 if args or options.events and not options.build: 201 parser.print_help() 202 return 1 203 204 if options.config_file and not os.path.exists(options.config_file): 205 logging.error('Specified config file %s does not exist.', 206 options.config_file) 207 return 1 208 209 config = forgiving_config_parser.ForgivingConfigParser() 210 config.read(options.config_file) 211 212 if options.list: 213 print 'Supported events:' 214 for event_class in driver.Driver.EVENT_CLASSES: 215 print ' ', event_class.KEYWORD 216 return 0 217 218 # If we're just sanity checking, we can stop after we've parsed the 219 # config file. 220 if options.sanity: 221 # config_file_getter generates a high amount of noise at DEBUG level 222 logging.getLogger().setLevel(logging.WARNING) 223 d = driver.Driver(None, None, True) 224 d.SetUpEventsAndTasks(config, None) 225 tasks_per_event = d.TasksFromConfig(config) 226 # flatten [[a]] -> [a] 227 tasks = [x for y in tasks_per_event.values() for x in y] 228 control_files_exist = sanity.CheckControlFileExistance(tasks) 229 return control_files_exist 230 231 logging_manager.configure_logging(SchedulerLoggingConfig(), 232 log_dir=options.log_dir) 233 if not options.log_dir: 234 logging.info('Not logging to a file, as --log_dir was not passed.') 235 236 # If server database is enabled, check if the server has role 237 # `suite_scheduler`. If the server does not have suite_scheduler role, 238 # exception will be raised and suite scheduler will not continue to run. 239 if not server_manager_utils: 240 raise ImportError( 241 'Could not import autotest_lib.site_utils.server_manager_utils') 242 if server_manager_utils.use_server_db(): 243 server_manager_utils.confirm_server_has_role(hostname='localhost', 244 role='suite_scheduler') 245 246 afe_server = global_config.global_config.get_config_value( 247 CONFIG_SECTION_SERVER, "suite_scheduler_afe", default=None) 248 249 afe = frontend_wrappers.RetryingAFE( 250 server=afe_server, timeout_min=10, delay_sec=5, debug=False) 251 logging.info('Connecting to: %s' , afe.server) 252 enumerator = board_enumerator.BoardEnumerator(afe) 253 scheduler = deduping_scheduler.DedupingScheduler(afe, options.file_bug) 254 mv = manifest_versions.ManifestVersions(options.tmp_repo_dir) 255 d = driver.Driver(scheduler, enumerator) 256 d.SetUpEventsAndTasks(config, mv) 257 258 try: 259 if options.events: 260 # Act as though listed events have just happened. 261 keywords = re.split('\s*,\s*', options.events) 262 if not options.tmp_repo_dir: 263 logging.warn('To run a list of events, you may need to use ' 264 '--repo_dir to specify a folder that already has ' 265 'manifest repo set up. This is needed for suites ' 266 'requiring firmware update.') 267 logging.info('Forcing events: %r', keywords) 268 d.ForceEventsOnceForBuild(keywords, options.build) 269 else: 270 if not options.tmp_repo_dir: 271 mv.Initialize() 272 d.RunForever(config, mv) 273 except Exception as e: 274 logging.error('Fatal exception in suite_scheduler: %r\n%s', e, 275 traceback.format_exc()) 276 return 1 277 278 if __name__ == "__main__": 279 sys.exit(main()) 280
