1 #!/usr/bin/python 2 # 3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """CrOS suite scheduler. Will schedule suites based on configured triggers. 8 9 The Scheduler understands two main primitives: Events and Tasks. Each stanza 10 in the config file specifies a Task that triggers on a given Event. 11 12 Events: 13 The scheduler supports two kinds of Events: timed events, and 14 build system events -- like a particular build artifact becoming available. 15 Every Event has a set of Tasks that get run whenever the event happens. 16 17 Tasks: 18 Basically, event handlers. A Task is specified in the config file like so: 19 [NightlyPower] 20 suite: power 21 run_on: nightly 22 pool: remote_power 23 branch_specs: >=R20,factory 24 25 This specifies a Task that gets run whenever the 'nightly' event occurs. 26 The Task schedules a suite of tests called 'power' on the pool of machines 27 called 'remote_power', for both the factory branch and all active release 28 branches from R20 on. 29 30 31 On startup, the scheduler reads in a config file that provides a few 32 parameters for certain supported Events (the time/day of the 'weekly' 33 and 'nightly' triggers, for example), and configures all the Tasks 34 that will be in play. 35 """ 36 37 import getpass, logging, logging.handlers, optparse, os, re, signal, sys 38 import traceback 39 import common 40 import board_enumerator, deduping_scheduler, driver, forgiving_config_parser 41 import manifest_versions, sanity, task 42 from autotest_lib.client.common_lib import global_config 43 from autotest_lib.client.common_lib import utils 44 from autotest_lib.client.common_lib import logging_config, logging_manager 45 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 46 try: 47 from autotest_lib.frontend import setup_django_environment 48 # server_manager_utils depend on django which 49 # may not be available when people run checks with --sanity 50 from autotest_lib.site_utils import server_manager_utils 51 except ImportError: 52 server_manager_utils = None 53 logging.debug('Could not load server_manager_utils module, expected ' 54 'if you are running sanity check or pre-submit hook') 55 56 try: 57 from chromite.lib import ts_mon_config 58 except ImportError: 59 ts_mon_config = utils.metrics_mock 60 61 62 CONFIG_SECTION = 'SCHEDULER' 63 64 CONFIG_SECTION_SERVER = 'SERVER' 65 66 67 def signal_handler(signal, frame): 68 """Singnal hanlder to exit gracefully. 69 70 @param signal: signum 71 @param frame: stack frame object 72 """ 73 logging.info('Signal %d received. Exiting gracefully...', signal) 74 sys.exit(0) 75 76 77 class SeverityFilter(logging.Filter): 78 """Filters out messages of anything other than self._level""" 79 def __init__(self, level): 80 self._level = level 81 82 83 def filter(self, record): 84 """Causes only messages of |self._level| severity to be logged.""" 85 return record.levelno == self._level 86 87 88 class SchedulerLoggingConfig(logging_config.LoggingConfig): 89 """Configure loggings for scheduler, e.g., email setup.""" 90 def __init__(self): 91 super(SchedulerLoggingConfig, self).__init__() 92 self._from_address = global_config.global_config.get_config_value( 93 CONFIG_SECTION, "notify_email_from", default=getpass.getuser()) 94 95 self._notify_address = global_config.global_config.get_config_value( 96 CONFIG_SECTION, "notify_email") 97 98 self._smtp_server = global_config.global_config.get_config_value( 99 CONFIG_SECTION_SERVER, "smtp_server", default='localhost') 100 101 self._smtp_port = global_config.global_config.get_config_value( 102 CONFIG_SECTION_SERVER, "smtp_port", default=None) 103 104 self._smtp_user = global_config.global_config.get_config_value( 105 CONFIG_SECTION_SERVER, "smtp_user", default='') 106 107 self._smtp_password = global_config.global_config.get_config_value( 108 CONFIG_SECTION_SERVER, "smtp_password", default='') 109 110 111 @classmethod 112 def get_log_name(cls): 113 """Get timestamped log name of suite_scheduler, e.g., 114 suite_scheduler.log.2013-2-1-02-05-06. 115 116 @param cls: class 117 """ 118 return cls.get_timestamped_log_name('suite_scheduler') 119 120 121 def add_smtp_handler(self, subject, level=logging.ERROR): 122 """Add smtp handler to logging handler to trigger email when logging 123 occurs. 124 125 @param subject: email subject. 126 @param level: level of logging to trigger smtp handler. 127 """ 128 if not self._smtp_user or not self._smtp_password: 129 creds = None 130 else: 131 creds = (self._smtp_user, self._smtp_password) 132 server = self._smtp_server 133 if self._smtp_port: 134 server = (server, self._smtp_port) 135 136 handler = logging.handlers.SMTPHandler(server, 137 self._from_address, 138 [self._notify_address], 139 subject, 140 creds) 141 handler.setLevel(level) 142 # We want to send mail for the given level, and only the given level. 143 # One can add more handlers to send messages for other levels. 144 handler.addFilter(SeverityFilter(level)) 145 handler.setFormatter( 146 logging.Formatter('%(asctime)s %(levelname)-5s %(message)s')) 147 self.logger.addHandler(handler) 148 return handler 149 150 151 def configure_logging(self, log_dir=None): 152 super(SchedulerLoggingConfig, self).configure_logging(use_console=True) 153 154 if not log_dir: 155 return 156 base = self.get_log_name() 157 158 self.add_file_handler(base + '.DEBUG', logging.DEBUG, log_dir=log_dir) 159 self.add_file_handler(base + '.INFO', logging.INFO, log_dir=log_dir) 160 self.add_smtp_handler('Suite scheduler ERROR', logging.ERROR) 161 self.add_smtp_handler('Suite scheduler WARNING', logging.WARN) 162 163 164 def parse_options(): 165 """Parse commandline options.""" 166 usage = "usage: %prog [options]" 167 parser = optparse.OptionParser(usage=usage) 168 parser.add_option('-f', '--config_file', dest='config_file', 169 metavar='/path/to/config', default='suite_scheduler.ini', 170 help='Scheduler config. Defaults to suite_scheduler.ini') 171 parser.add_option('-e', '--events', dest='events', 172 metavar='list,of,events', 173 help='Handle listed events once each, then exit. '\ 174 'Must also specify a build to test.') 175 parser.add_option('-i', '--build', dest='build', 176 help='If handling a list of events, the build to test.'\ 177 ' Ignored otherwise.') 178 parser.add_option('-o', '--os_type', dest='os_type', 179 default=task.OS_TYPE_CROS, 180 help='If handling a list of events, the OS type to test.'\ 181 ' Ignored otherwise. This argument allows the test to ' 182 'know if it\'s testing ChromeOS or Launch Control ' 183 'builds. suite scheduler that runs without a build ' 184 'specified(using -i), does not need this argument.') 185 parser.add_option('-d', '--log_dir', dest='log_dir', 186 help='Log to a file in the specified directory.') 187 parser.add_option('-l', '--list_events', dest='list', 188 action='store_true', default=False, 189 help='List supported events and exit.') 190 parser.add_option('-r', '--repo_dir', dest='tmp_repo_dir', default=None, 191 help=('Path to a tmpdir containing manifest versions. ' 192 'This option is only used for testing.')) 193 parser.add_option('-t', '--sanity', dest='sanity', action='store_true', 194 default=False, 195 help='Check the config file for any issues.') 196 parser.add_option('-b', '--file_bug', dest='file_bug', action='store_true', 197 default=False, 198 help='File bugs for known suite scheduling exceptions.') 199 200 201 options, args = parser.parse_args() 202 return parser, options, args 203 204 205 def main(): 206 """Entry point for suite_scheduler.py""" 207 signal.signal(signal.SIGINT, signal_handler) 208 signal.signal(signal.SIGHUP, signal_handler) 209 signal.signal(signal.SIGTERM, signal_handler) 210 211 parser, options, args = parse_options() 212 if args or options.events and not options.build: 213 parser.print_help() 214 return 1 215 216 if options.config_file and not os.path.exists(options.config_file): 217 logging.error('Specified config file %s does not exist.', 218 options.config_file) 219 return 1 220 221 config = forgiving_config_parser.ForgivingConfigParser() 222 config.read(options.config_file) 223 224 if options.list: 225 print 'Supported events:' 226 for event_class in driver.Driver.EVENT_CLASSES: 227 print ' ', event_class.KEYWORD 228 return 0 229 230 # If we're just sanity checking, we can stop after we've parsed the 231 # config file. 232 if options.sanity: 233 # config_file_getter generates a high amount of noise at DEBUG level 234 logging.getLogger().setLevel(logging.WARNING) 235 d = driver.Driver(None, None, True) 236 d.SetUpEventsAndTasks(config, None) 237 tasks_per_event = d.TasksFromConfig(config) 238 # flatten [[a]] -> [a] 239 tasks = [x for y in tasks_per_event.values() for x in y] 240 control_files_exist = sanity.CheckControlFileExistence(tasks) 241 return control_files_exist 242 243 logging_manager.configure_logging(SchedulerLoggingConfig(), 244 log_dir=options.log_dir) 245 if not options.log_dir: 246 logging.info('Not logging to a file, as --log_dir was not passed.') 247 248 # If server database is enabled, check if the server has role 249 # `suite_scheduler`. If the server does not have suite_scheduler role, 250 # exception will be raised and suite scheduler will not continue to run. 251 if not server_manager_utils: 252 raise ImportError( 253 'Could not import autotest_lib.site_utils.server_manager_utils') 254 if server_manager_utils.use_server_db(): 255 server_manager_utils.confirm_server_has_role(hostname='localhost', 256 role='suite_scheduler') 257 258 afe_server = global_config.global_config.get_config_value( 259 CONFIG_SECTION_SERVER, "suite_scheduler_afe", default=None) 260 261 afe = frontend_wrappers.RetryingAFE( 262 server=afe_server, timeout_min=10, delay_sec=5, debug=False) 263 logging.info('Connecting to: %s' , afe.server) 264 enumerator = board_enumerator.BoardEnumerator(afe) 265 scheduler = deduping_scheduler.DedupingScheduler(afe, options.file_bug) 266 mv = manifest_versions.ManifestVersions(options.tmp_repo_dir) 267 d = driver.Driver(scheduler, enumerator) 268 d.SetUpEventsAndTasks(config, mv) 269 270 # Set up metrics upload for Monarch. 271 ts_mon_config.SetupTsMonGlobalState('autotest_suite_scheduler') 272 273 try: 274 if options.events: 275 # Act as though listed events have just happened. 276 keywords = re.split('\s*,\s*', options.events) 277 if not options.tmp_repo_dir: 278 logging.warn('To run a list of events, you may need to use ' 279 '--repo_dir to specify a folder that already has ' 280 'manifest repo set up. This is needed for suites ' 281 'requiring firmware update.') 282 logging.info('Forcing events: %r', keywords) 283 d.ForceEventsOnceForBuild(keywords, options.build, options.os_type) 284 else: 285 if not options.tmp_repo_dir: 286 mv.Initialize() 287 d.RunForever(config, mv) 288 except Exception as e: 289 logging.error('Fatal exception in suite_scheduler: %r\n%s', e, 290 traceback.format_exc()) 291 return 1 292 293 if __name__ == "__main__": 294 sys.exit(main()) 295
