1 #!/usr/bin/python 2 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 3 # Use of this source code is governed by a BSD-style license that can be 4 # found in the LICENSE file. 5 6 """Runs on autotest servers from a cron job to self update them. 7 8 This script is designed to run on all autotest servers to allow them to 9 automatically self-update based on the manifests used to create their (existing) 10 repos. 11 """ 12 13 from __future__ import print_function 14 15 import ConfigParser 16 import argparse 17 import os 18 import re 19 import subprocess 20 import sys 21 import time 22 23 import common 24 25 from autotest_lib.client.common_lib import global_config 26 from autotest_lib.server import utils as server_utils 27 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 28 29 30 # How long after restarting a service do we watch it to see if it's stable. 31 SERVICE_STABILITY_TIMER = 60 32 33 # A dict to map update_commands defined in config file to repos or files that 34 # decide whether need to update these commands. E.g. if no changes under 35 # frontend repo, no need to update afe. 36 COMMANDS_TO_REPOS_DICT = {'afe': 'frontend/client/', 37 'tko': 'frontend/client/'} 38 BUILD_EXTERNALS_COMMAND = 'build_externals' 39 40 _RESTART_SERVICES_FILE = os.path.join(os.environ['HOME'], 41 'push_restart_services') 42 43 AFE = frontend_wrappers.RetryingAFE( 44 server=server_utils.get_global_afe_hostname(), timeout_min=5, 45 delay_sec=10) 46 47 class DirtyTreeException(Exception): 48 """Raised when the tree has been modified in an unexpected way.""" 49 50 51 class UnknownCommandException(Exception): 52 """Raised when we try to run a command name with no associated command.""" 53 54 55 class UnstableServices(Exception): 56 """Raised if a service appears unstable after restart.""" 57 58 59 def strip_terminal_codes(text): 60 """This function removes all terminal formatting codes from a string. 61 62 @param text: String of text to cleanup. 63 @returns String with format codes removed. 64 """ 65 ESC = '\x1b' 66 return re.sub(ESC+r'\[[^m]*m', '', text) 67 68 69 def _clean_pyc_files(): 70 print('Removing .pyc files') 71 try: 72 subprocess.check_output([ 73 'find', '.', 74 '(', 75 # These are ignored to reduce IO load (crbug.com/759780). 76 '-path', './site-packages', 77 '-o', '-path', './containers', 78 '-o', '-path', './logs', 79 '-o', '-path', './results', 80 ')', 81 '-prune', 82 '-o', '-name', '*.pyc', 83 '-exec', 'rm', '-f', '{}', '+']) 84 except Exception as e: 85 print('Warning: fail to remove .pyc! %s' % e) 86 87 88 def verify_repo_clean(): 89 """This function cleans the current repo then verifies that it is valid. 90 91 @raises DirtyTreeException if the repo is still not clean. 92 @raises subprocess.CalledProcessError on a repo command failure. 93 """ 94 subprocess.check_output(['git', 'reset', '--hard']) 95 # Forcefully blow away any non-gitignored files in the tree. 96 subprocess.check_output(['git', 'clean', '-fd']) 97 out = subprocess.check_output(['repo', 'status'], stderr=subprocess.STDOUT) 98 out = strip_terminal_codes(out).strip() 99 100 if not 'working directory clean' in out: 101 raise DirtyTreeException(out) 102 103 104 def _clean_externals(): 105 """Clean untracked files within ExternalSource and site-packages/ 106 107 @raises subprocess.CalledProcessError on a git command failure. 108 """ 109 dirs_to_clean = ['site-packages/', 'ExternalSource/'] 110 cmd = ['git', 'clean', '-fxd'] + dirs_to_clean 111 subprocess.check_output(cmd) 112 113 114 def repo_versions(): 115 """This function collects the versions of all git repos in the general repo. 116 117 @returns A dictionary mapping project names to git hashes for HEAD. 118 @raises subprocess.CalledProcessError on a repo command failure. 119 """ 120 cmd = ['repo', 'forall', '-p', '-c', 'pwd && git log -1 --format=%h'] 121 output = strip_terminal_codes(subprocess.check_output(cmd)) 122 123 # The expected output format is: 124 125 # project chrome_build/ 126 # /dir/holding/chrome_build 127 # 73dee9d 128 # 129 # project chrome_release/ 130 # /dir/holding/chrome_release 131 # 9f3a5d8 132 133 lines = output.splitlines() 134 135 PROJECT_PREFIX = 'project ' 136 137 project_heads = {} 138 for n in range(0, len(lines), 4): 139 project_line = lines[n] 140 project_dir = lines[n+1] 141 project_hash = lines[n+2] 142 # lines[n+3] is a blank line, but doesn't exist for the final block. 143 144 # Convert 'project chrome_build/' -> 'chrome_build' 145 assert project_line.startswith(PROJECT_PREFIX) 146 name = project_line[len(PROJECT_PREFIX):].rstrip('/') 147 148 project_heads[name] = (project_dir, project_hash) 149 150 return project_heads 151 152 153 def repo_versions_to_decide_whether_run_cmd_update(): 154 """Collect versions of repos/files defined in COMMANDS_TO_REPOS_DICT. 155 156 For the update_commands defined in config files, no need to run the command 157 every time. Only run it when the repos/files related to the commands have 158 been changed. 159 160 @returns A set of tuples: {(cmd, repo_version), ()...} 161 """ 162 results = set() 163 for cmd, repo in COMMANDS_TO_REPOS_DICT.iteritems(): 164 version = subprocess.check_output( 165 ['git', 'log', '-1', '--pretty=tformat:%h', 166 '%s/%s' % (common.autotest_dir, repo)]) 167 results.add((cmd, version.strip())) 168 return results 169 170 171 def repo_sync(update_push_servers=False): 172 """Perform a repo sync. 173 174 @param update_push_servers: If True, then update test_push servers to ToT. 175 Otherwise, update server to prod branch. 176 @raises subprocess.CalledProcessError on a repo command failure. 177 """ 178 subprocess.check_output(['repo', 'sync']) 179 if update_push_servers: 180 print('Updating push servers, checkout cros/master') 181 subprocess.check_output(['git', 'checkout', 'cros/master'], 182 stderr=subprocess.STDOUT) 183 else: 184 print('Updating server to prod branch') 185 subprocess.check_output(['git', 'checkout', 'cros/prod'], 186 stderr=subprocess.STDOUT) 187 _clean_pyc_files() 188 189 190 def discover_update_commands(): 191 """Lookup the commands to run on this server. 192 193 These commonly come from shadow_config.ini, since they vary by server type. 194 195 @returns List of command names in string format. 196 """ 197 try: 198 return global_config.global_config.get_config_value( 199 'UPDATE', 'commands', type=list) 200 201 except (ConfigParser.NoSectionError, global_config.ConfigError): 202 return [] 203 204 205 def get_restart_services(): 206 """Find the services that need restarting on the current server. 207 208 These commonly come from shadow_config.ini, since they vary by server type. 209 210 @returns Iterable of service names in string format. 211 """ 212 with open(_RESTART_SERVICES_FILE) as f: 213 for line in f: 214 yield line.rstrip() 215 216 217 def update_command(cmd_tag, dryrun=False, use_chromite_master=False): 218 """Restart a command. 219 220 The command name is looked up in global_config.ini to find the full command 221 to run, then it's executed. 222 223 @param cmd_tag: Which command to restart. 224 @param dryrun: If true print the command that would have been run. 225 @param use_chromite_master: True if updating chromite to master, rather 226 than prod. 227 228 @raises UnknownCommandException If cmd_tag can't be looked up. 229 @raises subprocess.CalledProcessError on a command failure. 230 """ 231 # Lookup the list of commands to consider. They are intended to be 232 # in global_config.ini so that they can be shared everywhere. 233 cmds = dict(global_config.global_config.config.items( 234 'UPDATE_COMMANDS')) 235 236 if cmd_tag not in cmds: 237 raise UnknownCommandException(cmd_tag, cmds) 238 239 expanded_command = cmds[cmd_tag].replace('AUTOTEST_REPO', 240 common.autotest_dir) 241 # When updating push servers, pass an arg to build_externals to update 242 # chromite to master branch for testing 243 if use_chromite_master and cmd_tag == BUILD_EXTERNALS_COMMAND: 244 expanded_command += ' --use_chromite_master' 245 246 print('Running: %s: %s' % (cmd_tag, expanded_command)) 247 if dryrun: 248 print('Skip: %s' % expanded_command) 249 else: 250 try: 251 subprocess.check_output(expanded_command, shell=True, 252 stderr=subprocess.STDOUT) 253 except subprocess.CalledProcessError as e: 254 print('FAILED:') 255 print(e.output) 256 raise 257 258 259 def restart_service(service_name, dryrun=False): 260 """Restart a service. 261 262 Restarts the standard service with "service <name> restart". 263 264 @param service_name: The name of the service to restart. 265 @param dryrun: Don't really run anything, just print out the command. 266 267 @raises subprocess.CalledProcessError on a command failure. 268 """ 269 cmd = ['sudo', 'service', service_name, 'restart'] 270 print('Restarting: %s' % service_name) 271 if dryrun: 272 print('Skip: %s' % ' '.join(cmd)) 273 else: 274 subprocess.check_call(cmd, stderr=subprocess.STDOUT) 275 276 277 def service_status(service_name): 278 """Return the results "status <name>" for a given service. 279 280 This string is expected to contain the pid, and so to change is the service 281 is shutdown or restarted for any reason. 282 283 @param service_name: The name of the service to check on. 284 285 @returns The output of the external command. 286 Ex: autofs start/running, process 1931 287 288 @raises subprocess.CalledProcessError on a command failure. 289 """ 290 return subprocess.check_output(['sudo', 'service', service_name, 'status']) 291 292 293 def restart_services(service_names, dryrun=False, skip_service_status=False): 294 """Restart services as needed for the current server type. 295 296 Restart the listed set of services, and watch to see if they are stable for 297 at least SERVICE_STABILITY_TIMER. It restarts all services quickly, 298 waits for that delay, then verifies the status of all of them. 299 300 @param service_names: The list of service to restart and monitor. 301 @param dryrun: Don't really restart the service, just print out the command. 302 @param skip_service_status: Set to True to skip service status check. 303 Default is False. 304 305 @raises subprocess.CalledProcessError on a command failure. 306 @raises UnstableServices if any services are unstable after restart. 307 """ 308 service_statuses = {} 309 310 if dryrun: 311 for name in service_names: 312 restart_service(name, dryrun=True) 313 return 314 315 # Restart each, and record the status (including pid). 316 for name in service_names: 317 restart_service(name) 318 319 # Skip service status check if --skip-service-status is specified. Used for 320 # servers in backup status. 321 if skip_service_status: 322 print('--skip-service-status is specified, skip checking services.') 323 return 324 325 # Wait for a while to let the services settle. 326 time.sleep(SERVICE_STABILITY_TIMER) 327 service_statuses = {name: service_status(name) for name in service_names} 328 time.sleep(SERVICE_STABILITY_TIMER) 329 # Look for any services that changed status. 330 unstable_services = [n for n in service_names 331 if service_status(n) != service_statuses[n]] 332 333 # Report any services having issues. 334 if unstable_services: 335 raise UnstableServices(unstable_services) 336 337 338 def run_deploy_actions(cmds_skip=set(), dryrun=False, 339 skip_service_status=False, use_chromite_master=False): 340 """Run arbitrary update commands specified in global.ini. 341 342 @param cmds_skip: cmds no need to run since the corresponding repo/file 343 does not change. 344 @param dryrun: Don't really restart the service, just print out the command. 345 @param skip_service_status: Set to True to skip service status check. 346 Default is False. 347 @param use_chromite_master: True if updating chromite to master, rather 348 than prod. 349 350 @raises subprocess.CalledProcessError on a command failure. 351 @raises UnstableServices if any services are unstable after restart. 352 """ 353 defined_cmds = set(discover_update_commands()) 354 cmds = defined_cmds - cmds_skip 355 if cmds: 356 print('Running update commands:', ', '.join(cmds)) 357 for cmd in cmds: 358 update_command(cmd, dryrun=dryrun, 359 use_chromite_master=use_chromite_master) 360 361 services = list(get_restart_services()) 362 if services: 363 print('Restarting Services:', ', '.join(services)) 364 restart_services(services, dryrun=dryrun, 365 skip_service_status=skip_service_status) 366 367 368 def report_changes(versions_before, versions_after): 369 """Produce a report describing what changed in all repos. 370 371 @param versions_before: Results of repo_versions() from before the update. 372 @param versions_after: Results of repo_versions() from after the update. 373 374 @returns string containing a human friendly changes report. 375 """ 376 result = [] 377 378 if versions_after: 379 for project in sorted(set(versions_before.keys() + versions_after.keys())): 380 result.append('%s:' % project) 381 382 _, before_hash = versions_before.get(project, (None, None)) 383 after_dir, after_hash = versions_after.get(project, (None, None)) 384 385 if project not in versions_before: 386 result.append('Added.') 387 388 elif project not in versions_after: 389 result.append('Removed.') 390 391 elif before_hash == after_hash: 392 result.append('No Change.') 393 394 else: 395 hashes = '%s..%s' % (before_hash, after_hash) 396 cmd = ['git', 'log', hashes, '--oneline'] 397 out = subprocess.check_output(cmd, cwd=after_dir, 398 stderr=subprocess.STDOUT) 399 result.append(out.strip()) 400 401 result.append('') 402 else: 403 for project in sorted(versions_before.keys()): 404 _, before_hash = versions_before[project] 405 result.append('%s: %s' % (project, before_hash)) 406 result.append('') 407 408 return '\n'.join(result) 409 410 411 def parse_arguments(args): 412 """Parse command line arguments. 413 414 @param args: The command line arguments to parse. (ususally sys.argsv[1:]) 415 416 @returns An argparse.Namespace populated with argument values. 417 """ 418 parser = argparse.ArgumentParser( 419 description='Command to update an autotest server.') 420 parser.add_argument('--skip-verify', action='store_false', 421 dest='verify', default=True, 422 help='Disable verification of a clean repository.') 423 parser.add_argument('--skip-update', action='store_false', 424 dest='update', default=True, 425 help='Skip the repository source code update.') 426 parser.add_argument('--skip-actions', action='store_false', 427 dest='actions', default=True, 428 help='Skip the post update actions.') 429 parser.add_argument('--skip-report', action='store_false', 430 dest='report', default=True, 431 help='Skip the git version report.') 432 parser.add_argument('--actions-only', action='store_true', 433 help='Run the post update actions (restart services).') 434 parser.add_argument('--dryrun', action='store_true', 435 help='Don\'t actually run any commands, just log.') 436 parser.add_argument('--skip-service-status', action='store_true', 437 help='Skip checking the service status.') 438 parser.add_argument('--update_push_servers', action='store_true', 439 help='Indicate to update test_push server. If not ' 440 'specify, then update server to production.') 441 parser.add_argument('--force-clean-externals', action='store_true', 442 default=False, 443 help='Force a cleanup of all untracked files within ' 444 'site-packages/ and ExternalSource/, so that ' 445 'build_externals will build from scratch.') 446 parser.add_argument('--force_update', action='store_true', 447 help='Force to run the update commands for afe, tko ' 448 'and build_externals') 449 450 results = parser.parse_args(args) 451 452 if results.actions_only: 453 results.verify = False 454 results.update = False 455 results.report = False 456 457 # TODO(dgarrett): Make these behaviors support dryrun. 458 if results.dryrun: 459 results.verify = False 460 results.update = False 461 results.force_clean_externals = False 462 463 if not results.update_push_servers: 464 print('Will skip service check for pushing servers in prod.') 465 results.skip_service_status = True 466 return results 467 468 469 class ChangeDir(object): 470 471 """Context manager for changing to a directory temporarily.""" 472 473 def __init__(self, dir): 474 self.new_dir = dir 475 self.old_dir = None 476 477 def __enter__(self): 478 self.old_dir = os.getcwd() 479 os.chdir(self.new_dir) 480 481 def __exit__(self, exc_type, exc_val, exc_tb): 482 os.chdir(self.old_dir) 483 484 485 def _sync_chromiumos_repo(): 486 """Update ~chromeos-test/chromiumos repo.""" 487 print('Updating ~chromeos-test/chromiumos') 488 with ChangeDir(os.path.expanduser('~chromeos-test/chromiumos')): 489 ret = subprocess.call(['repo', 'sync'], stderr=subprocess.STDOUT) 490 _clean_pyc_files() 491 if ret != 0: 492 print('Update failed, exited with status: %d' % ret) 493 494 495 def main(args): 496 """Main method.""" 497 os.chdir(common.autotest_dir) 498 global_config.global_config.parse_config_file() 499 500 behaviors = parse_arguments(args) 501 502 if behaviors.verify: 503 print('Checking tree status:') 504 verify_repo_clean() 505 print('Tree status: clean') 506 507 if behaviors.force_clean_externals: 508 print('Cleaning all external packages and their cache...') 509 _clean_externals() 510 print('...done.') 511 512 versions_before = repo_versions() 513 versions_after = set() 514 cmd_versions_before = repo_versions_to_decide_whether_run_cmd_update() 515 cmd_versions_after = set() 516 517 if behaviors.update: 518 print('Updating Repo.') 519 repo_sync(behaviors.update_push_servers) 520 versions_after = repo_versions() 521 cmd_versions_after = repo_versions_to_decide_whether_run_cmd_update() 522 _sync_chromiumos_repo() 523 524 if behaviors.actions: 525 # If the corresponding repo/file not change, no need to run the cmd. 526 cmds_skip = (set() if behaviors.force_update else 527 {t[0] for t in cmd_versions_before & cmd_versions_after}) 528 run_deploy_actions( 529 cmds_skip, behaviors.dryrun, behaviors.skip_service_status, 530 use_chromite_master=behaviors.update_push_servers) 531 532 if behaviors.report: 533 print('Changes:') 534 print(report_changes(versions_before, versions_after)) 535 536 537 if __name__ == '__main__': 538 sys.exit(main(sys.argv[1:])) 539