1 import os, time, socket, shutil, glob, logging, traceback, tempfile, re 2 import shlex 3 import subprocess 4 5 from autotest_lib.client.common_lib import error 6 from autotest_lib.client.common_lib.global_config import global_config 7 from autotest_lib.server import utils, autotest 8 from autotest_lib.server.hosts import host_info 9 from autotest_lib.server.hosts import remote 10 from autotest_lib.server.hosts import rpc_server_tracker 11 from autotest_lib.server.hosts import ssh_multiplex 12 13 # pylint: disable=C0111 14 15 get_value = global_config.get_config_value 16 enable_master_ssh = get_value('AUTOSERV', 'enable_master_ssh', type=bool, 17 default=False) 18 19 20 class AbstractSSHHost(remote.RemoteHost): 21 """ 22 This class represents a generic implementation of most of the 23 framework necessary for controlling a host via ssh. It implements 24 almost all of the abstract Host methods, except for the core 25 Host.run method. 26 """ 27 VERSION_PREFIX = '' 28 29 def _initialize(self, hostname, user="root", port=22, password="", 30 is_client_install_supported=True, afe_host=None, 31 host_info_store=None, connection_pool=None, 32 *args, **dargs): 33 super(AbstractSSHHost, self)._initialize(hostname=hostname, 34 *args, **dargs) 35 """ 36 @param hostname: The hostname of the host. 37 @param user: The username to use when ssh'ing into the host. 38 @param password: The password to use when ssh'ing into the host. 39 @param port: The port to use for ssh. 40 @param is_client_install_supported: Boolean to indicate if we can 41 install autotest on the host. 42 @param afe_host: The host object attained from the AFE (get_hosts). 43 @param host_info_store: Optional host_info.CachingHostInfoStore object 44 to obtain / update host information. 45 @param connection_pool: ssh_multiplex.ConnectionPool instance to share 46 the master ssh connection across control scripts. 47 """ 48 # IP address is retrieved only on demand. Otherwise the host 49 # initialization will fail for host is not online. 50 self._ip = None 51 self.user = user 52 self.port = port 53 self.password = password 54 self._is_client_install_supported = is_client_install_supported 55 self._use_rsync = None 56 self.known_hosts_file = tempfile.mkstemp()[1] 57 self._rpc_server_tracker = rpc_server_tracker.RpcServerTracker(self); 58 59 """ 60 Master SSH connection background job, socket temp directory and socket 61 control path option. If master-SSH is enabled, these fields will be 62 initialized by start_master_ssh when a new SSH connection is initiated. 63 """ 64 self._connection_pool = connection_pool 65 if connection_pool: 66 self._master_ssh = connection_pool.get(hostname, user, port) 67 else: 68 self._master_ssh = ssh_multiplex.MasterSsh(hostname, user, port) 69 70 self._afe_host = afe_host or utils.EmptyAFEHost() 71 self.host_info_store = (host_info_store or 72 host_info.InMemoryHostInfoStore()) 73 74 @property 75 def ip(self): 76 """@return IP address of the host. 77 """ 78 if not self._ip: 79 self._ip = socket.getaddrinfo(self.hostname, None)[0][4][0] 80 return self._ip 81 82 83 @property 84 def is_client_install_supported(self): 85 """" 86 Returns True if the host supports autotest client installs, False 87 otherwise. 88 """ 89 return self._is_client_install_supported 90 91 92 @property 93 def rpc_server_tracker(self): 94 """" 95 @return The RPC server tracker associated with this host. 96 """ 97 return self._rpc_server_tracker 98 99 100 def make_ssh_command(self, user="root", port=22, opts='', 101 hosts_file='/dev/null', 102 connect_timeout=30, alive_interval=300): 103 base_command = ("/usr/bin/ssh -a -x %s -o StrictHostKeyChecking=no " 104 "-o UserKnownHostsFile=%s -o BatchMode=yes " 105 "-o ConnectTimeout=%d -o ServerAliveInterval=%d " 106 "-l %s -p %d") 107 assert isinstance(connect_timeout, (int, long)) 108 assert connect_timeout > 0 # can't disable the timeout 109 return base_command % (opts, hosts_file, connect_timeout, 110 alive_interval, user, port) 111 112 113 def use_rsync(self): 114 if self._use_rsync is not None: 115 return self._use_rsync 116 117 # Check if rsync is available on the remote host. If it's not, 118 # don't try to use it for any future file transfers. 119 self._use_rsync = self.check_rsync() 120 if not self._use_rsync: 121 logging.warning("rsync not available on remote host %s -- disabled", 122 self.hostname) 123 return self._use_rsync 124 125 126 def check_rsync(self): 127 """ 128 Check if rsync is available on the remote host. 129 """ 130 try: 131 self.run("rsync --version", stdout_tee=None, stderr_tee=None) 132 except error.AutoservRunError: 133 return False 134 return True 135 136 137 def _encode_remote_paths(self, paths, escape=True, use_scp=False): 138 """ 139 Given a list of file paths, encodes it as a single remote path, in 140 the style used by rsync and scp. 141 escape: add \\ to protect special characters. 142 use_scp: encode for scp if true, rsync if false. 143 """ 144 if escape: 145 paths = [utils.scp_remote_escape(path) for path in paths] 146 147 remote = self.hostname 148 149 # rsync and scp require IPv6 brackets, even when there isn't any 150 # trailing port number (ssh doesn't support IPv6 brackets). 151 # In the Python >= 3.3 future, 'import ipaddress' will parse addresses. 152 if re.search(r':.*:', remote): 153 remote = '[%s]' % remote 154 155 if use_scp: 156 return '%s@%s:"%s"' % (self.user, remote, " ".join(paths)) 157 else: 158 return '%s@%s:%s' % ( 159 self.user, remote, 160 " :".join('"%s"' % p for p in paths)) 161 162 def _encode_local_paths(self, paths, escape=True): 163 """ 164 Given a list of file paths, encodes it as a single local path. 165 escape: add \\ to protect special characters. 166 """ 167 if escape: 168 paths = [utils.sh_escape(path) for path in paths] 169 170 return " ".join('"%s"' % p for p in paths) 171 172 def _make_rsync_cmd(self, sources, dest, delete_dest, 173 preserve_symlinks, safe_symlinks): 174 """ 175 Given a string of source paths and a destination path, produces the 176 appropriate rsync command for copying them. Remote paths must be 177 pre-encoded. 178 """ 179 ssh_cmd = self.make_ssh_command(user=self.user, port=self.port, 180 opts=self._master_ssh.ssh_option, 181 hosts_file=self.known_hosts_file) 182 if delete_dest: 183 delete_flag = "--delete" 184 else: 185 delete_flag = "" 186 if safe_symlinks: 187 symlink_flag = "-l --safe-links" 188 elif preserve_symlinks: 189 symlink_flag = "-l" 190 else: 191 symlink_flag = "-L" 192 command = ("rsync %s %s --timeout=1800 --rsh='%s' -az --no-o --no-g " 193 "%s \"%s\"") 194 return command % (symlink_flag, delete_flag, ssh_cmd, sources, dest) 195 196 197 def _make_ssh_cmd(self, cmd): 198 """ 199 Create a base ssh command string for the host which can be used 200 to run commands directly on the machine 201 """ 202 base_cmd = self.make_ssh_command(user=self.user, port=self.port, 203 opts=self._master_ssh.ssh_option, 204 hosts_file=self.known_hosts_file) 205 206 return '%s %s "%s"' % (base_cmd, self.hostname, utils.sh_escape(cmd)) 207 208 def _make_scp_cmd(self, sources, dest): 209 """ 210 Given a string of source paths and a destination path, produces the 211 appropriate scp command for encoding it. Remote paths must be 212 pre-encoded. 213 """ 214 command = ("scp -rq %s -o StrictHostKeyChecking=no " 215 "-o UserKnownHostsFile=%s -P %d %s '%s'") 216 return command % (self._master_ssh.ssh_option, self.known_hosts_file, 217 self.port, sources, dest) 218 219 220 def _make_rsync_compatible_globs(self, path, is_local): 221 """ 222 Given an rsync-style path, returns a list of globbed paths 223 that will hopefully provide equivalent behaviour for scp. Does not 224 support the full range of rsync pattern matching behaviour, only that 225 exposed in the get/send_file interface (trailing slashes). 226 227 The is_local param is flag indicating if the paths should be 228 interpreted as local or remote paths. 229 """ 230 231 # non-trailing slash paths should just work 232 if len(path) == 0 or path[-1] != "/": 233 return [path] 234 235 # make a function to test if a pattern matches any files 236 if is_local: 237 def glob_matches_files(path, pattern): 238 return len(glob.glob(path + pattern)) > 0 239 else: 240 def glob_matches_files(path, pattern): 241 result = self.run("ls \"%s\"%s" % (utils.sh_escape(path), 242 pattern), 243 stdout_tee=None, ignore_status=True) 244 return result.exit_status == 0 245 246 # take a set of globs that cover all files, and see which are needed 247 patterns = ["*", ".[!.]*"] 248 patterns = [p for p in patterns if glob_matches_files(path, p)] 249 250 # convert them into a set of paths suitable for the commandline 251 if is_local: 252 return ["\"%s\"%s" % (utils.sh_escape(path), pattern) 253 for pattern in patterns] 254 else: 255 return [utils.scp_remote_escape(path) + pattern 256 for pattern in patterns] 257 258 259 def _make_rsync_compatible_source(self, source, is_local): 260 """ 261 Applies the same logic as _make_rsync_compatible_globs, but 262 applies it to an entire list of sources, producing a new list of 263 sources, properly quoted. 264 """ 265 return sum((self._make_rsync_compatible_globs(path, is_local) 266 for path in source), []) 267 268 269 def _set_umask_perms(self, dest): 270 """ 271 Given a destination file/dir (recursively) set the permissions on 272 all the files and directories to the max allowed by running umask. 273 """ 274 275 # now this looks strange but I haven't found a way in Python to _just_ 276 # get the umask, apparently the only option is to try to set it 277 umask = os.umask(0) 278 os.umask(umask) 279 280 max_privs = 0777 & ~umask 281 282 def set_file_privs(filename): 283 """Sets mode of |filename|. Assumes |filename| exists.""" 284 file_stat = os.stat(filename) 285 286 file_privs = max_privs 287 # if the original file permissions do not have at least one 288 # executable bit then do not set it anywhere 289 if not file_stat.st_mode & 0111: 290 file_privs &= ~0111 291 292 os.chmod(filename, file_privs) 293 294 # try a bottom-up walk so changes on directory permissions won't cut 295 # our access to the files/directories inside it 296 for root, dirs, files in os.walk(dest, topdown=False): 297 # when setting the privileges we emulate the chmod "X" behaviour 298 # that sets to execute only if it is a directory or any of the 299 # owner/group/other already has execute right 300 for dirname in dirs: 301 os.chmod(os.path.join(root, dirname), max_privs) 302 303 # Filter out broken symlinks as we go. 304 for filename in filter(os.path.exists, files): 305 set_file_privs(os.path.join(root, filename)) 306 307 308 # now set privs for the dest itself 309 if os.path.isdir(dest): 310 os.chmod(dest, max_privs) 311 else: 312 set_file_privs(dest) 313 314 315 def get_file(self, source, dest, delete_dest=False, preserve_perm=True, 316 preserve_symlinks=False, retry=True, safe_symlinks=False): 317 """ 318 Copy files from the remote host to a local path. 319 320 Directories will be copied recursively. 321 If a source component is a directory with a trailing slash, 322 the content of the directory will be copied, otherwise, the 323 directory itself and its content will be copied. This 324 behavior is similar to that of the program 'rsync'. 325 326 Args: 327 source: either 328 1) a single file or directory, as a string 329 2) a list of one or more (possibly mixed) 330 files or directories 331 dest: a file or a directory (if source contains a 332 directory or more than one element, you must 333 supply a directory dest) 334 delete_dest: if this is true, the command will also clear 335 out any old files at dest that are not in the 336 source 337 preserve_perm: tells get_file() to try to preserve the sources 338 permissions on files and dirs 339 preserve_symlinks: try to preserve symlinks instead of 340 transforming them into files/dirs on copy 341 safe_symlinks: same as preserve_symlinks, but discard links 342 that may point outside the copied tree 343 Raises: 344 AutoservRunError: the scp command failed 345 """ 346 logging.debug('get_file. source: %s, dest: %s, delete_dest: %s,' 347 'preserve_perm: %s, preserve_symlinks:%s', source, dest, 348 delete_dest, preserve_perm, preserve_symlinks) 349 # Start a master SSH connection if necessary. 350 self.start_master_ssh() 351 352 if isinstance(source, basestring): 353 source = [source] 354 dest = os.path.abspath(dest) 355 356 # If rsync is disabled or fails, try scp. 357 try_scp = True 358 if self.use_rsync(): 359 logging.debug('Using Rsync.') 360 try: 361 remote_source = self._encode_remote_paths(source) 362 local_dest = utils.sh_escape(dest) 363 rsync = self._make_rsync_cmd(remote_source, local_dest, 364 delete_dest, preserve_symlinks, 365 safe_symlinks) 366 utils.run(rsync) 367 try_scp = False 368 except error.CmdError, e: 369 # retry on rsync exit values which may be caused by transient 370 # network problems: 371 # 372 # rc 10: Error in socket I/O 373 # rc 12: Error in rsync protocol data stream 374 # rc 23: Partial transfer due to error 375 # rc 255: Ssh error 376 # 377 # Note that rc 23 includes dangling symlinks. In this case 378 # retrying is useless, but not very damaging since rsync checks 379 # for those before starting the transfer (scp does not). 380 status = e.result_obj.exit_status 381 if status in [10, 12, 23, 255] and retry: 382 logging.warning('rsync status %d, retrying', status) 383 self.get_file(source, dest, delete_dest, preserve_perm, 384 preserve_symlinks, retry=False) 385 # The nested get_file() does all that's needed. 386 return 387 else: 388 logging.warning("trying scp, rsync failed: %s (%d)", 389 e, status) 390 391 if try_scp: 392 logging.debug('Trying scp.') 393 # scp has no equivalent to --delete, just drop the entire dest dir 394 if delete_dest and os.path.isdir(dest): 395 shutil.rmtree(dest) 396 os.mkdir(dest) 397 398 remote_source = self._make_rsync_compatible_source(source, False) 399 if remote_source: 400 # _make_rsync_compatible_source() already did the escaping 401 remote_source = self._encode_remote_paths( 402 remote_source, escape=False, use_scp=True) 403 local_dest = utils.sh_escape(dest) 404 scp = self._make_scp_cmd(remote_source, local_dest) 405 try: 406 utils.run(scp) 407 except error.CmdError, e: 408 logging.debug('scp failed: %s', e) 409 raise error.AutoservRunError(e.args[0], e.args[1]) 410 411 if not preserve_perm: 412 # we have no way to tell scp to not try to preserve the 413 # permissions so set them after copy instead. 414 # for rsync we could use "--no-p --chmod=ugo=rwX" but those 415 # options are only in very recent rsync versions 416 self._set_umask_perms(dest) 417 418 419 def send_file(self, source, dest, delete_dest=False, 420 preserve_symlinks=False): 421 """ 422 Copy files from a local path to the remote host. 423 424 Directories will be copied recursively. 425 If a source component is a directory with a trailing slash, 426 the content of the directory will be copied, otherwise, the 427 directory itself and its content will be copied. This 428 behavior is similar to that of the program 'rsync'. 429 430 Args: 431 source: either 432 1) a single file or directory, as a string 433 2) a list of one or more (possibly mixed) 434 files or directories 435 dest: a file or a directory (if source contains a 436 directory or more than one element, you must 437 supply a directory dest) 438 delete_dest: if this is true, the command will also clear 439 out any old files at dest that are not in the 440 source 441 preserve_symlinks: controls if symlinks on the source will be 442 copied as such on the destination or transformed into the 443 referenced file/directory 444 445 Raises: 446 AutoservRunError: the scp command failed 447 """ 448 logging.debug('send_file. source: %s, dest: %s, delete_dest: %s,' 449 'preserve_symlinks:%s', source, dest, 450 delete_dest, preserve_symlinks) 451 # Start a master SSH connection if necessary. 452 self.start_master_ssh() 453 454 if isinstance(source, basestring): 455 source = [source] 456 457 local_sources = self._encode_local_paths(source) 458 if not local_sources: 459 raise error.TestError('source |%s| yielded an empty string' % ( 460 source)) 461 if local_sources.find('\x00') != -1: 462 raise error.TestError('one or more sources include NUL char') 463 464 # If rsync is disabled or fails, try scp. 465 try_scp = True 466 if self.use_rsync(): 467 logging.debug('Using Rsync.') 468 remote_dest = self._encode_remote_paths([dest]) 469 try: 470 rsync = self._make_rsync_cmd(local_sources, remote_dest, 471 delete_dest, preserve_symlinks, 472 False) 473 utils.run(rsync) 474 try_scp = False 475 except error.CmdError, e: 476 logging.warning("trying scp, rsync failed: %s", e) 477 478 if try_scp: 479 logging.debug('Trying scp.') 480 # scp has no equivalent to --delete, just drop the entire dest dir 481 if delete_dest: 482 is_dir = self.run("ls -d %s/" % dest, 483 ignore_status=True).exit_status == 0 484 if is_dir: 485 cmd = "rm -rf %s && mkdir %s" 486 cmd %= (dest, dest) 487 self.run(cmd) 488 489 remote_dest = self._encode_remote_paths([dest], use_scp=True) 490 local_sources = self._make_rsync_compatible_source(source, True) 491 if local_sources: 492 sources = self._encode_local_paths(local_sources, escape=False) 493 scp = self._make_scp_cmd(sources, remote_dest) 494 try: 495 utils.run(scp) 496 except error.CmdError, e: 497 logging.debug('scp failed: %s', e) 498 raise error.AutoservRunError(e.args[0], e.args[1]) 499 else: 500 logging.debug('skipping scp for empty source list') 501 502 503 def verify_ssh_user_access(self): 504 """Verify ssh access to this host. 505 506 @returns False if ssh_ping fails due to Permissions error, True 507 otherwise. 508 """ 509 try: 510 self.ssh_ping() 511 except (error.AutoservSshPermissionDeniedError, 512 error.AutoservSshPingHostError): 513 return False 514 return True 515 516 517 def ssh_ping(self, timeout=60, connect_timeout=None, base_cmd='true'): 518 """ 519 Pings remote host via ssh. 520 521 @param timeout: Time in seconds before giving up. 522 Defaults to 60 seconds. 523 @param base_cmd: The base command to run with the ssh ping. 524 Defaults to true. 525 @raise AutoservSSHTimeout: If the ssh ping times out. 526 @raise AutoservSshPermissionDeniedError: If ssh ping fails due to 527 permissions. 528 @raise AutoservSshPingHostError: For other AutoservRunErrors. 529 """ 530 ctimeout = min(timeout, connect_timeout or timeout) 531 try: 532 self.run(base_cmd, timeout=timeout, connect_timeout=ctimeout, 533 ssh_failure_retry_ok=True) 534 except error.AutoservSSHTimeout: 535 msg = "Host (ssh) verify timed out (timeout = %d)" % timeout 536 raise error.AutoservSSHTimeout(msg) 537 except error.AutoservSshPermissionDeniedError: 538 #let AutoservSshPermissionDeniedError be visible to the callers 539 raise 540 except error.AutoservRunError, e: 541 # convert the generic AutoservRunError into something more 542 # specific for this context 543 raise error.AutoservSshPingHostError(e.description + '\n' + 544 repr(e.result_obj)) 545 546 547 def is_up(self, timeout=60, connect_timeout=None, base_cmd='true'): 548 """ 549 Check if the remote host is up by ssh-ing and running a base command. 550 551 @param timeout: timeout in seconds. 552 @param base_cmd: a base command to run with ssh. The default is 'true'. 553 @returns True if the remote host is up before the timeout expires, 554 False otherwise. 555 """ 556 try: 557 self.ssh_ping(timeout=timeout, 558 connect_timeout=connect_timeout, 559 base_cmd=base_cmd) 560 except error.AutoservError: 561 return False 562 else: 563 return True 564 565 566 def wait_up(self, timeout=None): 567 """ 568 Wait until the remote host is up or the timeout expires. 569 570 In fact, it will wait until an ssh connection to the remote 571 host can be established, and getty is running. 572 573 @param timeout time limit in seconds before returning even 574 if the host is not up. 575 576 @returns True if the host was found to be up before the timeout expires, 577 False otherwise 578 """ 579 if timeout: 580 current_time = int(time.time()) 581 end_time = current_time + timeout 582 583 autoserv_error_logged = False 584 while not timeout or current_time < end_time: 585 if self.is_up(timeout=end_time - current_time, 586 connect_timeout=20): 587 try: 588 if self.are_wait_up_processes_up(): 589 logging.debug('Host %s is now up', self.hostname) 590 return True 591 except error.AutoservError as e: 592 if not autoserv_error_logged: 593 logging.debug('Ignoring failure to reach %s: %s %s', 594 self.hostname, e, 595 '(and further similar failures)') 596 autoserv_error_logged = True 597 time.sleep(1) 598 current_time = int(time.time()) 599 600 logging.debug('Host %s is still down after waiting %d seconds', 601 self.hostname, int(timeout + time.time() - end_time)) 602 return False 603 604 605 def wait_down(self, timeout=None, warning_timer=None, old_boot_id=None): 606 """ 607 Wait until the remote host is down or the timeout expires. 608 609 If old_boot_id is provided, this will wait until either the machine 610 is unpingable or self.get_boot_id() returns a value different from 611 old_boot_id. If the boot_id value has changed then the function 612 returns true under the assumption that the machine has shut down 613 and has now already come back up. 614 615 If old_boot_id is None then until the machine becomes unreachable the 616 method assumes the machine has not yet shut down. 617 618 Based on this definition, the 4 possible permutations of timeout 619 and old_boot_id are: 620 1. timeout and old_boot_id: wait timeout seconds for either the 621 host to become unpingable, or the boot id 622 to change. In the latter case we've rebooted 623 and in the former case we've only shutdown, 624 but both cases return True. 625 2. only timeout: wait timeout seconds for the host to become unpingable. 626 If the host remains pingable throughout timeout seconds 627 we return False. 628 3. only old_boot_id: wait forever until either the host becomes 629 unpingable or the boot_id changes. Return true 630 when either of those conditions are met. 631 4. not timeout, not old_boot_id: wait forever till the host becomes 632 unpingable. 633 634 @param timeout Time limit in seconds before returning even 635 if the host is still up. 636 @param warning_timer Time limit in seconds that will generate 637 a warning if the host is not down yet. 638 @param old_boot_id A string containing the result of self.get_boot_id() 639 prior to the host being told to shut down. Can be None if this is 640 not available. 641 642 @returns True if the host was found to be down, False otherwise 643 """ 644 #TODO: there is currently no way to distinguish between knowing 645 #TODO: boot_id was unsupported and not knowing the boot_id. 646 current_time = int(time.time()) 647 if timeout: 648 end_time = current_time + timeout 649 650 if warning_timer: 651 warn_time = current_time + warning_timer 652 653 if old_boot_id is not None: 654 logging.debug('Host %s pre-shutdown boot_id is %s', 655 self.hostname, old_boot_id) 656 657 # Impose semi real-time deadline constraints, since some clients 658 # (eg: watchdog timer tests) expect strict checking of time elapsed. 659 # Each iteration of this loop is treated as though it atomically 660 # completes within current_time, this is needed because if we used 661 # inline time.time() calls instead then the following could happen: 662 # 663 # while not timeout or time.time() < end_time: [23 < 30] 664 # some code. [takes 10 secs] 665 # try: 666 # new_boot_id = self.get_boot_id(timeout=end_time - time.time()) 667 # [30 - 33] 668 # The last step will lead to a return True, when in fact the machine 669 # went down at 32 seconds (>30). Hence we need to pass get_boot_id 670 # the same time that allowed us into that iteration of the loop. 671 while not timeout or current_time < end_time: 672 try: 673 new_boot_id = self.get_boot_id(timeout=end_time-current_time) 674 except error.AutoservError: 675 logging.debug('Host %s is now unreachable over ssh, is down', 676 self.hostname) 677 return True 678 else: 679 # if the machine is up but the boot_id value has changed from 680 # old boot id, then we can assume the machine has gone down 681 # and then already come back up 682 if old_boot_id is not None and old_boot_id != new_boot_id: 683 logging.debug('Host %s now has boot_id %s and so must ' 684 'have rebooted', self.hostname, new_boot_id) 685 return True 686 687 if warning_timer and current_time > warn_time: 688 self.record("INFO", None, "shutdown", 689 "Shutdown took longer than %ds" % warning_timer) 690 # Print the warning only once. 691 warning_timer = None 692 # If a machine is stuck switching runlevels 693 # This may cause the machine to reboot. 694 self.run('kill -HUP 1', ignore_status=True) 695 696 time.sleep(1) 697 current_time = int(time.time()) 698 699 return False 700 701 702 # tunable constants for the verify & repair code 703 AUTOTEST_GB_DISKSPACE_REQUIRED = get_value("SERVER", 704 "gb_diskspace_required", 705 type=float, 706 default=20.0) 707 708 709 def verify_connectivity(self): 710 super(AbstractSSHHost, self).verify_connectivity() 711 712 logging.info('Pinging host ' + self.hostname) 713 self.ssh_ping() 714 logging.info("Host (ssh) %s is alive", self.hostname) 715 716 if self.is_shutting_down(): 717 raise error.AutoservHostIsShuttingDownError("Host is shutting down") 718 719 720 def verify_software(self): 721 super(AbstractSSHHost, self).verify_software() 722 try: 723 self.check_diskspace(autotest.Autotest.get_install_dir(self), 724 self.AUTOTEST_GB_DISKSPACE_REQUIRED) 725 except error.AutoservHostError: 726 raise # only want to raise if it's a space issue 727 except autotest.AutodirNotFoundError: 728 # autotest dir may not exist, etc. ignore 729 logging.debug('autodir space check exception, this is probably ' 730 'safe to ignore\n' + traceback.format_exc()) 731 732 733 def close(self): 734 super(AbstractSSHHost, self).close() 735 self.rpc_server_tracker.disconnect_all() 736 if not self._connection_pool: 737 self._master_ssh.close() 738 if os.path.exists(self.known_hosts_file): 739 os.remove(self.known_hosts_file) 740 741 742 def restart_master_ssh(self): 743 """ 744 Stop and restart the ssh master connection. This is meant as a last 745 resort when ssh commands fail and we don't understand why. 746 """ 747 logging.debug('Restarting master ssh connection') 748 self._master_ssh.close() 749 self._master_ssh.maybe_start(timeout=30) 750 751 752 753 def start_master_ssh(self, timeout=5): 754 """ 755 Called whenever a slave SSH connection needs to be initiated (e.g., by 756 run, rsync, scp). If master SSH support is enabled and a master SSH 757 connection is not active already, start a new one in the background. 758 Also, cleanup any zombie master SSH connections (e.g., dead due to 759 reboot). 760 761 timeout: timeout in seconds (default 5) to wait for master ssh 762 connection to be established. If timeout is reached, a 763 warning message is logged, but no other action is taken. 764 """ 765 if not enable_master_ssh: 766 return 767 self._master_ssh.maybe_start(timeout=timeout) 768 769 770 def clear_known_hosts(self): 771 """Clears out the temporary ssh known_hosts file. 772 773 This is useful if the test SSHes to the machine, then reinstalls it, 774 then SSHes to it again. It can be called after the reinstall to 775 reduce the spam in the logs. 776 """ 777 logging.info("Clearing known hosts for host '%s', file '%s'.", 778 self.hostname, self.known_hosts_file) 779 # Clear out the file by opening it for writing and then closing. 780 fh = open(self.known_hosts_file, "w") 781 fh.close() 782 783 784 def collect_logs(self, remote_src_dir, local_dest_dir, ignore_errors=True): 785 """Copy log directories from a host to a local directory. 786 787 @param remote_src_dir: A destination directory on the host. 788 @param local_dest_dir: A path to a local destination directory. 789 If it doesn't exist it will be created. 790 @param ignore_errors: If True, ignore exceptions. 791 792 @raises OSError: If there were problems creating the local_dest_dir and 793 ignore_errors is False. 794 @raises AutoservRunError, AutotestRunError: If something goes wrong 795 while copying the directories and ignore_errors is False. 796 """ 797 locally_created_dest = False 798 if (not os.path.exists(local_dest_dir) 799 or not os.path.isdir(local_dest_dir)): 800 try: 801 os.makedirs(local_dest_dir) 802 locally_created_dest = True 803 except OSError as e: 804 logging.warning('Unable to collect logs from host ' 805 '%s: %s', self.hostname, e) 806 if not ignore_errors: 807 raise 808 return 809 try: 810 self.get_file(remote_src_dir, local_dest_dir, safe_symlinks=True) 811 except (error.AutotestRunError, error.AutoservRunError, 812 error.AutoservSSHTimeout) as e: 813 logging.warning('Collection of %s to local dir %s from host %s ' 814 'failed: %s', remote_src_dir, local_dest_dir, 815 self.hostname, e) 816 if locally_created_dest: 817 shutil.rmtree(local_dest_dir, ignore_errors=ignore_errors) 818 if not ignore_errors: 819 raise 820 821 822 def create_ssh_tunnel(self, port, local_port): 823 """Create an ssh tunnel from local_port to port. 824 825 This is used to forward a port securely through a tunnel process from 826 the server to the DUT for RPC server connection. 827 828 @param port: remote port on the host. 829 @param local_port: local forwarding port. 830 831 @return: the tunnel process. 832 """ 833 tunnel_options = '-n -N -q -L %d:localhost:%d' % (local_port, port) 834 ssh_cmd = self.make_ssh_command(opts=tunnel_options) 835 tunnel_cmd = '%s %s' % (ssh_cmd, self.hostname) 836 logging.debug('Full tunnel command: %s', tunnel_cmd) 837 # Exec the ssh process directly here rather than using a shell. 838 # Using a shell leaves a dangling ssh process, because we deliver 839 # signals to the shell wrapping ssh, not the ssh process itself. 840 args = shlex.split(tunnel_cmd) 841 tunnel_proc = subprocess.Popen(args, close_fds=True) 842 logging.debug('Started ssh tunnel, local = %d' 843 ' remote = %d, pid = %d', 844 local_port, port, tunnel_proc.pid) 845 return tunnel_proc 846 847 848 def disconnect_ssh_tunnel(self, tunnel_proc, port): 849 """ 850 Disconnects a previously forwarded port from the server to the DUT for 851 RPC server connection. 852 853 @param tunnel_proc: a tunnel process returned from |create_ssh_tunnel|. 854 @param port: remote port on the DUT, used in ADBHost. 855 856 """ 857 if tunnel_proc.poll() is None: 858 tunnel_proc.terminate() 859 logging.debug('Terminated tunnel, pid %d', tunnel_proc.pid) 860 else: 861 logging.debug('Tunnel pid %d terminated early, status %d', 862 tunnel_proc.pid, tunnel_proc.returncode) 863 864 865 def get_os_type(self): 866 """Returns the host OS descriptor (to be implemented in subclasses). 867 868 @return A string describing the OS type. 869 """ 870 raise NotImplementedError 871
