Home | History | Annotate | Download | only in rpm_control_system
      1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import ctypes
      6 import datetime
      7 import logging
      8 import multiprocessing
      9 import os
     10 import pexpect
     11 import Queue
     12 import re
     13 import threading
     14 import time
     15 
     16 from config import rpm_config
     17 import dli_urllib
     18 import rpm_logging_config
     19 
     20 import common
     21 from autotest_lib.client.common_lib import error
     22 from autotest_lib.client.common_lib.cros import retry
     23 
     24 RPM_CALL_TIMEOUT_MINS = rpm_config.getint('RPM_INFRASTRUCTURE',
     25                                           'call_timeout_mins')
     26 SET_POWER_STATE_TIMEOUT_SECONDS = rpm_config.getint(
     27         'RPM_INFRASTRUCTURE', 'set_power_state_timeout_seconds')
     28 PROCESS_TIMEOUT_BUFFER = 30
     29 
     30 
     31 class RPMController(object):
     32     """
     33     This abstract class implements RPM request queueing and
     34     processes queued requests.
     35 
     36     The actual interaction with the RPM device will be implemented
     37     by the RPM specific subclasses.
     38 
     39     It assumes that you know the RPM hostname and that the device is on
     40     the specified RPM.
     41 
     42     This class also allows support for RPM devices that can be accessed
     43     directly or through a hydra serial concentrator device.
     44 
     45     Implementation details:
     46     This is an abstract class, subclasses must implement the methods
     47     listed here. You must not instantiate this class but should
     48     instantiate one of those leaf subclasses. Subclasses should
     49     also set TYPE class attribute to indicate device type.
     50 
     51     @var behind_hydra: boolean value to represent whether or not this RPM is
     52                         behind a hydra device.
     53     @var hostname: hostname for this rpm device.
     54     @var is_running_lock: lock used to control access to _running.
     55     @var request_queue: queue used to store requested outlet state changes.
     56     @var queue_lock: lock used to control access to request_queue.
     57     @var _running: boolean value to represent if this controller is currently
     58                    looping over queued requests.
     59     """
     60 
     61 
     62     SSH_LOGIN_CMD = ('ssh -l %s -o StrictHostKeyChecking=no '
     63                      '-o ConnectTimeout=90 -o UserKnownHostsFile=/dev/null %s')
     64     USERNAME_PROMPT = 'Username:'
     65     HYRDA_RETRY_SLEEP_SECS = 10
     66     HYDRA_MAX_CONNECT_RETRIES = 3
     67     LOGOUT_CMD = 'logout'
     68     CLI_CMD = 'CLI'
     69     CLI_HELD = 'The administrator \[root\] has an active .* session.'
     70     CLI_KILL_PREVIOUS = 'cancel'
     71     CLI_PROMPT = 'cli>'
     72     HYDRA_PROMPT = '#'
     73     PORT_STATUS_CMD = 'portStatus'
     74     QUIT_CMD = 'quit'
     75     SESSION_KILL_CMD_FORMAT = 'administration sessions kill %s'
     76     HYDRA_CONN_HELD_MSG_FORMAT = 'is being used'
     77     CYCLE_SLEEP_TIME = 5
     78 
     79     # Global Variables that will likely be changed by subclasses.
     80     DEVICE_PROMPT = '$'
     81     PASSWORD_PROMPT = 'Password:'
     82     # The state change command can be any string format but must accept 2 vars:
     83     # state followed by device/Plug name.
     84     SET_STATE_CMD = '%s %s'
     85     SUCCESS_MSG = None # Some RPM's may not return a success msg.
     86 
     87     NEW_STATE_ON = 'ON'
     88     NEW_STATE_OFF = 'OFF'
     89     NEW_STATE_CYCLE = 'CYCLE'
     90     TYPE = 'Should set TYPE in subclass.'
     91 
     92 
     93     def __init__(self, rpm_hostname, hydra_hostname=None):
     94         """
     95         RPMController Constructor.
     96         To be called by subclasses.
     97 
     98         @param rpm_hostname: hostname of rpm device to be controlled.
     99         """
    100         self._dns_zone = rpm_config.get('CROS', 'dns_zone')
    101         self.hostname = rpm_hostname
    102         self.request_queue = Queue.Queue()
    103         self._running = False
    104         self.is_running_lock = threading.Lock()
    105         # If a hydra name is provided by the subclass then we know we are
    106         # talking to an rpm behind a hydra device.
    107         self.hydra_hostname = hydra_hostname if hydra_hostname else None
    108         self.behind_hydra = hydra_hostname is not None
    109 
    110 
    111     def _start_processing_requests(self):
    112         """
    113         Check if there is a thread processing requests.
    114         If not start one.
    115         """
    116         with self.is_running_lock:
    117             if not self._running:
    118                 self._running = True
    119                 self._running_thread = threading.Thread(target=self._run)
    120                 self._running_thread.start()
    121 
    122 
    123     def _stop_processing_requests(self):
    124         """
    125         Called if the request request_queue is empty.
    126         Set running status to false.
    127         """
    128         with self.is_running_lock:
    129             logging.debug('Request queue is empty. RPM Controller for %s'
    130                           ' is terminating.', self.hostname)
    131             self._running = False
    132         if not self.request_queue.empty():
    133             # This can occur if an item was pushed into the queue after we
    134             # exited the while-check and before the _stop_processing_requests
    135             # call was made. Therefore we need to start processing again.
    136             self._start_processing_requests()
    137 
    138 
    139     def _run(self):
    140         """
    141         Processes all queued up requests for this RPM Controller.
    142         Callers should first request_queue up atleast one request and if this
    143         RPM Controller is not running then call run.
    144 
    145         Caller can either simply call run but then they will be blocked or
    146         can instantiate a new thread to process all queued up requests.
    147         For example:
    148           threading.Thread(target=rpm_controller.run).start()
    149 
    150         Requests are in the format of:
    151           [powerunit_info, new_state, condition_var, result]
    152         Run will set the result with the correct value.
    153         """
    154         while not self.request_queue.empty():
    155             try:
    156                 result = multiprocessing.Value(ctypes.c_bool, False)
    157                 request = self.request_queue.get()
    158                 device_hostname = request['powerunit_info'].device_hostname
    159                 if (datetime.datetime.utcnow() > (request['start_time'] +
    160                         datetime.timedelta(minutes=RPM_CALL_TIMEOUT_MINS))):
    161                     logging.error('The request was waited for too long to be '
    162                                   "processed. It is timed out and won't be "
    163                                   'processed.')
    164                     request['result_queue'].put(False)
    165                     continue
    166 
    167                 is_timeout = multiprocessing.Value(ctypes.c_bool, False)
    168                 process = multiprocessing.Process(target=self._process_request,
    169                                                   args=(request, result,
    170                                                         is_timeout))
    171                 process.start()
    172                 process.join(SET_POWER_STATE_TIMEOUT_SECONDS +
    173                              PROCESS_TIMEOUT_BUFFER)
    174                 if process.is_alive():
    175                     logging.debug('%s: process (%s) still running, will be '
    176                                   'terminated!', device_hostname, process.pid)
    177                     process.terminate()
    178                     is_timeout.value = True
    179 
    180                 if is_timeout.value:
    181                     raise error.TimeoutException(
    182                             'Attempt to set power state is timed out after %s '
    183                             'seconds.' % SET_POWER_STATE_TIMEOUT_SECONDS)
    184                 if not result.value:
    185                     logging.error('Request to change %s to state %s failed.',
    186                                   device_hostname, request['new_state'])
    187             except Exception as e:
    188                 logging.error('Request to change %s to state %s failed: '
    189                               'Raised exception: %s', device_hostname,
    190                               request['new_state'], e)
    191                 result.value = False
    192 
    193             # Put result inside the result Queue to allow the caller to resume.
    194             request['result_queue'].put(result.value)
    195         self._stop_processing_requests()
    196 
    197 
    198     def _process_request(self, request, result, is_timeout):
    199         """Process the request to change a device's outlet state.
    200 
    201         The call of set_power_state is made in a new running process. If it
    202         takes longer than SET_POWER_STATE_TIMEOUT_SECONDS, the request will be
    203         timed out.
    204 
    205         @param request: A request to change a device's outlet state.
    206         @param result: A Value object passed to the new process for the caller
    207                        thread to retrieve the result.
    208         @param is_timeout: A Value object passed to the new process for the
    209                            caller thread to retrieve the information about if
    210                            the set_power_state call timed out.
    211         """
    212         try:
    213             logging.getLogger().handlers = []
    214             kwargs = {'use_log_server': True}
    215             is_timeout_value, result_value = retry.timeout(
    216                      rpm_logging_config.set_up_logging,
    217                      args=(),
    218                      kwargs=kwargs,
    219                      timeout_sec=10)
    220             if is_timeout_value:
    221                 raise Exception('Setup local log server handler timed out.')
    222         except Exception as e:
    223             # Fail over to log to a new file.
    224             LOG_FILENAME_FORMAT = rpm_config.get('GENERAL',
    225                                                  'dispatcher_logname_format')
    226             log_filename_format = LOG_FILENAME_FORMAT.replace(
    227                     'dispatcher', 'controller_%d' % os.getpid())
    228             logging.getLogger().handlers = []
    229             rpm_logging_config.set_up_logging(
    230                     log_filename_format=log_filename_format,
    231                     use_log_server=False)
    232             logging.info('Failed to set up logging through log server: %s', e)
    233         kwargs = {'powerunit_info':request['powerunit_info'],
    234                   'new_state':request['new_state']}
    235         try:
    236             is_timeout_value, result_value = retry.timeout(
    237                     self.set_power_state,
    238                     args=(),
    239                     kwargs=kwargs,
    240                     timeout_sec=SET_POWER_STATE_TIMEOUT_SECONDS)
    241             result.value = result_value
    242             is_timeout.value = is_timeout_value
    243         except Exception as e:
    244             # This method runs in a subprocess. Must log the exception,
    245             # otherwise exceptions raised in set_power_state just get lost.
    246             # Need to convert e to a str type, because our logging server
    247             # code doesn't handle the conversion very well.
    248             logging.error('Request to change %s to state %s failed: '
    249                           'Raised exception: %s',
    250                           request['powerunit_info'].device_hostname,
    251                           request['new_state'], str(e))
    252             raise e
    253 
    254 
    255     def queue_request(self, powerunit_info, new_state):
    256         """
    257         Queues up a requested state change for a device's outlet.
    258 
    259         Requests are in the format of:
    260           [powerunit_info, new_state, condition_var, result]
    261         Run will set the result with the correct value.
    262 
    263         @param powerunit_info: And PowerUnitInfo instance.
    264         @param new_state: ON/OFF/CYCLE - state or action we want to perform on
    265                           the outlet.
    266         """
    267         request = {}
    268         request['powerunit_info'] = powerunit_info
    269         request['new_state'] = new_state
    270         request['start_time'] = datetime.datetime.utcnow()
    271         # Reserve a spot for the result to be stored.
    272         request['result_queue'] = Queue.Queue()
    273         # Place in request_queue
    274         self.request_queue.put(request)
    275         self._start_processing_requests()
    276         # Block until the request is processed.
    277         result = request['result_queue'].get(block=True)
    278         return result
    279 
    280 
    281     def _kill_previous_connection(self):
    282         """
    283         In case the port to the RPM through the hydra serial concentrator is in
    284         use, terminate the previous connection so we can log into the RPM.
    285 
    286         It logs into the hydra serial concentrator over ssh, launches the CLI
    287         command, gets the port number and then kills the current session.
    288         """
    289         ssh = self._authenticate_with_hydra(admin_override=True)
    290         if not ssh:
    291             return
    292         ssh.expect(RPMController.PASSWORD_PROMPT, timeout=60)
    293         ssh.sendline(rpm_config.get('HYDRA', 'admin_password'))
    294         ssh.expect(RPMController.HYDRA_PROMPT)
    295         ssh.sendline(RPMController.CLI_CMD)
    296         cli_prompt_re = re.compile(RPMController.CLI_PROMPT)
    297         cli_held_re = re.compile(RPMController.CLI_HELD)
    298         response = ssh.expect_list([cli_prompt_re, cli_held_re], timeout=60)
    299         if response == 1:
    300             # Need to kill the previous adminstator's session.
    301             logging.error("Need to disconnect previous administrator's CLI "
    302                           "session to release the connection to RPM device %s.",
    303                           self.hostname)
    304             ssh.sendline(RPMController.CLI_KILL_PREVIOUS)
    305             ssh.expect(RPMController.CLI_PROMPT)
    306         ssh.sendline(RPMController.PORT_STATUS_CMD)
    307         ssh.expect(': %s' % self.hostname)
    308         ports_status = ssh.before
    309         port_number = ports_status.split(' ')[-1]
    310         ssh.expect(RPMController.CLI_PROMPT)
    311         ssh.sendline(RPMController.SESSION_KILL_CMD_FORMAT % port_number)
    312         ssh.expect(RPMController.CLI_PROMPT)
    313         self._logout(ssh, admin_logout=True)
    314 
    315 
    316     def _hydra_login(self, ssh):
    317         """
    318         Perform the extra steps required to log into a hydra serial
    319         concentrator.
    320 
    321         @param ssh: pexpect.spawn object used to communicate with the hydra
    322                     serial concentrator.
    323 
    324         @return: True if the login procedure is successful. False if an error
    325                  occurred. The most common case would be if another user is
    326                  logged into the device.
    327         """
    328         try:
    329             response = ssh.expect_list(
    330                     [re.compile(RPMController.PASSWORD_PROMPT),
    331                      re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)],
    332                     timeout=15)
    333         except pexpect.TIMEOUT:
    334             # If there was a timeout, this ssh tunnel could be set up to
    335             # not require the hydra password.
    336             ssh.sendline('')
    337             try:
    338                 ssh.expect(re.compile(RPMController.USERNAME_PROMPT))
    339                 logging.debug('Connected to rpm through hydra. Logging in.')
    340                 return True
    341             except pexpect.ExceptionPexpect:
    342                 return False
    343         if response == 0:
    344             try:
    345                 ssh.sendline(rpm_config.get('HYDRA','password'))
    346                 ssh.sendline('')
    347                 response = ssh.expect_list(
    348                         [re.compile(RPMController.USERNAME_PROMPT),
    349                          re.compile(RPMController.HYDRA_CONN_HELD_MSG_FORMAT)],
    350                         timeout=60)
    351             except pexpect.EOF:
    352                 # Did not receive any of the expect responses, retry.
    353                 return False
    354             except pexpect.TIMEOUT:
    355                 logging.debug('Timeout occurred logging in to hydra.')
    356                 return False
    357         # Send the username that the subclass will have set in its
    358         # construction.
    359         if response == 1:
    360             logging.debug('SSH Terminal most likely serving another'
    361                           ' connection, retrying.')
    362             # Kill the connection for the next connection attempt.
    363             try:
    364                 self._kill_previous_connection()
    365             except pexpect.ExceptionPexpect:
    366                 logging.error('Failed to disconnect previous connection, '
    367                               'retrying.')
    368                 raise
    369             return False
    370         logging.debug('Connected to rpm through hydra. Logging in.')
    371         return True
    372 
    373 
    374     def _authenticate_with_hydra(self, admin_override=False):
    375         """
    376         Some RPM's are behind a hydra serial concentrator and require their ssh
    377         connection to be tunneled through this device. This can fail if another
    378         user is logged in; therefore this will retry multiple times.
    379 
    380         This function also allows us to authenticate directly to the
    381         administrator interface of the hydra device.
    382 
    383         @param admin_override: Set to True if we are trying to access the
    384                                administrator interface rather than tunnel
    385                                through to the RPM.
    386 
    387         @return: The connected pexpect.spawn instance if the login procedure is
    388                  successful. None if an error occurred. The most common case
    389                  would be if another user is logged into the device.
    390         """
    391         if admin_override:
    392             username = rpm_config.get('HYDRA', 'admin_username')
    393         else:
    394             username = '%s:%s' % (rpm_config.get('HYDRA','username'),
    395                                   self.hostname)
    396         cmd = RPMController.SSH_LOGIN_CMD % (username, self.hydra_hostname)
    397         num_attempts = 0
    398         while num_attempts < RPMController.HYDRA_MAX_CONNECT_RETRIES:
    399             try:
    400                 ssh = pexpect.spawn(cmd)
    401             except pexpect.ExceptionPexpect:
    402                 return None
    403             if admin_override:
    404                 return ssh
    405             if self._hydra_login(ssh):
    406                 return ssh
    407             # Authenticating with hydra failed. Sleep then retry.
    408             time.sleep(RPMController.HYRDA_RETRY_SLEEP_SECS)
    409             num_attempts += 1
    410         logging.error('Failed to connect to the hydra serial concentrator after'
    411                       ' %d attempts.', RPMController.HYDRA_MAX_CONNECT_RETRIES)
    412         return None
    413 
    414 
    415     def _login(self):
    416         """
    417         Log in into the RPM Device.
    418 
    419         The login process should be able to connect to the device whether or not
    420         it is behind a hydra serial concentrator.
    421 
    422         @return: ssh - a pexpect.spawn instance if the connection was successful
    423                  or None if it was not.
    424         """
    425         if self.behind_hydra:
    426             # Tunnel the connection through the hydra.
    427             ssh = self._authenticate_with_hydra()
    428             if not ssh:
    429                 return None
    430             ssh.sendline(self._username)
    431         else:
    432             # Connect directly to the RPM over SSH.
    433             hostname = '%s.%s' % (self.hostname, self._dns_zone)
    434             cmd = RPMController.SSH_LOGIN_CMD % (self._username, hostname)
    435             try:
    436                 ssh = pexpect.spawn(cmd)
    437             except pexpect.ExceptionPexpect:
    438                 return None
    439         # Wait for the password prompt
    440         try:
    441             ssh.expect(self.PASSWORD_PROMPT, timeout=60)
    442             ssh.sendline(self._password)
    443             ssh.expect(self.DEVICE_PROMPT, timeout=60)
    444         except pexpect.ExceptionPexpect:
    445             return None
    446         return ssh
    447 
    448 
    449     def _logout(self, ssh, admin_logout=False):
    450         """
    451         Log out of the RPM device.
    452 
    453         Send the device specific logout command and if the connection is through
    454         a hydra serial concentrator, kill the ssh connection.
    455 
    456         @param admin_logout: Set to True if we are trying to logout of the
    457                              administrator interface of a hydra serial
    458                              concentrator, rather than an RPM.
    459         @param ssh: pexpect.spawn instance to use to send the logout command.
    460         """
    461         if admin_logout:
    462             ssh.sendline(RPMController.QUIT_CMD)
    463             ssh.expect(RPMController.HYDRA_PROMPT)
    464         ssh.sendline(self.LOGOUT_CMD)
    465         if self.behind_hydra and not admin_logout:
    466             # Terminate the hydra session.
    467             ssh.sendline('~.')
    468             # Wait a bit so hydra disconnects completely. Launching another
    469             # request immediately can cause a timeout.
    470             time.sleep(5)
    471 
    472 
    473     def set_power_state(self, powerunit_info, new_state):
    474         """
    475         Set the state of the dut's outlet on this RPM.
    476 
    477         For ssh based devices, this will create the connection either directly
    478         or through a hydra tunnel and call the underlying _change_state function
    479         to be implemented by the subclass device.
    480 
    481         For non-ssh based devices, this method should be overloaded with the
    482         proper connection and state change code. And the subclass will handle
    483         accessing the RPM devices.
    484 
    485         @param powerunit_info: An instance of PowerUnitInfo.
    486         @param new_state: ON/OFF/CYCLE - state or action we want to perform on
    487                           the outlet.
    488 
    489         @return: True if the attempt to change power state was successful,
    490                  False otherwise.
    491         """
    492         ssh = self._login()
    493         if not ssh:
    494             return False
    495         if new_state == self.NEW_STATE_CYCLE:
    496             logging.debug('Beginning Power Cycle for device: %s',
    497                           powerunit_info.device_hostname)
    498             result = self._change_state(powerunit_info, self.NEW_STATE_OFF, ssh)
    499             if not result:
    500                 return result
    501             time.sleep(RPMController.CYCLE_SLEEP_TIME)
    502             result = self._change_state(powerunit_info, self.NEW_STATE_ON, ssh)
    503         else:
    504             # Try to change the state of the device's power outlet.
    505             result = self._change_state(powerunit_info, new_state, ssh)
    506 
    507         # Terminate hydra connection if necessary.
    508         self._logout(ssh)
    509         ssh.close(force=True)
    510         return result
    511 
    512 
    513     def _change_state(self, powerunit_info, new_state, ssh):
    514         """
    515         Perform the actual state change operation.
    516 
    517         Once we have established communication with the RPM this method is
    518         responsible for changing the state of the RPM outlet.
    519 
    520         @param powerunit_info: An instance of PowerUnitInfo.
    521         @param new_state: ON/OFF - state or action we want to perform on
    522                           the outlet.
    523         @param ssh: The ssh connection used to execute the state change commands
    524                     on the RPM device.
    525 
    526         @return: True if the attempt to change power state was successful,
    527                  False otherwise.
    528         """
    529         outlet = powerunit_info.outlet
    530         device_hostname = powerunit_info.device_hostname
    531         if not outlet:
    532             logging.error('Request to change outlet for device: %s to new '
    533                           'state %s failed: outlet is unknown, please '
    534                           'make sure POWERUNIT_OUTLET exist in the host\'s '
    535                           'attributes in afe.', device_hostname, new_state)
    536         ssh.sendline(self.SET_STATE_CMD % (new_state, outlet))
    537         if self.SUCCESS_MSG:
    538             # If this RPM device returns a success message check for it before
    539             # continuing.
    540             try:
    541                 ssh.expect(self.SUCCESS_MSG, timeout=60)
    542             except pexpect.ExceptionPexpect:
    543                 logging.error('Request to change outlet for device: %s to new '
    544                               'state %s failed.', device_hostname, new_state)
    545                 return False
    546         logging.debug('Outlet for device: %s set to %s', device_hostname,
    547                       new_state)
    548         return True
    549 
    550 
    551     def type(self):
    552         """
    553         Get the type of RPM device we are interacting with.
    554         Class attribute TYPE should be set by the subclasses.
    555 
    556         @return: string representation of RPM device type.
    557         """
    558         return self.TYPE
    559 
    560 
    561 class SentryRPMController(RPMController):
    562     """
    563     This class implements power control for Sentry Switched CDU
    564     http://www.servertech.com/products/switched-pdus/
    565 
    566     Example usage:
    567       rpm = SentrySwitchedCDU('chromeos-rack1-rpm1')
    568       rpm.queue_request('chromeos-rack1-host1', 'ON')
    569 
    570     @var _username: username used to access device.
    571     @var _password: password used to access device.
    572     """
    573 
    574 
    575     DEVICE_PROMPT = 'Switched CDU:'
    576     SET_STATE_CMD = '%s %s'
    577     SUCCESS_MSG = 'Command successful'
    578     NUM_OF_OUTLETS = 17
    579     TYPE = 'Sentry'
    580 
    581 
    582     def __init__(self, hostname, hydra_hostname=None):
    583         super(SentryRPMController, self).__init__(hostname, hydra_hostname)
    584         self._username = rpm_config.get('SENTRY', 'username')
    585         self._password = rpm_config.get('SENTRY', 'password')
    586 
    587 
    588     def _setup_test_user(self, ssh):
    589         """Configure the test user for the RPM
    590 
    591         @param ssh: Pexpect object to use to configure the RPM.
    592         """
    593         # Create and configure the testing user profile.
    594         testing_user = rpm_config.get('SENTRY','testing_user')
    595         testing_password = rpm_config.get('SENTRY','testing_password')
    596         ssh.sendline('create user %s' % testing_user)
    597         response = ssh.expect_list([re.compile('not unique'),
    598                                     re.compile(self.PASSWORD_PROMPT)])
    599         if not response:
    600             return
    601         # Testing user is not set up yet.
    602         ssh.sendline(testing_password)
    603         ssh.expect('Verify Password:')
    604         ssh.sendline(testing_password)
    605         ssh.expect(self.SUCCESS_MSG)
    606         ssh.expect(self.DEVICE_PROMPT)
    607         ssh.sendline('add outlettouser all %s' % testing_user)
    608         ssh.expect(self.SUCCESS_MSG)
    609         ssh.expect(self.DEVICE_PROMPT)
    610 
    611 
    612     def _clear_outlet_names(self, ssh):
    613         """
    614         Before setting the outlet names, we need to clear out all the old
    615         names so there are no conflicts. For example trying to assign outlet
    616         2 a name already assigned to outlet 9.
    617         """
    618         for outlet in range(1, self.NUM_OF_OUTLETS):
    619             outlet_name = 'Outlet_%d' % outlet
    620             ssh.sendline(self.SET_OUTLET_NAME_CMD % (outlet, outlet_name))
    621             ssh.expect(self.SUCCESS_MSG)
    622             ssh.expect(self.DEVICE_PROMPT)
    623 
    624 
    625     def setup(self, outlet_naming_map):
    626         """
    627         Configure the RPM by adding the test user and setting up the outlet
    628         names.
    629 
    630         Note the rpm infrastructure does not rely on the outlet name to map a
    631         device to its outlet any more. We keep this method in case there is
    632         a need to label outlets for other reasons. We may deprecate
    633         this method if it has been proved the outlet names will not be used
    634         in any scenario.
    635 
    636         @param outlet_naming_map: Dictionary used to map the outlet numbers to
    637                                   host names. Keys must be ints. And names are
    638                                   in the format of 'hostX'.
    639 
    640         @return: True if setup completed successfully, False otherwise.
    641         """
    642         ssh = self._login()
    643         if not ssh:
    644             logging.error('Could not connect to %s.', self.hostname)
    645             return False
    646         try:
    647             self._setup_test_user(ssh)
    648             # Set up the outlet names.
    649             # Hosts have the same name format as the RPM hostname except they
    650             # end in hostX instead of rpmX.
    651             dut_name_format = re.sub('-rpm[0-9]*', '', self.hostname)
    652             if self.behind_hydra:
    653                 # Remove "chromeosX" from DUTs behind the hydra due to a length
    654                 # constraint on the names we can store inside the RPM.
    655                 dut_name_format = re.sub('chromeos[0-9]*-', '', dut_name_format)
    656             dut_name_format = dut_name_format + '-%s'
    657             self._clear_outlet_names(ssh)
    658             for outlet, name in outlet_naming_map.items():
    659                 dut_name = dut_name_format % name
    660                 ssh.sendline(self.SET_OUTLET_NAME_CMD % (outlet, dut_name))
    661                 ssh.expect(self.SUCCESS_MSG)
    662                 ssh.expect(self.DEVICE_PROMPT)
    663         except pexpect.ExceptionPexpect as e:
    664             logging.error('Setup failed. %s', e)
    665             return False
    666         finally:
    667             self._logout(ssh)
    668         return True
    669 
    670 
    671 class WebPoweredRPMController(RPMController):
    672     """
    673     This class implements RPMController for the Web Powered units
    674     produced by Digital Loggers Inc.
    675 
    676     @var _rpm: dli_urllib.Powerswitch instance used to interact with RPM.
    677     """
    678 
    679 
    680     TYPE = 'Webpowered'
    681 
    682 
    683     def __init__(self, hostname, powerswitch=None):
    684         username = rpm_config.get('WEBPOWERED', 'username')
    685         password = rpm_config.get('WEBPOWERED', 'password')
    686         # Call the constructor in RPMController. However since this is a web
    687         # accessible device, there should not be a need to tunnel through a
    688         # hydra serial concentrator.
    689         super(WebPoweredRPMController, self).__init__(hostname)
    690         self.hostname = '%s.%s' % (self.hostname, self._dns_zone)
    691         if not powerswitch:
    692             self._rpm = dli_urllib.Powerswitch(hostname=self.hostname,
    693                                                userid=username,
    694                                                password=password)
    695         else:
    696             # Should only be used in unit_testing
    697             self._rpm = powerswitch
    698 
    699 
    700     def _get_outlet_state(self, outlet):
    701         """
    702         Look up the state for a given outlet on the RPM.
    703 
    704         @param outlet: the outlet to look up.
    705 
    706         @return state: the outlet's current state.
    707         """
    708         status_list = self._rpm.statuslist()
    709         for outlet_name, hostname, state in status_list:
    710             if outlet_name == outlet:
    711                 return state
    712         return None
    713 
    714 
    715     def set_power_state(self, powerunit_info, new_state):
    716         """
    717         Since this does not utilize SSH in any manner, this will overload the
    718         set_power_state in RPMController and completes all steps of changing
    719         the device's outlet state.
    720         """
    721         device_hostname = powerunit_info.device_hostname
    722         outlet = powerunit_info.outlet
    723         if not outlet:
    724             logging.error('Request to change outlet for device %s to '
    725                           'new state %s failed: outlet is unknown. Make sure '
    726                           'POWERUNIT_OUTLET exists in the host\'s '
    727                           'attributes in afe' , device_hostname, new_state)
    728             return False
    729         state = self._get_outlet_state(outlet)
    730         expected_state = new_state
    731         if new_state == self.NEW_STATE_CYCLE:
    732             logging.debug('Beginning Power Cycle for device: %s',
    733                           device_hostname)
    734             self._rpm.off(outlet)
    735             logging.debug('Outlet for device: %s set to OFF', device_hostname)
    736             # Pause for 5 seconds before restoring power.
    737             time.sleep(RPMController.CYCLE_SLEEP_TIME)
    738             self._rpm.on(outlet)
    739             logging.debug('Outlet for device: %s set to ON', device_hostname)
    740             expected_state = self.NEW_STATE_ON
    741         if new_state == self.NEW_STATE_OFF:
    742             self._rpm.off(outlet)
    743             logging.debug('Outlet for device: %s set to OFF', device_hostname)
    744         if new_state == self.NEW_STATE_ON:
    745             self._rpm.on(outlet)
    746             logging.debug('Outlet for device: %s set to ON', device_hostname)
    747         # Lookup the final state of the outlet
    748         return self._is_plug_state(powerunit_info, expected_state)
    749 
    750 
    751     def _is_plug_state(self, powerunit_info, expected_state):
    752         state = self._get_outlet_state(powerunit_info.outlet)
    753         if expected_state not in state:
    754             logging.error('Outlet for device: %s did not change to new state'
    755                           ' %s', powerunit_info.device_hostname, expected_state)
    756             return False
    757         return True
    758 
    759 
    760 class CiscoPOEController(RPMController):
    761     """
    762     This class implements power control for Cisco POE switch.
    763 
    764     Example usage:
    765       poe = CiscoPOEController('chromeos1-poe-switch1')
    766       poe.queue_request('chromeos1-rack5-host12-servo', 'ON')
    767     """
    768 
    769 
    770     SSH_LOGIN_CMD = ('ssh -o StrictHostKeyChecking=no '
    771                      '-o UserKnownHostsFile=/dev/null %s')
    772     POE_USERNAME_PROMPT = 'User Name:'
    773     POE_PROMPT = '%s#'
    774     EXIT_CMD = 'exit'
    775     END_CMD = 'end'
    776     CONFIG = 'configure terminal'
    777     CONFIG_PROMPT = '%s\(config\)#'
    778     CONFIG_IF = 'interface %s'
    779     CONFIG_IF_PROMPT = '%s\(config-if\)#'
    780     SET_STATE_ON = 'power inline auto'
    781     SET_STATE_OFF = 'power inline never'
    782     CHECK_INTERFACE_STATE = 'show interface status %s'
    783     INTERFACE_STATE_MSG = 'Port\s+.*%s(\s+(\S+)){6,6}'
    784     CHECK_STATE_TIMEOUT = 60
    785     CMD_TIMEOUT = 30
    786     LOGIN_TIMEOUT = 60
    787     PORT_UP = 'Up'
    788     PORT_DOWN = 'Down'
    789     TYPE = 'CiscoPOE'
    790 
    791 
    792     def __init__(self, hostname):
    793         """
    794         Initialize controller class for a Cisco POE switch.
    795 
    796         @param hostname: the Cisco POE switch host name.
    797         """
    798         super(CiscoPOEController, self).__init__(hostname)
    799         self._username = rpm_config.get('CiscoPOE', 'username')
    800         self._password = rpm_config.get('CiscoPOE', 'password')
    801         # For a switch, e.g. 'chromeos2-poe-switch8',
    802         # the device prompt looks like 'chromeos2-poe-sw8#'.
    803         short_hostname = self.hostname.replace('switch', 'sw')
    804         self.poe_prompt = self.POE_PROMPT % short_hostname
    805         self.config_prompt = self.CONFIG_PROMPT % short_hostname
    806         self.config_if_prompt = self.CONFIG_IF_PROMPT % short_hostname
    807 
    808 
    809     def _login(self):
    810         """
    811         Log in into the Cisco POE switch.
    812 
    813         Overload _login in RPMController, as it always prompts for a user name.
    814 
    815         @return: ssh - a pexpect.spawn instance if the connection was successful
    816                  or None if it was not.
    817         """
    818         hostname = '%s.%s' % (self.hostname, self._dns_zone)
    819         cmd = self.SSH_LOGIN_CMD % (hostname)
    820         try:
    821             ssh = pexpect.spawn(cmd)
    822         except pexpect.ExceptionPexpect:
    823             logging.error('Could not connect to switch %s', hostname)
    824             return None
    825         # Wait for the username and password prompt.
    826         try:
    827             ssh.expect(self.POE_USERNAME_PROMPT, timeout=self.LOGIN_TIMEOUT)
    828             ssh.sendline(self._username)
    829             ssh.expect(self.PASSWORD_PROMPT, timeout=self.LOGIN_TIMEOUT)
    830             ssh.sendline(self._password)
    831             ssh.expect(self.poe_prompt, timeout=self.LOGIN_TIMEOUT)
    832         except pexpect.ExceptionPexpect:
    833             logging.error('Could not log into switch %s', hostname)
    834             return None
    835         return ssh
    836 
    837 
    838     def _enter_configuration_terminal(self, interface, ssh):
    839         """
    840         Enter configuration terminal of |interface|.
    841 
    842         This function expects that we've already logged into the switch
    843         and the ssh is prompting the switch name. The work flow is
    844             chromeos1-poe-sw1#
    845             chromeos1-poe-sw1#configure terminal
    846             chromeos1-poe-sw1(config)#interface fa36
    847             chromeos1-poe-sw1(config-if)#
    848         On success, the function exits with 'config-if' prompt.
    849         On failure, the function exits with device prompt,
    850         e.g. 'chromeos1-poe-sw1#' in the above case.
    851 
    852         @param interface: the name of the interface.
    853         @param ssh: pexpect.spawn instance to use.
    854 
    855         @return: True on success otherwise False.
    856         """
    857         try:
    858             # Enter configure terminal.
    859             ssh.sendline(self.CONFIG)
    860             ssh.expect(self.config_prompt, timeout=self.CMD_TIMEOUT)
    861             # Enter configure terminal of the interface.
    862             ssh.sendline(self.CONFIG_IF % interface)
    863             ssh.expect(self.config_if_prompt, timeout=self.CMD_TIMEOUT)
    864             return True
    865         except pexpect.ExceptionPexpect, e:
    866             ssh.sendline(self.END_CMD)
    867             logging.exception(e)
    868         return False
    869 
    870 
    871     def _exit_configuration_terminal(self, ssh):
    872         """
    873         Exit interface configuration terminal.
    874 
    875         On success, the function exits with device prompt,
    876         e.g. 'chromeos1-poe-sw1#' in the above case.
    877         On failure, the function exists with 'config-if' prompt.
    878 
    879         @param ssh: pexpect.spawn instance to use.
    880 
    881         @return: True on success otherwise False.
    882         """
    883         try:
    884             ssh.sendline(self.END_CMD)
    885             ssh.expect(self.poe_prompt, timeout=self.CMD_TIMEOUT)
    886             return True
    887         except pexpect.ExceptionPexpect, e:
    888             logging.exception(e)
    889         return False
    890 
    891 
    892     def _verify_state(self, interface, expected_state, ssh):
    893         """
    894         Check whehter the current state of |interface| matches expected state.
    895 
    896         This function tries to check the state of |interface| multiple
    897         times until its state matches the expected state or time is out.
    898 
    899         After the command of changing state has been executed,
    900         the state of an interface doesn't always change immediately to
    901         the expected state but requires some time. As such, we need
    902         a retry logic here.
    903 
    904         @param interface: the name of the interface.
    905         @param expect_state: the expected state, 'ON' or 'OFF'
    906         @param ssh: pexpect.spawn instance to use.
    907 
    908         @return: True if the state of |interface| swiches to |expected_state|,
    909                  otherwise False.
    910         """
    911         expected_state = (self.PORT_UP if expected_state == self.NEW_STATE_ON
    912                           else self.PORT_DOWN)
    913         try:
    914             start = time.time()
    915             while((time.time() - start) < self.CHECK_STATE_TIMEOUT):
    916                 ssh.sendline(self.CHECK_INTERFACE_STATE % interface)
    917                 state_matcher = '.*'.join([self.INTERFACE_STATE_MSG % interface,
    918                                            self.poe_prompt])
    919                 ssh.expect(state_matcher, timeout=self.CMD_TIMEOUT)
    920                 state = ssh.match.group(2)
    921                 if state == expected_state:
    922                     return True
    923         except pexpect.ExceptionPexpect, e:
    924             logging.exception(e)
    925         return False
    926 
    927 
    928     def _logout(self, ssh, admin_logout=False):
    929         """
    930         Log out of the Cisco POE switch after changing state.
    931 
    932         Overload _logout in RPMController.
    933 
    934         @param admin_logout: ignored by this method.
    935         @param ssh: pexpect.spawn instance to use to send the logout command.
    936         """
    937         ssh.sendline(self.EXIT_CMD)
    938 
    939 
    940     def _change_state(self, powerunit_info, new_state, ssh):
    941         """
    942         Perform the actual state change operation.
    943 
    944         Overload _change_state in RPMController.
    945 
    946         @param powerunit_info: An PowerUnitInfo instance.
    947         @param new_state: ON/OFF - state or action we want to perform on
    948                           the outlet.
    949         @param ssh: The ssh connection used to execute the state change commands
    950                     on the POE switch.
    951 
    952         @return: True if the attempt to change power state was successful,
    953                  False otherwise.
    954         """
    955         interface = powerunit_info.outlet
    956         device_hostname = powerunit_info.device_hostname
    957         if not interface:
    958             logging.error('Could not change state: the interface on %s for %s '
    959                           'was not given.', self.hostname, device_hostname)
    960             return False
    961 
    962         # Enter configuration terminal.
    963         if not self._enter_configuration_terminal(interface, ssh):
    964             logging.error('Could not enter configuration terminal for %s',
    965                           interface)
    966             return False
    967         # Change the state.
    968         if new_state == self.NEW_STATE_ON:
    969             ssh.sendline(self.SET_STATE_ON)
    970         elif new_state == self.NEW_STATE_OFF:
    971             ssh.sendline(self.SET_STATE_OFF)
    972         else:
    973             logging.error('Unknown state request: %s', new_state)
    974             return False
    975         # Exit configuraiton terminal.
    976         if not self._exit_configuration_terminal(ssh):
    977             logging.error('Skipping verifying outlet state for device: %s, '
    978                           'because could not exit configuration terminal.',
    979                           device_hostname)
    980             return False
    981         # Verify if the state has changed successfully.
    982         if not self._verify_state(interface, new_state, ssh):
    983             logging.error('Could not verify state on interface %s', interface)
    984             return False
    985 
    986         logging.debug('Outlet for device: %s set to %s',
    987                       device_hostname, new_state)
    988         return True
    989