1 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 import common 5 import logging 6 import os 7 import re 8 import tempfile 9 import time 10 import urllib2 11 12 from autotest_lib.client.common_lib import error, global_config 13 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 14 from autotest_lib.server.hosts import cros_host 15 16 17 AUTOTEST_INSTALL_DIR = global_config.global_config.get_config_value( 18 'SCHEDULER', 'drone_installation_directory') 19 #'/usr/local/autotest' 20 SHADOW_CONFIG_PATH = '%s/shadow_config.ini' % AUTOTEST_INSTALL_DIR 21 ATEST_PATH = '%s/cli/atest' % AUTOTEST_INSTALL_DIR 22 SUBNET_DUT_SEARCH_RE = ( 23 r'/?.*\((?P<ip>192.168.231.*)\) at ' 24 '(?P<mac>[0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])') 25 MOBLAB_IMAGE_STORAGE = '/mnt/moblab/static' 26 MOBLAB_BOTO_LOCATION = '/home/moblab/.boto' 27 MOBLAB_LAUNCH_CONTROL_KEY_LOCATION = '/home/moblab/.launch_control_key' 28 MOBLAB_AUTODIR = '/usr/local/autodir' 29 DHCPD_LEASE_FILE = '/var/lib/dhcp/dhcpd.leases' 30 MOBLAB_SERVICES = ['moblab-scheduler-init', 31 'moblab-database-init', 32 'moblab-devserver-init', 33 'moblab-gsoffloader-init', 34 'moblab-gsoffloader_s-init'] 35 MOBLAB_PROCESSES = ['apache2', 'dhcpd'] 36 DUT_VERIFY_SLEEP_SECS = 5 37 DUT_VERIFY_TIMEOUT = 15 * 60 38 MOBLAB_TMP_DIR = '/mnt/moblab/tmp' 39 40 41 class MoblabHost(cros_host.CrosHost): 42 """Moblab specific host class.""" 43 44 45 def _initialize(self, *args, **dargs): 46 super(MoblabHost, self)._initialize(*args, **dargs) 47 # Clear the Moblab Image Storage so that staging an image is properly 48 # tested. 49 if dargs.get('retain_image_storage') is not True: 50 self.run('rm -rf %s/*' % MOBLAB_IMAGE_STORAGE) 51 self._dhcpd_leasefile = None 52 self.web_address = dargs.get('web_address', self.hostname) 53 timeout_min = dargs.get('rpc_timeout_min', 1) 54 self.afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min, 55 user='moblab', 56 server=self.web_address) 57 self.tko = frontend_wrappers.RetryingTKO(timeout_min=timeout_min, 58 user='moblab', 59 server=self.web_address) 60 61 62 @staticmethod 63 def check_host(host, timeout=10): 64 """ 65 Check if the given host is an moblab host. 66 67 @param host: An ssh host representing a device. 68 @param timeout: The timeout for the run command. 69 70 71 @return: True if the host device has adb. 72 73 @raises AutoservRunError: If the command failed. 74 @raises AutoservSSHTimeout: Ssh connection has timed out. 75 """ 76 try: 77 result = host.run( 78 'grep -q moblab /etc/lsb-release && ' 79 '! test -f /mnt/stateful_partition/.android_tester', 80 ignore_status=True, timeout=timeout) 81 except (error.AutoservRunError, error.AutoservSSHTimeout): 82 return False 83 return result.exit_status == 0 84 85 86 def install_boto_file(self, boto_path=''): 87 """Install a boto file on the Moblab device. 88 89 @param boto_path: Path to the boto file to install. If None, sends the 90 boto file in the current HOME directory. 91 92 @raises error.TestError if the boto file does not exist. 93 """ 94 if not boto_path: 95 boto_path = os.path.join(os.getenv('HOME'), '.boto') 96 if not os.path.exists(boto_path): 97 raise error.TestError('Boto File:%s does not exist.' % boto_path) 98 self.send_file(boto_path, MOBLAB_BOTO_LOCATION) 99 self.run('chown moblab:moblab %s' % MOBLAB_BOTO_LOCATION) 100 101 102 def get_autodir(self): 103 """Return the directory to install autotest for client side tests.""" 104 return self.autodir or MOBLAB_AUTODIR 105 106 107 def run_as_moblab(self, command, **kwargs): 108 """Moblab commands should be ran as the moblab user not root. 109 110 @param command: Command to run as user moblab. 111 """ 112 command = "su - moblab -c '%s'" % command 113 return self.run(command, **kwargs) 114 115 116 def reboot(self, **dargs): 117 """Reboot the Moblab Host and wait for its services to restart.""" 118 super(MoblabHost, self).reboot(**dargs) 119 # In general after a reboot, we want to wait till the web frontend 120 # and other Autotest services are up before executing. However should 121 # something be wrong with these services, repair needs to be able 122 # to continue and reimage the device. 123 try: 124 self.wait_afe_up() 125 except (urllib2.HTTPError, urllib2.URLError) as e: 126 logging.error('DUT has rebooted but AFE has failed to load.: %s', 127 e) 128 129 130 def wait_afe_up(self, timeout_min=5): 131 """Wait till the AFE is up and loaded. 132 133 Attempt to reach the Moblab's AFE and database through its RPC 134 interface. 135 136 @param timeout_min: Minutes to wait for the AFE to respond. Default is 137 5 minutes. 138 139 @raises urllib2.HTTPError if AFE does not respond within the timeout. 140 """ 141 # Use a new AFE object with a longer timeout to wait for the AFE to 142 # load. 143 afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min, 144 server=self.hostname) 145 # Verify the AFE can handle a simple request. 146 afe.get_hosts() 147 148 149 def _wake_devices(self): 150 """Search the subnet and attempt to ping any available duts. 151 152 Fills up the arp table with entries about devices on the subnet. 153 154 Either uses fping or directly pings devices listed in the dhcpd lease 155 file. 156 """ 157 fping_result = self.run('fping -g 192.168.231.100 192.168.231.120', 158 ignore_status=True) 159 # If fping is not on the system, ping entries in the dhcpd lease file. 160 if fping_result.exit_status == 127: 161 leases = set(self.run('grep ^lease %s' % DHCPD_LEASE_FILE, 162 ignore_status=True).stdout.splitlines()) 163 for lease in leases: 164 ip = re.match('lease (?P<ip>.*) {', lease).groups('ip') 165 self.run('ping %s -w 1' % ip, ignore_status=True) 166 167 168 def add_dut(self, hostname): 169 """Add a DUT hostname to the AFE. 170 171 @param hostname: DUT hostname to add. 172 """ 173 result = self.run_as_moblab('%s host create %s' % (ATEST_PATH, 174 hostname)) 175 logging.debug('atest host create output for host %s:\n%s', 176 hostname, result.stdout) 177 178 179 def find_and_add_duts(self): 180 """Discover DUTs on the testing subnet and add them to the AFE. 181 182 Runs 'arp -a' on the Moblab host and parses the output to discover DUTs 183 and if they are not already in the AFE, adds them. 184 """ 185 self._wake_devices() 186 existing_hosts = [host.hostname for host in self.afe.get_hosts()] 187 arp_command = self.run('arp -a') 188 for line in arp_command.stdout.splitlines(): 189 match = re.match(SUBNET_DUT_SEARCH_RE, line) 190 if match: 191 dut_hostname = match.group('ip') 192 if dut_hostname in existing_hosts: 193 break 194 self.add_dut(dut_hostname) 195 196 197 def verify_software(self): 198 """Verify working software on a Chrome OS system. 199 200 Tests for the following conditions: 201 1. All conditions tested by the parent version of this 202 function. 203 2. Ensures that Moblab services are running. 204 3. Ensures that both DUTs successfully run Verify. 205 206 """ 207 # In case cleanup or powerwash wiped the autodir, create an empty 208 # directory. 209 self.run('mkdir -p %s' % MOBLAB_AUTODIR) 210 super(MoblabHost, self).verify_software() 211 self._verify_moblab_services() 212 self._verify_duts() 213 214 215 def _verify_moblab_services(self): 216 """Verify the required Moblab services are up and running. 217 218 @raises AutoservError if any moblab service is not running. 219 """ 220 for service in MOBLAB_SERVICES: 221 if not self.upstart_status(service): 222 raise error.AutoservError('Moblab service: %s is not running.' 223 % service) 224 for process in MOBLAB_PROCESSES: 225 try: 226 self.run('pgrep %s' % process) 227 except error.AutoservRunError: 228 raise error.AutoservError('Moblab process: %s is not running.' 229 % process) 230 231 232 def _verify_duts(self): 233 """Verify the Moblab DUTs are up and running. 234 235 @raises AutoservError if no DUTs are in the Ready State. 236 """ 237 # Add the DUTs if they have not yet been added. 238 self.find_and_add_duts() 239 # Ensure a boto file is installed in case this Moblab was wiped in 240 # repair. 241 self.install_boto_file() 242 hosts = self.afe.reverify_hosts() 243 logging.debug('DUTs scheduled for reverification: %s', hosts) 244 # Wait till all pending special tasks are completed. 245 total_time = 0 246 while (self.afe.get_special_tasks(is_complete=False) and 247 total_time < DUT_VERIFY_TIMEOUT): 248 total_time = total_time + DUT_VERIFY_SLEEP_SECS 249 time.sleep(DUT_VERIFY_SLEEP_SECS) 250 if not self.afe.get_hosts(status='Ready'): 251 for host in self.afe.get_hosts(): 252 logging.error('DUT: %s Status: %s', host, host.status) 253 raise error.AutoservError('Moblab has 0 Ready DUTs') 254 255 256 def check_device(self): 257 """Moblab specific check_device. 258 259 Runs after a repair method has been attempted: 260 * Reboots the moblab to start its services. 261 * Creates the autotest client directory in case powerwash was used to 262 wipe stateful and repair. 263 * Reinstall the dhcp lease file if it was preserved. 264 """ 265 # Moblab requires a reboot to initialize it's services prior to 266 # verification. 267 self.reboot() 268 self.wait_afe_up() 269 # Stateful could have been wiped so setup an empty autotest client 270 # directory. 271 self.run('mkdir -p %s' % self.get_autodir(), ignore_status=True) 272 # Restore the dhcpd lease file if it was backed up. 273 # TODO (sbasi) - Currently this is required for repairs but may need 274 # to be expanded to regular installs as well. 275 if self._dhcpd_leasefile: 276 self.send_file(self._dhcpd_leasefile.name, DHCPD_LEASE_FILE) 277 self.run('chown dhcp:dhcp %s' % DHCPD_LEASE_FILE) 278 super(MoblabHost, self).check_device() 279 280 281 def repair(self): 282 """Moblab specific repair. 283 284 Preserves the dhcp lease file prior to repairing the device. 285 """ 286 try: 287 temp = tempfile.TemporaryFile() 288 self.get_file(DHCPD_LEASE_FILE, temp.name) 289 self._dhcpd_leasefile = temp 290 except error.AutoservRunError: 291 logging.debug('Failed to retrieve dhcpd lease file from host.') 292 super(MoblabHost, self).repair() 293 294 295 def get_platform(self): 296 """Determine the correct platform label for this host. 297 298 For Moblab devices '_moblab' is appended. 299 300 @returns a string representing this host's platform. 301 """ 302 return super(MoblabHost, self).get_platform() + '_moblab' 303 304 305 def make_tmp_dir(self, base=MOBLAB_TMP_DIR): 306 """Creates a temporary directory. 307 308 @param base: The directory where it should be created. 309 310 @return Path to a newly created temporary directory. 311 """ 312 self.run('mkdir -p %s' % base) 313 return self.run('mktemp -d -p %s' % base).stdout.strip() 314
