1 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import re 8 import time 9 10 import common 11 from autotest_lib.client.common_lib import error, global_config 12 from autotest_lib.client.common_lib.cros import retry 13 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 14 from autotest_lib.server.hosts import cros_host 15 from autotest_lib.server.hosts import cros_repair 16 17 from chromite.lib import timeout_util 18 19 AUTOTEST_INSTALL_DIR = global_config.global_config.get_config_value( 20 'SCHEDULER', 'drone_installation_directory') 21 22 #'/usr/local/autotest' 23 SHADOW_CONFIG_PATH = '%s/shadow_config.ini' % AUTOTEST_INSTALL_DIR 24 ATEST_PATH = '%s/cli/atest' % AUTOTEST_INSTALL_DIR 25 SUBNET_DUT_SEARCH_RE = ( 26 r'/?.*\((?P<ip>192.168.231.*)\) at ' 27 '(?P<mac>[0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])') 28 MOBLAB_HOME = '/home/moblab' 29 MOBLAB_BOTO_LOCATION = '%s/.boto' % MOBLAB_HOME 30 MOBLAB_LAUNCH_CONTROL_KEY_LOCATION = '%s/.launch_control_key' % MOBLAB_HOME 31 MOBLAB_SERVICE_ACCOUNT_LOCATION = '%s/.service_account.json' % MOBLAB_HOME 32 MOBLAB_AUTODIR = '/usr/local/autodir' 33 DHCPD_LEASE_FILE = '/var/lib/dhcp/dhcpd.leases' 34 MOBLAB_SERVICES = ['moblab-scheduler-init', 35 'moblab-database-init', 36 'moblab-devserver-init', 37 'moblab-gsoffloader-init', 38 'moblab-gsoffloader_s-init'] 39 MOBLAB_PROCESSES = ['apache2', 'dhcpd'] 40 DUT_VERIFY_SLEEP_SECS = 5 41 DUT_VERIFY_TIMEOUT = 15 * 60 42 MOBLAB_TMP_DIR = '/mnt/moblab/tmp' 43 MOBLAB_PORT = 80 44 45 46 class UpstartServiceNotRunning(error.AutoservError): 47 """An expected upstart service was not in the expected state.""" 48 49 def __init__(self, service_name): 50 """Create us. 51 @param service_name: Name of the service_name that was in the worng 52 state. 53 """ 54 super(UpstartServiceNotRunning, self).__init__( 55 'Upstart service %s not in running state.' % service_name) 56 57 58 class MoblabHost(cros_host.CrosHost): 59 """Moblab specific host class.""" 60 61 62 def _initialize_frontend_rpcs(self, timeout_min): 63 """Initialize frontends for AFE and TKO for a moblab host. 64 65 We tunnel all communication to the frontends through an SSH tunnel as 66 many testing environments block everything except SSH access to the 67 moblab DUT. 68 69 @param timeout_min: The timeout minuties for AFE services. 70 """ 71 web_address = self.rpc_server_tracker.tunnel_connect(MOBLAB_PORT) 72 # Pass timeout_min to self.afe 73 self.afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min, 74 user='moblab', 75 server=web_address) 76 # Use default timeout_min of MoblabHost for self.tko 77 self.tko = frontend_wrappers.RetryingTKO(timeout_min=self.timeout_min, 78 user='moblab', 79 server=web_address) 80 81 82 def _initialize(self, *args, **dargs): 83 super(MoblabHost, self)._initialize(*args, **dargs) 84 # TODO(jrbarnette): Our superclass already initialized 85 # _repair_strategy, and now we're re-initializing it here. 86 # That's awkward, if not actually wrong. 87 self._repair_strategy = cros_repair.create_moblab_repair_strategy() 88 self.timeout_min = dargs.get('rpc_timeout_min', 1) 89 self._initialize_frontend_rpcs(self.timeout_min) 90 91 92 @staticmethod 93 def check_host(host, timeout=10): 94 """ 95 Check if the given host is an moblab host. 96 97 @param host: An ssh host representing a device. 98 @param timeout: The timeout for the run command. 99 100 101 @return: True if the host device has adb. 102 103 @raises AutoservRunError: If the command failed. 104 @raises AutoservSSHTimeout: Ssh connection has timed out. 105 """ 106 try: 107 result = host.run( 108 'grep -q moblab /etc/lsb-release && ' 109 '! test -f /mnt/stateful_partition/.android_tester', 110 ignore_status=True, timeout=timeout) 111 except (error.AutoservRunError, error.AutoservSSHTimeout): 112 return False 113 return result.exit_status == 0 114 115 116 def install_boto_file(self, boto_path=''): 117 """Install a boto file on the Moblab device. 118 119 @param boto_path: Path to the boto file to install. If None, sends the 120 boto file in the current HOME directory. 121 122 @raises error.TestError if the boto file does not exist. 123 """ 124 if not boto_path: 125 boto_path = os.path.join(os.getenv('HOME'), '.boto') 126 if not os.path.exists(boto_path): 127 raise error.TestError('Boto File:%s does not exist.' % boto_path) 128 self.send_file(boto_path, MOBLAB_BOTO_LOCATION) 129 self.run('chown moblab:moblab %s' % MOBLAB_BOTO_LOCATION) 130 131 132 def get_autodir(self): 133 """Return the directory to install autotest for client side tests.""" 134 return self.autodir or MOBLAB_AUTODIR 135 136 137 def run_as_moblab(self, command, **kwargs): 138 """Moblab commands should be ran as the moblab user not root. 139 140 @param command: Command to run as user moblab. 141 """ 142 command = "su - moblab -c '%s'" % command 143 return self.run(command, **kwargs) 144 145 146 def wait_afe_up(self, timeout_min=5): 147 """Wait till the AFE is up and loaded. 148 149 Attempt to reach the Moblab's AFE and database through its RPC 150 interface. 151 152 @param timeout_min: Minutes to wait for the AFE to respond. Default is 153 5 minutes. 154 155 @raises urllib2.HTTPError if AFE does not respond within the timeout. 156 """ 157 # Use moblabhost's own AFE object with a longer timeout to wait for the 158 # AFE to load. Also re-create the ssh tunnel for connections to moblab. 159 # Set the timeout_min to be longer than self.timeout_min for rebooting. 160 self._initialize_frontend_rpcs(timeout_min) 161 # Verify the AFE can handle a simple request. 162 self._check_afe() 163 # Reset the timeout_min after rebooting checks for afe services. 164 self.afe.set_timeout(self.timeout_min) 165 166 167 def _wake_devices(self): 168 """Search the subnet and attempt to ping any available duts. 169 170 Fills up the arp table with entries about devices on the subnet. 171 172 Either uses fping or directly pings devices listed in the dhcpd lease 173 file. 174 """ 175 fping_result = self.run(('fping -g 192.168.231.100 192.168.231.110 ' 176 '-a -c 10 -p 30 -q'), 177 ignore_status=True) 178 # If fping is not on the system, ping entries in the dhcpd lease file. 179 if fping_result.exit_status == 127: 180 leases = set(self.run('grep ^lease %s' % DHCPD_LEASE_FILE, 181 ignore_status=True).stdout.splitlines()) 182 for lease in leases: 183 ip = re.match('lease (?P<ip>.*) {', lease).groups('ip') 184 self.run('ping %s -w 1' % ip, ignore_status=True) 185 186 187 def add_dut(self, hostname): 188 """Add a DUT hostname to the AFE. 189 190 @param hostname: DUT hostname to add. 191 """ 192 result = self.run_as_moblab('%s host create %s' % (ATEST_PATH, 193 hostname)) 194 logging.debug('atest host create output for host %s:\n%s', 195 hostname, result.stdout) 196 197 198 def find_and_add_duts(self): 199 """Discover DUTs on the testing subnet and add them to the AFE. 200 201 Runs 'arp -a' on the Moblab host and parses the output to discover DUTs 202 and if they are not already in the AFE, adds them. 203 """ 204 self._wake_devices() 205 existing_hosts = [host.hostname for host in self.afe.get_hosts()] 206 arp_command = self.run('arp -a') 207 for line in arp_command.stdout.splitlines(): 208 match = re.match(SUBNET_DUT_SEARCH_RE, line) 209 if match: 210 dut_hostname = match.group('ip') 211 if dut_hostname in existing_hosts: 212 break 213 # SSP package ip's start at 150 for the moblab, so it is not 214 # a DUT 215 if int(dut_hostname.split('.')[-1]) > 150: 216 break 217 self.add_dut(dut_hostname) 218 219 220 def verify_software(self): 221 """Create the autodir then do standard verify.""" 222 # In case cleanup or powerwash wiped the autodir, create an empty 223 # directory. 224 # Removing this mkdir command will result in the disk size check 225 # not being performed. 226 self.run('mkdir -p %s' % MOBLAB_AUTODIR) 227 super(MoblabHost, self).verify_software() 228 229 230 def _verify_upstart_service(self, service, timeout_m): 231 """Verify that the given moblab service is running. 232 233 @param service: The upstart service to check for. 234 @timeout_m: Timeout (in minuts) before giving up. 235 @raises TimeoutException or UpstartServiceNotRunning if service isn't 236 running. 237 """ 238 @retry.retry(error.AutoservError, timeout_min=timeout_m, delay_sec=10) 239 def _verify(): 240 if not self.upstart_status(service): 241 raise UpstartServiceNotRunning(service) 242 _verify() 243 244 def verify_moblab_services(self, timeout_m): 245 """Verify the required Moblab services are up and running. 246 247 @param timeout_m: Timeout (in minutes) for how long to wait for services 248 to start. Actual time taken may be slightly more than this. 249 @raises AutoservError if any moblab service is not running. 250 """ 251 if not MOBLAB_SERVICES: 252 return 253 254 service = MOBLAB_SERVICES[0] 255 try: 256 # First service can take a long time to start, especially on first 257 # boot where container setup can take 5-10 minutes, depending on the 258 # device. 259 self._verify_upstart_service(service, timeout_m) 260 except error.TimeoutException: 261 raise error.UpstartServiceNotRunning(service) 262 263 for service in MOBLAB_SERVICES[1:]: 264 try: 265 # Follow up services should come up quickly. 266 self._verify_upstart_service(service, 0.5) 267 except error.TimeoutException: 268 raise error.UpstartServiceNotRunning(service) 269 270 for process in MOBLAB_PROCESSES: 271 try: 272 self.run('pgrep %s' % process) 273 except error.AutoservRunError: 274 raise error.AutoservError('Moblab process: %s is not running.' 275 % process) 276 277 278 def _check_afe(self): 279 """Verify whether afe of moblab works before verifying its DUTs. 280 281 Verifying moblab sometimes happens after a successful provision, in 282 which case moblab is restarted but tunnel of afe is not re-connected. 283 This func is used to check whether afe is working now. 284 285 @return True if afe works. 286 @raises error.AutoservError if AFE is down; other exceptions are passed 287 through. 288 """ 289 try: 290 self.afe.get_hosts() 291 except (error.TimeoutException, timeout_util.TimeoutError) as e: 292 raise error.AutoservError('Moblab AFE is not responding: %s' % 293 str(e)) 294 except Exception as e: 295 logging.error('Unknown exception when checking moblab AFE: %s', e) 296 raise 297 298 return True 299 300 301 def verify_duts(self): 302 """Verify the Moblab DUTs are up and running. 303 304 @raises AutoservError if no DUTs are in the Ready State. 305 """ 306 hosts = self.afe.reverify_hosts() 307 logging.debug('DUTs scheduled for reverification: %s', hosts) 308 309 310 def verify_special_tasks_complete(self): 311 """Wait till the special tasks on the moblab host are complete.""" 312 total_time = 0 313 while (self.afe.get_special_tasks(is_complete=False) and 314 total_time < DUT_VERIFY_TIMEOUT): 315 total_time = total_time + DUT_VERIFY_SLEEP_SECS 316 time.sleep(DUT_VERIFY_SLEEP_SECS) 317 if not self.afe.get_hosts(status='Ready'): 318 for host in self.afe.get_hosts(): 319 logging.error('DUT: %s Status: %s', host, host.status) 320 raise error.AutoservError('Moblab has 0 Ready DUTs') 321 322 323 def get_platform(self): 324 """Determine the correct platform label for this host. 325 326 For Moblab devices '_moblab' is appended. 327 328 @returns a string representing this host's platform. 329 """ 330 return super(MoblabHost, self).get_platform() + '_moblab' 331 332 333 def make_tmp_dir(self, base=MOBLAB_TMP_DIR): 334 """Creates a temporary directory. 335 336 @param base: The directory where it should be created. 337 338 @return Path to a newly created temporary directory. 339 """ 340 self.run('mkdir -p %s' % base) 341 return self.run('mktemp -d -p %s' % base).stdout.strip() 342 343 344 def get_os_type(self): 345 return 'moblab' 346
