1 # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 5 import logging 6 import os 7 import re 8 import time 9 10 import common 11 from autotest_lib.client.common_lib import error, global_config 12 from autotest_lib.client.common_lib.cros import retry 13 from autotest_lib.server.cros.dynamic_suite import frontend_wrappers 14 from autotest_lib.server.hosts import cros_host 15 from autotest_lib.server.hosts import cros_repair 16 17 from chromite.lib import timeout_util 18 19 AUTOTEST_INSTALL_DIR = global_config.global_config.get_config_value( 20 'SCHEDULER', 'drone_installation_directory') 21 22 #'/usr/local/autotest' 23 SHADOW_CONFIG_PATH = '%s/shadow_config.ini' % AUTOTEST_INSTALL_DIR 24 ATEST_PATH = '%s/cli/atest' % AUTOTEST_INSTALL_DIR 25 26 # Sample output of fping that we are matching against, the fping command 27 # will return 10 lines but they will be one of these two formats. 28 # We want to get the IP address for the first line and not match the 29 # second line that has a non 0 %loss. 30 #192.168.231.100 : xmt/rcv/%loss = 10/10/0%, min/avg/max = 0.68/0.88/1.13 31 #192.168.231.102 : xmt/rcv/%loss = 10/0/100% 32 SUBNET_DUT_SEARCH_RE = (r'(?P<ip>192.168.231.1[0-1][0-9]) : ' 33 'xmt\/rcv\/%loss = [0-9]+\/[0-9]+\/0%') 34 35 MOBLAB_HOME = '/home/moblab' 36 MOBLAB_BOTO_LOCATION = '%s/.boto' % MOBLAB_HOME 37 MOBLAB_LAUNCH_CONTROL_KEY_LOCATION = '%s/.launch_control_key' % MOBLAB_HOME 38 MOBLAB_SERVICE_ACCOUNT_LOCATION = '%s/.service_account.json' % MOBLAB_HOME 39 MOBLAB_AUTODIR = '/usr/local/autodir' 40 DHCPD_LEASE_FILE = '/var/lib/dhcp/dhcpd.leases' 41 MOBLAB_SERVICES = ['moblab-scheduler-init', 42 'moblab-database-init', 43 'moblab-devserver-init', 44 'moblab-gsoffloader-init', 45 'moblab-gsoffloader_s-init'] 46 MOBLAB_PROCESSES = ['apache2', 'dhcpd'] 47 DUT_VERIFY_SLEEP_SECS = 5 48 DUT_VERIFY_TIMEOUT = 15 * 60 49 MOBLAB_TMP_DIR = '/mnt/moblab/tmp' 50 MOBLAB_PORT = 80 51 52 53 class UpstartServiceNotRunning(error.AutoservError): 54 """An expected upstart service was not in the expected state.""" 55 56 def __init__(self, service_name): 57 """Create us. 58 @param service_name: Name of the service_name that was in the worng 59 state. 60 """ 61 super(UpstartServiceNotRunning, self).__init__( 62 'Upstart service %s not in running state. Most likely this ' 63 'means moblab did not boot correctly, check the boot logs ' 64 'for detailed error messages as to see why this service was ' 65 'not started.' % 66 service_name) 67 68 69 class MoblabHost(cros_host.CrosHost): 70 """Moblab specific host class.""" 71 72 73 def _initialize_frontend_rpcs(self, timeout_min): 74 """Initialize frontends for AFE and TKO for a moblab host. 75 76 We tunnel all communication to the frontends through an SSH tunnel as 77 many testing environments block everything except SSH access to the 78 moblab DUT. 79 80 @param timeout_min: The timeout minuties for AFE services. 81 """ 82 web_address = self.rpc_server_tracker.tunnel_connect(MOBLAB_PORT) 83 # Pass timeout_min to self.afe 84 self.afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min, 85 user='moblab', 86 server=web_address) 87 # Use default timeout_min of MoblabHost for self.tko 88 self.tko = frontend_wrappers.RetryingTKO(timeout_min=self.timeout_min, 89 user='moblab', 90 server=web_address) 91 92 93 def _initialize(self, *args, **dargs): 94 super(MoblabHost, self)._initialize(*args, **dargs) 95 # TODO(jrbarnette): Our superclass already initialized 96 # _repair_strategy, and now we're re-initializing it here. 97 # That's awkward, if not actually wrong. 98 self._repair_strategy = cros_repair.create_moblab_repair_strategy() 99 self.timeout_min = dargs.get('rpc_timeout_min', 1) 100 self._initialize_frontend_rpcs(self.timeout_min) 101 102 103 @staticmethod 104 def check_host(host, timeout=10): 105 """ 106 Check if the given host is an moblab host. 107 108 @param host: An ssh host representing a device. 109 @param timeout: The timeout for the run command. 110 111 112 @return: True if the host device has adb. 113 114 @raises AutoservRunError: If the command failed. 115 @raises AutoservSSHTimeout: Ssh connection has timed out. 116 """ 117 try: 118 result = host.run( 119 'grep -q moblab /etc/lsb-release && ' 120 '! test -f /mnt/stateful_partition/.android_tester', 121 ignore_status=True, timeout=timeout) 122 except (error.AutoservRunError, error.AutoservSSHTimeout): 123 return False 124 return result.exit_status == 0 125 126 127 def install_boto_file(self, boto_path=''): 128 """Install a boto file on the Moblab device. 129 130 @param boto_path: Path to the boto file to install. If None, sends the 131 boto file in the current HOME directory. 132 133 @raises error.TestError if the boto file does not exist. 134 """ 135 if not boto_path: 136 boto_path = os.path.join(os.getenv('HOME'), '.boto') 137 if not os.path.exists(boto_path): 138 raise error.TestError('Boto File:%s does not exist.' % boto_path) 139 self.send_file(boto_path, MOBLAB_BOTO_LOCATION) 140 self.run('chown moblab:moblab %s' % MOBLAB_BOTO_LOCATION) 141 142 143 def get_autodir(self): 144 """Return the directory to install autotest for client side tests.""" 145 return self.autodir or MOBLAB_AUTODIR 146 147 148 def run_as_moblab(self, command, **kwargs): 149 """Moblab commands should be ran as the moblab user not root. 150 151 @param command: Command to run as user moblab. 152 """ 153 command = "su - moblab -c '%s'" % command 154 return self.run(command, **kwargs) 155 156 157 def wait_afe_up(self, timeout_min=5): 158 """Wait till the AFE is up and loaded. 159 160 Attempt to reach the Moblab's AFE and database through its RPC 161 interface. 162 163 @param timeout_min: Minutes to wait for the AFE to respond. Default is 164 5 minutes. 165 166 @raises urllib2.HTTPError if AFE does not respond within the timeout. 167 """ 168 # Use moblabhost's own AFE object with a longer timeout to wait for the 169 # AFE to load. Also re-create the ssh tunnel for connections to moblab. 170 # Set the timeout_min to be longer than self.timeout_min for rebooting. 171 self._initialize_frontend_rpcs(timeout_min) 172 # Verify the AFE can handle a simple request. 173 self._check_afe() 174 # Reset the timeout_min after rebooting checks for afe services. 175 self.afe.set_timeout(self.timeout_min) 176 177 178 def add_dut(self, hostname): 179 """Add a DUT hostname to the AFE. 180 181 @param hostname: DUT hostname to add. 182 """ 183 result = self.run_as_moblab('%s host create %s' % (ATEST_PATH, 184 hostname)) 185 logging.debug('atest host create output for host %s:\n%s', 186 hostname, result.stdout) 187 188 189 def find_and_add_duts(self): 190 """Discover DUTs on the testing subnet and add them to the AFE. 191 192 Pings the range of IP's a DUT might be assigned by moblab, then 193 parses the output to discover connected DUTs, connected means 194 they have 0% dropped pings. 195 If they are not already in the AFE, adds them to AFE. 196 """ 197 existing_hosts = [host.hostname for host in self.afe.get_hosts()] 198 fping_result = self.run('fping -g 192.168.231.100 192.168.231.110 ' 199 '-a -c 10 -p 30 -q', ignore_status=True) 200 for line in fping_result.stderr.splitlines(): 201 match = re.match(SUBNET_DUT_SEARCH_RE, line) 202 if match: 203 dut_ip = match.group('ip') 204 if dut_ip in existing_hosts: 205 break 206 self.add_dut(dut_ip) 207 208 209 def verify_software(self): 210 """Create the autodir then do standard verify.""" 211 # In case cleanup or powerwash wiped the autodir, create an empty 212 # directory. 213 # Removing this mkdir command will result in the disk size check 214 # not being performed. 215 self.run('mkdir -p %s' % MOBLAB_AUTODIR) 216 super(MoblabHost, self).verify_software() 217 218 219 def _verify_upstart_service(self, service, timeout_m): 220 """Verify that the given moblab service is running. 221 222 @param service: The upstart service to check for. 223 @timeout_m: Timeout (in minuts) before giving up. 224 @raises TimeoutException or UpstartServiceNotRunning if service isn't 225 running. 226 """ 227 @retry.retry(error.AutoservError, timeout_min=timeout_m, delay_sec=10) 228 def _verify(): 229 if not self.upstart_status(service): 230 raise UpstartServiceNotRunning(service) 231 _verify() 232 233 def verify_moblab_services(self, timeout_m): 234 """Verify the required Moblab services are up and running. 235 236 @param timeout_m: Timeout (in minutes) for how long to wait for services 237 to start. Actual time taken may be slightly more than this. 238 @raises AutoservError if any moblab service is not running. 239 """ 240 if not MOBLAB_SERVICES: 241 return 242 243 service = MOBLAB_SERVICES[0] 244 try: 245 # First service can take a long time to start, especially on first 246 # boot where container setup can take 5-10 minutes, depending on the 247 # device. 248 self._verify_upstart_service(service, timeout_m) 249 except error.TimeoutException: 250 raise UpstartServiceNotRunning(service) 251 252 for service in MOBLAB_SERVICES[1:]: 253 try: 254 # Follow up services should come up quickly. 255 self._verify_upstart_service(service, 0.5) 256 except error.TimeoutException: 257 raise UpstartServiceNotRunning(service) 258 259 for process in MOBLAB_PROCESSES: 260 try: 261 self.run('pgrep %s' % process) 262 except error.AutoservRunError: 263 raise error.AutoservError('Moblab process: %s is not running.' 264 % process) 265 266 267 def _check_afe(self): 268 """Verify whether afe of moblab works before verifying its DUTs. 269 270 Verifying moblab sometimes happens after a successful provision, in 271 which case moblab is restarted but tunnel of afe is not re-connected. 272 This func is used to check whether afe is working now. 273 274 @return True if afe works. 275 @raises error.AutoservError if AFE is down; other exceptions are passed 276 through. 277 """ 278 try: 279 self.afe.get_hosts() 280 except (error.TimeoutException, timeout_util.TimeoutError) as e: 281 raise error.AutoservError('Moblab AFE is not responding: %s' % 282 str(e)) 283 except Exception as e: 284 logging.error('Unknown exception when checking moblab AFE: %s', e) 285 raise 286 287 return True 288 289 290 def verify_duts(self): 291 """Verify the Moblab DUTs are up and running. 292 293 @raises AutoservError if no DUTs are in the Ready State. 294 """ 295 hosts = self.afe.reverify_hosts() 296 logging.debug('DUTs scheduled for reverification: %s', hosts) 297 298 299 def verify_special_tasks_complete(self): 300 """Wait till the special tasks on the moblab host are complete.""" 301 total_time = 0 302 while (self.afe.get_special_tasks(is_complete=False) and 303 total_time < DUT_VERIFY_TIMEOUT): 304 total_time = total_time + DUT_VERIFY_SLEEP_SECS 305 time.sleep(DUT_VERIFY_SLEEP_SECS) 306 if not self.afe.get_hosts(status='Ready'): 307 for host in self.afe.get_hosts(): 308 logging.error('DUT: %s Status: %s', host, host.status) 309 raise error.AutoservError('Moblab has 0 Ready DUTs') 310 311 312 def get_platform(self): 313 """Determine the correct platform label for this host. 314 315 For Moblab devices '_moblab' is appended. 316 317 @returns a string representing this host's platform. 318 """ 319 return super(MoblabHost, self).get_platform() + '_moblab' 320 321 322 def make_tmp_dir(self, base=MOBLAB_TMP_DIR): 323 """Creates a temporary directory. 324 325 @param base: The directory where it should be created. 326 327 @return Path to a newly created temporary directory. 328 """ 329 self.run('mkdir -p %s' % base) 330 return self.run('mktemp -d -p %s' % base).stdout.strip() 331 332 333 def get_os_type(self): 334 return 'moblab' 335
