Home | History | Annotate | Download | only in hosts
      1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
      2 # Use of this source code is governed by a BSD-style license that can be
      3 # found in the LICENSE file.
      4 
      5 import datetime
      6 import errno
      7 import logging
      8 import os
      9 import re
     10 import signal
     11 import stat
     12 import time
     13 
     14 import common
     15 
     16 from autotest_lib.client.bin import utils as client_utils
     17 from autotest_lib.client.common_lib import android_utils
     18 from autotest_lib.client.common_lib import error
     19 from autotest_lib.client.common_lib import global_config
     20 from autotest_lib.client.common_lib.cros import retry
     21 from autotest_lib.server import constants as server_constants
     22 from autotest_lib.server import utils
     23 from autotest_lib.server.cros.dynamic_suite import constants
     24 from autotest_lib.server.hosts import abstract_ssh
     25 from autotest_lib.server.hosts import adb_label
     26 from autotest_lib.server.hosts import base_label
     27 from autotest_lib.server.hosts import teststation_host
     28 
     29 
     30 CONFIG = global_config.global_config
     31 
     32 ADB_CMD = 'adb'
     33 FASTBOOT_CMD = 'fastboot'
     34 SHELL_CMD = 'shell'
     35 # Some devices have no serial, then `adb serial` has output such as:
     36 # (no serial number)  device
     37 # ??????????          device
     38 DEVICE_NO_SERIAL_MSG = '(no serial number)'
     39 DEVICE_NO_SERIAL_TAG = '<NO_SERIAL>'
     40 # Regex to find an adb device. Examples:
     41 # 0146B5580B01801B    device
     42 # 018e0ecb20c97a62    device
     43 # 172.22.75.141:5555  device
     44 # localhost:22        device
     45 DEVICE_FINDER_REGEX = (r'^(?P<SERIAL>([\w-]+)|((tcp:)?' +
     46                        '\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}([:]5555)?)|' +
     47                        '((tcp:)?localhost([:]22)?)|' +
     48                        re.escape(DEVICE_NO_SERIAL_MSG) +
     49                        r')[ \t]+(?:device|fastboot)')
     50 CMD_OUTPUT_PREFIX = 'ADB_CMD_OUTPUT'
     51 CMD_OUTPUT_REGEX = ('(?P<OUTPUT>[\s\S]*)%s:(?P<EXIT_CODE>\d{1,3})' %
     52                     CMD_OUTPUT_PREFIX)
     53 RELEASE_FILE = 'ro.build.version.release'
     54 BOARD_FILE = 'ro.product.device'
     55 SDK_FILE = 'ro.build.version.sdk'
     56 LOGCAT_FILE_FMT = 'logcat_%s.log'
     57 TMP_DIR = '/data/local/tmp'
     58 # Regex to pull out file type, perms and symlink. Example:
     59 # lrwxrwx--- 1 6 root system 2015-09-12 19:21 blah_link -> ./blah
     60 FILE_INFO_REGEX = '^(?P<TYPE>[dl-])(?P<PERMS>[rwx-]{9})'
     61 FILE_SYMLINK_REGEX = '^.*-> (?P<SYMLINK>.+)'
     62 # List of the perm stats indexed by the order they are listed in the example
     63 # supplied above.
     64 FILE_PERMS_FLAGS = [stat.S_IRUSR, stat.S_IWUSR, stat.S_IXUSR,
     65                     stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP,
     66                     stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH]
     67 
     68 # Default maximum number of seconds to wait for a device to be down.
     69 DEFAULT_WAIT_DOWN_TIME_SECONDS = 10
     70 # Default maximum number of seconds to wait for a device to be up.
     71 DEFAULT_WAIT_UP_TIME_SECONDS = 300
     72 
     73 # Default timeout for retrying adb/fastboot command.
     74 DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS = 10
     75 
     76 OS_TYPE_ANDROID = 'android'
     77 OS_TYPE_BRILLO = 'brillo'
     78 
     79 ADB_DEVICE_PREFIXES = ['product:', 'model:', 'device:']
     80 
     81 # Default permissions for files/dirs copied from the device.
     82 _DEFAULT_FILE_PERMS = 0o600
     83 _DEFAULT_DIR_PERMS = 0o700
     84 
     85 # Constants for getprop return value for a given property.
     86 PROPERTY_VALUE_TRUE = '1'
     87 
     88 # Timeout used for retrying installing apk. After reinstall apk failed, we try
     89 # to reboot the device and try again.
     90 APK_INSTALL_TIMEOUT_MIN = 5
     91 
     92 # The amount of time to wait for package verification to be turned off.
     93 DISABLE_PACKAGE_VERIFICATION_TIMEOUT_MIN = 1
     94 
     95 # Directory where (non-Brillo) Android stores tombstone crash logs.
     96 ANDROID_TOMBSTONE_CRASH_LOG_DIR = '/data/tombstones'
     97 # Directory where Brillo stores crash logs for native (non-Java) crashes.
     98 BRILLO_NATIVE_CRASH_LOG_DIR = '/data/misc/crash_reporter/crash'
     99 
    100 # A specific string value to return when a timeout has occurred.
    101 TIMEOUT_MSG = 'TIMEOUT_OCCURRED'
    102 
    103 class ADBHost(abstract_ssh.AbstractSSHHost):
    104     """This class represents a host running an ADB server."""
    105 
    106     @staticmethod
    107     def check_host(host, timeout=10):
    108         """
    109         Check if the given host is an adb host.
    110 
    111         If SSH connectivity can't be established, check_host will try to use
    112         user 'adb' as well. If SSH connectivity still can't be established
    113         then the original SSH user is restored.
    114 
    115         @param host: An ssh host representing a device.
    116         @param timeout: The timeout for the run command.
    117 
    118 
    119         @return: True if the host device has adb.
    120 
    121         @raises AutoservRunError: If the command failed.
    122         @raises AutoservSSHTimeout: Ssh connection has timed out.
    123         """
    124         # host object may not have user attribute if it's a LocalHost object.
    125         current_user = host.user if hasattr(host, 'user') else None
    126         try:
    127             if not (host.hostname == 'localhost' or
    128                     host.verify_ssh_user_access()):
    129                 host.user = 'adb'
    130             result = host.run(
    131                     'test -f %s' % server_constants.ANDROID_TESTER_FILEFLAG,
    132                     timeout=timeout)
    133         except (error.GenericHostRunError, error.AutoservSSHTimeout):
    134             if current_user is not None:
    135                 host.user = current_user
    136             return False
    137         return result.exit_status == 0
    138 
    139 
    140     def _initialize(self, hostname='localhost', serials=None,
    141                     adb_serial=None, fastboot_serial=None,
    142                     teststation=None, *args, **dargs):
    143         """Initialize an ADB Host.
    144 
    145         This will create an ADB Host. Hostname should always refer to the
    146         test station connected to an Android DUT. This will be the DUT
    147         to test with.  If there are multiple, serial must be specified or an
    148         exception will be raised.
    149 
    150         @param hostname: Hostname of the machine running ADB.
    151         @param serials: DEPRECATED (to be removed)
    152         @param adb_serial: An ADB device serial. If None, assume a single
    153                            device is attached (and fail otherwise).
    154         @param fastboot_serial: A fastboot device serial. If None, defaults to
    155                                 the ADB serial (or assumes a single device if
    156                                 the latter is None).
    157         @param teststation: The teststation object ADBHost should use.
    158         """
    159         # Sets up the is_client_install_supported field.
    160         super(ADBHost, self)._initialize(hostname=hostname,
    161                                          is_client_install_supported=False,
    162                                          *args, **dargs)
    163 
    164         self.tmp_dirs = []
    165         self.labels = base_label.LabelRetriever(adb_label.ADB_LABELS)
    166         adb_serial = adb_serial or self._afe_host.attributes.get('serials')
    167         fastboot_serial = (fastboot_serial or
    168                 self._afe_host.attributes.get('fastboot_serial'))
    169 
    170         self.adb_serial = adb_serial
    171         if adb_serial:
    172             adb_prefix = any(adb_serial.startswith(p)
    173                              for p in ADB_DEVICE_PREFIXES)
    174             self.fastboot_serial = (fastboot_serial or
    175                     ('tcp:%s' % adb_serial.split(':')[0] if
    176                     ':' in adb_serial and not adb_prefix else adb_serial))
    177             self._use_tcpip = ':' in adb_serial and not adb_prefix
    178         else:
    179             self.fastboot_serial = fastboot_serial or adb_serial
    180             self._use_tcpip = False
    181         self.teststation = (teststation if teststation
    182                 else teststation_host.create_teststationhost(
    183                         hostname=hostname,
    184                         user=self.user,
    185                         password=self.password,
    186                         port=self.port
    187                 ))
    188 
    189         msg ='Initializing ADB device on host: %s' % hostname
    190         if self.adb_serial:
    191             msg += ', ADB serial: %s' % self.adb_serial
    192         if self.fastboot_serial:
    193             msg += ', fastboot serial: %s' % self.fastboot_serial
    194         logging.debug(msg)
    195 
    196         self._os_type = None
    197 
    198 
    199     def _connect_over_tcpip_as_needed(self):
    200         """Connect to the ADB device over TCP/IP if so configured."""
    201         if not self._use_tcpip:
    202             return
    203         logging.debug('Connecting to device over TCP/IP')
    204         self.adb_run('connect %s' % self.adb_serial)
    205 
    206 
    207     def _restart_adbd_with_root_permissions(self):
    208         """Restarts the adb daemon with root permissions."""
    209         @retry.retry(error.GenericHostRunError, timeout_min=20/60.0,
    210                      delay_sec=1)
    211         def run_adb_root():
    212             """Run command `adb root`."""
    213             self.adb_run('root')
    214 
    215         # adb command may flake with error "device not found". Retry the root
    216         # command to reduce the chance of flake.
    217         run_adb_root()
    218         # TODO(ralphnathan): Remove this sleep once b/19749057 is resolved.
    219         time.sleep(1)
    220         self._connect_over_tcpip_as_needed()
    221         self.adb_run('wait-for-device')
    222 
    223 
    224     def _set_tcp_port(self):
    225         """Ensure the device remains in tcp/ip mode after a reboot."""
    226         if not self._use_tcpip:
    227             return
    228         port = self.adb_serial.split(':')[-1]
    229         self.run('setprop persist.adb.tcp.port %s' % port)
    230 
    231 
    232     def _reset_adbd_connection(self):
    233         """Resets adbd connection to the device after a reboot/initialization"""
    234         self._connect_over_tcpip_as_needed()
    235         self._restart_adbd_with_root_permissions()
    236         self._set_tcp_port()
    237 
    238 
    239     # pylint: disable=missing-docstring
    240     def adb_run(self, command, **kwargs):
    241         """Runs an adb command.
    242 
    243         This command will launch on the test station.
    244 
    245         Refer to _device_run method for docstring for parameters.
    246         """
    247         # Suppresses 'adb devices' from printing to the logs, which often
    248         # causes large log files.
    249         if command == "devices":
    250             kwargs['verbose'] = False
    251         return self._device_run(ADB_CMD, command, **kwargs)
    252 
    253 
    254     # pylint: disable=missing-docstring
    255     def fastboot_run(self, command, **kwargs):
    256         """Runs an fastboot command.
    257 
    258         This command will launch on the test station.
    259 
    260         Refer to _device_run method for docstring for parameters.
    261         """
    262         return self._device_run(FASTBOOT_CMD, command, **kwargs)
    263 
    264 
    265     def _log_adb_pid(self):
    266         """Log the pid of adb server.
    267 
    268         adb's server is known to have bugs and randomly restart. BY logging
    269         the server's pid it will allow us to better debug random adb failures.
    270         """
    271         adb_pid = self.teststation.run('pgrep -f "adb.*server"')
    272         logging.debug('ADB Server PID: %s', adb_pid.stdout)
    273 
    274 
    275     def _device_run(self, function, command, shell=False,
    276                     timeout=3600, ignore_status=False, ignore_timeout=False,
    277                     stdout=utils.TEE_TO_LOGS, stderr=utils.TEE_TO_LOGS,
    278                     connect_timeout=30, options='', stdin=None, verbose=True,
    279                     require_sudo=False, args=()):
    280         """Runs a command named `function` on the test station.
    281 
    282         This command will launch on the test station.
    283 
    284         @param command: Command to run.
    285         @param shell: If true the command runs in the adb shell otherwise if
    286                       False it will be passed directly to adb. For example
    287                       reboot with shell=False will call 'adb reboot'. This
    288                       option only applies to function adb.
    289         @param timeout: Time limit in seconds before attempting to
    290                         kill the running process. The run() function
    291                         will take a few seconds longer than 'timeout'
    292                         to complete if it has to kill the process.
    293         @param ignore_status: Do not raise an exception, no matter
    294                               what the exit code of the command is.
    295         @param ignore_timeout: Bool True if command timeouts should be
    296                                ignored.  Will return None on command timeout.
    297         @param stdout: Redirect stdout.
    298         @param stderr: Redirect stderr.
    299         @param connect_timeout: Connection timeout (in seconds)
    300         @param options: String with additional ssh command options
    301         @param stdin: Stdin to pass (a string) to the executed command
    302         @param require_sudo: True to require sudo to run the command. Default is
    303                              False.
    304         @param args: Sequence of strings to pass as arguments to command by
    305                      quoting them in " and escaping their contents if
    306                      necessary.
    307 
    308         @returns a CMDResult object.
    309         """
    310         if function == ADB_CMD:
    311             serial = self.adb_serial
    312         elif function == FASTBOOT_CMD:
    313             serial = self.fastboot_serial
    314         else:
    315             raise NotImplementedError('Mode %s is not supported' % function)
    316 
    317         if function != ADB_CMD and shell:
    318             raise error.CmdError('shell option is only applicable to `adb`.')
    319 
    320         client_side_cmd = 'timeout --signal=%d %d %s' % (signal.SIGKILL,
    321                                                          timeout + 1, function)
    322         cmd = '%s%s ' % ('sudo -n ' if require_sudo else '', client_side_cmd)
    323 
    324         if serial:
    325             cmd += '-s %s ' % serial
    326 
    327         if shell:
    328             cmd += '%s ' % SHELL_CMD
    329         cmd += command
    330 
    331         self._log_adb_pid()
    332 
    333         if verbose:
    334             logging.debug('Command: %s', cmd)
    335 
    336         return self.teststation.run(cmd, timeout=timeout,
    337                 ignore_status=ignore_status,
    338                 ignore_timeout=ignore_timeout, stdout_tee=stdout,
    339                 stderr_tee=stderr, options=options, stdin=stdin,
    340                 connect_timeout=connect_timeout, args=args)
    341 
    342 
    343     def _run_output_with_retry(self, cmd):
    344         """Call run_output method for the given command with retry.
    345 
    346         adb command can be flaky some time, and the command may fail or return
    347         empty string. It may take several retries until a value can be returned.
    348 
    349         @param cmd: The command to run.
    350 
    351         @return: Return value from the command after retry.
    352         """
    353         try:
    354             return client_utils.poll_for_condition(
    355                     lambda: self.run_output(cmd, ignore_status=True),
    356                     timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS,
    357                     sleep_interval=0.5,
    358                     desc='Get return value for command `%s`' % cmd)
    359         except client_utils.TimeoutError:
    360             return ''
    361 
    362 
    363     def get_product_name(self):
    364         """Get the product name of the device, eg., shamu, bat"""
    365         return self.run_output('getprop %s' % BOARD_FILE)
    366 
    367     def get_board_name(self):
    368         """Get the name of the board, e.g., shamu, bat_land etc.
    369         """
    370         product = self.get_product_name()
    371         return android_utils.AndroidAliases.get_board_name(product)
    372 
    373 
    374     def get_board(self):
    375         """Determine the correct board label for the device.
    376 
    377         @returns a string representing this device's board.
    378         """
    379         board = self.get_board_name()
    380         board_os = self.get_os_type()
    381         return constants.BOARD_PREFIX + '-'.join([board_os, board])
    382 
    383 
    384     def job_start(self):
    385         """Overload of parent which intentionally doesn't log certain files.
    386 
    387         The parent implementation attempts to log certain Linux files, such as
    388         /var/log, which do not exist on Android, thus there is no call to the
    389         parent's job_start().  The sync call is made so that logcat logs can be
    390         approximately matched to server logs.
    391         """
    392         # Try resetting the ADB daemon on the device, however if we are
    393         # creating the host to do a repair job, the device maybe inaccesible
    394         # via ADB.
    395         try:
    396             self._reset_adbd_connection()
    397         except error.GenericHostRunError as e:
    398             logging.error('Unable to reset the device adb daemon connection: '
    399                           '%s.', e)
    400 
    401         if self.is_up():
    402             self._sync_time()
    403             self._enable_native_crash_logging()
    404 
    405 
    406     def run(self, command, timeout=3600, ignore_status=False,
    407             ignore_timeout=False, stdout_tee=utils.TEE_TO_LOGS,
    408             stderr_tee=utils.TEE_TO_LOGS, connect_timeout=30, options='',
    409             stdin=None, verbose=True, args=()):
    410         """Run a command on the adb device.
    411 
    412         The command given will be ran directly on the adb device; for example
    413         'ls' will be ran as: 'abd shell ls'
    414 
    415         @param command: The command line string.
    416         @param timeout: Time limit in seconds before attempting to
    417                         kill the running process. The run() function
    418                         will take a few seconds longer than 'timeout'
    419                         to complete if it has to kill the process.
    420         @param ignore_status: Do not raise an exception, no matter
    421                               what the exit code of the command is.
    422         @param ignore_timeout: Bool True if command timeouts should be
    423                                ignored.  Will return None on command timeout.
    424         @param stdout_tee: Redirect stdout.
    425         @param stderr_tee: Redirect stderr.
    426         @param connect_timeout: Connection timeout (in seconds).
    427         @param options: String with additional ssh command options.
    428         @param stdin: Stdin to pass (a string) to the executed command
    429         @param args: Sequence of strings to pass as arguments to command by
    430                      quoting them in " and escaping their contents if
    431                      necessary.
    432 
    433         @returns A CMDResult object or None if the call timed out and
    434                  ignore_timeout is True.
    435 
    436         @raises AutoservRunError: If the command failed.
    437         @raises AutoservSSHTimeout: Ssh connection has timed out.
    438         """
    439         command = ('"%s; echo %s:\$?"' %
    440                    (utils.sh_escape(command), CMD_OUTPUT_PREFIX))
    441 
    442         def _run():
    443             """Run the command and try to parse the exit code.
    444             """
    445             result = self.adb_run(
    446                     command, shell=True, timeout=timeout,
    447                     ignore_status=ignore_status, ignore_timeout=ignore_timeout,
    448                     stdout=stdout_tee, stderr=stderr_tee,
    449                     connect_timeout=connect_timeout, options=options,
    450                     stdin=stdin, verbose=verbose, args=args)
    451             if not result:
    452                 # In case of timeouts. Set the return to a specific string
    453                 # value. That way the caller of poll_for_condition knows
    454                 # a timeout occurs and should return None. Return None here will
    455                 # lead to the command to be retried.
    456                 return TIMEOUT_MSG
    457             parse_output = re.match(CMD_OUTPUT_REGEX, result.stdout)
    458             if not parse_output and not ignore_status:
    459                 logging.error('Failed to parse the exit code for command: `%s`.'
    460                               ' result: `%s`', command, result.stdout)
    461                 return None
    462             elif parse_output:
    463                 result.stdout = parse_output.group('OUTPUT')
    464                 result.exit_status = int(parse_output.group('EXIT_CODE'))
    465                 if result.exit_status != 0 and not ignore_status:
    466                     raise error.AutoservRunError(command, result)
    467             return result
    468 
    469         result = client_utils.poll_for_condition(
    470                 lambda: _run(),
    471                 timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS,
    472                 sleep_interval=0.5,
    473                 desc='Run command `%s`' % command)
    474         return None if result == TIMEOUT_MSG else result
    475 
    476 
    477     def check_boot_to_adb_complete(self, exception_type=error.TimeoutException):
    478         """Check if the device has finished booting and accessible by adb.
    479 
    480         @param exception_type: Type of exception to raise. Default is set to
    481                 error.TimeoutException for retry.
    482 
    483         @raise exception_type: If the device has not finished booting yet, raise
    484                 an exception of type `exception_type`.
    485         """
    486         bootcomplete = self._run_output_with_retry('getprop dev.bootcomplete')
    487         if bootcomplete != PROPERTY_VALUE_TRUE:
    488             raise exception_type('dev.bootcomplete is %s.' % bootcomplete)
    489         if self.get_os_type() == OS_TYPE_ANDROID:
    490             boot_completed = self._run_output_with_retry(
    491                     'getprop sys.boot_completed')
    492             if boot_completed != PROPERTY_VALUE_TRUE:
    493                 raise exception_type('sys.boot_completed is %s.' %
    494                                      boot_completed)
    495 
    496 
    497     def wait_up(self, timeout=DEFAULT_WAIT_UP_TIME_SECONDS, command=ADB_CMD):
    498         """Wait until the remote host is up or the timeout expires.
    499 
    500         Overrides wait_down from AbstractSSHHost.
    501 
    502         @param timeout: Time limit in seconds before returning even if the host
    503                 is not up.
    504         @param command: The command used to test if a device is up, i.e.,
    505                 accessible by the given command. Default is set to `adb`.
    506 
    507         @returns True if the host was found to be up before the timeout expires,
    508                  False otherwise.
    509         """
    510         @retry.retry(error.TimeoutException, timeout_min=timeout/60.0,
    511                      delay_sec=1)
    512         def _wait_up():
    513             if not self.is_up(command=command):
    514                 raise error.TimeoutException('Device is still down.')
    515             if command == ADB_CMD:
    516                 self.check_boot_to_adb_complete()
    517             return True
    518 
    519         try:
    520             _wait_up()
    521             logging.debug('Host %s is now up, and can be accessed by %s.',
    522                           self.hostname, command)
    523             return True
    524         except error.TimeoutException:
    525             logging.debug('Host %s is still down after waiting %d seconds',
    526                           self.hostname, timeout)
    527             return False
    528 
    529 
    530     def wait_down(self, timeout=DEFAULT_WAIT_DOWN_TIME_SECONDS,
    531                   warning_timer=None, old_boot_id=None, command=ADB_CMD,
    532                   boot_id=None):
    533         """Wait till the host goes down.
    534 
    535         Return when the host is down (not accessible via the command) OR when
    536         the device's boot_id changes (if a boot_id was provided).
    537 
    538         Overrides wait_down from AbstractSSHHost.
    539 
    540         @param timeout: Time in seconds to wait for the host to go down.
    541         @param warning_timer: Time limit in seconds that will generate
    542                               a warning if the host is not down yet.
    543                               Currently ignored.
    544         @param old_boot_id: Not applicable for adb_host.
    545         @param command: `adb`, test if the device can be accessed by adb
    546                 command, or `fastboot`, test if the device can be accessed by
    547                 fastboot command. Default is set to `adb`.
    548         @param boot_id: UUID of previous boot (consider the device down when the
    549                         boot_id changes from this value). Ignored if None.
    550 
    551         @returns True if the device goes down before the timeout, False
    552                  otherwise.
    553         """
    554         @retry.retry(error.TimeoutException, timeout_min=timeout/60.0,
    555                      delay_sec=1)
    556         def _wait_down():
    557             up = self.is_up(command=command)
    558             if not up:
    559                 return True
    560             if boot_id:
    561                 try:
    562                     new_boot_id = self.get_boot_id()
    563                     if new_boot_id != boot_id:
    564                         return True
    565                 except error.GenericHostRunError:
    566                     pass
    567             raise error.TimeoutException('Device is still up.')
    568 
    569         try:
    570             _wait_down()
    571             logging.debug('Host %s is now down', self.hostname)
    572             return True
    573         except error.TimeoutException:
    574             logging.debug('Host %s is still up after waiting %d seconds',
    575                           self.hostname, timeout)
    576             return False
    577 
    578 
    579     def reboot(self):
    580         """Reboot the android device via adb.
    581 
    582         @raises AutoservRebootError if reboot failed.
    583         """
    584         # Not calling super.reboot() as we want to reboot the ADB device not
    585         # the test station we are running ADB on.
    586         boot_id = self.get_boot_id()
    587         self.adb_run('reboot', timeout=10, ignore_timeout=True)
    588         if not self.wait_down(boot_id=boot_id):
    589             raise error.AutoservRebootError(
    590                     'ADB Device %s is still up after reboot' % self.adb_serial)
    591         if not self.wait_up():
    592             raise error.AutoservRebootError(
    593                     'ADB Device %s failed to return from reboot.' %
    594                     self.adb_serial)
    595         self._reset_adbd_connection()
    596 
    597 
    598     def fastboot_reboot(self):
    599         """Do a fastboot reboot to go back to adb.
    600 
    601         @raises AutoservRebootError if reboot failed.
    602         """
    603         self.fastboot_run('reboot')
    604         if not self.wait_down(command=FASTBOOT_CMD):
    605             raise error.AutoservRebootError(
    606                     'Device %s is still in fastboot mode after reboot' %
    607                     self.fastboot_serial)
    608         if not self.wait_up():
    609             raise error.AutoservRebootError(
    610                     'Device %s failed to boot to adb after fastboot reboot.' %
    611                     self.adb_serial)
    612         self._reset_adbd_connection()
    613 
    614 
    615     def remount(self):
    616         """Remounts paritions on the device read-write.
    617 
    618         Specifically, the /system, /vendor (if present) and /oem (if present)
    619         partitions on the device are remounted read-write.
    620         """
    621         self.adb_run('remount')
    622 
    623 
    624     @staticmethod
    625     def parse_device_serials(devices_output):
    626         """Return a list of parsed serials from the output.
    627 
    628         @param devices_output: Output from either an adb or fastboot command.
    629 
    630         @returns List of device serials
    631         """
    632         devices = []
    633         for line in devices_output.splitlines():
    634             match = re.search(DEVICE_FINDER_REGEX, line)
    635             if match:
    636                 serial = match.group('SERIAL')
    637                 if serial == DEVICE_NO_SERIAL_MSG or re.match(r'^\?+$', serial):
    638                     serial = DEVICE_NO_SERIAL_TAG
    639                 logging.debug('Found Device: %s', serial)
    640                 devices.append(serial)
    641         return devices
    642 
    643 
    644     def _get_devices(self, use_adb):
    645         """Get a list of devices currently attached to the test station.
    646 
    647         @params use_adb: True to get adb accessible devices. Set to False to
    648                          get fastboot accessible devices.
    649 
    650         @returns a list of devices attached to the test station.
    651         """
    652         if use_adb:
    653             result = self.adb_run('devices').stdout
    654             if self.adb_serial and self.adb_serial not in result:
    655                 self._connect_over_tcpip_as_needed()
    656         else:
    657             result = self.fastboot_run('devices').stdout
    658             if (self.fastboot_serial and
    659                 self.fastboot_serial not in result):
    660                 # fastboot devices won't list the devices using TCP
    661                 try:
    662                     if 'product' in self.fastboot_run('getvar product',
    663                                                       timeout=2).stderr:
    664                         result += '\n%s\tfastboot' % self.fastboot_serial
    665                 # The main reason we do a general Exception catch here instead
    666                 # of setting ignore_timeout/status to True is because even when
    667                 # the fastboot process has been nuked, it still stays around and
    668                 # so bgjob wants to warn us of this and tries to read the
    669                 # /proc/<pid>/stack file which then promptly returns an
    670                 # 'Operation not permitted' error since we're running as moblab
    671                 # and we don't have permission to read those files.
    672                 except Exception:
    673                     pass
    674         return self.parse_device_serials(result)
    675 
    676 
    677     def adb_devices(self):
    678         """Get a list of devices currently attached to the test station and
    679         accessible with the adb command."""
    680         devices = self._get_devices(use_adb=True)
    681         if self.adb_serial is None and len(devices) > 1:
    682             raise error.AutoservError(
    683                     'Not given ADB serial but multiple devices detected')
    684         return devices
    685 
    686 
    687     def fastboot_devices(self):
    688         """Get a list of devices currently attached to the test station and
    689         accessible by fastboot command.
    690         """
    691         devices = self._get_devices(use_adb=False)
    692         if self.fastboot_serial is None and len(devices) > 1:
    693             raise error.AutoservError(
    694                     'Not given fastboot serial but multiple devices detected')
    695         return devices
    696 
    697 
    698     def is_up(self, timeout=0, command=ADB_CMD):
    699         """Determine if the specified adb device is up with expected mode.
    700 
    701         @param timeout: Not currently used.
    702         @param command: `adb`, the device can be accessed by adb command,
    703                 or `fastboot`, the device can be accessed by fastboot command.
    704                 Default is set to `adb`.
    705 
    706         @returns True if the device is detectable by given command, False
    707                  otherwise.
    708 
    709         """
    710         if command == ADB_CMD:
    711             devices = self.adb_devices()
    712             serial = self.adb_serial
    713             # ADB has a device state, if the device is not online, no
    714             # subsequent ADB command will complete.
    715             # DUT with single device connected may not have adb_serial set.
    716             # Therefore, skip checking if serial is in the list of adb devices
    717             # if self.adb_serial is not set.
    718             if (serial and serial not in devices) or not self.is_device_ready():
    719                 logging.debug('Waiting for device to enter the ready state.')
    720                 return False
    721         elif command == FASTBOOT_CMD:
    722             devices = self.fastboot_devices()
    723             serial = self.fastboot_serial
    724         else:
    725             raise NotImplementedError('Mode %s is not supported' % command)
    726 
    727         return bool(devices and (not serial or serial in devices))
    728 
    729 
    730     def stop_loggers(self):
    731         """Inherited stop_loggers function.
    732 
    733         Calls parent function and captures logcat, since the end of the run
    734         is logically the end/stop of the logcat log.
    735         """
    736         super(ADBHost, self).stop_loggers()
    737 
    738         # When called from atest and tools like it there will be no job.
    739         if not self.job:
    740             return
    741 
    742         # Record logcat log to a temporary file on the teststation.
    743         tmp_dir = self.teststation.get_tmp_dir()
    744         logcat_filename = LOGCAT_FILE_FMT % self.adb_serial
    745         teststation_filename = os.path.join(tmp_dir, logcat_filename)
    746         try:
    747             self.adb_run('logcat -v time -d > "%s"' % (teststation_filename),
    748                          timeout=20)
    749         except (error.GenericHostRunError, error.AutoservSSHTimeout,
    750                 error.CmdTimeoutError):
    751             return
    752         # Copy-back the log to the drone's results directory.
    753         results_logcat_filename = os.path.join(self.job.resultdir,
    754                                                logcat_filename)
    755         self.teststation.get_file(teststation_filename,
    756                                   results_logcat_filename)
    757         try:
    758             self.teststation.run('rm -rf %s' % tmp_dir)
    759         except (error.GenericHostRunError, error.AutoservSSHTimeout) as e:
    760             logging.warn('failed to remove dir %s: %s', tmp_dir, e)
    761 
    762         self._collect_crash_logs()
    763 
    764 
    765     def close(self):
    766         """Close the ADBHost object.
    767 
    768         Called as the test ends. Will return the device to USB mode and kill
    769         the ADB server.
    770         """
    771         super(ADBHost, self).close()
    772         self.teststation.close()
    773 
    774 
    775     def syslog(self, message, tag='autotest'):
    776         """Logs a message to syslog on the device.
    777 
    778         @param message String message to log into syslog
    779         @param tag String tag prefix for syslog
    780 
    781         """
    782         self.run('log -t "%s" "%s"' % (tag, message))
    783 
    784 
    785     def get_autodir(self):
    786         """Return the directory to install autotest for client side tests."""
    787         return '/data/autotest'
    788 
    789 
    790     def is_device_ready(self):
    791         """Return the if the device is ready for ADB commands."""
    792         try:
    793             # Retry to avoid possible flakes.
    794             is_ready = client_utils.poll_for_condition(
    795                 lambda: self.adb_run('get-state').stdout.strip() == 'device',
    796                 timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS, sleep_interval=1,
    797                 desc='Waiting for device state to be `device`')
    798         except client_utils.TimeoutError:
    799             is_ready = False
    800 
    801         logging.debug('Device state is %sready', '' if is_ready else 'NOT ')
    802         return is_ready
    803 
    804 
    805     def send_file(self, source, dest, delete_dest=False,
    806                   preserve_symlinks=False, excludes=None):
    807         """Copy files from the drone to the device.
    808 
    809         Just a note, there is the possibility the test station is localhost
    810         which makes some of these steps redundant (e.g. creating tmp dir) but
    811         that scenario will undoubtedly be a development scenario (test station
    812         is also the moblab) and not the typical live test running scenario so
    813         the redundancy I think is harmless.
    814 
    815         @param source: The file/directory on the drone to send to the device.
    816         @param dest: The destination path on the device to copy to.
    817         @param delete_dest: A flag set to choose whether or not to delete
    818                             dest on the device if it exists.
    819         @param preserve_symlinks: Controls if symlinks on the source will be
    820                                   copied as such on the destination or
    821                                   transformed into the referenced
    822                                   file/directory.
    823         @param excludes: A list of file pattern that matches files not to be
    824                          sent. `send_file` will fail if exclude is set, since
    825                          local copy does not support --exclude, e.g., when
    826                          using scp to copy file.
    827         """
    828         # If we need to preserve symlinks, let's check if the source is a
    829         # symlink itself and if so, just create it on the device.
    830         if preserve_symlinks:
    831             symlink_target = None
    832             try:
    833                 symlink_target = os.readlink(source)
    834             except OSError:
    835                 # Guess it's not a symlink.
    836                 pass
    837 
    838             if symlink_target is not None:
    839                 # Once we create the symlink, let's get out of here.
    840                 self.run('ln -s %s %s' % (symlink_target, dest))
    841                 return
    842 
    843         # Stage the files on the test station.
    844         tmp_dir = self.teststation.get_tmp_dir()
    845         src_path = os.path.join(tmp_dir, os.path.basename(dest))
    846         # Now copy the file over to the test station so you can reference the
    847         # file in the push command.
    848         self.teststation.send_file(
    849                 source, src_path, preserve_symlinks=preserve_symlinks,
    850                 excludes=excludes)
    851 
    852         if delete_dest:
    853             self.run('rm -rf %s' % dest)
    854 
    855         self.adb_run('push %s %s' % (src_path, dest))
    856 
    857         # Cleanup the test station.
    858         try:
    859             self.teststation.run('rm -rf %s' % tmp_dir)
    860         except (error.GenericHostRunError, error.AutoservSSHTimeout) as e:
    861             logging.warn('failed to remove dir %s: %s', tmp_dir, e)
    862 
    863 
    864     def _get_file_info(self, dest):
    865         """Get permission and possible symlink info about file on the device.
    866 
    867         These files are on the device so we only have shell commands (via adb)
    868         to get the info we want.  We'll use 'ls' to get it all.
    869 
    870         @param dest: File to get info about.
    871 
    872         @returns a dict of the file permissions and symlink.
    873         """
    874         # Grab file info.
    875         file_info = self.run_output('ls -ld %s' % dest)
    876         symlink = None
    877         perms = 0
    878         match = re.match(FILE_INFO_REGEX, file_info)
    879         if match:
    880             # Check if it's a symlink and grab the linked dest if it is.
    881             if match.group('TYPE') == 'l':
    882                 symlink_match = re.match(FILE_SYMLINK_REGEX, file_info)
    883                 if symlink_match:
    884                     symlink = symlink_match.group('SYMLINK')
    885 
    886             # Set the perms.
    887             for perm, perm_flag in zip(match.group('PERMS'), FILE_PERMS_FLAGS):
    888                 if perm != '-':
    889                     perms |= perm_flag
    890 
    891         return {'perms': perms,
    892                 'symlink': symlink}
    893 
    894 
    895     def get_file(self, source, dest, delete_dest=False, preserve_perm=True,
    896                  preserve_symlinks=False):
    897         """Copy files from the device to the drone.
    898 
    899         Just a note, there is the possibility the test station is localhost
    900         which makes some of these steps redundant (e.g. creating tmp dir) but
    901         that scenario will undoubtedly be a development scenario (test station
    902         is also the moblab) and not the typical live test running scenario so
    903         the redundancy I think is harmless.
    904 
    905         @param source: The file/directory on the device to copy back to the
    906                        drone.
    907         @param dest: The destination path on the drone to copy to.
    908         @param delete_dest: A flag set to choose whether or not to delete
    909                             dest on the drone if it exists.
    910         @param preserve_perm: Tells get_file() to try to preserve the sources
    911                               permissions on files and dirs.
    912         @param preserve_symlinks: Try to preserve symlinks instead of
    913                                   transforming them into files/dirs on copy.
    914         """
    915         # Stage the files on the test station under teststation_temp_dir.
    916         teststation_temp_dir = self.teststation.get_tmp_dir()
    917         teststation_dest = os.path.join(teststation_temp_dir,
    918                                         os.path.basename(source))
    919 
    920         source_info = {}
    921         if preserve_symlinks or preserve_perm:
    922             source_info = self._get_file_info(source)
    923 
    924         # If we want to preserve symlinks, just create it here, otherwise pull
    925         # the file off the device.
    926         #
    927         # TODO(sadmac): Directories containing symlinks won't behave as
    928         # expected.
    929         if preserve_symlinks and source_info['symlink']:
    930             os.symlink(source_info['symlink'], dest)
    931         else:
    932             self.adb_run('pull %s %s' % (source, teststation_temp_dir))
    933 
    934             # Copy over the file from the test station and clean up.
    935             self.teststation.get_file(teststation_dest, dest,
    936                                       delete_dest=delete_dest)
    937             try:
    938                 self.teststation.run('rm -rf %s' % teststation_temp_dir)
    939             except (error.GenericHostRunError, error.AutoservSSHTimeout) as e:
    940                 logging.warn('failed to remove dir %s: %s',
    941                              teststation_temp_dir, e)
    942 
    943             # Source will be copied under dest if either:
    944             #  1. Source is a directory and doesn't end with /.
    945             #  2. Source is a file and dest is a directory.
    946             command = '[ -d %s ]' % source
    947             source_is_dir = self.run(command,
    948                                      ignore_status=True).exit_status == 0
    949             logging.debug('%s on the device %s a directory', source,
    950                           'is' if source_is_dir else 'is not')
    951 
    952             if ((source_is_dir and not source.endswith(os.sep)) or
    953                 (not source_is_dir and os.path.isdir(dest))):
    954                 receive_path = os.path.join(dest, os.path.basename(source))
    955             else:
    956                 receive_path = dest
    957 
    958             if not os.path.exists(receive_path):
    959                 logging.warning('Expected file %s does not exist; skipping'
    960                                 ' permissions copy', receive_path)
    961                 return
    962 
    963             # Set the permissions of the received file/dirs.
    964             if os.path.isdir(receive_path):
    965                 for root, _dirs, files in os.walk(receive_path):
    966                     def process(rel_path, default_perm):
    967                         info = self._get_file_info(os.path.join(source,
    968                                                                 rel_path))
    969                         if info['perms'] != 0:
    970                             target = os.path.join(receive_path, rel_path)
    971                             if preserve_perm:
    972                                 os.chmod(target, info['perms'])
    973                             else:
    974                                 os.chmod(target, default_perm)
    975 
    976                     rel_root = os.path.relpath(root, receive_path)
    977                     process(rel_root, _DEFAULT_DIR_PERMS)
    978                     for f in files:
    979                         process(os.path.join(rel_root, f), _DEFAULT_FILE_PERMS)
    980             elif preserve_perm:
    981                 os.chmod(receive_path, source_info['perms'])
    982             else:
    983                 os.chmod(receive_path, _DEFAULT_FILE_PERMS)
    984 
    985 
    986     def get_release_version(self):
    987         """Get the release version from the RELEASE_FILE on the device.
    988 
    989         @returns The release string in the RELEASE_FILE.
    990 
    991         """
    992         return self.run_output('getprop %s' % RELEASE_FILE)
    993 
    994 
    995     def get_tmp_dir(self, parent=''):
    996         """Return a suitable temporary directory on the device.
    997 
    998         We ensure this is a subdirectory of /data/local/tmp.
    999 
   1000         @param parent: Parent directory of the returned tmp dir.
   1001 
   1002         @returns a path to the temp directory on the host.
   1003         """
   1004         # TODO(kevcheng): Refactor the cleanup of tmp dir to be inherited
   1005         #                 from the parent.
   1006         if not parent.startswith(TMP_DIR):
   1007             parent = os.path.join(TMP_DIR, parent.lstrip(os.path.sep))
   1008         self.run('mkdir -p %s' % parent)
   1009         tmp_dir = self.run_output('mktemp -d -p %s' % parent)
   1010         self.tmp_dirs.append(tmp_dir)
   1011         return tmp_dir
   1012 
   1013 
   1014     def get_platform(self):
   1015         """Determine the correct platform label for this host.
   1016 
   1017         @returns a string representing this host's platform.
   1018         """
   1019         return 'adb'
   1020 
   1021 
   1022     def get_os_type(self):
   1023         """Get the OS type of the DUT, e.g., android or brillo.
   1024         """
   1025         if not self._os_type:
   1026             if self.run_output('getprop ro.product.brand') == 'Brillo':
   1027                 self._os_type = OS_TYPE_BRILLO
   1028             else:
   1029                 self._os_type = OS_TYPE_ANDROID
   1030 
   1031         return self._os_type
   1032 
   1033 
   1034     def _forward(self, reverse, args):
   1035         """Execute a forwarding command.
   1036 
   1037         @param reverse: Whether this is reverse forwarding (Boolean).
   1038         @param args: List of command arguments.
   1039         """
   1040         cmd = '%s %s' % ('reverse' if reverse else 'forward', ' '.join(args))
   1041         self.adb_run(cmd)
   1042 
   1043 
   1044     def add_forwarding(self, src, dst, reverse=False, rebind=True):
   1045         """Forward a port between the ADB host and device.
   1046 
   1047         Port specifications are any strings accepted as such by ADB, for
   1048         example 'tcp:8080'.
   1049 
   1050         @param src: Port specification to forward from.
   1051         @param dst: Port specification to forward to.
   1052         @param reverse: Do reverse forwarding from device to host (Boolean).
   1053         @param rebind: Allow rebinding an already bound port (Boolean).
   1054         """
   1055         args = []
   1056         if not rebind:
   1057             args.append('--no-rebind')
   1058         args += [src, dst]
   1059         self._forward(reverse, args)
   1060 
   1061 
   1062     def remove_forwarding(self, src=None, reverse=False):
   1063         """Removes forwarding on port.
   1064 
   1065         @param src: Port specification, or None to remove all forwarding.
   1066         @param reverse: Whether this is reverse forwarding (Boolean).
   1067         """
   1068         args = []
   1069         if src is None:
   1070             args.append('--remove-all')
   1071         else:
   1072             args += ['--remove', src]
   1073         self._forward(reverse, args)
   1074 
   1075 
   1076     def create_ssh_tunnel(self, port, local_port):
   1077         """
   1078         Forwards a port securely through a tunnel process from the server
   1079         to the DUT for RPC server connection.
   1080         Add a 'ADB forward' rule to forward the RPC packets from the AdbHost
   1081         to the DUT.
   1082 
   1083         @param port: remote port on the DUT.
   1084         @param local_port: local forwarding port.
   1085 
   1086         @return: the tunnel process.
   1087         """
   1088         self.add_forwarding('tcp:%s' % port, 'tcp:%s' % port)
   1089         return super(ADBHost, self).create_ssh_tunnel(port, local_port)
   1090 
   1091 
   1092     def disconnect_ssh_tunnel(self, tunnel_proc, port):
   1093         """
   1094         Disconnects a previously forwarded port from the server to the DUT for
   1095         RPC server connection.
   1096         Remove the previously added 'ADB forward' rule to forward the RPC
   1097         packets from the AdbHost to the DUT.
   1098 
   1099         @param tunnel_proc: the original tunnel process returned from
   1100                             |create_ssh_tunnel|.
   1101         @param port: remote port on the DUT.
   1102 
   1103         """
   1104         self.remove_forwarding('tcp:%s' % port)
   1105         super(ADBHost, self).disconnect_ssh_tunnel(tunnel_proc, port)
   1106 
   1107 
   1108     def ensure_adb_mode(self, timeout=DEFAULT_WAIT_UP_TIME_SECONDS):
   1109         """Ensure the device is up and can be accessed by adb command.
   1110 
   1111         @param timeout: Time limit in seconds before returning even if the host
   1112                         is not up.
   1113 
   1114         @raise: error.AutoservError if the device failed to reboot into
   1115                 adb mode.
   1116         """
   1117         if self.is_up():
   1118             return
   1119         # Ignore timeout error to allow `fastboot reboot` to fail quietly and
   1120         # check if the device is in adb mode.
   1121         self.fastboot_run('reboot', timeout=timeout, ignore_timeout=True)
   1122         if not self.wait_up(timeout=timeout):
   1123             raise error.AutoservError(
   1124                     'Device %s failed to reboot into adb mode.' %
   1125                     self.adb_serial)
   1126         self._reset_adbd_connection()
   1127 
   1128 
   1129     @retry.retry(error.GenericHostRunError, timeout_min=10)
   1130     def download_file(self, build_url, file, dest_dir, unzip=False,
   1131                       unzip_dest=None):
   1132         """Download the given file from the build url.
   1133 
   1134         @param build_url: The url to use for downloading Android artifacts.
   1135                 pattern: http://$devserver:###/static/branch/target/build_id
   1136         @param file: Name of the file to be downloaded, e.g., boot.img.
   1137         @param dest_dir: Destination folder for the file to be downloaded to.
   1138         @param unzip: If True, unzip the downloaded file.
   1139         @param unzip_dest: Location to unzip the downloaded file to. If not
   1140                            provided, dest_dir is used.
   1141         """
   1142         # Append the file name to the url if build_url is linked to the folder
   1143         # containing the file.
   1144         if not build_url.endswith('/%s' % file):
   1145             src_url = os.path.join(build_url, file)
   1146         else:
   1147             src_url = build_url
   1148         dest_file = os.path.join(dest_dir, file)
   1149         try:
   1150             self.teststation.run('wget -q -O "%s" "%s"' % (dest_file, src_url))
   1151             if unzip:
   1152                 unzip_dest = unzip_dest or dest_dir
   1153                 self.teststation.run('unzip "%s/%s" -x -d "%s"' %
   1154                                      (dest_dir, file, unzip_dest))
   1155         except:
   1156             # Delete the destination file if download failed.
   1157             self.teststation.run('rm -f "%s"' % dest_file)
   1158             raise
   1159 
   1160 
   1161     @property
   1162     def job_repo_url_attribute(self):
   1163         """Get the host attribute name for job_repo_url, which should append the
   1164         adb serial.
   1165         """
   1166         return '%s_%s' % (constants.JOB_REPO_URL, self.adb_serial)
   1167 
   1168 
   1169     def list_files_glob(self, path_glob):
   1170         """Get a list of files on the device given glob pattern path.
   1171 
   1172         @param path_glob: The path glob that we want to return the list of
   1173                 files that match the glob.  Relative paths will not work as
   1174                 expected.  Supply an absolute path to get the list of files
   1175                 you're hoping for.
   1176 
   1177         @returns List of files that match the path_glob.
   1178         """
   1179         # This is just in case path_glob has no path separator.
   1180         base_path = os.path.dirname(path_glob) or '.'
   1181         result = self.run('find %s -path \'%s\' -print' %
   1182                           (base_path, path_glob), ignore_status=True)
   1183         if result.exit_status != 0:
   1184             return []
   1185         return result.stdout.splitlines()
   1186 
   1187 
   1188     @retry.retry(error.GenericHostRunError,
   1189                  timeout_min=DISABLE_PACKAGE_VERIFICATION_TIMEOUT_MIN)
   1190     def disable_package_verification(self):
   1191         """Disables package verification on an android device.
   1192 
   1193         Disables the package verificatoin manager allowing any package to be
   1194         installed without checking
   1195         """
   1196         logging.info('Disabling package verification on %s.', self.adb_serial)
   1197         self.check_boot_to_adb_complete()
   1198         self.run('am broadcast -a '
   1199                  'com.google.gservices.intent.action.GSERVICES_OVERRIDE -e '
   1200                  'global:package_verifier_enable 0')
   1201 
   1202 
   1203     @retry.retry(error.GenericHostRunError, timeout_min=APK_INSTALL_TIMEOUT_MIN)
   1204     def install_apk(self, apk, force_reinstall=True):
   1205         """Install the specified apk.
   1206 
   1207         This will install the apk and override it if it's already installed and
   1208         will also allow for downgraded apks.
   1209 
   1210         @param apk: The path to apk file.
   1211         @param force_reinstall: True to reinstall the apk even if it's already
   1212                 installed. Default is set to True.
   1213 
   1214         @returns a CMDResult object.
   1215         """
   1216         try:
   1217             client_utils.poll_for_condition(
   1218                     lambda: self.run('pm list packages',
   1219                                      ignore_status=True).exit_status == 0,
   1220                     timeout=120)
   1221             client_utils.poll_for_condition(
   1222                     lambda: self.run('service list | grep mount',
   1223                                      ignore_status=True).exit_status == 0,
   1224                     timeout=120)
   1225             return self.adb_run('install %s -d %s' %
   1226                                 ('-r' if force_reinstall else '', apk))
   1227         except error.GenericHostRunError:
   1228             self.reboot()
   1229             raise
   1230 
   1231 
   1232     def uninstall_package(self, package):
   1233         """Remove the specified package.
   1234 
   1235         @param package: Android package name.
   1236 
   1237         @raises GenericHostRunError: uninstall failed
   1238         """
   1239         result = self.adb_run('uninstall %s' % package)
   1240 
   1241         if self.is_apk_installed(package):
   1242             raise error.GenericHostRunError('Uninstall of "%s" failed.'
   1243                                             % package, result)
   1244 
   1245     @retry.retry(error.GenericHostRunError, timeout_min=0.2)
   1246     def _confirm_apk_installed(self, package_name):
   1247         """Confirm if apk is already installed with the given name.
   1248 
   1249         `pm list packages` command is not reliable some time. The retry helps to
   1250         reduce the chance of false negative.
   1251 
   1252         @param package_name: Name of the package, e.g., com.android.phone.
   1253 
   1254         @raise AutoservRunError: If the package is not found or pm list command
   1255                 failed for any reason.
   1256         """
   1257         name = 'package:%s' % package_name
   1258         self.adb_run('shell pm list packages | grep -w "%s"' % name)
   1259 
   1260 
   1261     def is_apk_installed(self, package_name):
   1262         """Check if apk is already installed with the given name.
   1263 
   1264         @param package_name: Name of the package, e.g., com.android.phone.
   1265 
   1266         @return: True if package is installed. False otherwise.
   1267         """
   1268         try:
   1269             self._confirm_apk_installed(package_name)
   1270             return True
   1271         except:
   1272             return False
   1273 
   1274     def get_attributes_to_clear_before_provision(self):
   1275         """Get a list of attributes to be cleared before machine_install starts.
   1276         """
   1277         return [self.job_repo_url_attribute]
   1278 
   1279 
   1280     def get_labels(self):
   1281         """Return a list of the labels gathered from the devices connected.
   1282 
   1283         @return: A list of strings that denote the labels from all the devices
   1284                  connected.
   1285         """
   1286         return self.labels.get_labels(self)
   1287 
   1288 
   1289     def _sync_time(self):
   1290         """Approximate synchronization of time between host and ADB device.
   1291 
   1292         This sets the ADB/Android device's clock to approximately the same
   1293         time as the Autotest host for the purposes of comparing Android system
   1294         logs such as logcat to logs from the Autotest host system.
   1295         """
   1296         command = 'date '
   1297         sdk_version = int(self.run('getprop %s' % SDK_FILE).stdout)
   1298         if sdk_version < 23:
   1299             # Android L and earlier use this format: date -s (format).
   1300             command += ('-s %s' %
   1301                         datetime.datetime.now().strftime('%Y%m%d.%H%M%S'))
   1302         else:
   1303             # Android M and later use this format: date -u (format).
   1304             command += ('-u %s' %
   1305                         datetime.datetime.utcnow().strftime('%m%d%H%M%Y.%S'))
   1306         self.run(command, timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS,
   1307                  ignore_timeout=True)
   1308 
   1309 
   1310     def _enable_native_crash_logging(self):
   1311         """Enable native (non-Java) crash logging.
   1312         """
   1313         if self.get_os_type() == OS_TYPE_ANDROID:
   1314             self._enable_android_native_crash_logging()
   1315 
   1316 
   1317     def _enable_brillo_native_crash_logging(self):
   1318         """Enables native crash logging for a Brillo DUT.
   1319         """
   1320         try:
   1321             self.run('touch /data/misc/metrics/enabled',
   1322                      timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS,
   1323                      ignore_timeout=True)
   1324             # If running, crash_sender will delete crash files every hour.
   1325             self.run('stop crash_sender',
   1326                      timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS,
   1327                      ignore_timeout=True)
   1328         except error.GenericHostRunError as e:
   1329             logging.warn(e)
   1330             logging.warn('Failed to enable Brillo native crash logging.')
   1331 
   1332 
   1333     def _enable_android_native_crash_logging(self):
   1334         """Enables native crash logging for an Android DUT.
   1335         """
   1336         # debuggerd should be enabled by default on Android.
   1337         result = self.run('pgrep debuggerd',
   1338                           timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS,
   1339                           ignore_timeout=True, ignore_status=True)
   1340         if not result or result.exit_status != 0:
   1341             logging.debug('Unable to confirm that debuggerd is running.')
   1342 
   1343 
   1344     def _collect_crash_logs(self):
   1345         """Copies crash log files from the DUT to the drone.
   1346         """
   1347         if self.get_os_type() == OS_TYPE_BRILLO:
   1348             self._collect_crash_logs_dut(BRILLO_NATIVE_CRASH_LOG_DIR)
   1349         elif self.get_os_type() == OS_TYPE_ANDROID:
   1350             self._collect_crash_logs_dut(ANDROID_TOMBSTONE_CRASH_LOG_DIR)
   1351 
   1352 
   1353     def _collect_crash_logs_dut(self, log_directory):
   1354         """Copies native crash logs from the Android/Brillo DUT to the drone.
   1355 
   1356         @param log_directory: absolute path of the directory on the DUT where
   1357                 log files are stored.
   1358         """
   1359         files = None
   1360         try:
   1361             result = self.run('find %s -maxdepth 1 -type f' % log_directory,
   1362                               timeout=DEFAULT_COMMAND_RETRY_TIMEOUT_SECONDS)
   1363             files = result.stdout.strip().split()
   1364         except (error.GenericHostRunError, error.AutoservSSHTimeout,
   1365                 error.CmdTimeoutError):
   1366             logging.debug('Unable to call find %s, unable to find crash logs',
   1367                           log_directory)
   1368         if not files:
   1369             logging.debug('There are no crash logs on the DUT.')
   1370             return
   1371 
   1372         crash_dir = os.path.join(self.job.resultdir, 'crash')
   1373         try:
   1374             os.mkdir(crash_dir)
   1375         except OSError as e:
   1376             if e.errno != errno.EEXIST:
   1377                 raise e
   1378 
   1379         for f in files:
   1380             logging.debug('DUT native crash file produced: %s', f)
   1381             dest = os.path.join(crash_dir, os.path.basename(f))
   1382             # We've had cases where the crash file on the DUT has permissions
   1383             # "000". Let's override permissions to make them sane for the user
   1384             # collecting the crashes.
   1385             self.get_file(source=f, dest=dest, preserve_perm=False)
   1386