Home | History | Annotate | Download | only in tools 
      1 #!/usr/bin/env python
      2 # Copyright 2016 The Chromium Authors. All rights reserved.
      3 # Use of this source code is governed by a BSD-style license that can be
      4 # found in the LICENSE file.
      5 
      6 """A script to recover devices in a known bad state."""
      7 
      8 import argparse
      9 import logging
     10 import os
     11 import psutil
     12 import signal
     13 import sys
     14 
     15 if __name__ == '__main__':
     16   sys.path.append(
     17       os.path.abspath(os.path.join(os.path.dirname(__file__),
     18                                    '..', '..', '..')))
     19 from devil import devil_env
     20 from devil.android import device_blacklist
     21 from devil.android import device_errors
     22 from devil.android import device_utils
     23 from devil.android.tools import device_status
     24 from devil.utils import lsusb
     25 # TODO(jbudorick): Resolve this after experimenting w/ disabling the USB reset.
     26 from devil.utils import reset_usb  # pylint: disable=unused-import
     27 from devil.utils import run_tests_helper
     28 
     29 logger = logging.getLogger(__name__)
     30 
     31 
     32 def KillAllAdb():
     33   def get_all_adb():
     34     for p in psutil.process_iter():
     35       try:
     36         if 'adb' in p.name:
     37           yield p
     38       except (psutil.NoSuchProcess, psutil.AccessDenied):
     39         pass
     40 
     41   for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]:
     42     for p in get_all_adb():
     43       try:
     44         logger.info('kill %d %d (%s [%s])', sig, p.pid, p.name,
     45                     ' '.join(p.cmdline))
     46         p.send_signal(sig)
     47       except (psutil.NoSuchProcess, psutil.AccessDenied):
     48         pass
     49   for p in get_all_adb():
     50     try:
     51       logger.error('Unable to kill %d (%s [%s])', p.pid, p.name,
     52                    ' '.join(p.cmdline))
     53     except (psutil.NoSuchProcess, psutil.AccessDenied):
     54       pass
     55 
     56 
     57 def RecoverDevice(device, blacklist, should_reboot=lambda device: True):
     58   if device_status.IsBlacklisted(device.adb.GetDeviceSerial(),
     59                                  blacklist):
     60     logger.debug('%s is blacklisted, skipping recovery.', str(device))
     61     return
     62 
     63   if should_reboot(device):
     64     try:
     65       device.WaitUntilFullyBooted(retries=0)
     66     except (device_errors.CommandTimeoutError,
     67             device_errors.CommandFailedError,
     68             device_errors.DeviceUnreachableError):
     69       logger.exception('Failure while waiting for %s. '
     70                        'Attempting to recover.', str(device))
     71     try:
     72       try:
     73         device.Reboot(block=False, timeout=5, retries=0)
     74       except device_errors.CommandTimeoutError:
     75         logger.warning('Timed out while attempting to reboot %s normally.'
     76                        'Attempting alternative reboot.', str(device))
     77         # The device drops offline before we can grab the exit code, so
     78         # we don't check for status.
     79         try:
     80           device.adb.Root()
     81         finally:
     82           # We are already in a failure mode, attempt to reboot regardless of
     83           # what device.adb.Root() returns. If the sysrq reboot fails an
     84           # exception willbe thrown at that level.
     85           device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None,
     86                            timeout=5, retries=0)
     87     except (device_errors.CommandFailedError,
     88             device_errors.DeviceUnreachableError):
     89       logger.exception('Failed to reboot %s.', str(device))
     90       if blacklist:
     91         blacklist.Extend([device.adb.GetDeviceSerial()],
     92                          reason='reboot_failure')
     93     except device_errors.CommandTimeoutError:
     94       logger.exception('Timed out while rebooting %s.', str(device))
     95       if blacklist:
     96         blacklist.Extend([device.adb.GetDeviceSerial()],
     97                          reason='reboot_timeout')
     98 
     99     try:
    100       device.WaitUntilFullyBooted(
    101           retries=0, timeout=device.REBOOT_DEFAULT_TIMEOUT)
    102     except (device_errors.CommandFailedError,
    103             device_errors.DeviceUnreachableError):
    104       logger.exception('Failure while waiting for %s.', str(device))
    105       if blacklist:
    106         blacklist.Extend([device.adb.GetDeviceSerial()],
    107                          reason='reboot_failure')
    108     except device_errors.CommandTimeoutError:
    109       logger.exception('Timed out while waiting for %s.', str(device))
    110       if blacklist:
    111         blacklist.Extend([device.adb.GetDeviceSerial()],
    112                          reason='reboot_timeout')
    113 
    114 
    115 def RecoverDevices(devices, blacklist, enable_usb_reset=False):
    116   """Attempts to recover any inoperable devices in the provided list.
    117 
    118   Args:
    119     devices: The list of devices to attempt to recover.
    120     blacklist: The current device blacklist, which will be used then
    121       reset.
    122   """
    123 
    124   statuses = device_status.DeviceStatus(devices, blacklist)
    125 
    126   should_restart_usb = set(
    127       status['serial'] for status in statuses
    128       if (not status['usb_status']
    129           or status['adb_status'] in ('offline', 'missing')))
    130   should_restart_adb = should_restart_usb.union(set(
    131       status['serial'] for status in statuses
    132       if status['adb_status'] == 'unauthorized'))
    133   should_reboot_device = should_restart_adb.union(set(
    134       status['serial'] for status in statuses
    135       if status['blacklisted']))
    136 
    137   logger.debug('Should restart USB for:')
    138   for d in should_restart_usb:
    139     logger.debug('  %s', d)
    140   logger.debug('Should restart ADB for:')
    141   for d in should_restart_adb:
    142     logger.debug('  %s', d)
    143   logger.debug('Should reboot:')
    144   for d in should_reboot_device:
    145     logger.debug('  %s', d)
    146 
    147   if blacklist:
    148     blacklist.Reset()
    149 
    150   if should_restart_adb:
    151     KillAllAdb()
    152   for serial in should_restart_usb:
    153     try:
    154       # TODO(crbug.com/642194): Resetting may be causing more harm
    155       # (specifically, kernel panics) than it does good.
    156       if enable_usb_reset:
    157         reset_usb.reset_android_usb(serial)
    158       else:
    159         logger.warning('USB reset disabled for %s (crbug.com/642914)',
    160                        serial)
    161     except IOError:
    162       logger.exception('Unable to reset USB for %s.', serial)
    163       if blacklist:
    164         blacklist.Extend([serial], reason='USB failure')
    165     except device_errors.DeviceUnreachableError:
    166       logger.exception('Unable to reset USB for %s.', serial)
    167       if blacklist:
    168         blacklist.Extend([serial], reason='offline')
    169 
    170   device_utils.DeviceUtils.parallel(devices).pMap(
    171       RecoverDevice, blacklist,
    172       should_reboot=lambda device: device.serial in should_reboot_device)
    173 
    174 
    175 def main():
    176   parser = argparse.ArgumentParser()
    177   parser.add_argument('--adb-path',
    178                       help='Absolute path to the adb binary to use.')
    179   parser.add_argument('--blacklist-file', help='Device blacklist JSON file.')
    180   parser.add_argument('--known-devices-file', action='append', default=[],
    181                       dest='known_devices_files',
    182                       help='Path to known device lists.')
    183   parser.add_argument('--enable-usb-reset', action='store_true',
    184                       help='Reset USB if necessary.')
    185   parser.add_argument('-v', '--verbose', action='count', default=1,
    186                       help='Log more information.')
    187 
    188   args = parser.parse_args()
    189   run_tests_helper.SetLogLevel(args.verbose)
    190 
    191   devil_dynamic_config = devil_env.EmptyConfig()
    192   if args.adb_path:
    193     devil_dynamic_config['dependencies'].update(
    194         devil_env.LocalConfigItem(
    195             'adb', devil_env.GetPlatform(), args.adb_path))
    196   devil_env.config.Initialize(configs=[devil_dynamic_config])
    197 
    198   blacklist = (device_blacklist.Blacklist(args.blacklist_file)
    199                if args.blacklist_file
    200                else None)
    201 
    202   expected_devices = device_status.GetExpectedDevices(args.known_devices_files)
    203   usb_devices = set(lsusb.get_android_devices())
    204   devices = [device_utils.DeviceUtils(s)
    205              for s in expected_devices.union(usb_devices)]
    206 
    207   RecoverDevices(devices, blacklist, enable_usb_reset=args.enable_usb_reset)
    208 
    209 
    210 if __name__ == '__main__':
    211   sys.exit(main())
    212