1 #!/usr/bin/env python 2 # 3 # Copyright 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """A class to keep track of devices across builds and report state.""" 8 9 import argparse 10 import json 11 import logging 12 import os 13 import psutil 14 import re 15 import signal 16 import sys 17 18 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) 19 import devil_chromium 20 from devil import devil_env 21 from devil.android import battery_utils 22 from devil.android import device_blacklist 23 from devil.android import device_errors 24 from devil.android import device_list 25 from devil.android import device_utils 26 from devil.android.sdk import adb_wrapper 27 from devil.constants import exit_codes 28 from devil.utils import lsusb 29 from devil.utils import reset_usb 30 from devil.utils import run_tests_helper 31 from pylib.constants import host_paths 32 33 _RE_DEVICE_ID = re.compile(r'Device ID = (\d+)') 34 35 36 def KillAllAdb(): 37 def GetAllAdb(): 38 for p in psutil.process_iter(): 39 try: 40 if 'adb' in p.name: 41 yield p 42 except (psutil.NoSuchProcess, psutil.AccessDenied): 43 pass 44 45 for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]: 46 for p in GetAllAdb(): 47 try: 48 logging.info('kill %d %d (%s [%s])', sig, p.pid, p.name, 49 ' '.join(p.cmdline)) 50 p.send_signal(sig) 51 except (psutil.NoSuchProcess, psutil.AccessDenied): 52 pass 53 for p in GetAllAdb(): 54 try: 55 logging.error('Unable to kill %d (%s [%s])', p.pid, p.name, 56 ' '.join(p.cmdline)) 57 except (psutil.NoSuchProcess, psutil.AccessDenied): 58 pass 59 60 61 def _IsBlacklisted(serial, blacklist): 62 return blacklist and serial in blacklist.Read() 63 64 65 def _BatteryStatus(device, blacklist): 66 battery_info = {} 67 try: 68 battery = battery_utils.BatteryUtils(device) 69 battery_info = battery.GetBatteryInfo(timeout=5) 70 battery_level = int(battery_info.get('level', 100)) 71 72 if battery_level < 15: 73 logging.error('Critically low battery level (%d)', battery_level) 74 battery = battery_utils.BatteryUtils(device) 75 if not battery.GetCharging(): 76 battery.SetCharging(True) 77 if blacklist: 78 blacklist.Extend([device.adb.GetDeviceSerial()], reason='low_battery') 79 80 except device_errors.CommandFailedError: 81 logging.exception('Failed to get battery information for %s', 82 str(device)) 83 84 return battery_info 85 86 87 def _IMEISlice(device): 88 imei_slice = '' 89 try: 90 for l in device.RunShellCommand(['dumpsys', 'iphonesubinfo'], 91 check_return=True, timeout=5): 92 m = _RE_DEVICE_ID.match(l) 93 if m: 94 imei_slice = m.group(1)[-6:] 95 except device_errors.CommandFailedError: 96 logging.exception('Failed to get IMEI slice for %s', str(device)) 97 98 return imei_slice 99 100 101 def DeviceStatus(devices, blacklist): 102 """Generates status information for the given devices. 103 104 Args: 105 devices: The devices to generate status for. 106 blacklist: The current device blacklist. 107 Returns: 108 A dict of the following form: 109 { 110 '<serial>': { 111 'serial': '<serial>', 112 'adb_status': str, 113 'usb_status': bool, 114 'blacklisted': bool, 115 # only if the device is connected and not blacklisted 116 'type': ro.build.product, 117 'build': ro.build.id, 118 'build_detail': ro.build.fingerprint, 119 'battery': { 120 ... 121 }, 122 'imei_slice': str, 123 'wifi_ip': str, 124 }, 125 ... 126 } 127 """ 128 adb_devices = { 129 a[0].GetDeviceSerial(): a 130 for a in adb_wrapper.AdbWrapper.Devices(desired_state=None, long_list=True) 131 } 132 usb_devices = set(lsusb.get_android_devices()) 133 134 def blacklisting_device_status(device): 135 serial = device.adb.GetDeviceSerial() 136 adb_status = ( 137 adb_devices[serial][1] if serial in adb_devices 138 else 'missing') 139 usb_status = bool(serial in usb_devices) 140 141 device_status = { 142 'serial': serial, 143 'adb_status': adb_status, 144 'usb_status': usb_status, 145 } 146 147 if not _IsBlacklisted(serial, blacklist): 148 if adb_status == 'device': 149 try: 150 build_product = device.build_product 151 build_id = device.build_id 152 build_fingerprint = device.GetProp('ro.build.fingerprint', cache=True) 153 wifi_ip = device.GetProp('dhcp.wlan0.ipaddress') 154 battery_info = _BatteryStatus(device, blacklist) 155 imei_slice = _IMEISlice(device) 156 157 if (device.product_name == 'mantaray' and 158 battery_info.get('AC powered', None) != 'true'): 159 logging.error('Mantaray device not connected to AC power.') 160 161 device_status.update({ 162 'ro.build.product': build_product, 163 'ro.build.id': build_id, 164 'ro.build.fingerprint': build_fingerprint, 165 'battery': battery_info, 166 'imei_slice': imei_slice, 167 'wifi_ip': wifi_ip, 168 169 # TODO(jbudorick): Remove these once no clients depend on them. 170 'type': build_product, 171 'build': build_id, 172 'build_detail': build_fingerprint, 173 }) 174 175 except device_errors.CommandFailedError: 176 logging.exception('Failure while getting device status for %s.', 177 str(device)) 178 if blacklist: 179 blacklist.Extend([serial], reason='status_check_failure') 180 181 except device_errors.CommandTimeoutError: 182 logging.exception('Timeout while getting device status for %s.', 183 str(device)) 184 if blacklist: 185 blacklist.Extend([serial], reason='status_check_timeout') 186 187 elif blacklist: 188 blacklist.Extend([serial], 189 reason=adb_status if usb_status else 'offline') 190 191 device_status['blacklisted'] = _IsBlacklisted(serial, blacklist) 192 193 return device_status 194 195 parallel_devices = device_utils.DeviceUtils.parallel(devices) 196 statuses = parallel_devices.pMap(blacklisting_device_status).pGet(None) 197 return statuses 198 199 200 def RecoverDevices(devices, blacklist): 201 """Attempts to recover any inoperable devices in the provided list. 202 203 Args: 204 devices: The list of devices to attempt to recover. 205 blacklist: The current device blacklist, which will be used then 206 reset. 207 Returns: 208 Nothing. 209 """ 210 211 statuses = DeviceStatus(devices, blacklist) 212 213 should_restart_usb = set( 214 status['serial'] for status in statuses 215 if (not status['usb_status'] 216 or status['adb_status'] in ('offline', 'missing'))) 217 should_restart_adb = should_restart_usb.union(set( 218 status['serial'] for status in statuses 219 if status['adb_status'] == 'unauthorized')) 220 should_reboot_device = should_restart_adb.union(set( 221 status['serial'] for status in statuses 222 if status['blacklisted'])) 223 224 logging.debug('Should restart USB for:') 225 for d in should_restart_usb: 226 logging.debug(' %s', d) 227 logging.debug('Should restart ADB for:') 228 for d in should_restart_adb: 229 logging.debug(' %s', d) 230 logging.debug('Should reboot:') 231 for d in should_reboot_device: 232 logging.debug(' %s', d) 233 234 if blacklist: 235 blacklist.Reset() 236 237 if should_restart_adb: 238 KillAllAdb() 239 for serial in should_restart_usb: 240 try: 241 reset_usb.reset_android_usb(serial) 242 except IOError: 243 logging.exception('Unable to reset USB for %s.', serial) 244 if blacklist: 245 blacklist.Extend([serial], reason='usb_failure') 246 except device_errors.DeviceUnreachableError: 247 logging.exception('Unable to reset USB for %s.', serial) 248 if blacklist: 249 blacklist.Extend([serial], reason='offline') 250 251 def blacklisting_recovery(device): 252 if _IsBlacklisted(device.adb.GetDeviceSerial(), blacklist): 253 logging.debug('%s is blacklisted, skipping recovery.', str(device)) 254 return 255 256 if str(device) in should_reboot_device: 257 try: 258 device.WaitUntilFullyBooted(retries=0) 259 return 260 except (device_errors.CommandTimeoutError, 261 device_errors.CommandFailedError): 262 logging.exception('Failure while waiting for %s. ' 263 'Attempting to recover.', str(device)) 264 265 try: 266 try: 267 device.Reboot(block=False, timeout=5, retries=0) 268 except device_errors.CommandTimeoutError: 269 logging.warning('Timed out while attempting to reboot %s normally.' 270 'Attempting alternative reboot.', str(device)) 271 # The device drops offline before we can grab the exit code, so 272 # we don't check for status. 273 device.adb.Root() 274 device.adb.Shell('echo b > /proc/sysrq-trigger', expect_status=None, 275 timeout=5, retries=0) 276 except device_errors.CommandFailedError: 277 logging.exception('Failed to reboot %s.', str(device)) 278 if blacklist: 279 blacklist.Extend([device.adb.GetDeviceSerial()], 280 reason='reboot_failure') 281 except device_errors.CommandTimeoutError: 282 logging.exception('Timed out while rebooting %s.', str(device)) 283 if blacklist: 284 blacklist.Extend([device.adb.GetDeviceSerial()], 285 reason='reboot_timeout') 286 287 try: 288 device.WaitUntilFullyBooted(retries=0) 289 except device_errors.CommandFailedError: 290 logging.exception('Failure while waiting for %s.', str(device)) 291 if blacklist: 292 blacklist.Extend([device.adb.GetDeviceSerial()], 293 reason='reboot_failure') 294 except device_errors.CommandTimeoutError: 295 logging.exception('Timed out while waiting for %s.', str(device)) 296 if blacklist: 297 blacklist.Extend([device.adb.GetDeviceSerial()], 298 reason='reboot_timeout') 299 300 device_utils.DeviceUtils.parallel(devices).pMap(blacklisting_recovery) 301 302 303 def main(): 304 parser = argparse.ArgumentParser() 305 parser.add_argument('--out-dir', 306 help='Directory where the device path is stored', 307 default=os.path.join(host_paths.DIR_SOURCE_ROOT, 'out')) 308 parser.add_argument('--restart-usb', action='store_true', 309 help='DEPRECATED. ' 310 'This script now always tries to reset USB.') 311 parser.add_argument('--json-output', 312 help='Output JSON information into a specified file.') 313 parser.add_argument('--adb-path', 314 help='Absolute path to the adb binary to use.') 315 parser.add_argument('--blacklist-file', help='Device blacklist JSON file.') 316 parser.add_argument('--known-devices-file', action='append', default=[], 317 dest='known_devices_files', 318 help='Path to known device lists.') 319 parser.add_argument('-v', '--verbose', action='count', default=1, 320 help='Log more information.') 321 322 args = parser.parse_args() 323 324 run_tests_helper.SetLogLevel(args.verbose) 325 326 devil_custom_deps = None 327 if args.adb_path: 328 devil_custom_deps = { 329 'adb': { 330 devil_env.GetPlatform(): [args.adb_path], 331 }, 332 } 333 334 devil_chromium.Initialize(custom_deps=devil_custom_deps) 335 336 blacklist = (device_blacklist.Blacklist(args.blacklist_file) 337 if args.blacklist_file 338 else None) 339 340 last_devices_path = os.path.join( 341 args.out_dir, device_list.LAST_DEVICES_FILENAME) 342 args.known_devices_files.append(last_devices_path) 343 344 expected_devices = set() 345 try: 346 for path in args.known_devices_files: 347 if os.path.exists(path): 348 expected_devices.update(device_list.GetPersistentDeviceList(path)) 349 except IOError: 350 logging.warning('Problem reading %s, skipping.', path) 351 352 logging.info('Expected devices:') 353 for device in expected_devices: 354 logging.info(' %s', device) 355 356 usb_devices = set(lsusb.get_android_devices()) 357 devices = [device_utils.DeviceUtils(s) 358 for s in expected_devices.union(usb_devices)] 359 360 RecoverDevices(devices, blacklist) 361 statuses = DeviceStatus(devices, blacklist) 362 363 # Log the state of all devices. 364 for status in statuses: 365 logging.info(status['serial']) 366 adb_status = status.get('adb_status') 367 blacklisted = status.get('blacklisted') 368 logging.info(' USB status: %s', 369 'online' if status.get('usb_status') else 'offline') 370 logging.info(' ADB status: %s', adb_status) 371 logging.info(' Blacklisted: %s', str(blacklisted)) 372 if adb_status == 'device' and not blacklisted: 373 logging.info(' Device type: %s', status.get('ro.build.product')) 374 logging.info(' OS build: %s', status.get('ro.build.id')) 375 logging.info(' OS build fingerprint: %s', 376 status.get('ro.build.fingerprint')) 377 logging.info(' Battery state:') 378 for k, v in status.get('battery', {}).iteritems(): 379 logging.info(' %s: %s', k, v) 380 logging.info(' IMEI slice: %s', status.get('imei_slice')) 381 logging.info(' WiFi IP: %s', status.get('wifi_ip')) 382 383 # Update the last devices file(s). 384 for path in args.known_devices_files: 385 device_list.WritePersistentDeviceList( 386 path, [status['serial'] for status in statuses]) 387 388 # Write device info to file for buildbot info display. 389 if os.path.exists('/home/chrome-bot'): 390 with open('/home/chrome-bot/.adb_device_info', 'w') as f: 391 for status in statuses: 392 try: 393 if status['adb_status'] == 'device': 394 f.write('{serial} {adb_status} {build_product} {build_id} ' 395 '{temperature:.1f}C {level}%\n'.format( 396 serial=status['serial'], 397 adb_status=status['adb_status'], 398 build_product=status['type'], 399 build_id=status['build'], 400 temperature=float(status['battery']['temperature']) / 10, 401 level=status['battery']['level'] 402 )) 403 elif status.get('usb_status', False): 404 f.write('{serial} {adb_status}\n'.format( 405 serial=status['serial'], 406 adb_status=status['adb_status'] 407 )) 408 else: 409 f.write('{serial} offline\n'.format( 410 serial=status['serial'] 411 )) 412 except Exception: # pylint: disable=broad-except 413 pass 414 415 # Dump the device statuses to JSON. 416 if args.json_output: 417 with open(args.json_output, 'wb') as f: 418 f.write(json.dumps(statuses, indent=4)) 419 420 live_devices = [status['serial'] for status in statuses 421 if (status['adb_status'] == 'device' 422 and not _IsBlacklisted(status['serial'], blacklist))] 423 424 # If all devices failed, or if there are no devices, it's an infra error. 425 return 0 if live_devices else exit_codes.INFRA 426 427 428 if __name__ == '__main__': 429 sys.exit(main()) 430
