1 #!/usr/bin/env python 2 # 3 # Copyright 2013 The Chromium Authors. All rights reserved. 4 # Use of this source code is governed by a BSD-style license that can be 5 # found in the LICENSE file. 6 7 """A class to keep track of devices across builds and report state.""" 8 import json 9 import logging 10 import optparse 11 import os 12 import psutil 13 import re 14 import signal 15 import smtplib 16 import subprocess 17 import sys 18 import time 19 import urllib 20 21 import bb_annotations 22 import bb_utils 23 24 sys.path.append(os.path.join(os.path.dirname(__file__), 25 os.pardir, os.pardir, 'util', 'lib', 26 'common')) 27 import perf_tests_results_helper # pylint: disable=F0401 28 29 sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 30 from pylib import android_commands 31 from pylib import constants 32 from pylib.cmd_helper import GetCmdOutput 33 from pylib.device import device_blacklist 34 from pylib.device import device_errors 35 from pylib.device import device_list 36 from pylib.device import device_utils 37 38 def DeviceInfo(serial, options): 39 """Gathers info on a device via various adb calls. 40 41 Args: 42 serial: The serial of the attached device to construct info about. 43 44 Returns: 45 Tuple of device type, build id, report as a string, error messages, and 46 boolean indicating whether or not device can be used for testing. 47 """ 48 49 device_adb = device_utils.DeviceUtils(serial) 50 device_type = device_adb.GetProp('ro.build.product') 51 device_build = device_adb.GetProp('ro.build.id') 52 device_build_type = device_adb.GetProp('ro.build.type') 53 device_product_name = device_adb.GetProp('ro.product.name') 54 55 try: 56 battery_info = device_adb.old_interface.GetBatteryInfo() 57 except Exception as e: 58 battery_info = {} 59 logging.error('Unable to obtain battery info for %s, %s', serial, e) 60 61 def _GetData(re_expression, line, lambda_function=lambda x:x): 62 if not line: 63 return 'Unknown' 64 found = re.findall(re_expression, line) 65 if found and len(found): 66 return lambda_function(found[0]) 67 return 'Unknown' 68 69 battery_level = int(battery_info.get('level', 100)) 70 imei_slice = _GetData('Device ID = (\d+)', 71 device_adb.old_interface.GetSubscriberInfo(), 72 lambda x: x[-6:]) 73 report = ['Device %s (%s)' % (serial, device_type), 74 ' Build: %s (%s)' % 75 (device_build, device_adb.GetProp('ro.build.fingerprint')), 76 ' Current Battery Service state: ', 77 '\n'.join([' %s: %s' % (k, v) 78 for k, v in battery_info.iteritems()]), 79 ' IMEI slice: %s' % imei_slice, 80 ' Wifi IP: %s' % device_adb.GetProp('dhcp.wlan0.ipaddress'), 81 ''] 82 83 errors = [] 84 dev_good = True 85 if battery_level < 15: 86 errors += ['Device critically low in battery. Turning off device.'] 87 dev_good = False 88 if not options.no_provisioning_check: 89 setup_wizard_disabled = ( 90 device_adb.GetProp('ro.setupwizard.mode') == 'DISABLED') 91 if not setup_wizard_disabled and device_build_type != 'user': 92 errors += ['Setup wizard not disabled. Was it provisioned correctly?'] 93 if (device_product_name == 'mantaray' and 94 battery_info.get('AC powered', None) != 'true'): 95 errors += ['Mantaray device not connected to AC power.'] 96 97 # Turn off devices with low battery. 98 if battery_level < 15: 99 try: 100 device_adb.EnableRoot() 101 except device_errors.CommandFailedError as e: 102 # Attempt shutdown anyway. 103 # TODO(jbudorick) Handle this exception appropriately after interface 104 # conversions are finished. 105 logging.error(str(e)) 106 device_adb.old_interface.Shutdown() 107 full_report = '\n'.join(report) 108 return device_type, device_build, battery_level, full_report, errors, dev_good 109 110 111 def CheckForMissingDevices(options, adb_online_devs): 112 """Uses file of previous online devices to detect broken phones. 113 114 Args: 115 options: out_dir parameter of options argument is used as the base 116 directory to load and update the cache file. 117 adb_online_devs: A list of serial numbers of the currently visible 118 and online attached devices. 119 """ 120 # TODO(navabi): remove this once the bug that causes different number 121 # of devices to be detected between calls is fixed. 122 logger = logging.getLogger() 123 logger.setLevel(logging.INFO) 124 125 out_dir = os.path.abspath(options.out_dir) 126 127 # last_devices denotes all known devices prior to this run 128 last_devices_path = os.path.join(out_dir, device_list.LAST_DEVICES_FILENAME) 129 last_missing_devices_path = os.path.join(out_dir, 130 device_list.LAST_MISSING_DEVICES_FILENAME) 131 try: 132 last_devices = device_list.GetPersistentDeviceList(last_devices_path) 133 except IOError: 134 # Ignore error, file might not exist 135 last_devices = [] 136 137 try: 138 last_missing_devices = device_list.GetPersistentDeviceList( 139 last_missing_devices_path) 140 except IOError: 141 last_missing_devices = [] 142 143 missing_devs = list(set(last_devices) - set(adb_online_devs)) 144 new_missing_devs = list(set(missing_devs) - set(last_missing_devices)) 145 146 if new_missing_devs and os.environ.get('BUILDBOT_SLAVENAME'): 147 logging.info('new_missing_devs %s' % new_missing_devs) 148 devices_missing_msg = '%d devices not detected.' % len(missing_devs) 149 bb_annotations.PrintSummaryText(devices_missing_msg) 150 151 from_address = 'chrome-bot (at] chromium.org' 152 to_addresses = ['chrome-labs-tech-ticket (at] google.com', 153 'chrome-android-device-alert (at] google.com'] 154 cc_addresses = ['chrome-android-device-alert (at] google.com'] 155 subject = 'Devices offline on %s, %s, %s' % ( 156 os.environ.get('BUILDBOT_SLAVENAME'), 157 os.environ.get('BUILDBOT_BUILDERNAME'), 158 os.environ.get('BUILDBOT_BUILDNUMBER')) 159 msg = ('Please reboot the following devices:\n%s' % 160 '\n'.join(map(str,new_missing_devs))) 161 SendEmail(from_address, to_addresses, cc_addresses, subject, msg) 162 163 all_known_devices = list(set(adb_online_devs) | set(last_devices)) 164 device_list.WritePersistentDeviceList(last_devices_path, all_known_devices) 165 device_list.WritePersistentDeviceList(last_missing_devices_path, missing_devs) 166 167 if not all_known_devices: 168 # This can happen if for some reason the .last_devices file is not 169 # present or if it was empty. 170 return ['No online devices. Have any devices been plugged in?'] 171 if missing_devs: 172 devices_missing_msg = '%d devices not detected.' % len(missing_devs) 173 bb_annotations.PrintSummaryText(devices_missing_msg) 174 175 # TODO(navabi): Debug by printing both output from GetCmdOutput and 176 # GetAttachedDevices to compare results. 177 crbug_link = ('https://code.google.com/p/chromium/issues/entry?summary=' 178 '%s&comment=%s&labels=Restrict-View-Google,OS-Android,Infra' % 179 (urllib.quote('Device Offline'), 180 urllib.quote('Buildbot: %s %s\n' 181 'Build: %s\n' 182 '(please don\'t change any labels)' % 183 (os.environ.get('BUILDBOT_BUILDERNAME'), 184 os.environ.get('BUILDBOT_SLAVENAME'), 185 os.environ.get('BUILDBOT_BUILDNUMBER'))))) 186 return ['Current online devices: %s' % adb_online_devs, 187 '%s are no longer visible. Were they removed?\n' % missing_devs, 188 'SHERIFF:\n', 189 '@@@STEP_LINK@Click here to file a bug@%s@@@\n' % crbug_link, 190 'Cache file: %s\n\n' % last_devices_path, 191 'adb devices: %s' % GetCmdOutput(['adb', 'devices']), 192 'adb devices(GetAttachedDevices): %s' % adb_online_devs] 193 else: 194 new_devs = set(adb_online_devs) - set(last_devices) 195 if new_devs and os.path.exists(last_devices_path): 196 bb_annotations.PrintWarning() 197 bb_annotations.PrintSummaryText( 198 '%d new devices detected' % len(new_devs)) 199 print ('New devices detected %s. And now back to your ' 200 'regularly scheduled program.' % list(new_devs)) 201 202 203 def SendEmail(from_address, to_addresses, cc_addresses, subject, msg): 204 msg_body = '\r\n'.join(['From: %s' % from_address, 205 'To: %s' % ', '.join(to_addresses), 206 'CC: %s' % ', '.join(cc_addresses), 207 'Subject: %s' % subject, '', msg]) 208 try: 209 server = smtplib.SMTP('localhost') 210 server.sendmail(from_address, to_addresses, msg_body) 211 server.quit() 212 except Exception as e: 213 print 'Failed to send alert email. Error: %s' % e 214 215 216 def RestartUsb(): 217 if not os.path.isfile('/usr/bin/restart_usb'): 218 print ('ERROR: Could not restart usb. /usr/bin/restart_usb not installed ' 219 'on host (see BUG=305769).') 220 return False 221 222 lsusb_proc = bb_utils.SpawnCmd(['lsusb'], stdout=subprocess.PIPE) 223 lsusb_output, _ = lsusb_proc.communicate() 224 if lsusb_proc.returncode: 225 print ('Error: Could not get list of USB ports (i.e. lsusb).') 226 return lsusb_proc.returncode 227 228 usb_devices = [re.findall('Bus (\d\d\d) Device (\d\d\d)', lsusb_line)[0] 229 for lsusb_line in lsusb_output.strip().split('\n')] 230 231 all_restarted = True 232 # Walk USB devices from leaves up (i.e reverse sorted) restarting the 233 # connection. If a parent node (e.g. usb hub) is restarted before the 234 # devices connected to it, the (bus, dev) for the hub can change, making the 235 # output we have wrong. This way we restart the devices before the hub. 236 for (bus, dev) in reversed(sorted(usb_devices)): 237 # Can not restart root usb connections 238 if dev != '001': 239 return_code = bb_utils.RunCmd(['/usr/bin/restart_usb', bus, dev]) 240 if return_code: 241 print 'Error restarting USB device /dev/bus/usb/%s/%s' % (bus, dev) 242 all_restarted = False 243 else: 244 print 'Restarted USB device /dev/bus/usb/%s/%s' % (bus, dev) 245 246 return all_restarted 247 248 249 def KillAllAdb(): 250 def GetAllAdb(): 251 for p in psutil.process_iter(): 252 try: 253 if 'adb' in p.name: 254 yield p 255 except (psutil.error.NoSuchProcess, psutil.error.AccessDenied): 256 pass 257 258 for sig in [signal.SIGTERM, signal.SIGQUIT, signal.SIGKILL]: 259 for p in GetAllAdb(): 260 try: 261 print 'kill %d %d (%s [%s])' % (sig, p.pid, p.name, 262 ' '.join(p.cmdline)) 263 p.send_signal(sig) 264 except (psutil.error.NoSuchProcess, psutil.error.AccessDenied): 265 pass 266 for p in GetAllAdb(): 267 try: 268 print 'Unable to kill %d (%s [%s])' % (p.pid, p.name, ' '.join(p.cmdline)) 269 except (psutil.error.NoSuchProcess, psutil.error.AccessDenied): 270 pass 271 272 273 def main(): 274 parser = optparse.OptionParser() 275 parser.add_option('', '--out-dir', 276 help='Directory where the device path is stored', 277 default=os.path.join(constants.DIR_SOURCE_ROOT, 'out')) 278 parser.add_option('--no-provisioning-check', action='store_true', 279 help='Will not check if devices are provisioned properly.') 280 parser.add_option('--device-status-dashboard', action='store_true', 281 help='Output device status data for dashboard.') 282 parser.add_option('--restart-usb', action='store_true', 283 help='Restart USB ports before running device check.') 284 parser.add_option('--json-output', 285 help='Output JSON information into a specified file.') 286 287 options, args = parser.parse_args() 288 if args: 289 parser.error('Unknown options %s' % args) 290 291 # Remove the last build's "bad devices" before checking device statuses. 292 device_blacklist.ResetBlacklist() 293 294 try: 295 expected_devices = device_list.GetPersistentDeviceList( 296 os.path.join(options.out_dir, device_list.LAST_DEVICES_FILENAME)) 297 except IOError: 298 expected_devices = [] 299 devices = android_commands.GetAttachedDevices() 300 # Only restart usb if devices are missing. 301 if set(expected_devices) != set(devices): 302 print 'expected_devices: %s, devices: %s' % (expected_devices, devices) 303 KillAllAdb() 304 retries = 5 305 usb_restarted = True 306 if options.restart_usb: 307 if not RestartUsb(): 308 usb_restarted = False 309 bb_annotations.PrintWarning() 310 print 'USB reset stage failed, wait for any device to come back.' 311 while retries: 312 print 'retry adb devices...' 313 time.sleep(1) 314 devices = android_commands.GetAttachedDevices() 315 if set(expected_devices) == set(devices): 316 # All devices are online, keep going. 317 break 318 if not usb_restarted and devices: 319 # The USB wasn't restarted, but there's at least one device online. 320 # No point in trying to wait for all devices. 321 break 322 retries -= 1 323 324 # TODO(navabi): Test to make sure this fails and then fix call 325 offline_devices = android_commands.GetAttachedDevices( 326 hardware=False, emulator=False, offline=True) 327 328 types, builds, batteries, reports, errors = [], [], [], [], [] 329 fail_step_lst = [] 330 if devices: 331 types, builds, batteries, reports, errors, fail_step_lst = ( 332 zip(*[DeviceInfo(dev, options) for dev in devices])) 333 334 err_msg = CheckForMissingDevices(options, devices) or [] 335 336 unique_types = list(set(types)) 337 unique_builds = list(set(builds)) 338 339 bb_annotations.PrintMsg('Online devices: %d. Device types %s, builds %s' 340 % (len(devices), unique_types, unique_builds)) 341 print '\n'.join(reports) 342 343 for serial, dev_errors in zip(devices, errors): 344 if dev_errors: 345 err_msg += ['%s errors:' % serial] 346 err_msg += [' %s' % error for error in dev_errors] 347 348 if err_msg: 349 bb_annotations.PrintWarning() 350 msg = '\n'.join(err_msg) 351 print msg 352 from_address = 'buildbot (at] chromium.org' 353 to_addresses = ['chromium-android-device-alerts (at] google.com'] 354 bot_name = os.environ.get('BUILDBOT_BUILDERNAME') 355 slave_name = os.environ.get('BUILDBOT_SLAVENAME') 356 subject = 'Device status check errors on %s, %s.' % (slave_name, bot_name) 357 SendEmail(from_address, to_addresses, [], subject, msg) 358 359 if options.device_status_dashboard: 360 perf_tests_results_helper.PrintPerfResult('BotDevices', 'OnlineDevices', 361 [len(devices)], 'devices') 362 perf_tests_results_helper.PrintPerfResult('BotDevices', 'OfflineDevices', 363 [len(offline_devices)], 'devices', 364 'unimportant') 365 for serial, battery in zip(devices, batteries): 366 perf_tests_results_helper.PrintPerfResult('DeviceBattery', serial, 367 [battery], '%', 368 'unimportant') 369 370 if options.json_output: 371 with open(options.json_output, 'wb') as f: 372 f.write(json.dumps({ 373 'online_devices': devices, 374 'offline_devices': offline_devices, 375 'expected_devices': expected_devices, 376 'unique_types': unique_types, 377 'unique_builds': unique_builds, 378 })) 379 380 if False in fail_step_lst: 381 # TODO(navabi): Build fails on device status check step if there exists any 382 # devices with critically low battery. Remove those devices from testing, 383 # allowing build to continue with good devices. 384 return 2 385 386 if not devices: 387 return 1 388 389 390 if __name__ == '__main__': 391 sys.exit(main()) 392
