Home | History | Annotate | Download | only in device
      1 /*
      2  * Copyright (C) 2010 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.tradefed.device;
     18 
     19 import com.android.ddmlib.AndroidDebugBridge.IDeviceChangeListener;
     20 import com.android.ddmlib.DdmPreferences;
     21 import com.android.ddmlib.EmulatorConsole;
     22 import com.android.ddmlib.IDevice;
     23 import com.android.ddmlib.IDevice.DeviceState;
     24 import com.android.ddmlib.Log.LogLevel;
     25 import com.android.tradefed.command.remote.DeviceDescriptor;
     26 import com.android.tradefed.config.GlobalConfiguration;
     27 import com.android.tradefed.config.IGlobalConfiguration;
     28 import com.android.tradefed.config.Option;
     29 import com.android.tradefed.config.OptionClass;
     30 import com.android.tradefed.device.IDeviceMonitor.DeviceLister;
     31 import com.android.tradefed.device.IManagedTestDevice.DeviceEventResponse;
     32 import com.android.tradefed.host.IHostOptions;
     33 import com.android.tradefed.log.ILogRegistry.EventType;
     34 import com.android.tradefed.log.LogRegistry;
     35 import com.android.tradefed.log.LogUtil.CLog;
     36 import com.android.tradefed.util.ArrayUtil;
     37 import com.android.tradefed.util.CommandResult;
     38 import com.android.tradefed.util.CommandStatus;
     39 import com.android.tradefed.util.IRunUtil;
     40 import com.android.tradefed.util.RunUtil;
     41 import com.android.tradefed.util.SizeLimitedOutputStream;
     42 import com.android.tradefed.util.StreamUtil;
     43 import com.android.tradefed.util.TableFormatter;
     44 import com.android.tradefed.util.hostmetric.IHostMonitor;
     45 
     46 import com.google.common.annotations.VisibleForTesting;
     47 
     48 import java.io.IOException;
     49 import java.io.PrintWriter;
     50 import java.lang.reflect.Field;
     51 import java.util.ArrayList;
     52 import java.util.Arrays;
     53 import java.util.Collection;
     54 import java.util.Collections;
     55 import java.util.Comparator;
     56 import java.util.HashMap;
     57 import java.util.HashSet;
     58 import java.util.List;
     59 import java.util.Map;
     60 import java.util.Set;
     61 import java.util.concurrent.CountDownLatch;
     62 import java.util.concurrent.Semaphore;
     63 import java.util.concurrent.TimeUnit;
     64 import java.util.regex.Pattern;
     65 
     66 @OptionClass(alias = "dmgr", global_namespace = false)
     67 public class DeviceManager implements IDeviceManager {
     68 
     69     /** Display string for unknown properties */
     70     public static final String UNKNOWN_DISPLAY_STRING = "unknown";
     71 
     72     /** max wait time in ms for fastboot devices command to complete */
     73     private static final long FASTBOOT_CMD_TIMEOUT = 1 * 60 * 1000;
     74     /** time to wait in ms between fastboot devices requests */
     75     private static final long FASTBOOT_POLL_WAIT_TIME = 5 * 1000;
     76     /**
     77      * time to wait for device adb shell responsive connection before declaring it unavailable for
     78      * testing
     79      */
     80     private static final int CHECK_WAIT_DEVICE_AVAIL_MS = 30 * 1000;
     81 
     82     /* the max size of the emulator output in bytes */
     83     private static final long MAX_EMULATOR_OUTPUT = 20 * 1024 * 1024;
     84 
     85     /* the emulator output log name */
     86     private static final String EMULATOR_OUTPUT = "emulator_log";
     87 
     88     /** a {@link DeviceSelectionOptions} that matches any device. Visible for testing. */
     89     static final IDeviceSelection ANY_DEVICE_OPTIONS = new DeviceSelectionOptions();
     90     private static final String NULL_DEVICE_SERIAL_PREFIX = "null-device";
     91     private static final String EMULATOR_SERIAL_PREFIX = "emulator";
     92     private static final String TCP_DEVICE_SERIAL_PREFIX = "tcp-device";
     93 
     94     /**
     95      * Pattern for a device listed by 'adb devices':
     96      *
     97      * <p>List of devices attached
     98      *
     99      * <p>serial1 device
    100      *
    101      * <p>serial2 device
    102      */
    103     private static final String DEVICE_LIST_PATTERN = "(.*)(\n)(%s)(\\s+)(device)(.*?)";
    104 
    105     private Semaphore mConcurrentFlashLock = null;
    106 
    107     /**
    108      * This serves both as an indication of whether the flash lock should be used, and as an
    109      * indicator of whether or not the flash lock has been initialized -- if this is true
    110      * and {@code mConcurrentFlashLock} is {@code null}, then it has not yet been initialized.
    111      */
    112     private Boolean mShouldCheckFlashLock = true;
    113 
    114     protected DeviceMonitorMultiplexer mDvcMon = new DeviceMonitorMultiplexer();
    115     private Boolean mDvcMonRunning = false;
    116 
    117     private boolean mIsInitialized = false;
    118 
    119     private ManagedDeviceList mManagedDeviceList;
    120 
    121     private IAndroidDebugBridge mAdbBridge;
    122     private ManagedDeviceListener mManagedDeviceListener;
    123     protected boolean mFastbootEnabled;
    124     private Set<IFastbootListener> mFastbootListeners;
    125     private FastbootMonitor mFastbootMonitor;
    126     private boolean mIsTerminated = false;
    127     private IDeviceSelection mGlobalDeviceFilter;
    128 
    129     @Option(name = "max-emulators",
    130             description = "the maximum number of emulators that can be allocated at one time")
    131     private int mNumEmulatorSupported = 1;
    132     @Option(name = "max-null-devices",
    133             description = "the maximum number of no device runs that can be allocated at one time.")
    134     private int mNumNullDevicesSupported = 5;
    135     @Option(name = "max-tcp-devices",
    136             description = "the maximum number of tcp devices that can be allocated at one time")
    137     private int mNumTcpDevicesSupported = 1;
    138 
    139     private boolean mSynchronousMode = false;
    140 
    141     @Option(name = "device-recovery-interval",
    142             description = "the interval in ms between attempts to recover unavailable devices.",
    143             isTimeVal = true)
    144     private long mDeviceRecoveryInterval = 30 * 60 * 1000;
    145 
    146     @Option(name = "adb-path", description = "path of the adb binary to use, "
    147             + "default use the one in $PATH.")
    148     private String mAdbPath = "adb";
    149 
    150     @Option(name = "fastboot-path", description = "path of the fastboot binary to use, "
    151             + "default use the one in $PATH.")
    152     private String mFastbootPath = "fastboot";
    153 
    154     private DeviceRecoverer mDeviceRecoverer;
    155 
    156     private List<IHostMonitor> mGlobalHostMonitors = null;
    157 
    158     /** Counter to wait for the first physical connection before proceeding **/
    159     private CountDownLatch mFirstDeviceAdded = new CountDownLatch(1);
    160 
    161     /** Flag to remember if adb bridge has been disconnected and needs to be reset * */
    162     private boolean mAdbBridgeNeedRestart = false;
    163 
    164     /**
    165      * The DeviceManager should be retrieved from the {@link GlobalConfiguration}
    166      */
    167     public DeviceManager() {
    168     }
    169 
    170     @Override
    171     public void init() {
    172         init(null, null);
    173     }
    174 
    175     /**
    176      * Initialize the device manager. This must be called once and only once before any other
    177      * methods are called.
    178      */
    179     @Override
    180     public void init(IDeviceSelection globalDeviceFilter,
    181             List<IDeviceMonitor> globalDeviceMonitors) {
    182         init(globalDeviceFilter, globalDeviceMonitors,
    183                 new ManagedTestDeviceFactory(mFastbootEnabled, DeviceManager.this, mDvcMon));
    184     }
    185 
    186     /**
    187      * Initialize the device manager. This must be called once and only once before any other
    188      * methods are called.
    189      */
    190     public synchronized void init(IDeviceSelection globalDeviceFilter,
    191             List<IDeviceMonitor> globalDeviceMonitors, IManagedTestDeviceFactory deviceFactory) {
    192         if (mIsInitialized) {
    193             throw new IllegalStateException("already initialized");
    194         }
    195 
    196         if (globalDeviceFilter == null) {
    197             globalDeviceFilter = getGlobalConfig().getDeviceRequirements();
    198         }
    199 
    200         if (globalDeviceMonitors == null) {
    201             globalDeviceMonitors = getGlobalConfig().getDeviceMonitors();
    202         }
    203 
    204         mGlobalHostMonitors = getGlobalConfig().getHostMonitors();
    205         if (mGlobalHostMonitors != null) {
    206             for (IHostMonitor hm : mGlobalHostMonitors) {
    207                 hm.start();
    208             }
    209         }
    210 
    211         mIsInitialized = true;
    212         mGlobalDeviceFilter = globalDeviceFilter;
    213         if (globalDeviceMonitors != null) {
    214             mDvcMon.addMonitors(globalDeviceMonitors);
    215         }
    216         mManagedDeviceList = new ManagedDeviceList(deviceFactory);
    217 
    218         final FastbootHelper fastboot = new FastbootHelper(getRunUtil(), mFastbootPath);
    219         if (fastboot.isFastbootAvailable()) {
    220             mFastbootListeners = Collections.synchronizedSet(new HashSet<IFastbootListener>());
    221             mFastbootMonitor = new FastbootMonitor();
    222             startFastbootMonitor();
    223             // don't set fastboot enabled bit until mFastbootListeners has been initialized
    224             mFastbootEnabled = true;
    225             deviceFactory.setFastbootEnabled(mFastbootEnabled);
    226             // Populate the fastboot devices
    227             // TODO: remove when refactoring fastboot handling
    228             addFastbootDevices();
    229         } else {
    230             CLog.w("Fastboot is not available.");
    231             mFastbootListeners = null;
    232             mFastbootMonitor = null;
    233             mFastbootEnabled = false;
    234             deviceFactory.setFastbootEnabled(mFastbootEnabled);
    235         }
    236 
    237         // don't start adding devices until fastboot support has been established
    238         startAdbBridgeAndDependentServices();
    239     }
    240 
    241     /** Initialize adb connection and services depending on adb connection. */
    242     private synchronized void startAdbBridgeAndDependentServices() {
    243         // TODO: Temporarily increase default timeout as workaround for syncFiles timeouts
    244         DdmPreferences.setTimeOut(120 * 1000);
    245         mAdbBridge = createAdbBridge();
    246         mManagedDeviceListener = new ManagedDeviceListener();
    247         // It's important to add the listener before initializing the ADB bridge to avoid a race
    248         // condition when detecting devices.
    249         mAdbBridge.addDeviceChangeListener(mManagedDeviceListener);
    250         if (mDvcMon != null && !mDvcMonRunning) {
    251             mDvcMon.setDeviceLister(new DeviceLister() {
    252                 @Override
    253                 public List<DeviceDescriptor> listDevices() {
    254                     return listAllDevices();
    255                 }
    256             });
    257             mDvcMon.run();
    258             mDvcMonRunning = true;
    259         }
    260 
    261         mAdbBridge.init(false /* client support */, mAdbPath);
    262         addEmulators();
    263         addNullDevices();
    264         addTcpDevices();
    265 
    266         List<IMultiDeviceRecovery> recoverers = getGlobalConfig().getMultiDeviceRecoveryHandlers();
    267         if (recoverers != null) {
    268             for (IMultiDeviceRecovery recoverer : recoverers) {
    269                 recoverer.setFastbootPath(mFastbootPath);
    270             }
    271             mDeviceRecoverer = new DeviceRecoverer(recoverers);
    272             startDeviceRecoverer();
    273         }
    274     }
    275 
    276 
    277     /**
    278      * Return if adb bridge has been stopped and needs restart.
    279      *
    280      * <p>Exposed for unit testing.
    281      */
    282     @VisibleForTesting
    283     boolean shouldAdbBridgeBeRestarted() {
    284         return mAdbBridgeNeedRestart;
    285     }
    286 
    287     /** {@inheritDoc} */
    288     @Override
    289     public synchronized void restartAdbBridge() {
    290         if (mAdbBridgeNeedRestart) {
    291             mAdbBridgeNeedRestart = false;
    292             startAdbBridgeAndDependentServices();
    293         }
    294     }
    295 
    296     /**
    297      * Instruct DeviceManager whether to use background threads or not.
    298      * <p/>
    299      * Exposed to make unit tests more deterministic.
    300      *
    301      * @param syncMode
    302      */
    303     void setSynchronousMode(boolean syncMode) {
    304         mSynchronousMode = syncMode;
    305     }
    306 
    307     private void checkInit() {
    308         if (!mIsInitialized) {
    309             throw new IllegalStateException("DeviceManager has not been initialized");
    310         }
    311     }
    312 
    313     /**
    314      * Start fastboot monitoring.
    315      * <p/>
    316      * Exposed for unit testing.
    317      */
    318     void startFastbootMonitor() {
    319         mFastbootMonitor.start();
    320     }
    321 
    322     /**
    323      * Start device recovery.
    324      * <p/>
    325      * Exposed for unit testing.
    326      */
    327     void startDeviceRecoverer() {
    328         mDeviceRecoverer.start();
    329     }
    330 
    331     /**
    332      * Get the {@link IGlobalConfiguration} instance to use.
    333      * <p />
    334      * Exposed for unit testing.
    335      */
    336     IGlobalConfiguration getGlobalConfig() {
    337         return GlobalConfiguration.getInstance();
    338     }
    339 
    340     /**
    341      * Gets the {@link IHostOptions} instance to use.
    342      * <p/>
    343      * Exposed for unit testing
    344      */
    345     IHostOptions getHostOptions() {
    346         return getGlobalConfig().getHostOptions();
    347     }
    348 
    349     /**
    350      * Get the {@link RunUtil} instance to use.
    351      * <p/>
    352      * Exposed for unit testing.
    353      */
    354     IRunUtil getRunUtil() {
    355         return RunUtil.getDefault();
    356     }
    357 
    358     /**
    359      * Create a {@link RunUtil} instance to use.
    360      * <p/>
    361      * Exposed for unit testing.
    362      */
    363     IRunUtil createRunUtil() {
    364         return new RunUtil();
    365     }
    366 
    367     /**
    368      * Asynchronously checks if device is available, and adds to queue
    369      *
    370      * @param testDevice
    371      */
    372     private void checkAndAddAvailableDevice(final IManagedTestDevice testDevice) {
    373         if (mGlobalDeviceFilter != null && !mGlobalDeviceFilter.matches(testDevice.getIDevice())) {
    374             CLog.logAndDisplay(LogLevel.INFO, "device %s doesn't match global filter, ignoring",
    375                     testDevice.getSerialNumber());
    376             mManagedDeviceList.handleDeviceEvent(testDevice, DeviceEvent.AVAILABLE_CHECK_IGNORED);
    377             return;
    378         }
    379 
    380         final String threadName = String.format("Check device %s", testDevice.getSerialNumber());
    381         Runnable checkRunnable = new Runnable() {
    382             @Override
    383             public void run() {
    384                 CLog.d("checking new '%s' '%s' responsiveness", testDevice.getClass().getName(),
    385                         testDevice.getSerialNumber());
    386                 if (testDevice.getMonitor().waitForDeviceShell(CHECK_WAIT_DEVICE_AVAIL_MS)) {
    387                     DeviceEventResponse r = mManagedDeviceList.handleDeviceEvent(testDevice,
    388                             DeviceEvent.AVAILABLE_CHECK_PASSED);
    389                     if (r.stateChanged && r.allocationState == DeviceAllocationState.Available) {
    390                         CLog.logAndDisplay(LogLevel.INFO, "Detected new device %s",
    391                                 testDevice.getSerialNumber());
    392                     } else {
    393                         CLog.d("Device %s failed or ignored responsiveness check, ",
    394                                 testDevice.getSerialNumber());
    395                     }
    396                 } else {
    397                     DeviceEventResponse r = mManagedDeviceList.handleDeviceEvent(testDevice,
    398                             DeviceEvent.AVAILABLE_CHECK_FAILED);
    399                     if (r.stateChanged && r.allocationState == DeviceAllocationState.Unavailable) {
    400                         CLog.w("Device %s is unresponsive, will not be available for testing",
    401                                 testDevice.getSerialNumber());
    402                     }
    403                 }
    404             }
    405         };
    406         if (mSynchronousMode) {
    407             checkRunnable.run();
    408         } else {
    409             Thread checkThread = new Thread(checkRunnable, threadName);
    410             // Device checking threads shouldn't hold the JVM open
    411             checkThread.setName("DeviceManager-checkRunnable");
    412             checkThread.setDaemon(true);
    413             checkThread.start();
    414         }
    415     }
    416 
    417     /**
    418      * Add placeholder objects for the max number of 'no device required' concurrent allocations
    419      */
    420     private void addNullDevices() {
    421         for (int i = 0; i < mNumNullDevicesSupported; i++) {
    422             addAvailableDevice(new NullDevice(String.format("%s-%d", NULL_DEVICE_SERIAL_PREFIX, i)));
    423         }
    424     }
    425 
    426     /**
    427      * Add placeholder objects for the max number of emulators that can be allocated
    428      */
    429     private void addEmulators() {
    430         // TODO currently this means 'additional emulators not already running'
    431         int port = 5554;
    432         for (int i = 0; i < mNumEmulatorSupported; i++) {
    433             addAvailableDevice(new StubDevice(String.format("%s-%d", EMULATOR_SERIAL_PREFIX, port),
    434                     true));
    435             port += 2;
    436         }
    437     }
    438 
    439     /**
    440      * Add placeholder objects for the max number of tcp devices that can be connected
    441      */
    442     private void addTcpDevices() {
    443         for (int i = 0; i < mNumTcpDevicesSupported; i++) {
    444             addAvailableDevice(new TcpDevice(String.format("%s-%d", TCP_DEVICE_SERIAL_PREFIX, i)));
    445         }
    446     }
    447 
    448     public void addAvailableDevice(IDevice stubDevice) {
    449         IManagedTestDevice d = mManagedDeviceList.findOrCreate(stubDevice);
    450         if (d != null) {
    451             mManagedDeviceList.handleDeviceEvent(d, DeviceEvent.FORCE_AVAILABLE);
    452         } else {
    453             CLog.e("Could not create stub device");
    454         }
    455     }
    456 
    457     private void addFastbootDevices() {
    458         final FastbootHelper fastboot = new FastbootHelper(getRunUtil(), mFastbootPath);
    459         Set<String> serials = fastboot.getDevices();
    460         for (String serial : serials) {
    461             FastbootDevice d = new FastbootDevice(serial);
    462             if (mGlobalDeviceFilter != null && mGlobalDeviceFilter.matches(d)) {
    463                 addAvailableDevice(d);
    464             }
    465         }
    466     }
    467 
    468     public static class FastbootDevice extends StubDevice {
    469         public FastbootDevice(String serial) {
    470             super(serial, false);
    471         }
    472     }
    473 
    474     /**
    475      * Creates a {@link IDeviceStateMonitor} to use.
    476      * <p/>
    477      * Exposed so unit tests can mock
    478      */
    479     IDeviceStateMonitor createStateMonitor(IDevice device) {
    480         return new DeviceStateMonitor(this, device, mFastbootEnabled);
    481     }
    482 
    483     /**
    484      * {@inheritDoc}
    485      */
    486     @Override
    487     public ITestDevice allocateDevice() {
    488         return allocateDevice(ANY_DEVICE_OPTIONS);
    489     }
    490 
    491     /**
    492      * {@inheritDoc}
    493      */
    494     @Override
    495     public ITestDevice allocateDevice(IDeviceSelection options) {
    496         checkInit();
    497         return mManagedDeviceList.allocate(options);
    498     }
    499 
    500     /**
    501      * {@inheritDoc}
    502      */
    503     @Override
    504     public ITestDevice forceAllocateDevice(String serial) {
    505         checkInit();
    506         IManagedTestDevice d = mManagedDeviceList.findOrCreate(new StubDevice(serial, false));
    507         if (d != null) {
    508             DeviceEventResponse r = d.handleAllocationEvent(DeviceEvent.FORCE_ALLOCATE_REQUEST);
    509             if (r.stateChanged && r.allocationState == DeviceAllocationState.Allocated) {
    510                 return d;
    511             }
    512         }
    513         return null;
    514     }
    515 
    516     /**
    517      * Creates the {@link IAndroidDebugBridge} to use.
    518      * <p/>
    519      * Exposed so tests can mock this.
    520      * @return the {@link IAndroidDebugBridge}
    521      */
    522     synchronized IAndroidDebugBridge createAdbBridge() {
    523         return new AndroidDebugBridgeWrapper();
    524     }
    525 
    526     /**
    527      * {@inheritDoc}
    528      */
    529     @Override
    530     public void freeDevice(ITestDevice device, FreeDeviceState deviceState) {
    531         checkInit();
    532         IManagedTestDevice managedDevice = (IManagedTestDevice)device;
    533         // force stop capturing logcat just to be sure
    534         managedDevice.stopLogcat();
    535         IDevice ideviceToReturn = device.getIDevice();
    536         // don't kill emulator if it wasn't launched by launchEmulator (ie emulatorProcess is null).
    537         if (ideviceToReturn.isEmulator() && managedDevice.getEmulatorProcess() != null) {
    538             try {
    539                 killEmulator(device);
    540                 // stop emulator output log
    541                 device.stopEmulatorOutput();
    542                 // emulator killed - return a stub device
    543                 // TODO: this is a bit of a hack. Consider having DeviceManager inject a StubDevice
    544                 // when deviceDisconnected event is received
    545                 ideviceToReturn = new StubDevice(ideviceToReturn.getSerialNumber(), true);
    546                 deviceState = FreeDeviceState.AVAILABLE;
    547                 managedDevice.setIDevice(ideviceToReturn);
    548             } catch (DeviceNotAvailableException e) {
    549                 CLog.e(e);
    550                 deviceState = FreeDeviceState.UNAVAILABLE;
    551             }
    552         }
    553         if (ideviceToReturn instanceof TcpDevice) {
    554             // Make sure the device goes back to the original state.
    555             managedDevice.setDeviceState(TestDeviceState.NOT_AVAILABLE);
    556         }
    557         DeviceEventResponse r = mManagedDeviceList.handleDeviceEvent(managedDevice,
    558                 getEventFromFree(managedDevice, deviceState));
    559         if (r != null && !r.stateChanged) {
    560             CLog.e("Device %s was in unexpected state %s when freeing", device.getSerialNumber(),
    561                     r.allocationState.toString());
    562         }
    563     }
    564 
    565     /**
    566      * Helper method to convert from a {@link com.android.tradefed.device.FreeDeviceState} to a
    567      * {@link com.android.tradefed.device.DeviceEvent}
    568      *
    569      * @param managedDevice
    570      */
    571     private DeviceEvent getEventFromFree(
    572             IManagedTestDevice managedDevice, FreeDeviceState deviceState) {
    573         switch (deviceState) {
    574             case UNRESPONSIVE:
    575                 return DeviceEvent.FREE_UNRESPONSIVE;
    576             case AVAILABLE:
    577                 return DeviceEvent.FREE_AVAILABLE;
    578             case UNAVAILABLE:
    579                 // We double check if device is still showing in adb or not to confirm the
    580                 // connection is gone.
    581                 if (TestDeviceState.NOT_AVAILABLE.equals(managedDevice.getDeviceState())) {
    582                     String devices = executeGlobalAdbCommand("devices");
    583                     Pattern p =
    584                             Pattern.compile(
    585                                     String.format(
    586                                             DEVICE_LIST_PATTERN, managedDevice.getSerialNumber()));
    587                     if (devices == null || !p.matcher(devices).find()) {
    588                         return DeviceEvent.FREE_UNKNOWN;
    589                     }
    590                 }
    591                 return DeviceEvent.FREE_UNAVAILABLE;
    592             case IGNORE:
    593                 return DeviceEvent.FREE_UNKNOWN;
    594         }
    595         throw new IllegalStateException("unknown FreeDeviceState");
    596     }
    597 
    598     /**
    599      * {@inheritDoc}
    600      */
    601     @Override
    602     public void launchEmulator(ITestDevice device, long bootTimeout, IRunUtil runUtil,
    603             List<String> emulatorArgs)
    604             throws DeviceNotAvailableException {
    605         if (!device.getIDevice().isEmulator()) {
    606             throw new IllegalStateException(String.format("Device %s is not an emulator",
    607                     device.getSerialNumber()));
    608         }
    609         if (!device.getDeviceState().equals(TestDeviceState.NOT_AVAILABLE)) {
    610             throw new IllegalStateException(String.format(
    611                     "Emulator device %s is in state %s. Expected: %s", device.getSerialNumber(),
    612                     device.getDeviceState(), TestDeviceState.NOT_AVAILABLE));
    613         }
    614         List<String> fullArgs = new ArrayList<String>(emulatorArgs);
    615 
    616         try {
    617             CLog.i("launching emulator with %s", fullArgs.toString());
    618             SizeLimitedOutputStream emulatorOutput = new SizeLimitedOutputStream(
    619                     MAX_EMULATOR_OUTPUT, EMULATOR_OUTPUT, ".txt");
    620             Process p = runUtil.runCmdInBackground(fullArgs, emulatorOutput);
    621             // sleep a small amount to wait for process to start successfully
    622             getRunUtil().sleep(500);
    623             assertEmulatorProcessAlive(p, device);
    624             TestDevice testDevice = (TestDevice) device;
    625             testDevice.setEmulatorProcess(p);
    626             testDevice.setEmulatorOutputStream(emulatorOutput);
    627         } catch (IOException e) {
    628             // TODO: is this the most appropriate exception to throw?
    629             throw new DeviceNotAvailableException("Failed to start emulator process", e,
    630                     device.getSerialNumber());
    631         }
    632 
    633         device.waitForDeviceAvailable(bootTimeout);
    634     }
    635 
    636     private void assertEmulatorProcessAlive(Process p, ITestDevice device)
    637             throws DeviceNotAvailableException {
    638         if (!isProcessRunning(p)) {
    639             try {
    640                 CLog.e("Emulator process has died . stdout: '%s', stderr: '%s'",
    641                         StreamUtil.getStringFromStream(p.getInputStream()),
    642                         StreamUtil.getStringFromStream(p.getErrorStream()));
    643             } catch (IOException e) {
    644                 // ignore
    645             }
    646             throw new DeviceNotAvailableException("emulator died after launch",
    647                     device.getSerialNumber());
    648         }
    649     }
    650 
    651     /**
    652      * Check if emulator process has died
    653      *
    654      * @param p the {@link Process} to check
    655      * @return true if process is running, false otherwise
    656      */
    657     private boolean isProcessRunning(Process p) {
    658         try {
    659             p.exitValue();
    660         } catch (IllegalThreadStateException e) {
    661             // expected if process is still alive
    662             return true;
    663         }
    664         return false;
    665     }
    666 
    667     /**
    668      * {@inheritDoc}
    669      */
    670     @Override
    671     public void killEmulator(ITestDevice device) throws DeviceNotAvailableException {
    672         EmulatorConsole console = EmulatorConsole.getConsole(device.getIDevice());
    673         if (console != null) {
    674             console.kill();
    675             // check and wait for device to become not avail
    676             device.waitForDeviceNotAvailable(5 * 1000);
    677             // lets ensure process is killed too - fall through
    678         } else {
    679             CLog.w("Could not get emulator console for %s", device.getSerialNumber());
    680         }
    681         // lets try killing the process
    682         Process emulatorProcess = ((IManagedTestDevice) device).getEmulatorProcess();
    683         if (emulatorProcess != null) {
    684             emulatorProcess.destroy();
    685             if (isProcessRunning(emulatorProcess)) {
    686                 CLog.w("Emulator process still running after destroy for %s",
    687                         device.getSerialNumber());
    688                 forceKillProcess(emulatorProcess, device.getSerialNumber());
    689             }
    690         }
    691         if (!device.waitForDeviceNotAvailable(20 * 1000)) {
    692             throw new DeviceNotAvailableException(String.format("Failed to kill emulator %s",
    693                     device.getSerialNumber()), device.getSerialNumber());
    694         }
    695     }
    696 
    697     /**
    698      * Disgusting hack alert! Attempt to force kill given process.
    699      * Relies on implementation details. Only works on linux
    700      *
    701      * @param emulatorProcess the {@link Process} to kill
    702      * @param emulatorSerial the serial number of emulator. Only used for logging
    703      */
    704     private void forceKillProcess(Process emulatorProcess, String emulatorSerial) {
    705         if (emulatorProcess.getClass().getName().equals("java.lang.UNIXProcess")) {
    706             try {
    707                 CLog.i("Attempting to force kill emulator process for %s", emulatorSerial);
    708                 Field f = emulatorProcess.getClass().getDeclaredField("pid");
    709                 f.setAccessible(true);
    710                 Integer pid = (Integer)f.get(emulatorProcess);
    711                 if (pid != null) {
    712                     RunUtil.getDefault().runTimedCmd(5 * 1000, "kill", "-9", pid.toString());
    713                 }
    714             } catch (NoSuchFieldException e) {
    715                 CLog.d("got NoSuchFieldException when attempting to read process pid");
    716             } catch (IllegalAccessException e) {
    717                 CLog.d("got IllegalAccessException when attempting to read process pid");
    718             }
    719         }
    720     }
    721 
    722     /**
    723      * {@inheritDoc}
    724      */
    725     @Override
    726     public ITestDevice connectToTcpDevice(String ipAndPort) {
    727         ITestDevice tcpDevice = forceAllocateDevice(ipAndPort);
    728         if (tcpDevice == null) {
    729             return null;
    730         }
    731         if (doAdbConnect(ipAndPort)) {
    732             try {
    733                 tcpDevice.setRecovery(new WaitDeviceRecovery());
    734                 tcpDevice.waitForDeviceOnline();
    735                 return tcpDevice;
    736             } catch (DeviceNotAvailableException e) {
    737                 CLog.w("Device with tcp serial %s did not come online", ipAndPort);
    738             }
    739         }
    740         freeDevice(tcpDevice, FreeDeviceState.IGNORE);
    741         return null;
    742     }
    743 
    744     /**
    745      * {@inheritDoc}
    746      */
    747     @Override
    748     public ITestDevice reconnectDeviceToTcp(ITestDevice usbDevice)
    749             throws DeviceNotAvailableException {
    750         CLog.i("Reconnecting device %s to adb over tcpip", usbDevice.getSerialNumber());
    751         ITestDevice tcpDevice = null;
    752         if (usbDevice instanceof IManagedTestDevice) {
    753             IManagedTestDevice managedUsbDevice = (IManagedTestDevice) usbDevice;
    754             String ipAndPort = managedUsbDevice.switchToAdbTcp();
    755             if (ipAndPort != null) {
    756                 CLog.d("Device %s was switched to adb tcp on %s", usbDevice.getSerialNumber(),
    757                         ipAndPort);
    758                 tcpDevice = connectToTcpDevice(ipAndPort);
    759                 if (tcpDevice == null) {
    760                     // ruh roh, could not connect to device
    761                     // Try to re-establish connection back to usb device
    762                     managedUsbDevice.recoverDevice();
    763                 }
    764             }
    765         } else {
    766             CLog.e("reconnectDeviceToTcp: unrecognized device type.");
    767         }
    768         return tcpDevice;
    769     }
    770 
    771     @Override
    772     public boolean disconnectFromTcpDevice(ITestDevice tcpDevice) {
    773         CLog.i("Disconnecting and freeing tcp device %s", tcpDevice.getSerialNumber());
    774         boolean result = false;
    775         try {
    776             result = tcpDevice.switchToAdbUsb();
    777         } catch (DeviceNotAvailableException e) {
    778             CLog.w("Failed to switch device %s to usb mode: %s", tcpDevice.getSerialNumber(),
    779                     e.getMessage());
    780         }
    781         freeDevice(tcpDevice, FreeDeviceState.IGNORE);
    782         return result;
    783     }
    784 
    785     private boolean doAdbConnect(String ipAndPort) {
    786         final String resultSuccess = String.format("connected to %s", ipAndPort);
    787         for (int i = 1; i <= 3; i++) {
    788             String adbConnectResult = executeGlobalAdbCommand("connect", ipAndPort);
    789             // runcommand "adb connect ipAndPort"
    790             if (adbConnectResult != null && adbConnectResult.startsWith(resultSuccess)) {
    791                 return true;
    792             }
    793             CLog.w("Failed to connect to device on %s, attempt %d of 3. Response: %s.",
    794                     ipAndPort, i, adbConnectResult);
    795             getRunUtil().sleep(5 * 1000);
    796         }
    797         return false;
    798     }
    799 
    800     /**
    801      * Execute a adb command not targeted to a particular device eg. 'adb connect'
    802      *
    803      * @param cmdArgs
    804      * @return std output if the command succeedm null otherwise.
    805      */
    806     public String executeGlobalAdbCommand(String... cmdArgs) {
    807         String[] fullCmd = ArrayUtil.buildArray(new String[] {"adb"}, cmdArgs);
    808         CommandResult result = getRunUtil().runTimedCmd(FASTBOOT_CMD_TIMEOUT, fullCmd);
    809         if (CommandStatus.SUCCESS.equals(result.getStatus())) {
    810             return result.getStdout();
    811         }
    812         CLog.w("adb %s failed", cmdArgs[0]);
    813         return null;
    814     }
    815 
    816     /**
    817      * {@inheritDoc}
    818      */
    819     @Override
    820     public synchronized void terminate() {
    821         checkInit();
    822         if (!mIsTerminated) {
    823             mIsTerminated = true;
    824             stopAdbBridgeAndDependentServices();
    825             // We are not terminating mFastbootMonitor here since it is a daemon thread.
    826             // Early terminating it can cause other threads to be blocked if they check
    827             // fastboot state of a device.
    828             if (mGlobalHostMonitors != null ) {
    829                 for (IHostMonitor hm : mGlobalHostMonitors) {
    830                     hm.terminate();
    831                 }
    832             }
    833         }
    834     }
    835 
    836     /** Stop adb bridge and services depending on adb connection. */
    837     private synchronized void stopAdbBridgeAndDependentServices() {
    838         terminateDeviceRecovery();
    839         mAdbBridge.removeDeviceChangeListener(mManagedDeviceListener);
    840         mAdbBridge.terminate();
    841     }
    842 
    843     /** {@inheritDoc} */
    844     @Override
    845     public synchronized void stopAdbBridge() {
    846         stopAdbBridgeAndDependentServices();
    847         mAdbBridgeNeedRestart = true;
    848     }
    849 
    850     /** {@inheritDoc} */
    851     @Override
    852     public synchronized void terminateDeviceRecovery() {
    853         if (mDeviceRecoverer != null) {
    854             mDeviceRecoverer.terminate();
    855         }
    856     }
    857 
    858     /** {@inheritDoc} */
    859     @Override
    860     public synchronized void terminateDeviceMonitor() {
    861         mDvcMon.stop();
    862     }
    863 
    864     /** {@inheritDoc} */
    865     @Override
    866     public synchronized void terminateHard() {
    867         checkInit();
    868         if (!mIsTerminated ) {
    869             for (IManagedTestDevice device : mManagedDeviceList) {
    870                 device.setRecovery(new AbortRecovery());
    871             }
    872             mAdbBridge.disconnectBridge();
    873             terminate();
    874         }
    875     }
    876 
    877     private static class AbortRecovery implements IDeviceRecovery {
    878 
    879         /**
    880          * {@inheritDoc}
    881          */
    882         @Override
    883         public void recoverDevice(IDeviceStateMonitor monitor, boolean recoverUntilOnline)
    884                 throws DeviceNotAvailableException {
    885             throw new DeviceNotAvailableException("aborted test session",
    886                     monitor.getSerialNumber());
    887         }
    888 
    889         /**
    890          * {@inheritDoc}
    891          */
    892         @Override
    893         public void recoverDeviceBootloader(IDeviceStateMonitor monitor)
    894                 throws DeviceNotAvailableException {
    895             throw new DeviceNotAvailableException("aborted test session",
    896                     monitor.getSerialNumber());
    897         }
    898 
    899         /**
    900          * {@inheritDoc}
    901          */
    902         @Override
    903         public void recoverDeviceRecovery(IDeviceStateMonitor monitor)
    904                 throws DeviceNotAvailableException {
    905             throw new DeviceNotAvailableException("aborted test session",
    906                     monitor.getSerialNumber());
    907         }
    908     }
    909 
    910 
    911     /**
    912      * {@inheritDoc}
    913      */
    914     @Override
    915     public List<DeviceDescriptor> listAllDevices() {
    916         final List<DeviceDescriptor> serialStates = new ArrayList<DeviceDescriptor>();
    917         IDeviceSelection selector = getDeviceSelectionOptions();
    918         for (IManagedTestDevice d : mManagedDeviceList) {
    919             IDevice idevice = d.getIDevice();
    920             serialStates.add(
    921                     new DeviceDescriptor(
    922                             idevice.getSerialNumber(),
    923                             idevice instanceof StubDevice,
    924                             idevice.getState(),
    925                             d.getAllocationState(),
    926                             getDisplay(selector.getDeviceProductType(idevice)),
    927                             getDisplay(selector.getDeviceProductVariant(idevice)),
    928                             getDisplay(idevice.getProperty("ro.build.version.sdk")),
    929                             getDisplay(idevice.getProperty("ro.build.id")),
    930                             getDisplay(selector.getBatteryLevel(idevice)),
    931                             d.getDeviceClass(),
    932                             getDisplay(d.getMacAddress()),
    933                             getDisplay(d.getSimState()),
    934                             getDisplay(d.getSimOperator()),
    935                             idevice));
    936         }
    937         return serialStates;
    938     }
    939 
    940     @Override
    941     public void displayDevicesInfo(PrintWriter stream) {
    942         ArrayList<List<String>> displayRows = new ArrayList<List<String>>();
    943         displayRows.add(Arrays.asList("Serial", "State", "Allocation", "Product", "Variant",
    944                 "Build", "Battery"));
    945         List<DeviceDescriptor> deviceList = listAllDevices();
    946         sortDeviceList(deviceList);
    947         addDevicesInfo(displayRows, deviceList);
    948         new TableFormatter().displayTable(displayRows, stream);
    949     }
    950 
    951     /**
    952      * Sorts list by state, then by serial.
    953      */
    954     @VisibleForTesting
    955     static List<DeviceDescriptor> sortDeviceList(List<DeviceDescriptor> deviceList) {
    956 
    957         Comparator<DeviceDescriptor> c = new Comparator<DeviceDescriptor>() {
    958 
    959             @Override
    960             public int compare(DeviceDescriptor o1, DeviceDescriptor o2) {
    961                 if (o1.getState() != o2.getState()) {
    962                     // sort by state
    963                     return o1.getState().toString()
    964                             .compareTo(o2.getState().toString());
    965                 }
    966                 // states are equal, sort by serial
    967                 return o1.getSerial().compareTo(o2.getSerial());
    968             }
    969 
    970         };
    971         Collections.sort(deviceList, c);
    972         return deviceList;
    973     }
    974 
    975     /**
    976      * Get the {@link IDeviceSelection} to use to display device info
    977      * <p/>
    978      * Exposed for unit testing.
    979      */
    980     IDeviceSelection getDeviceSelectionOptions() {
    981         return new DeviceSelectionOptions();
    982     }
    983 
    984     private void addDevicesInfo(List<List<String>> displayRows,
    985             List<DeviceDescriptor> sortedDeviceList) {
    986         for (DeviceDescriptor desc : sortedDeviceList) {
    987             if (desc.isStubDevice() &&
    988                     desc.getState() != DeviceAllocationState.Allocated) {
    989                 // don't add placeholder devices
    990                 continue;
    991             }
    992             displayRows.add(Arrays.asList(
    993                     desc.getSerial(),
    994                     desc.getDeviceState().toString(),
    995                     desc.getState().toString(),
    996                     desc.getProduct(),
    997                     desc.getProductVariant(),
    998                     desc.getBuildId(),
    999                     desc.getBatteryLevel())
   1000                     );
   1001         }
   1002     }
   1003 
   1004     /**
   1005      * Return the displayable string for given object
   1006      */
   1007     private String getDisplay(Object o) {
   1008         return o == null ? UNKNOWN_DISPLAY_STRING : o.toString();
   1009     }
   1010 
   1011     /**
   1012      * A class to listen for and act on device presence updates from ddmlib
   1013      */
   1014     private class ManagedDeviceListener implements IDeviceChangeListener {
   1015 
   1016         /**
   1017          * {@inheritDoc}
   1018          */
   1019         @Override
   1020         public void deviceChanged(IDevice idevice, int changeMask) {
   1021             if ((changeMask & IDevice.CHANGE_STATE) != 0) {
   1022                 IManagedTestDevice testDevice = mManagedDeviceList.findOrCreate(idevice);
   1023                 if (testDevice == null) {
   1024                     return;
   1025                 }
   1026                 TestDeviceState newState = TestDeviceState.getStateByDdms(idevice.getState());
   1027                 testDevice.setDeviceState(newState);
   1028                 if (newState == TestDeviceState.ONLINE) {
   1029                     DeviceEventResponse r = mManagedDeviceList.handleDeviceEvent(testDevice,
   1030                             DeviceEvent.STATE_CHANGE_ONLINE);
   1031                     if (r.stateChanged && r.allocationState ==
   1032                             DeviceAllocationState.Checking_Availability) {
   1033                         checkAndAddAvailableDevice(testDevice);
   1034                     }
   1035                 } else if (DeviceState.OFFLINE.equals(idevice.getState()) ||
   1036                         DeviceState.UNAUTHORIZED.equals(idevice.getState())) {
   1037                     // handle device changing to offline or unauthorized.
   1038                     mManagedDeviceList.handleDeviceEvent(testDevice,
   1039                             DeviceEvent.STATE_CHANGE_OFFLINE);
   1040                 }
   1041             }
   1042         }
   1043 
   1044         /**
   1045          * {@inheritDoc}
   1046          */
   1047         @Override
   1048         public void deviceConnected(IDevice idevice) {
   1049             CLog.d("Detected device connect %s, id %d", idevice.getSerialNumber(),
   1050                     idevice.hashCode());
   1051             String threadName = String.format("Connected device %s", idevice.getSerialNumber());
   1052             Runnable connectedRunnable =
   1053                     new Runnable() {
   1054                         @Override
   1055                         public void run() {
   1056                             IManagedTestDevice testDevice =
   1057                                     mManagedDeviceList.findOrCreate(idevice);
   1058                             if (testDevice == null) {
   1059                                 return;
   1060                             }
   1061                             // DDMS will allocate a new IDevice, so need
   1062                             // to update the TestDevice record with the new device
   1063                             CLog.d("Updating IDevice for device %s", idevice.getSerialNumber());
   1064                             testDevice.setIDevice(idevice);
   1065                             TestDeviceState newState =
   1066                                     TestDeviceState.getStateByDdms(idevice.getState());
   1067                             testDevice.setDeviceState(newState);
   1068                             if (newState == TestDeviceState.ONLINE) {
   1069                                 DeviceEventResponse r =
   1070                                         mManagedDeviceList.handleDeviceEvent(
   1071                                                 testDevice, DeviceEvent.CONNECTED_ONLINE);
   1072                                 if (r.stateChanged
   1073                                         && r.allocationState
   1074                                                 == DeviceAllocationState.Checking_Availability) {
   1075                                     checkAndAddAvailableDevice(testDevice);
   1076                                 }
   1077                                 logDeviceEvent(
   1078                                         EventType.DEVICE_CONNECTED, testDevice.getSerialNumber());
   1079                             } else if (DeviceState.OFFLINE.equals(idevice.getState())
   1080                                     || DeviceState.UNAUTHORIZED.equals(idevice.getState())) {
   1081                                 mManagedDeviceList.handleDeviceEvent(
   1082                                         testDevice, DeviceEvent.CONNECTED_OFFLINE);
   1083                                 logDeviceEvent(
   1084                                         EventType.DEVICE_CONNECTED_OFFLINE,
   1085                                         testDevice.getSerialNumber());
   1086                             }
   1087                             mFirstDeviceAdded.countDown();
   1088                         }
   1089                     };
   1090 
   1091             if (mSynchronousMode) {
   1092                 connectedRunnable.run();
   1093             } else {
   1094                 // Device creation step can take a little bit of time, so do it in a thread to
   1095                 // avoid blocking following events of new devices
   1096                 Thread checkThread = new Thread(connectedRunnable, threadName);
   1097                 // Device checking threads shouldn't hold the JVM open
   1098                 checkThread.setDaemon(true);
   1099                 checkThread.start();
   1100             }
   1101         }
   1102 
   1103         /**
   1104          * {@inheritDoc}
   1105          */
   1106         @Override
   1107         public void deviceDisconnected(IDevice disconnectedDevice) {
   1108             IManagedTestDevice d = mManagedDeviceList.find(disconnectedDevice.getSerialNumber());
   1109             if (d != null) {
   1110                 mManagedDeviceList.handleDeviceEvent(d, DeviceEvent.DISCONNECTED);
   1111                 d.setDeviceState(TestDeviceState.NOT_AVAILABLE);
   1112                 logDeviceEvent(EventType.DEVICE_DISCONNECTED, disconnectedDevice.getSerialNumber());
   1113             }
   1114         }
   1115     }
   1116 
   1117     /** Helper to log the device events. */
   1118     @VisibleForTesting
   1119     void logDeviceEvent(EventType event, String serial) {
   1120         Map<String, String> args = new HashMap<>();
   1121         args.put("serial", serial);
   1122         LogRegistry.getLogRegistry().logEvent(LogLevel.DEBUG, event, args);
   1123     }
   1124 
   1125     /** {@inheritDoc} */
   1126     @Override
   1127     public boolean waitForFirstDeviceAdded(long timeout) {
   1128         try {
   1129             return mFirstDeviceAdded.await(timeout, TimeUnit.MILLISECONDS);
   1130         } catch (InterruptedException e) {
   1131             throw new RuntimeException(e);
   1132         }
   1133     }
   1134 
   1135     /**
   1136      * {@inheritDoc}
   1137      */
   1138     @Override
   1139     public void addFastbootListener(IFastbootListener listener) {
   1140         checkInit();
   1141         if (mFastbootEnabled) {
   1142             mFastbootListeners.add(listener);
   1143         } else {
   1144             throw new UnsupportedOperationException("fastboot is not enabled");
   1145         }
   1146     }
   1147 
   1148     /**
   1149      * {@inheritDoc}
   1150      */
   1151     @Override
   1152     public void removeFastbootListener(IFastbootListener listener) {
   1153         checkInit();
   1154         if (mFastbootEnabled) {
   1155             mFastbootListeners.remove(listener);
   1156         }
   1157     }
   1158 
   1159     /**
   1160      * A class to monitor and update fastboot state of devices.
   1161      */
   1162     private class FastbootMonitor extends Thread {
   1163 
   1164         private boolean mQuit = false;
   1165 
   1166         FastbootMonitor() {
   1167             super("FastbootMonitor");
   1168             setDaemon(true);
   1169         }
   1170 
   1171         @Override
   1172         public void interrupt() {
   1173             mQuit = true;
   1174             super.interrupt();
   1175         }
   1176 
   1177         @Override
   1178         public void run() {
   1179             final FastbootHelper fastboot = new FastbootHelper(getRunUtil(), mFastbootPath);
   1180             while (!mQuit) {
   1181                 Set<String> serials = fastboot.getDevices();
   1182                 if (serials != null) {
   1183                     // Update known fastboot devices state
   1184                     mManagedDeviceList.updateFastbootStates(serials);
   1185                     // Add new fastboot devices.
   1186                     for (String serial : serials) {
   1187                         FastbootDevice d = new FastbootDevice(serial);
   1188                         if (mGlobalDeviceFilter != null && mGlobalDeviceFilter.matches(d)) {
   1189                             addAvailableDevice(d);
   1190                         }
   1191                     }
   1192                 }
   1193                 if (!mFastbootListeners.isEmpty()) {
   1194                     // create a copy of listeners for notification to prevent deadlocks
   1195                     Collection<IFastbootListener> listenersCopy =
   1196                             new ArrayList<IFastbootListener>(mFastbootListeners.size());
   1197                     listenersCopy.addAll(mFastbootListeners);
   1198                     for (IFastbootListener listener : listenersCopy) {
   1199                         listener.stateUpdated();
   1200                     }
   1201                 }
   1202                 getRunUtil().sleep(FASTBOOT_POLL_WAIT_TIME);
   1203             }
   1204         }
   1205     }
   1206 
   1207     /**
   1208      * A class for a thread which performs periodic device recovery operations.
   1209      */
   1210     private class DeviceRecoverer extends Thread {
   1211 
   1212         private boolean mQuit = false;
   1213         private List<IMultiDeviceRecovery> mMultiDeviceRecoverers;
   1214 
   1215         public DeviceRecoverer(List<IMultiDeviceRecovery> multiDeviceRecoverers) {
   1216             super("DeviceRecoverer");
   1217             mMultiDeviceRecoverers = multiDeviceRecoverers;
   1218             // Ensure that this thread doesn't prevent TF from terminating
   1219             setDaemon(true);
   1220         }
   1221 
   1222         @Override
   1223         public void run() {
   1224             while (!mQuit) {
   1225                 getRunUtil().sleep(mDeviceRecoveryInterval);
   1226                 if (mQuit) {
   1227                     // After the sleep time, we check if we should run or not.
   1228                     return;
   1229                 }
   1230                 if (mMultiDeviceRecoverers != null && !mMultiDeviceRecoverers.isEmpty()) {
   1231                     for (IMultiDeviceRecovery m : mMultiDeviceRecoverers) {
   1232                         m.recoverDevices(getDeviceList());
   1233                     }
   1234                 }
   1235             }
   1236         }
   1237 
   1238         public void terminate() {
   1239             mQuit = true;
   1240             interrupt();
   1241         }
   1242     }
   1243 
   1244     @VisibleForTesting
   1245     List<IManagedTestDevice> getDeviceList() {
   1246         return mManagedDeviceList.getCopy();
   1247     }
   1248 
   1249     @VisibleForTesting
   1250     void setMaxEmulators(int numEmulators) {
   1251         mNumEmulatorSupported = numEmulators;
   1252     }
   1253 
   1254     @VisibleForTesting
   1255     void setMaxNullDevices(int nullDevices) {
   1256         mNumNullDevicesSupported = nullDevices;
   1257     }
   1258 
   1259     @VisibleForTesting
   1260     void setMaxTcpDevices(int tcpDevices) {
   1261         mNumTcpDevicesSupported = tcpDevices;
   1262     }
   1263 
   1264     @Override
   1265     public boolean isNullDevice(String serial) {
   1266         return serial.startsWith(NULL_DEVICE_SERIAL_PREFIX);
   1267     }
   1268 
   1269     @Override
   1270     public boolean isEmulator(String serial) {
   1271         return serial.startsWith(EMULATOR_SERIAL_PREFIX);
   1272     }
   1273 
   1274     @Override
   1275     public void addDeviceMonitor(IDeviceMonitor mon) {
   1276         mDvcMon.addMonitor(mon);
   1277     }
   1278 
   1279     @Override
   1280     public void removeDeviceMonitor(IDeviceMonitor mon) {
   1281         mDvcMon.removeMonitor(mon);
   1282     }
   1283 
   1284     @Override
   1285     public String getFastbootPath() {
   1286         return mFastbootPath;
   1287     }
   1288 
   1289     /**
   1290      * Set the state of the concurrent flash limit implementation
   1291      *
   1292      * Exposed for unit testing
   1293      */
   1294     void setConcurrentFlashSettings(Semaphore flashLock, boolean shouldCheck) {
   1295         synchronized (mShouldCheckFlashLock) {
   1296             mConcurrentFlashLock = flashLock;
   1297             mShouldCheckFlashLock = shouldCheck;
   1298         }
   1299     }
   1300 
   1301     Semaphore getConcurrentFlashLock() {
   1302         return mConcurrentFlashLock;
   1303     }
   1304 
   1305     /** Initialize the concurrent flash lock semaphore **/
   1306     private void initConcurrentFlashLock() {
   1307         if (!mShouldCheckFlashLock) return;
   1308         // The logic below is to avoid multi-thread race conditions while initializing
   1309         // mConcurrentFlashLock when we hit this condition.
   1310         if (mConcurrentFlashLock == null) {
   1311             // null with mShouldCheckFlashLock == true means initialization hasn't been done yet
   1312             synchronized(mShouldCheckFlashLock) {
   1313                 // Check all state again, since another thread might have gotten here first
   1314                 if (!mShouldCheckFlashLock) return;
   1315 
   1316                 IHostOptions hostOptions = getHostOptions();
   1317                 Integer concurrentFlashingLimit = hostOptions.getConcurrentFlasherLimit();
   1318 
   1319                 if (concurrentFlashingLimit == null) {
   1320                     mShouldCheckFlashLock = false;
   1321                     return;
   1322                 }
   1323 
   1324                 if (mConcurrentFlashLock == null) {
   1325                     mConcurrentFlashLock = new Semaphore(concurrentFlashingLimit, true /* fair */);
   1326                 }
   1327             }
   1328         }
   1329     }
   1330 
   1331     /** {@inheritDoc} */
   1332     @Override
   1333     public int getAvailableFlashingPermits() {
   1334         initConcurrentFlashLock();
   1335         if (mConcurrentFlashLock != null) {
   1336             return mConcurrentFlashLock.availablePermits();
   1337         }
   1338         IHostOptions hostOptions = getHostOptions();
   1339         if (hostOptions.getConcurrentFlasherLimit() != null) {
   1340             return hostOptions.getConcurrentFlasherLimit();
   1341         }
   1342         return Integer.MAX_VALUE;
   1343     }
   1344 
   1345     /** {@inheritDoc} */
   1346     @Override
   1347     public void takeFlashingPermit() {
   1348         initConcurrentFlashLock();
   1349         if (!mShouldCheckFlashLock) return;
   1350 
   1351         IHostOptions hostOptions = getHostOptions();
   1352         Integer concurrentFlashingLimit = hostOptions.getConcurrentFlasherLimit();
   1353         CLog.i(
   1354                 "Requesting a flashing permit out of the max limit of %s. Current queue "
   1355                         + "length: %s",
   1356                 concurrentFlashingLimit,
   1357                 mConcurrentFlashLock.getQueueLength());
   1358         mConcurrentFlashLock.acquireUninterruptibly();
   1359     }
   1360 
   1361     /** {@inheritDoc} */
   1362     @Override
   1363     public void returnFlashingPermit() {
   1364         if (mConcurrentFlashLock != null) {
   1365             mConcurrentFlashLock.release();
   1366         }
   1367     }
   1368 
   1369     /** {@inheritDoc} */
   1370     @Override
   1371     public String getAdbVersion() {
   1372         return mAdbBridge.getAdbVersion(mAdbPath);
   1373     }
   1374 }
   1375