junit/runtime/RemoteAdtTestRunner.java

/*
 * Copyright (C) 2009 The Android Open Source Project
 *
 * Licensed under the Eclipse Public License, Version 1.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.eclipse.org/org/documents/epl-v10.php
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.ide.eclipse.adt.internal.launch.junit.runtime;

import com.android.ddmlib.AdbCommandRejectedException;
import com.android.ddmlib.IDevice;
import com.android.ddmlib.ShellCommandUnresponsiveException;
import com.android.ddmlib.TimeoutException;
import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner.TestSize;
import com.android.ddmlib.testrunner.ITestRunListener;
import com.android.ddmlib.testrunner.RemoteAndroidTestRunner;
import com.android.ddmlib.testrunner.TestIdentifier;
import com.android.ide.eclipse.adt.AdtPlugin;
import com.android.ide.eclipse.adt.internal.launch.LaunchMessages;

import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.jdt.internal.junit.runner.IListensToTestExecutions;
import org.eclipse.jdt.internal.junit.runner.ITestReference;
import org.eclipse.jdt.internal.junit.runner.MessageIds;
import org.eclipse.jdt.internal.junit.runner.RemoteTestRunner;
import org.eclipse.jdt.internal.junit.runner.TestExecution;
import org.eclipse.jdt.internal.junit.runner.TestReferenceFailure;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * Supports Eclipse JUnit execution of Android tests.
 * <p/>
 * Communicates back to a Eclipse JDT JUnit client via a socket connection.
 *
 * @see org.eclipse.jdt.internal.junit.runner.RemoteTestRunner for more details on the protocol
 */
@SuppressWarnings("restriction")
public class RemoteAdtTestRunner extends RemoteTestRunner {

    private static final String DELAY_MSEC_KEY = "delay_msec";
    /** the delay between each test execution when in collecting test info */
    private static final String COLLECT_TEST_DELAY_MS = "15";

    private AndroidJUnitLaunchInfo mLaunchInfo;
    private TestExecution mExecution;

    /**
     * Initialize the JDT JUnit test runner parameters from the {@code args}.
     *
     * @param args name-value pair of arguments to pass to parent JUnit runner.
     * @param launchInfo the Android specific test launch info
     */
    protected void init(String[] args, AndroidJUnitLaunchInfo launchInfo) {
        defaultInit(args);
        mLaunchInfo = launchInfo;
    }

    /**
     * Runs a set of tests, and reports back results using parent class.
     * <p/>
     * JDT Unit expects to be sent data in the following sequence:
     * <ol>
     *   <li>The total number of tests to be executed.</li>
     *   <li>The test 'tree' data about the tests to be executed, which is composed of the set of
     *   test class names, the number of tests in each class, and the names of each test in the
     *   class.</li>
     *   <li>The test execution result for each test method. Expects individual notifications of
     *   the test execution start, any failures, and the end of the test execution.</li>
     *   <li>The end of the test run, with its elapsed time.</li>
     * </ol>
     * <p/>
     * In order to satisfy this, this method performs two actual Android instrumentation runs.
     * The first is a 'log only' run that will collect the test tree data, without actually
     * executing the tests,  and send it back to JDT JUnit. The second is the actual test execution,
     * whose results will be communicated back in real-time to JDT JUnit.
     *
     * The tests are run concurrently on all devices. The overall structure is as follows:
     * <ol>
     *   <li> First, a separate job per device is run to collect test tree data. A per device
     *        {@link TestCollector} records information regarding the tests run on the device.
     *        </li>
     *   <li> Once all the devices have finished collecting the test tree data, the tree info is
     *        collected from all of them and passed to the Junit UI </li>
     *   <li> A job per device is again launched to do the actual test run. A per device
     *        {@link TestRunListener} notifies the shared {@link TestResultsNotifier} of test
     *        status. </li>
     *   <li> As tests complete, the test run listener updates the Junit UI </li>
     * </ol>
     *
     * @param testClassNames ignored - the AndroidJUnitLaunchInfo will be used to determine which
     *     tests to run.
     * @param testName ignored
     * @param execution used to report test progress
     */
    @Override
    public void runTests(String[] testClassNames, String testName, TestExecution execution) {
        // hold onto this execution reference so it can be used to report test progress
        mExecution = execution;

        List<IDevice> devices = new ArrayList<IDevice>(mLaunchInfo.getDevices());
        List<RemoteAndroidTestRunner> runners =
                new ArrayList<RemoteAndroidTestRunner>(devices.size());

        for (IDevice device : devices) {
            RemoteAndroidTestRunner runner = new RemoteAndroidTestRunner(
                    mLaunchInfo.getAppPackage(), mLaunchInfo.getRunner(), device);

            if (mLaunchInfo.getTestClass() != null) {
                if (mLaunchInfo.getTestMethod() != null) {
                    runner.setMethodName(mLaunchInfo.getTestClass(), mLaunchInfo.getTestMethod());
                } else {
                    runner.setClassName(mLaunchInfo.getTestClass());
                }
            }

            if (mLaunchInfo.getTestPackage() != null) {
                runner.setTestPackageName(mLaunchInfo.getTestPackage());
            }

            TestSize size = mLaunchInfo.getTestSize();
            if (size != null) {
                runner.setTestSize(size);
            }

            runners.add(runner);
        }

        // Launch all test info collector jobs
        List<TestTreeCollectorJob> collectorJobs =
                new ArrayList<TestTreeCollectorJob>(devices.size());
        List<TestCollector> perDeviceCollectors = new ArrayList<TestCollector>(devices.size());
        for (int i = 0; i < devices.size(); i++) {
            RemoteAndroidTestRunner runner = runners.get(i);
            String deviceName = devices.get(i).getName();
            TestCollector collector = new TestCollector(deviceName);
            perDeviceCollectors.add(collector);

            TestTreeCollectorJob job = new TestTreeCollectorJob(
                    "Test Tree Collector for " + deviceName,
                    runner, mLaunchInfo.isDebugMode(), collector);
            job.setPriority(Job.INTERACTIVE);
            job.schedule();

            collectorJobs.add(job);
        }

        // wait for all test info collector jobs to complete
        int totalTests = 0;
        for (TestTreeCollectorJob job : collectorJobs) {
            try {
                job.join();
            } catch (InterruptedException e) {
                endTestRunWithError(e.getMessage());
                return;
            }

            if (!job.getResult().isOK()) {
                endTestRunWithError(job.getResult().getMessage());
                return;
            }

            TestCollector collector = job.getCollector();
            String err = collector.getErrorMessage();
            if (err != null) {
                endTestRunWithError(err);
                return;
            }

            totalTests += collector.getTestCaseCount();
        }

        AdtPlugin.printToConsole(mLaunchInfo.getProject(), "Sending test information to Eclipse");
        notifyTestRunStarted(totalTests);
        sendTestTrees(perDeviceCollectors);

        List<TestRunnerJob> instrumentationRunnerJobs =
                new ArrayList<TestRunnerJob>(devices.size());

        TestResultsNotifier notifier = new TestResultsNotifier(mExecution.getListener(),
                devices.size());

        // Spawn all instrumentation runner jobs
        for (int i = 0; i < devices.size(); i++) {
            RemoteAndroidTestRunner runner = runners.get(i);
            String deviceName = devices.get(i).getName();
            TestRunListener testRunListener = new TestRunListener(deviceName, notifier);
            InstrumentationRunJob job = new InstrumentationRunJob(
                    "Test Tree Collector for " + deviceName,
                    runner, mLaunchInfo.isDebugMode(), testRunListener);
            job.setPriority(Job.INTERACTIVE);
            job.schedule();

            instrumentationRunnerJobs.add(job);
        }

        // Wait for all jobs to complete
        for (TestRunnerJob job : instrumentationRunnerJobs) {
            try {
                job.join();
            } catch (InterruptedException e) {
                endTestRunWithError(e.getMessage());
                return;
            }

            if (!job.getResult().isOK()) {
                endTestRunWithError(job.getResult().getMessage());
                return;
            }
        }
    }

    /** Sends info about the test tree to be executed (ie the suites and their enclosed tests) */
    private void sendTestTrees(List<TestCollector> perDeviceCollectors) {
        for (TestCollector c : perDeviceCollectors) {
            ITestReference ref = c.getDeviceSuite();
            ref.sendTree(this);
        }
    }

    private static abstract class TestRunnerJob extends Job {
        private ITestRunListener mListener;
        private RemoteAndroidTestRunner mRunner;
        private boolean mIsDebug;

        public TestRunnerJob(String name, RemoteAndroidTestRunner runner,
                boolean isDebug, ITestRunListener listener) {
            super(name);

            mRunner = runner;
            mIsDebug = isDebug;
            mListener = listener;
        }

        @Override
        protected IStatus run(IProgressMonitor monitor) {
            try {
                setupRunner();
                mRunner.run(mListener);
            } catch (TimeoutException e) {
                return new Status(Status.ERROR, AdtPlugin.PLUGIN_ID,
                        LaunchMessages.RemoteAdtTestRunner_RunTimeoutException,
                        e);
            } catch (IOException e) {
                return new Status(Status.ERROR, AdtPlugin.PLUGIN_ID,
                        String.format(LaunchMessages.RemoteAdtTestRunner_RunIOException_s,
                                e.getMessage()),
                        e);
            } catch (AdbCommandRejectedException e) {
                return new Status(Status.ERROR, AdtPlugin.PLUGIN_ID,
                        String.format(
                                LaunchMessages.RemoteAdtTestRunner_RunAdbCommandRejectedException_s,
                                e.getMessage()),
                        e);
            } catch (ShellCommandUnresponsiveException e) {
                return new Status(Status.ERROR, AdtPlugin.PLUGIN_ID,
                        LaunchMessages.RemoteAdtTestRunner_RunTimeoutException,
                        e);
            }

            return Status.OK_STATUS;
        }

        public RemoteAndroidTestRunner getRunner() {
            return mRunner;
        }

        public boolean isDebug() {
            return mIsDebug;
        }

        public ITestRunListener getListener() {
            return mListener;
        }

        protected abstract void setupRunner();
    }

    private static class TestTreeCollectorJob extends TestRunnerJob {
        public TestTreeCollectorJob(String name, RemoteAndroidTestRunner runner, boolean isDebug,
                TestCollector listener) {
            super(name, runner, isDebug, listener);
        }

        @Override
        protected void setupRunner() {
            RemoteAndroidTestRunner runner = getRunner();

            // set log only to just collect test case info,
            // so Eclipse has correct test case count/tree info
            runner.setLogOnly(true);

            // add a small delay between each test. Otherwise for large test suites framework may
            // report Binder transaction failures
            runner.addInstrumentationArg(DELAY_MSEC_KEY, COLLECT_TEST_DELAY_MS);
        }

        public TestCollector getCollector() {
            return (TestCollector) getListener();
        }
    }

    private static class InstrumentationRunJob extends TestRunnerJob {
        public InstrumentationRunJob(String name, RemoteAndroidTestRunner runner, boolean isDebug,
                ITestRunListener listener) {
            super(name, runner, isDebug, listener);
        }

        @Override
        protected void setupRunner() {
            RemoteAndroidTestRunner runner = getRunner();
            runner.setLogOnly(false);
            runner.removeInstrumentationArg(DELAY_MSEC_KEY);
            if (isDebug()) {
                runner.setDebug(true);
            }
        }
    }

    /**
     * Main entry method to run tests
     *
     * @param programArgs JDT JUnit program arguments to be processed by parent
     * @param junitInfo the {@link AndroidJUnitLaunchInfo} containing info about this test ru
     */
    public void runTests(String[] programArgs, AndroidJUnitLaunchInfo junitInfo) {
        init(programArgs, junitInfo);
        run();
    }

    /**
     * Stop the current test run.
     */
    public void terminate() {
        stop();
    }

    @Override
    protected void stop() {
        if (mExecution != null) {
            mExecution.stop();
        }
    }

    private void notifyTestRunEnded(long elapsedTime) {
        // copy from parent - not ideal, but method is private
        sendMessage(MessageIds.TEST_RUN_END + elapsedTime);
        flush();
        //shutDown();
    }

    /**
     * @param errorMessage
     */
    private void reportError(String errorMessage) {
        AdtPlugin.printErrorToConsole(mLaunchInfo.getProject(),
                String.format(LaunchMessages.RemoteAdtTestRunner_RunFailedMsg_s, errorMessage));
        // is this needed?
        //notifyTestRunStopped(-1);
    }

    private void endTestRunWithError(String message) {
        reportError(message);
        notifyTestRunEnded(0);
    }

    /**
     * This class provides the interface to notify the JDT UI regarding the status of tests.
     * When running tests on multiple devices, there is a {@link TestRunListener} that listens
     * to results from each device. Rather than all such listeners directly notifying JDT
     * from different threads, they all notify this class which notifies JDT. In addition,
     * the {@link #testRunEnded(String, long)} method make sure that JDT is notified that the
     * test run has completed only when tests on all devices have completed.
     * */
    private class TestResultsNotifier {
        private final IListensToTestExecutions mListener;
        private final int mDeviceCount;

        private int mCompletedRuns;
        private long mMaxElapsedTime;

        public TestResultsNotifier(IListensToTestExecutions listener, int nDevices) {
            mListener = listener;
            mDeviceCount = nDevices;
        }

        public synchronized void testEnded(TestCaseReference ref) {
            mListener.notifyTestEnded(ref);
        }

        public synchronized void testFailed(TestReferenceFailure ref) {
            mListener.notifyTestFailed(ref);
        }

        public synchronized void testRunEnded(String mDeviceName, long elapsedTime) {
            mCompletedRuns++;

            if (elapsedTime > mMaxElapsedTime) {
                mMaxElapsedTime = elapsedTime;
            }

            if (mCompletedRuns == mDeviceCount) {
                notifyTestRunEnded(mMaxElapsedTime);
            }
        }

        public synchronized void testStarted(TestCaseReference testId) {
            mListener.notifyTestStarted(testId);
        }
    }

    /**
     * TestRunListener that communicates results in real-time back to JDT JUnit via the
     * {@link TestResultsNotifier}.
     * */
    private class TestRunListener implements ITestRunListener {
        private final String mDeviceName;
        private TestResultsNotifier mNotifier;

        /**
         * Constructs a {@link ITestRunListener} that listens for test results on given device.
         * @param deviceName device on which the tests are being run
         * @param notifier notifier to inform of test status
         */
        public TestRunListener(String deviceName, TestResultsNotifier notifier) {
            mDeviceName = deviceName;
            mNotifier = notifier;
        }

        @Override
        public void testEnded(TestIdentifier test, Map<String, String> ignoredTestMetrics) {
            mNotifier.testEnded(new TestCaseReference(mDeviceName, test));
        }

        @Override
        public void testFailed(TestFailure status, TestIdentifier test, String trace) {
            String statusString;
            if (status == TestFailure.ERROR) {
                statusString = MessageIds.TEST_ERROR;
            } else {
                statusString = MessageIds.TEST_FAILED;
            }
            TestReferenceFailure failure =
                new TestReferenceFailure(new TestCaseReference(mDeviceName, test),
                        statusString, trace, null);
            mNotifier.testFailed(failure);
        }

        @Override
        public synchronized void testRunEnded(long elapsedTime, Map<String, String> runMetrics) {
            mNotifier.testRunEnded(mDeviceName, elapsedTime);
            AdtPlugin.printToConsole(mLaunchInfo.getProject(),
                    LaunchMessages.RemoteAdtTestRunner_RunCompleteMsg);
        }

        @Override
        public synchronized void testRunFailed(String errorMessage) {
            reportError(errorMessage);
        }

        @Override
        public synchronized void testRunStarted(String runName, int testCount) {
            // ignore
        }

        @Override
        public synchronized void testRunStopped(long elapsedTime) {
            notifyTestRunStopped(elapsedTime);
            AdtPlugin.printToConsole(mLaunchInfo.getProject(),
                    LaunchMessages.RemoteAdtTestRunner_RunStoppedMsg);
        }

        @Override
        public synchronized void testStarted(TestIdentifier test) {
            TestCaseReference testId = new TestCaseReference(mDeviceName, test);
            mNotifier.testStarted(testId);
        }
    }

    /** Override parent to get extra logs. */
    @Override
    protected boolean connect() {
        boolean result = super.connect();
        if (!result) {
            AdtPlugin.printErrorToConsole(mLaunchInfo.getProject(),
                    "Connect to Eclipse test result listener failed");
        }
        return result;
    }

    /** Override parent to dump error message to console. */
    @Override
    public void runFailed(String message, Exception exception) {
        if (exception != null) {
            AdtPlugin.logAndPrintError(exception, mLaunchInfo.getProject().getName(),
                    "Test launch failed: %s", message);
        } else {
            AdtPlugin.printErrorToConsole(mLaunchInfo.getProject(), "Test launch failed: %s",
                    message);
        }
    }
}