testtype/suite/ITestSuite.java

/*
 * Copyright (C) 2016 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.android.tradefed.testtype.suite;

import com.android.ddmlib.Log.LogLevel;
import com.android.tradefed.build.IBuildInfo;
import com.android.tradefed.config.ConfigurationException;
import com.android.tradefed.config.IConfiguration;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionCopier;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.invoker.IInvocationContext;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.result.ITestLoggerReceiver;
import com.android.tradefed.result.InputStreamSource;
import com.android.tradefed.result.LogDataType;
import com.android.tradefed.suite.checker.ISystemStatusChecker;
import com.android.tradefed.suite.checker.ISystemStatusCheckerReceiver;
import com.android.tradefed.testtype.IBuildReceiver;
import com.android.tradefed.testtype.IDeviceTest;
import com.android.tradefed.testtype.IInvocationContextReceiver;
import com.android.tradefed.testtype.IMultiDeviceTest;
import com.android.tradefed.testtype.IRemoteTest;
import com.android.tradefed.testtype.IRuntimeHintProvider;
import com.android.tradefed.testtype.IShardableTest;
import com.android.tradefed.testtype.ITestCollector;
import com.android.tradefed.util.TimeUtil;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;

/**
 * Abstract class used to run Test Suite. This class provide the base of how the Suite will be run.
 * Each implementation can define the list of tests via the {@link #loadTests()} method.
 */
public abstract class ITestSuite
        implements IRemoteTest,
                IDeviceTest,
                IBuildReceiver,
                ISystemStatusCheckerReceiver,
                IShardableTest,
                ITestCollector,
                IInvocationContextReceiver,
                IRuntimeHintProvider {

    public static final String MODULE_CHECKER_PRE = "PreModuleChecker";
    public static final String MODULE_CHECKER_POST = "PostModuleChecker";

    // Options for test failure case
    @Option(
        name = "bugreport-on-failure",
        description =
                "Take a bugreport on every test failure. Warning: This may require a lot"
                        + "of storage space of the machine running the tests."
    )
    private boolean mBugReportOnFailure = false;

    @Option(name = "logcat-on-failure",
            description = "Take a logcat snapshot on every test failure.")
    private boolean mLogcatOnFailure = false;

    @Option(name = "logcat-on-failure-size",
            description = "The max number of logcat data in bytes to capture when "
            + "--logcat-on-failure is on. Should be an amount that can comfortably fit in memory.")
    private int mMaxLogcatBytes = 500 * 1024; // 500K

    @Option(name = "screenshot-on-failure",
            description = "Take a screenshot on every test failure.")
    private boolean mScreenshotOnFailure = false;

    @Option(name = "reboot-on-failure",
            description = "Reboot the device after every test failure.")
    private boolean mRebootOnFailure = false;

    // Options for suite runner behavior
    @Option(name = "reboot-per-module", description = "Reboot the device before every module run.")
    private boolean mRebootPerModule = false;

    @Option(name = "skip-all-system-status-check",
            description = "Whether all system status check between modules should be skipped")
    private boolean mSkipAllSystemStatusCheck = false;

    @Option(
        name = "report-system-checkers",
        description = "Whether reporting system checkers as test or not."
    )
    private boolean mReportSystemChecker = false;

    @Option(
        name = "collect-tests-only",
        description =
                "Only invoke the suite to collect list of applicable test cases. All "
                        + "test run callbacks will be triggered, but test execution will not be "
                        + "actually carried out."
    )
    private boolean mCollectTestsOnly = false;

    private ITestDevice mDevice;
    private IBuildInfo mBuildInfo;
    private List<ISystemStatusChecker> mSystemStatusCheckers;
    private IInvocationContext mContext;

    // Sharding attributes
    private boolean mIsSharded = false;
    private ModuleDefinition mDirectModule = null;
    private boolean mShouldMakeDynamicModule = true;

    /**
     * Abstract method to load the tests configuration that will be run. Each tests is defined by a
     * {@link IConfiguration} and a unique name under which it will report results.
     */
    public abstract LinkedHashMap<String, IConfiguration> loadTests();

    /**
     * Return an instance of the class implementing {@link ITestSuite}.
     */
    private ITestSuite createInstance() {
        try {
            return this.getClass().newInstance();
        } catch (InstantiationException | IllegalAccessException e) {
            throw new RuntimeException(e);
        }
    }

    /** Helper that creates and returns the list of {@link ModuleDefinition} to be executed. */
    private List<ModuleDefinition> createExecutionList() {
        List<ModuleDefinition> runModules = new ArrayList<>();
        if (mDirectModule != null) {
            // If we are sharded and already know what to run then we just do it.
            runModules.add(mDirectModule);
            mDirectModule.setDevice(mDevice);
            mDirectModule.setDeviceInfos(mContext.getDeviceBuildMap());
            mDirectModule.setBuild(mBuildInfo);
            return runModules;
        }

        LinkedHashMap<String, IConfiguration> runConfig = loadTests();
        if (runConfig.isEmpty()) {
            CLog.i("No config were loaded. Nothing to run.");
            return runModules;
        }

        for (Entry<String, IConfiguration> config : runConfig.entrySet()) {
            if (!ValidateSuiteConfigHelper.validateConfig(config.getValue())) {
                throw new RuntimeException(
                        new ConfigurationException(
                                String.format(
                                        "Configuration %s cannot be run in a suite.",
                                        config.getValue().getName())));
            }
            ModuleDefinition module =
                    new ModuleDefinition(
                            config.getKey(),
                            config.getValue().getTests(),
                            config.getValue().getTargetPreparers(),
                            config.getValue().getMultiTargetPreparers(),
                            config.getValue().getConfigurationDescription());
            module.setDevice(mDevice);
            module.setDeviceInfos(mContext.getDeviceBuildMap());
            module.setBuild(mBuildInfo);
            runModules.add(module);
        }
        // Free the map once we are done with it.
        runConfig = null;
        return runModules;
    }

    /** Generic run method for all test loaded from {@link #loadTests()}. */
    @Override
    public final void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
        List<ModuleDefinition> runModules = createExecutionList();
        // Check if we have something to run.
        if (runModules.isEmpty()) {
            CLog.i("No tests to be run.");
            return;
        }

        // Allow checkers to log files for easier debbuging.
        for (ISystemStatusChecker checker : mSystemStatusCheckers) {
            if (checker instanceof ITestLoggerReceiver) {
                ((ITestLoggerReceiver) checker).setTestLogger(listener);
            }
        }

        /** Setup a special listener to take actions on test failures. */
        TestFailureListener failureListener =
                new TestFailureListener(
                        listener,
                        mContext.getDevices(),
                        mBugReportOnFailure,
                        mLogcatOnFailure,
                        mScreenshotOnFailure,
                        mRebootOnFailure,
                        mMaxLogcatBytes);

        // Only print the running log if we are going to run something.
        if (runModules.get(0).hasTests()) {
            CLog.logAndDisplay(
                    LogLevel.INFO,
                    "%s running %s modules: %s",
                    mDevice.getSerialNumber(),
                    runModules.size(),
                    runModules);
        }

        /** Run all the module, make sure to reduce the list to release resources as we go. */
        try {
            while (!runModules.isEmpty()) {
                ModuleDefinition module = runModules.remove(0);
                // Before running the module we ensure it has tests at this point or skip completely
                // to avoid running SystemCheckers and preparation for nothing.
                if (module.hasTests()) {
                    continue;
                }

                try {
                    listener.testModuleStarted(module.getModuleInvocationContext());
                    // Populate the module context with devices and builds
                    for (String deviceName : mContext.getDeviceConfigNames()) {
                        module.getModuleInvocationContext()
                                .addAllocatedDevice(deviceName, mContext.getDevice(deviceName));
                        module.getModuleInvocationContext()
                                .addDeviceBuildInfo(deviceName, mContext.getBuildInfo(deviceName));
                    }
                    runSingleModule(module, listener, failureListener);
                } finally {
                    // clear out module invocation context since we are now done with module
                    // execution
                    listener.testModuleEnded();
                }
            }
        } catch (DeviceNotAvailableException e) {
            CLog.e(
                    "A DeviceNotAvailableException occurred, following modules did not run: %s",
                    runModules);
            for (ModuleDefinition module : runModules) {
                listener.testRunStarted(module.getId(), 0);
                listener.testRunFailed("Module did not run due to device not available.");
                listener.testRunEnded(0, Collections.emptyMap());
            }
            throw e;
        }
    }

    /**
     * Helper method that handle running a single module logic.
     *
     * @param module The {@link ModuleDefinition} to be ran.
     * @param listener The {@link ITestInvocationListener} where to report results
     * @param failureListener The {@link TestFailureListener} that collect infos on failures.
     * @throws DeviceNotAvailableException
     */
    private void runSingleModule(
            ModuleDefinition module,
            ITestInvocationListener listener,
            TestFailureListener failureListener)
            throws DeviceNotAvailableException {
        if (mRebootPerModule) {
            if ("user".equals(mDevice.getProperty("ro.build.type"))) {
                CLog.e(
                        "reboot-per-module should only be used during development, "
                                + "this is a\" user\" build device");
            } else {
                CLog.d("Rebooting device before starting next module");
                mDevice.reboot();
            }
        }

        if (!mSkipAllSystemStatusCheck) {
            runPreModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener);
        }
        if (mCollectTestsOnly) {
            module.setCollectTestsOnly(mCollectTestsOnly);
        }
        // Actually run the module
        module.run(listener, failureListener);

        if (!mSkipAllSystemStatusCheck) {
            runPostModuleCheck(module.getId(), mSystemStatusCheckers, mDevice, listener);
        }
    }

    /**
     * Helper to run the System Status checkers preExecutionChecks defined for the test and log
     * their failures.
     */
    private void runPreModuleCheck(
            String moduleName,
            List<ISystemStatusChecker> checkers,
            ITestDevice device,
            ITestInvocationListener listener)
            throws DeviceNotAvailableException {
        long startTime = System.currentTimeMillis();
        CLog.i("Running system status checker before module execution: %s", moduleName);
        List<String> failures = new ArrayList<>();
        for (ISystemStatusChecker checker : checkers) {
            boolean result = checker.preExecutionCheck(device);
            if (!result) {
                failures.add(checker.getClass().getCanonicalName());
                CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName());
            }
        }
        if (!failures.isEmpty()) {
            CLog.w("There are failed system status checkers: %s capturing a bugreport",
                    failures.toString());
            try (InputStreamSource bugSource = device.getBugreport()) {
                listener.testLog(
                        String.format("bugreport-checker-pre-module-%s", moduleName),
                        LogDataType.BUGREPORT,
                        bugSource);
            }
        }

        // We report System checkers like tests.
        reportModuleCheckerResult(MODULE_CHECKER_PRE, moduleName, failures, startTime, listener);
    }

    /**
     * Helper to run the System Status checkers postExecutionCheck defined for the test and log
     * their failures.
     */
    private void runPostModuleCheck(
            String moduleName,
            List<ISystemStatusChecker> checkers,
            ITestDevice device,
            ITestInvocationListener listener)
            throws DeviceNotAvailableException {
        long startTime = System.currentTimeMillis();
        CLog.i("Running system status checker after module execution: %s", moduleName);
        List<String> failures = new ArrayList<>();
        for (ISystemStatusChecker checker : checkers) {
            boolean result = checker.postExecutionCheck(device);
            if (!result) {
                failures.add(checker.getClass().getCanonicalName());
                CLog.w("System status checker [%s] failed", checker.getClass().getCanonicalName());
            }
        }
        if (!failures.isEmpty()) {
            CLog.w("There are failed system status checkers: %s capturing a bugreport",
                    failures.toString());
            try (InputStreamSource bugSource = device.getBugreport()) {
                listener.testLog(
                        String.format("bugreport-checker-post-module-%s", moduleName),
                        LogDataType.BUGREPORT,
                        bugSource);
            }
        }

        // We report System checkers like tests.
        reportModuleCheckerResult(MODULE_CHECKER_POST, moduleName, failures, startTime, listener);
    }

    /** Helper to report status checker results as test results. */
    private void reportModuleCheckerResult(
            String identifier,
            String moduleName,
            List<String> failures,
            long startTime,
            ITestInvocationListener listener) {
        if (!mReportSystemChecker) {
            // do not log here, otherwise it could be very verbose.
            return;
        }
        // Avoid messing with the final test count by making them empty runs.
        listener.testRunStarted(identifier + "_" + moduleName, 0);
        if (!failures.isEmpty()) {
            listener.testRunFailed(String.format("%s failed '%s' checkers", moduleName, failures));
        }
        listener.testRunEnded(System.currentTimeMillis() - startTime, Collections.emptyMap());
    }

    /** {@inheritDoc} */
    @Override
    public Collection<IRemoteTest> split(int shardCountHint) {
        if (shardCountHint <= 1 || mIsSharded) {
            // cannot shard or already sharded
            return null;
        }

        LinkedHashMap<String, IConfiguration> runConfig = loadTests();
        if (runConfig.isEmpty()) {
            CLog.i("No config were loaded. Nothing to run.");
            return null;
        }
        injectInfo(runConfig);

        // We split individual tests on double the shardCountHint to provide better average.
        // The test pool mechanism prevent this from creating too much overhead.
        List<ModuleDefinition> splitModules =
                ModuleSplitter.splitConfiguration(
                        runConfig, shardCountHint, mShouldMakeDynamicModule);
        runConfig.clear();
        runConfig = null;
        // create an association of one ITestSuite <=> one ModuleDefinition as the smallest
        // execution unit supported.
        List<IRemoteTest> splitTests = new ArrayList<>();
        for (ModuleDefinition m : splitModules) {
            ITestSuite suite = createInstance();
            OptionCopier.copyOptionsNoThrow(this, suite);
            suite.mIsSharded = true;
            suite.mDirectModule = m;
            splitTests.add(suite);
        }
        // return the list of ITestSuite with their ModuleDefinition assigned
        return splitTests;
    }

    /**
     * Inject {@link ITestDevice} and {@link IBuildInfo} to the {@link IRemoteTest}s in the config
     * before sharding since they may be needed.
     */
    private void injectInfo(LinkedHashMap<String, IConfiguration> runConfig) {
        for (IConfiguration config : runConfig.values()) {
            for (IRemoteTest test : config.getTests()) {
                if (test instanceof IBuildReceiver) {
                    ((IBuildReceiver) test).setBuild(mBuildInfo);
                }
                if (test instanceof IDeviceTest) {
                    ((IDeviceTest) test).setDevice(mDevice);
                }
                if (test instanceof IMultiDeviceTest) {
                    ((IMultiDeviceTest) test).setDeviceInfos(mContext.getDeviceBuildMap());
                }
            }
        }
    }

    /** {@inheritDoc} */
    @Override
    public void setDevice(ITestDevice device) {
        mDevice = device;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public ITestDevice getDevice() {
        return mDevice;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void setBuild(IBuildInfo buildInfo) {
        mBuildInfo = buildInfo;
    }

    /**
     * Implementation of {@link ITestSuite} may require the build info to load the tests.
     */
    public IBuildInfo getBuildInfo() {
        return mBuildInfo;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void setSystemStatusChecker(List<ISystemStatusChecker> systemCheckers) {
        mSystemStatusCheckers = systemCheckers;
    }

    /**
     * Run the test suite in collector only mode, this requires all the sub-tests to implements this
     * interface too.
     */
    @Override
    public void setCollectTestsOnly(boolean shouldCollectTest) {
        mCollectTestsOnly = shouldCollectTest;
    }

    /**
     * When doing distributed sharding, we cannot have ModuleDefinition that shares tests in a pool
     * otherwise intra-module sharding will not work, so we allow to disable it.
     */
    public void setShouldMakeDynamicModule(boolean dynamicModule) {
        mShouldMakeDynamicModule = dynamicModule;
    }

    /** {@inheritDoc} */
    @Override
    public void setInvocationContext(IInvocationContext invocationContext) {
        mContext = invocationContext;
    }

    /** {@inheritDoc} */
    @Override
    public long getRuntimeHint() {
        if (mDirectModule != null) {
            CLog.d(
                    "    %s: %s",
                    mDirectModule.getId(),
                    TimeUtil.formatElapsedTime(mDirectModule.getRuntimeHint()));
            return mDirectModule.getRuntimeHint();
        }
        return 0l;
    }

    /**
     * Returns the {@link ModuleDefinition} to be executed directly, or null if none yet (when the
     * ITestSuite has not been sharded yet).
     */
    public ModuleDefinition getDirectModule() {
        return mDirectModule;
    }
}