google/coretests/CoreTestRunner.java

/*
 * Copyright (C) 2009 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.coretests;

import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;

import junit.framework.Test;
import junit.framework.TestResult;
import junit.framework.TestSuite;
import junit.textui.ResultPrinter;
import junit.textui.TestRunner;

/**
 * A special TestRunner implementation that is able to filter out annotated
 * tests and handles our known failures properly (expects them to fail).
 * Handy when running the Core Libraries tests on Android, the bare-metal
 * Dalvik VM, or the RI.
 */
public class CoreTestRunner extends TestRunner {

    /**
     * Reflects our environment.
     */
    private static boolean IS_DALVIK = "Dalvik".equals(
            System.getProperty("java.vm.name"));

    /**
     * Defines the default flags for running on Dalvik.
     */
    private static final int DEFAULT_FLAGS_DALVIK =
            CoreTestSuite.RUN_ANDROID_ONLY |
            CoreTestSuite.RUN_NORMAL_TESTS |
            CoreTestSuite.RUN_KNOWN_FAILURES |
            CoreTestSuite.RUN_SIDE_EFFECTS |
            CoreTestSuite.INVERT_KNOWN_FAILURES;

    /**
     * Defines the default flags for running on an RI.
     */
    private static final int DEFAULT_FLAGS_NON_DALVIK =
            CoreTestSuite.RUN_NORMAL_TESTS |
            CoreTestSuite.RUN_KNOWN_FAILURES |
            CoreTestSuite.RUN_SIDE_EFFECTS;

    /**
     * Holds the flags specified by the user on the command line.
     */
    private int fFlags;

    /**
     * Holds the timeout value specified by the user on the command line.
     */
    private int fTimeout;

    private int fStep = 1;

    /**
     * The path to write XML reports to, or {@code null} for no reports.
     */
    private String xmlReportsDirectory;

    /**
     * Creates a new instance of our CoreTestRunner.
     */
    public CoreTestRunner() {
        super();
    }

    @Override
    protected TestResult createTestResult() {
        return new CoreTestResult(fFlags, fTimeout);
    }

    protected ResultPrinter createPrinter() {
        return new CoreTestPrinter(System.out, fFlags);
    }

    /**
     * Provides our main entry point.
     */
    public static void main(String args[]) {
        Logger.global.setLevel(Level.OFF);

        System.out.println(
                "--------------------------------------------------");
        System.out.println("Android Core Libraries Test Suite");
        System.out.println("Version 1.0");
        System.out.println(
                "Copyright (c) 2009 The Android Open Source Project");
        System.out.println("");

        CoreTestRunner testRunner = new CoreTestRunner();
        try {
            TestResult r = testRunner.start(args);

            System.out.println(
            "--------------------------------------------------");

            if (!r.wasSuccessful()) {
                System.exit(FAILURE_EXIT);
            } else {
                System.exit(SUCCESS_EXIT);
            }
        } catch(Exception e) {
            System.err.println(e.getMessage());
            System.exit(EXCEPTION_EXIT);
        }

    }

    @Override
    public TestResult doRun(Test suite, boolean wait) {
        setPrinter(createPrinter());

        /*
         * Make sure the original suite is unreachable after we have
         * created the new one, so GC can dispose terminated tests.
         */
        CoreTestSuite coreTestSuite = new CoreTestSuite(suite, fFlags, fStep, null);

        XmlReportPrinter xmlReportPrinter = xmlReportsDirectory != null
                ? new XmlReportPrinter(coreTestSuite)
                : null;

        TestResult result = super.doRun(coreTestSuite, wait);

        if (xmlReportPrinter != null) {
            System.out.print("Printing XML Reports... ");
            xmlReportPrinter.setResults(result);
            int numFiles = xmlReportPrinter.generateReports(xmlReportsDirectory);
            System.out.println(numFiles + " files written.");
        }

        return result;
    }

    /**
     * Prints a help screen on the console.
     */
    private void showHelp() {
        System.out.println("Usage: run-core-tests [OPTION]... [TEST]...");
        System.out.println();
        System.out.println("Where each TEST is a class name, optionally followed");
        System.out.println("by \"#\" and a method name, and each OPTION is one of");
        System.out.println("the following:");
        System.out.println();
        System.out.println("    --include-all");
        System.out.println("    --exclude-all");
        System.out.println("    --include-android-only");
        System.out.println("    --exclude-android-only");
        System.out.println("    --include-broken-tests");
        System.out.println("    --exclude-broken-tests");
        System.out.println("    --include-known-failures");
        System.out.println("    --exclude-known-failures");
        System.out.println("    --include-normal-tests");
        System.out.println("    --exclude-normal-tests");
        System.out.println("    --include-side-effects");
        System.out.println("    --exclude-side-effects");
        System.out.println();
        System.out.println("    --known-failures-must-fail");
        System.out.println("    --known-failures-must-pass");
        System.out.println("    --timeout <seconds>");
        // System.out.println("    --find-side-effect <test>");
        System.out.println("    --isolate-all");
        System.out.println("    --isolate-none");
        System.out.println("    --verbose");
        System.out.println("    --xml-reports-directory <path>");
        System.out.println("    --help");
        System.out.println();
        System.out.println("Default parameters are:");
        System.out.println();

        if (IS_DALVIK) {
            System.out.println("    --include-android-only");
            System.out.println("    --exclude-broken-tests");
            System.out.println("    --include-known-failures");
            System.out.println("    --include-normal-tests");
            System.out.println("    --include-side-effects");
            System.out.println("    --known-failures-must-fail");
        } else {
            System.out.println("    --exclude-android-only");
            System.out.println("    --exclude-broken-tests");
            System.out.println("    --include-known-failures");
            System.out.println("    --include-normal-tests");
            System.out.println("    --include-side-effects");
            System.out.println("    --known-failures-must-pass");
        }

        System.out.println();
    }

    /**
     * Tries to create a Test instance from the given strings. The strings might
     * either specify a class only or a class plus a method name, separated by
     * a "#".
     */
    private Test createTest(List<String> testCases) throws Exception {
        TestSuite result = new TestSuite();
        for (String testCase : testCases) {
            int p = testCase.indexOf("#");
            if (p != -1) {
                String testName = testCase.substring(p + 1);
                testCase = testCase.substring(0, p);

                result.addTest(TestSuite.createTest(Class.forName(testCase), testName));
            } else {
                result.addTest(getTest(testCase));
            }
        }
        return result;
    }

    @Override
    protected TestResult start(String args[]) throws Exception {
        List<String> testNames = new ArrayList<String>();
        // String victimName = null;

        boolean wait = false;

        if (IS_DALVIK) {
            fFlags = DEFAULT_FLAGS_DALVIK;
        } else {
            fFlags = DEFAULT_FLAGS_NON_DALVIK;
        }

        for (int i= 0; i < args.length; i++) {
            if (args[i].startsWith("--")) {
                if (args[i].equals("--wait")) {
                    wait = true;
                } else if (args[i].equals("--include-all")) {
                    fFlags = fFlags | CoreTestSuite.RUN_ALL_TESTS;
                } else if (args[i].equals("--exclude-all")) {
                    fFlags = fFlags & ~CoreTestSuite.RUN_ALL_TESTS;
                } else if (args[i].equals("--include-android-only")) {
                    fFlags = fFlags | CoreTestSuite.RUN_ANDROID_ONLY;
                } else if (args[i].equals("--exclude-android-only")) {
                    fFlags = fFlags & ~CoreTestSuite.RUN_ANDROID_ONLY;
                } else if (args[i].equals("--include-broken-tests")) {
                    fFlags = fFlags | CoreTestSuite.RUN_BROKEN_TESTS;
                } else if (args[i].equals("--exclude-broken-tests")) {
                    fFlags = fFlags & ~CoreTestSuite.RUN_BROKEN_TESTS;
                } else if (args[i].equals("--include-known-failures")) {
                    fFlags = fFlags | CoreTestSuite.RUN_KNOWN_FAILURES;
                } else if (args[i].equals("--exclude-known-failures")) {
                    fFlags = fFlags & ~CoreTestSuite.RUN_KNOWN_FAILURES;
                } else if (args[i].equals("--include-normal-tests")) {
                    fFlags = fFlags | CoreTestSuite.RUN_NORMAL_TESTS;
                } else if (args[i].equals("--exclude-normal-tests")) {
                    fFlags = fFlags & ~CoreTestSuite.RUN_NORMAL_TESTS;
                } else if (args[i].equals("--include-side-effects")) {
                    fFlags = fFlags | CoreTestSuite.RUN_SIDE_EFFECTS;
                } else if (args[i].equals("--exclude-side-effects")) {
                    fFlags = fFlags & ~CoreTestSuite.RUN_SIDE_EFFECTS;
                } else if (args[i].equals("--known-failures-must-fail")) {
                    fFlags = fFlags | CoreTestSuite.INVERT_KNOWN_FAILURES;
                } else if (args[i].equals("--known-failures-must-pass")) {
                    fFlags = fFlags & ~CoreTestSuite.INVERT_KNOWN_FAILURES;
                } else if (args[i].equals("--timeout")) {
                    fTimeout = Integer.parseInt(args[++i]);
                } else if (args[i].equals("--reverse")) {
                    fFlags = fFlags | CoreTestSuite.REVERSE;
                } else if (args[i].equals("--step")) {
                    fStep = Integer.parseInt(args[++i]);
                } else if (args[i].equals("--isolate-all")) {
                    fFlags = (fFlags | CoreTestSuite.ISOLATE_ALL) &
                                   ~CoreTestSuite.ISOLATE_NONE;
                } else if (args[i].equals("--isolate-none")) {
                    fFlags = (fFlags | CoreTestSuite.ISOLATE_NONE) &
                                   ~CoreTestSuite.ISOLATE_ALL;
                } else if (args[i].equals("--verbose")) {
                    fFlags = fFlags | CoreTestSuite.VERBOSE;
                // } else if (args[i].equals("--find-side-effect")) {
                //    victimName = args[++i];
                } else if (args[i].equals("--dry-run")) {
                    fFlags = fFlags | CoreTestSuite.DRY_RUN;
                } else if (args[i].equals("--xml-reports-directory")) {
                    xmlReportsDirectory = args[++i];
                } else if (args[i].equals("--help")) {
                    showHelp();
                    System.exit(1);
                } else {
                    unknownArgument(args[i]);
                }
            } else if (args[i].startsWith("-")) {
                unknownArgument(args[i]);
            } else {
                testNames.add(args[i]);
            }
        }

        if (IS_DALVIK) {
            System.out.println("Using Dalvik VM version " +
                    System.getProperty("java.vm.version"));
        } else {
            System.out.println("Using Java VM version " +
                    System.getProperty("java.version"));
        }
        System.out.println();

        try {
            return doRun(createTest(testNames), wait);
        }
        catch(Exception e) {
            e.printStackTrace();
            throw new Exception("Could not create and run test suite: " + e);
        }
    }

    private static void unknownArgument(String arg) {
        System.err.println("Unknown argument " + arg + ", try --help");
        System.exit(1);
    }
}