Home | History | Annotate | Download | only in testrunner 
      1 /*
      2  * Copyright (C) 2012 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 
     17 package com.android.uiautomator.testrunner;
     18 
     19 import android.app.Activity;
     20 import android.app.IInstrumentationWatcher;
     21 import android.app.Instrumentation;
     22 import android.content.ComponentName;
     23 import android.os.Bundle;
     24 import android.os.Debug;
     25 import android.os.HandlerThread;
     26 import android.os.IBinder;
     27 import android.os.SystemClock;
     28 import android.test.RepetitiveTest;
     29 import android.util.Log;
     30 
     31 import com.android.uiautomator.core.ShellUiAutomatorBridge;
     32 import com.android.uiautomator.core.Tracer;
     33 import com.android.uiautomator.core.UiAutomationShellWrapper;
     34 import com.android.uiautomator.core.UiDevice;
     35 
     36 import java.io.ByteArrayOutputStream;
     37 import java.io.PrintStream;
     38 import java.lang.Thread.UncaughtExceptionHandler;
     39 import java.lang.reflect.Method;
     40 import java.util.ArrayList;
     41 import java.util.List;
     42 
     43 import junit.framework.AssertionFailedError;
     44 import junit.framework.Test;
     45 import junit.framework.TestCase;
     46 import junit.framework.TestListener;
     47 import junit.framework.TestResult;
     48 import junit.runner.BaseTestRunner;
     49 import junit.textui.ResultPrinter;
     50 
     51 /**
     52  * @hide
     53  */
     54 public class UiAutomatorTestRunner {
     55 
     56     private static final String LOGTAG = UiAutomatorTestRunner.class.getSimpleName();
     57     private static final int EXIT_OK = 0;
     58     private static final int EXIT_EXCEPTION = -1;
     59 
     60     private static final String HANDLER_THREAD_NAME = "UiAutomatorHandlerThread";
     61 
     62     private boolean mDebug;
     63     private boolean mMonkey;
     64     private Bundle mParams = null;
     65     private UiDevice mUiDevice;
     66     private List<String> mTestClasses = null;
     67     private final FakeInstrumentationWatcher mWatcher = new FakeInstrumentationWatcher();
     68     private final IAutomationSupport mAutomationSupport = new IAutomationSupport() {
     69         @Override
     70         public void sendStatus(int resultCode, Bundle status) {
     71             mWatcher.instrumentationStatus(null, resultCode, status);
     72         }
     73     };
     74     private final List<TestListener> mTestListeners = new ArrayList<TestListener>();
     75 
     76     private HandlerThread mHandlerThread;
     77 
     78     public void run(List<String> testClasses, Bundle params, boolean debug, boolean monkey) {
     79         Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler() {
     80             @Override
     81             public void uncaughtException(Thread thread, Throwable ex) {
     82                 Log.e(LOGTAG, "uncaught exception", ex);
     83                 Bundle results = new Bundle();
     84                 results.putString("shortMsg", ex.getClass().getName());
     85                 results.putString("longMsg", ex.getMessage());
     86                 mWatcher.instrumentationFinished(null, 0, results);
     87                 // bailing on uncaught exception
     88                 System.exit(EXIT_EXCEPTION);
     89             }
     90         });
     91 
     92         mTestClasses = testClasses;
     93         mParams = params;
     94         mDebug = debug;
     95         mMonkey = monkey;
     96         start();
     97         System.exit(EXIT_OK);
     98     }
     99 
    100     /**
    101      * Called after all test classes are in place, ready to test
    102      */
    103     protected void start() {
    104         TestCaseCollector collector = getTestCaseCollector(this.getClass().getClassLoader());
    105         try {
    106             collector.addTestClasses(mTestClasses);
    107         } catch (ClassNotFoundException e) {
    108             // will be caught by uncaught handler
    109             throw new RuntimeException(e.getMessage(), e);
    110         }
    111         if (mDebug) {
    112             Debug.waitForDebugger();
    113         }
    114         mHandlerThread = new HandlerThread(HANDLER_THREAD_NAME);
    115         mHandlerThread.setDaemon(true);
    116         mHandlerThread.start();
    117         UiAutomationShellWrapper automationWrapper = new UiAutomationShellWrapper();
    118         automationWrapper.connect();
    119 
    120         long startTime = SystemClock.uptimeMillis();
    121         TestResult testRunResult = new TestResult();
    122         ResultReporter resultPrinter;
    123         String outputFormat = mParams.getString("outputFormat");
    124         List<TestCase> testCases = collector.getTestCases();
    125         Bundle testRunOutput = new Bundle();
    126         if ("simple".equals(outputFormat)) {
    127             resultPrinter = new SimpleResultPrinter(System.out, true);
    128         } else {
    129             resultPrinter = new WatcherResultPrinter(testCases.size());
    130         }
    131         try {
    132             automationWrapper.setRunAsMonkey(mMonkey);
    133             mUiDevice = UiDevice.getInstance();
    134             mUiDevice.initialize(new ShellUiAutomatorBridge(automationWrapper.getUiAutomation()));
    135 
    136             String traceType = mParams.getString("traceOutputMode");
    137             if(traceType != null) {
    138                 Tracer.Mode mode = Tracer.Mode.valueOf(Tracer.Mode.class, traceType);
    139                 if (mode == Tracer.Mode.FILE || mode == Tracer.Mode.ALL) {
    140                     String filename = mParams.getString("traceLogFilename");
    141                     if (filename == null) {
    142                         throw new RuntimeException("Name of log file not specified. " +
    143                                 "Please specify it using traceLogFilename parameter");
    144                     }
    145                     Tracer.getInstance().setOutputFilename(filename);
    146                 }
    147                 Tracer.getInstance().setOutputMode(mode);
    148             }
    149 
    150             // add test listeners
    151             testRunResult.addListener(resultPrinter);
    152             // add all custom listeners
    153             for (TestListener listener : mTestListeners) {
    154                 testRunResult.addListener(listener);
    155             }
    156 
    157             // run tests for realz!
    158             for (TestCase testCase : testCases) {
    159                 prepareTestCase(testCase);
    160                 testCase.run(testRunResult);
    161             }
    162         } catch (Throwable t) {
    163             // catch all exceptions so a more verbose error message can be outputted
    164             resultPrinter.printUnexpectedError(t);
    165             testRunOutput.putString("shortMsg", t.getMessage());
    166         } finally {
    167             long runTime = SystemClock.uptimeMillis() - startTime;
    168             resultPrinter.print(testRunResult, runTime, testRunOutput);
    169             automationWrapper.disconnect();
    170             automationWrapper.setRunAsMonkey(false);
    171             mHandlerThread.quit();
    172         }
    173     }
    174 
    175     // copy & pasted from com.android.commands.am.Am
    176     private class FakeInstrumentationWatcher implements IInstrumentationWatcher {
    177 
    178         private final boolean mRawMode = true;
    179 
    180         @Override
    181         public IBinder asBinder() {
    182             throw new UnsupportedOperationException("I'm just a fake!");
    183         }
    184 
    185         @Override
    186         public void instrumentationStatus(ComponentName name, int resultCode, Bundle results) {
    187             synchronized (this) {
    188                 // pretty printer mode?
    189                 String pretty = null;
    190                 if (!mRawMode && results != null) {
    191                     pretty = results.getString(Instrumentation.REPORT_KEY_STREAMRESULT);
    192                 }
    193                 if (pretty != null) {
    194                     System.out.print(pretty);
    195                 } else {
    196                     if (results != null) {
    197                         for (String key : results.keySet()) {
    198                             System.out.println("INSTRUMENTATION_STATUS: " + key + "="
    199                                     + results.get(key));
    200                         }
    201                     }
    202                     System.out.println("INSTRUMENTATION_STATUS_CODE: " + resultCode);
    203                 }
    204                 notifyAll();
    205             }
    206         }
    207 
    208         @Override
    209         public void instrumentationFinished(ComponentName name, int resultCode, Bundle results) {
    210             synchronized (this) {
    211                 // pretty printer mode?
    212                 String pretty = null;
    213                 if (!mRawMode && results != null) {
    214                     pretty = results.getString(Instrumentation.REPORT_KEY_STREAMRESULT);
    215                 }
    216                 if (pretty != null) {
    217                     System.out.println(pretty);
    218                 } else {
    219                     if (results != null) {
    220                         for (String key : results.keySet()) {
    221                             System.out.println("INSTRUMENTATION_RESULT: " + key + "="
    222                                     + results.get(key));
    223                         }
    224                     }
    225                     System.out.println("INSTRUMENTATION_CODE: " + resultCode);
    226                 }
    227                 notifyAll();
    228             }
    229         }
    230     }
    231 
    232     private interface ResultReporter extends TestListener {
    233         public void print(TestResult result, long runTime, Bundle testOutput);
    234         public void printUnexpectedError(Throwable t);
    235     }
    236 
    237     // Copy & pasted from InstrumentationTestRunner.WatcherResultPrinter
    238     private class WatcherResultPrinter implements ResultReporter {
    239 
    240         private static final String REPORT_KEY_NUM_TOTAL = "numtests";
    241         private static final String REPORT_KEY_NAME_CLASS = "class";
    242         private static final String REPORT_KEY_NUM_CURRENT = "current";
    243         private static final String REPORT_KEY_NAME_TEST = "test";
    244         private static final String REPORT_KEY_NUM_ITERATIONS = "numiterations";
    245         private static final String REPORT_VALUE_ID = "UiAutomatorTestRunner";
    246         private static final String REPORT_KEY_STACK = "stack";
    247 
    248         private static final int REPORT_VALUE_RESULT_START = 1;
    249         private static final int REPORT_VALUE_RESULT_ERROR = -1;
    250         private static final int REPORT_VALUE_RESULT_FAILURE = -2;
    251 
    252         private final Bundle mResultTemplate;
    253         Bundle mTestResult;
    254         int mTestNum = 0;
    255         int mTestResultCode = 0;
    256         String mTestClass = null;
    257 
    258         private final SimpleResultPrinter mPrinter;
    259         private final ByteArrayOutputStream mStream;
    260         private final PrintStream mWriter;
    261 
    262         public WatcherResultPrinter(int numTests) {
    263             mResultTemplate = new Bundle();
    264             mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID);
    265             mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests);
    266 
    267             mStream = new ByteArrayOutputStream();
    268             mWriter = new PrintStream(mStream);
    269             mPrinter = new SimpleResultPrinter(mWriter, false);
    270         }
    271 
    272         /**
    273          * send a status for the start of a each test, so long tests can be seen
    274          * as "running"
    275          */
    276         @Override
    277         public void startTest(Test test) {
    278             String testClass = test.getClass().getName();
    279             String testName = ((TestCase) test).getName();
    280             mTestResult = new Bundle(mResultTemplate);
    281             mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass);
    282             mTestResult.putString(REPORT_KEY_NAME_TEST, testName);
    283             mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum);
    284             // pretty printing
    285             if (testClass != null && !testClass.equals(mTestClass)) {
    286                 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
    287                         String.format("\n%s:", testClass));
    288                 mTestClass = testClass;
    289             } else {
    290                 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "");
    291             }
    292 
    293             Method testMethod = null;
    294             try {
    295                 testMethod = test.getClass().getMethod(testName);
    296                 // Report total number of iterations, if test is repetitive
    297                 if (testMethod.isAnnotationPresent(RepetitiveTest.class)) {
    298                     int numIterations = testMethod.getAnnotation(RepetitiveTest.class)
    299                             .numIterations();
    300                     mTestResult.putInt(REPORT_KEY_NUM_ITERATIONS, numIterations);
    301                 }
    302             } catch (NoSuchMethodException e) {
    303                 // ignore- the test with given name does not exist. Will be
    304                 // handled during test
    305                 // execution
    306             }
    307 
    308             mAutomationSupport.sendStatus(REPORT_VALUE_RESULT_START, mTestResult);
    309             mTestResultCode = 0;
    310 
    311             mPrinter.startTest(test);
    312         }
    313 
    314         @Override
    315         public void addError(Test test, Throwable t) {
    316             mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t));
    317             mTestResultCode = REPORT_VALUE_RESULT_ERROR;
    318             // pretty printing
    319             mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
    320                 String.format("\nError in %s:\n%s",
    321                     ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t)));
    322 
    323             mPrinter.addError(test, t);
    324         }
    325 
    326         @Override
    327         public void addFailure(Test test, AssertionFailedError t) {
    328             mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t));
    329             mTestResultCode = REPORT_VALUE_RESULT_FAILURE;
    330             // pretty printing
    331             mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
    332                 String.format("\nFailure in %s:\n%s",
    333                     ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t)));
    334 
    335             mPrinter.addFailure(test, t);
    336         }
    337 
    338         @Override
    339         public void endTest(Test test) {
    340             if (mTestResultCode == 0) {
    341                 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ".");
    342             }
    343             mAutomationSupport.sendStatus(mTestResultCode, mTestResult);
    344 
    345             mPrinter.endTest(test);
    346         }
    347 
    348         @Override
    349         public void print(TestResult result, long runTime, Bundle testOutput) {
    350             mPrinter.print(result, runTime, testOutput);
    351             testOutput.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
    352                   String.format("\nTest results for %s=%s",
    353                   getClass().getSimpleName(),
    354                   mStream.toString()));
    355             mWriter.close();
    356             mAutomationSupport.sendStatus(Activity.RESULT_OK, testOutput);
    357         }
    358 
    359         @Override
    360         public void printUnexpectedError(Throwable t) {
    361             mWriter.println(String.format("Test run aborted due to unexpected exception: %s",
    362                     t.getMessage()));
    363             t.printStackTrace(mWriter);
    364         }
    365     }
    366 
    367     /**
    368      * Class that produces the same output as JUnit when running from command line. Can be
    369      * used when default UiAutomator output is too verbose.
    370      */
    371     private class SimpleResultPrinter extends ResultPrinter implements ResultReporter {
    372         private final boolean mFullOutput;
    373         public SimpleResultPrinter(PrintStream writer, boolean fullOutput) {
    374             super(writer);
    375             mFullOutput = fullOutput;
    376         }
    377 
    378         @Override
    379         public void print(TestResult result, long runTime, Bundle testOutput) {
    380             printHeader(runTime);
    381             if (mFullOutput) {
    382                 printErrors(result);
    383                 printFailures(result);
    384             }
    385             printFooter(result);
    386         }
    387 
    388         @Override
    389         public void printUnexpectedError(Throwable t) {
    390             if (mFullOutput) {
    391                 getWriter().printf("Test run aborted due to unexpected exeption: %s",
    392                         t.getMessage());
    393                 t.printStackTrace(getWriter());
    394             }
    395         }
    396     }
    397 
    398     protected TestCaseCollector getTestCaseCollector(ClassLoader classLoader) {
    399         return new TestCaseCollector(classLoader, getTestCaseFilter());
    400     }
    401 
    402     /**
    403      * Returns an object which determines if the class and its methods should be
    404      * accepted into the test suite.
    405      * @return
    406      */
    407     public UiAutomatorTestCaseFilter getTestCaseFilter() {
    408         return new UiAutomatorTestCaseFilter();
    409     }
    410 
    411     protected void addTestListener(TestListener listener) {
    412         if (!mTestListeners.contains(listener)) {
    413             mTestListeners.add(listener);
    414         }
    415     }
    416 
    417     protected void removeTestListener(TestListener listener) {
    418         mTestListeners.remove(listener);
    419     }
    420 
    421     /**
    422      * subclass may override this method to perform further preparation
    423      *
    424      * @param testCase
    425      */
    426     protected void prepareTestCase(TestCase testCase) {
    427         ((UiAutomatorTestCase)testCase).setAutomationSupport(mAutomationSupport);
    428         ((UiAutomatorTestCase)testCase).setUiDevice(mUiDevice);
    429         ((UiAutomatorTestCase)testCase).setParams(mParams);
    430     }
    431 }
    432