1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.uiautomator.testrunner; 18 19 import android.app.Activity; 20 import android.app.IInstrumentationWatcher; 21 import android.app.Instrumentation; 22 import android.content.ComponentName; 23 import android.os.Bundle; 24 import android.os.Debug; 25 import android.os.HandlerThread; 26 import android.os.IBinder; 27 import android.os.SystemClock; 28 import android.test.RepetitiveTest; 29 import android.util.Log; 30 31 import com.android.uiautomator.core.ShellUiAutomatorBridge; 32 import com.android.uiautomator.core.Tracer; 33 import com.android.uiautomator.core.UiAutomationShellWrapper; 34 import com.android.uiautomator.core.UiDevice; 35 36 import java.io.ByteArrayOutputStream; 37 import java.io.PrintStream; 38 import java.lang.Thread.UncaughtExceptionHandler; 39 import java.lang.reflect.Method; 40 import java.util.ArrayList; 41 import java.util.List; 42 43 import junit.framework.AssertionFailedError; 44 import junit.framework.Test; 45 import junit.framework.TestCase; 46 import junit.framework.TestListener; 47 import junit.framework.TestResult; 48 import junit.runner.BaseTestRunner; 49 import junit.textui.ResultPrinter; 50 51 /** 52 * @hide 53 */ 54 public class UiAutomatorTestRunner { 55 56 private static final String LOGTAG = UiAutomatorTestRunner.class.getSimpleName(); 57 private static final int EXIT_OK = 0; 58 private static final int EXIT_EXCEPTION = -1; 59 60 private static final String HANDLER_THREAD_NAME = "UiAutomatorHandlerThread"; 61 62 private boolean mDebug; 63 private boolean mMonkey; 64 private Bundle mParams = null; 65 private UiDevice mUiDevice; 66 private List<String> mTestClasses = null; 67 private final FakeInstrumentationWatcher mWatcher = new FakeInstrumentationWatcher(); 68 private final IAutomationSupport mAutomationSupport = new IAutomationSupport() { 69 @Override 70 public void sendStatus(int resultCode, Bundle status) { 71 mWatcher.instrumentationStatus(null, resultCode, status); 72 } 73 }; 74 private final List<TestListener> mTestListeners = new ArrayList<TestListener>(); 75 76 private HandlerThread mHandlerThread; 77 78 public void run(List<String> testClasses, Bundle params, boolean debug, boolean monkey) { 79 Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler() { 80 @Override 81 public void uncaughtException(Thread thread, Throwable ex) { 82 Log.e(LOGTAG, "uncaught exception", ex); 83 Bundle results = new Bundle(); 84 results.putString("shortMsg", ex.getClass().getName()); 85 results.putString("longMsg", ex.getMessage()); 86 mWatcher.instrumentationFinished(null, 0, results); 87 // bailing on uncaught exception 88 System.exit(EXIT_EXCEPTION); 89 } 90 }); 91 92 mTestClasses = testClasses; 93 mParams = params; 94 mDebug = debug; 95 mMonkey = monkey; 96 start(); 97 System.exit(EXIT_OK); 98 } 99 100 /** 101 * Called after all test classes are in place, ready to test 102 */ 103 protected void start() { 104 TestCaseCollector collector = getTestCaseCollector(this.getClass().getClassLoader()); 105 try { 106 collector.addTestClasses(mTestClasses); 107 } catch (ClassNotFoundException e) { 108 // will be caught by uncaught handler 109 throw new RuntimeException(e.getMessage(), e); 110 } 111 if (mDebug) { 112 Debug.waitForDebugger(); 113 } 114 mHandlerThread = new HandlerThread(HANDLER_THREAD_NAME); 115 mHandlerThread.setDaemon(true); 116 mHandlerThread.start(); 117 UiAutomationShellWrapper automationWrapper = new UiAutomationShellWrapper(); 118 automationWrapper.connect(); 119 120 long startTime = SystemClock.uptimeMillis(); 121 TestResult testRunResult = new TestResult(); 122 ResultReporter resultPrinter; 123 String outputFormat = mParams.getString("outputFormat"); 124 List<TestCase> testCases = collector.getTestCases(); 125 Bundle testRunOutput = new Bundle(); 126 if ("simple".equals(outputFormat)) { 127 resultPrinter = new SimpleResultPrinter(System.out, true); 128 } else { 129 resultPrinter = new WatcherResultPrinter(testCases.size()); 130 } 131 try { 132 automationWrapper.setRunAsMonkey(mMonkey); 133 mUiDevice = UiDevice.getInstance(); 134 mUiDevice.initialize(new ShellUiAutomatorBridge(automationWrapper.getUiAutomation())); 135 136 String traceType = mParams.getString("traceOutputMode"); 137 if(traceType != null) { 138 Tracer.Mode mode = Tracer.Mode.valueOf(Tracer.Mode.class, traceType); 139 if (mode == Tracer.Mode.FILE || mode == Tracer.Mode.ALL) { 140 String filename = mParams.getString("traceLogFilename"); 141 if (filename == null) { 142 throw new RuntimeException("Name of log file not specified. " + 143 "Please specify it using traceLogFilename parameter"); 144 } 145 Tracer.getInstance().setOutputFilename(filename); 146 } 147 Tracer.getInstance().setOutputMode(mode); 148 } 149 150 // add test listeners 151 testRunResult.addListener(resultPrinter); 152 // add all custom listeners 153 for (TestListener listener : mTestListeners) { 154 testRunResult.addListener(listener); 155 } 156 157 // run tests for realz! 158 for (TestCase testCase : testCases) { 159 prepareTestCase(testCase); 160 testCase.run(testRunResult); 161 } 162 } catch (Throwable t) { 163 // catch all exceptions so a more verbose error message can be outputted 164 resultPrinter.printUnexpectedError(t); 165 testRunOutput.putString("shortMsg", t.getMessage()); 166 } finally { 167 long runTime = SystemClock.uptimeMillis() - startTime; 168 resultPrinter.print(testRunResult, runTime, testRunOutput); 169 automationWrapper.disconnect(); 170 automationWrapper.setRunAsMonkey(false); 171 mHandlerThread.quit(); 172 } 173 } 174 175 // copy & pasted from com.android.commands.am.Am 176 private class FakeInstrumentationWatcher implements IInstrumentationWatcher { 177 178 private final boolean mRawMode = true; 179 180 @Override 181 public IBinder asBinder() { 182 throw new UnsupportedOperationException("I'm just a fake!"); 183 } 184 185 @Override 186 public void instrumentationStatus(ComponentName name, int resultCode, Bundle results) { 187 synchronized (this) { 188 // pretty printer mode? 189 String pretty = null; 190 if (!mRawMode && results != null) { 191 pretty = results.getString(Instrumentation.REPORT_KEY_STREAMRESULT); 192 } 193 if (pretty != null) { 194 System.out.print(pretty); 195 } else { 196 if (results != null) { 197 for (String key : results.keySet()) { 198 System.out.println("INSTRUMENTATION_STATUS: " + key + "=" 199 + results.get(key)); 200 } 201 } 202 System.out.println("INSTRUMENTATION_STATUS_CODE: " + resultCode); 203 } 204 notifyAll(); 205 } 206 } 207 208 @Override 209 public void instrumentationFinished(ComponentName name, int resultCode, Bundle results) { 210 synchronized (this) { 211 // pretty printer mode? 212 String pretty = null; 213 if (!mRawMode && results != null) { 214 pretty = results.getString(Instrumentation.REPORT_KEY_STREAMRESULT); 215 } 216 if (pretty != null) { 217 System.out.println(pretty); 218 } else { 219 if (results != null) { 220 for (String key : results.keySet()) { 221 System.out.println("INSTRUMENTATION_RESULT: " + key + "=" 222 + results.get(key)); 223 } 224 } 225 System.out.println("INSTRUMENTATION_CODE: " + resultCode); 226 } 227 notifyAll(); 228 } 229 } 230 } 231 232 private interface ResultReporter extends TestListener { 233 public void print(TestResult result, long runTime, Bundle testOutput); 234 public void printUnexpectedError(Throwable t); 235 } 236 237 // Copy & pasted from InstrumentationTestRunner.WatcherResultPrinter 238 private class WatcherResultPrinter implements ResultReporter { 239 240 private static final String REPORT_KEY_NUM_TOTAL = "numtests"; 241 private static final String REPORT_KEY_NAME_CLASS = "class"; 242 private static final String REPORT_KEY_NUM_CURRENT = "current"; 243 private static final String REPORT_KEY_NAME_TEST = "test"; 244 private static final String REPORT_KEY_NUM_ITERATIONS = "numiterations"; 245 private static final String REPORT_VALUE_ID = "UiAutomatorTestRunner"; 246 private static final String REPORT_KEY_STACK = "stack"; 247 248 private static final int REPORT_VALUE_RESULT_START = 1; 249 private static final int REPORT_VALUE_RESULT_ERROR = -1; 250 private static final int REPORT_VALUE_RESULT_FAILURE = -2; 251 252 private final Bundle mResultTemplate; 253 Bundle mTestResult; 254 int mTestNum = 0; 255 int mTestResultCode = 0; 256 String mTestClass = null; 257 258 private final SimpleResultPrinter mPrinter; 259 private final ByteArrayOutputStream mStream; 260 private final PrintStream mWriter; 261 262 public WatcherResultPrinter(int numTests) { 263 mResultTemplate = new Bundle(); 264 mResultTemplate.putString(Instrumentation.REPORT_KEY_IDENTIFIER, REPORT_VALUE_ID); 265 mResultTemplate.putInt(REPORT_KEY_NUM_TOTAL, numTests); 266 267 mStream = new ByteArrayOutputStream(); 268 mWriter = new PrintStream(mStream); 269 mPrinter = new SimpleResultPrinter(mWriter, false); 270 } 271 272 /** 273 * send a status for the start of a each test, so long tests can be seen 274 * as "running" 275 */ 276 @Override 277 public void startTest(Test test) { 278 String testClass = test.getClass().getName(); 279 String testName = ((TestCase) test).getName(); 280 mTestResult = new Bundle(mResultTemplate); 281 mTestResult.putString(REPORT_KEY_NAME_CLASS, testClass); 282 mTestResult.putString(REPORT_KEY_NAME_TEST, testName); 283 mTestResult.putInt(REPORT_KEY_NUM_CURRENT, ++mTestNum); 284 // pretty printing 285 if (testClass != null && !testClass.equals(mTestClass)) { 286 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 287 String.format("\n%s:", testClass)); 288 mTestClass = testClass; 289 } else { 290 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, ""); 291 } 292 293 Method testMethod = null; 294 try { 295 testMethod = test.getClass().getMethod(testName); 296 // Report total number of iterations, if test is repetitive 297 if (testMethod.isAnnotationPresent(RepetitiveTest.class)) { 298 int numIterations = testMethod.getAnnotation(RepetitiveTest.class) 299 .numIterations(); 300 mTestResult.putInt(REPORT_KEY_NUM_ITERATIONS, numIterations); 301 } 302 } catch (NoSuchMethodException e) { 303 // ignore- the test with given name does not exist. Will be 304 // handled during test 305 // execution 306 } 307 308 mAutomationSupport.sendStatus(REPORT_VALUE_RESULT_START, mTestResult); 309 mTestResultCode = 0; 310 311 mPrinter.startTest(test); 312 } 313 314 @Override 315 public void addError(Test test, Throwable t) { 316 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 317 mTestResultCode = REPORT_VALUE_RESULT_ERROR; 318 // pretty printing 319 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 320 String.format("\nError in %s:\n%s", 321 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 322 323 mPrinter.addError(test, t); 324 } 325 326 @Override 327 public void addFailure(Test test, AssertionFailedError t) { 328 mTestResult.putString(REPORT_KEY_STACK, BaseTestRunner.getFilteredTrace(t)); 329 mTestResultCode = REPORT_VALUE_RESULT_FAILURE; 330 // pretty printing 331 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 332 String.format("\nFailure in %s:\n%s", 333 ((TestCase)test).getName(), BaseTestRunner.getFilteredTrace(t))); 334 335 mPrinter.addFailure(test, t); 336 } 337 338 @Override 339 public void endTest(Test test) { 340 if (mTestResultCode == 0) { 341 mTestResult.putString(Instrumentation.REPORT_KEY_STREAMRESULT, "."); 342 } 343 mAutomationSupport.sendStatus(mTestResultCode, mTestResult); 344 345 mPrinter.endTest(test); 346 } 347 348 @Override 349 public void print(TestResult result, long runTime, Bundle testOutput) { 350 mPrinter.print(result, runTime, testOutput); 351 testOutput.putString(Instrumentation.REPORT_KEY_STREAMRESULT, 352 String.format("\nTest results for %s=%s", 353 getClass().getSimpleName(), 354 mStream.toString())); 355 mWriter.close(); 356 mAutomationSupport.sendStatus(Activity.RESULT_OK, testOutput); 357 } 358 359 @Override 360 public void printUnexpectedError(Throwable t) { 361 mWriter.println(String.format("Test run aborted due to unexpected exception: %s", 362 t.getMessage())); 363 t.printStackTrace(mWriter); 364 } 365 } 366 367 /** 368 * Class that produces the same output as JUnit when running from command line. Can be 369 * used when default UiAutomator output is too verbose. 370 */ 371 private class SimpleResultPrinter extends ResultPrinter implements ResultReporter { 372 private final boolean mFullOutput; 373 public SimpleResultPrinter(PrintStream writer, boolean fullOutput) { 374 super(writer); 375 mFullOutput = fullOutput; 376 } 377 378 @Override 379 public void print(TestResult result, long runTime, Bundle testOutput) { 380 printHeader(runTime); 381 if (mFullOutput) { 382 printErrors(result); 383 printFailures(result); 384 } 385 printFooter(result); 386 } 387 388 @Override 389 public void printUnexpectedError(Throwable t) { 390 if (mFullOutput) { 391 getWriter().printf("Test run aborted due to unexpected exeption: %s", 392 t.getMessage()); 393 t.printStackTrace(getWriter()); 394 } 395 } 396 } 397 398 protected TestCaseCollector getTestCaseCollector(ClassLoader classLoader) { 399 return new TestCaseCollector(classLoader, getTestCaseFilter()); 400 } 401 402 /** 403 * Returns an object which determines if the class and its methods should be 404 * accepted into the test suite. 405 * @return 406 */ 407 public UiAutomatorTestCaseFilter getTestCaseFilter() { 408 return new UiAutomatorTestCaseFilter(); 409 } 410 411 protected void addTestListener(TestListener listener) { 412 if (!mTestListeners.contains(listener)) { 413 mTestListeners.add(listener); 414 } 415 } 416 417 protected void removeTestListener(TestListener listener) { 418 mTestListeners.remove(listener); 419 } 420 421 /** 422 * subclass may override this method to perform further preparation 423 * 424 * @param testCase 425 */ 426 protected void prepareTestCase(TestCase testCase) { 427 ((UiAutomatorTestCase)testCase).setAutomationSupport(mAutomationSupport); 428 ((UiAutomatorTestCase)testCase).setUiDevice(mUiDevice); 429 ((UiAutomatorTestCase)testCase).setParams(mParams); 430 } 431 } 432