1 /* 2 * Copyright (C) 2006 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.test; 18 19 import android.content.Context; 20 import android.util.Log; 21 import android.os.Debug; 22 import android.os.SystemClock; 23 24 import java.io.File; 25 import java.lang.reflect.InvocationTargetException; 26 import java.lang.reflect.Method; 27 import java.lang.reflect.Modifier; 28 import java.util.ArrayList; 29 import java.util.List; 30 31 import junit.framework.TestSuite; 32 import junit.framework.TestListener; 33 import junit.framework.Test; 34 import junit.framework.TestResult; 35 import com.google.android.collect.Lists; 36 37 /** 38 * Support class that actually runs a test. Android uses this class, 39 * and you probably will not need to instantiate, extend, or call this 40 * class yourself. See the full {@link android.test} package description 41 * to learn more about testing Android applications. 42 * 43 * {@hide} Not needed for 1.0 SDK. 44 */ 45 public class TestRunner implements PerformanceTestCase.Intermediates { 46 public static final int REGRESSION = 0; 47 public static final int PERFORMANCE = 1; 48 public static final int PROFILING = 2; 49 50 public static final int CLEARSCREEN = 0; 51 private static final String TAG = "TestHarness"; 52 private Context mContext; 53 54 private int mMode = REGRESSION; 55 56 private List<Listener> mListeners = Lists.newArrayList(); 57 private int mPassed; 58 private int mFailed; 59 60 private int mInternalIterations; 61 private long mStartTime; 62 private long mEndTime; 63 64 private String mClassName; 65 66 List<IntermediateTime> mIntermediates = null; 67 68 private static Class mRunnableClass; 69 private static Class mJUnitClass; 70 71 static { 72 try { 73 mRunnableClass = Class.forName("java.lang.Runnable", false, null); 74 mJUnitClass = Class.forName("junit.framework.TestCase", false, null); 75 } catch (ClassNotFoundException ex) { 76 throw new RuntimeException("shouldn't happen", ex); 77 } 78 } 79 80 public class JunitTestSuite extends TestSuite implements TestListener { 81 boolean mError = false; 82 83 public JunitTestSuite() { 84 super(); 85 } 86 87 @Override 88 public void run(TestResult result) { 89 result.addListener(this); 90 super.run(result); 91 result.removeListener(this); 92 } 93 94 /** 95 * Implemented method of the interface TestListener which will listen for the 96 * start of a test. 97 * 98 * @param test 99 */ 100 public void startTest(Test test) { 101 started(test.toString()); 102 } 103 104 /** 105 * Implemented method of the interface TestListener which will listen for the 106 * end of the test. 107 * 108 * @param test 109 */ 110 public void endTest(Test test) { 111 finished(test.toString()); 112 if (!mError) { 113 passed(test.toString()); 114 } 115 } 116 117 /** 118 * Implemented method of the interface TestListener which will listen for an 119 * mError while running the test. 120 * 121 * @param test 122 */ 123 public void addError(Test test, Throwable t) { 124 mError = true; 125 failed(test.toString(), t); 126 } 127 128 public void addFailure(Test test, junit.framework.AssertionFailedError t) { 129 mError = true; 130 failed(test.toString(), t); 131 } 132 } 133 134 /** 135 * Listener.performance() 'intermediates' argument is a list of these. 136 */ 137 public static class IntermediateTime { 138 public IntermediateTime(String name, long timeInNS) { 139 this.name = name; 140 this.timeInNS = timeInNS; 141 } 142 143 public String name; 144 public long timeInNS; 145 } 146 147 /** 148 * Support class that receives status on test progress. You should not need to 149 * extend this interface yourself. 150 */ 151 public interface Listener { 152 void started(String className); 153 void finished(String className); 154 void performance(String className, 155 long itemTimeNS, int iterations, 156 List<IntermediateTime> itermediates); 157 void passed(String className); 158 void failed(String className, Throwable execption); 159 } 160 161 public TestRunner(Context context) { 162 mContext = context; 163 } 164 165 public void addListener(Listener listener) { 166 mListeners.add(listener); 167 } 168 169 public void startProfiling() { 170 File file = new File("/tmp/trace"); 171 file.mkdir(); 172 String base = "/tmp/trace/" + mClassName + ".dmtrace"; 173 Debug.startMethodTracing(base, 8 * 1024 * 1024); 174 } 175 176 public void finishProfiling() { 177 Debug.stopMethodTracing(); 178 } 179 180 private void started(String className) { 181 182 int count = mListeners.size(); 183 for (int i = 0; i < count; i++) { 184 mListeners.get(i).started(className); 185 } 186 } 187 188 private void finished(String className) { 189 int count = mListeners.size(); 190 for (int i = 0; i < count; i++) { 191 mListeners.get(i).finished(className); 192 } 193 } 194 195 private void performance(String className, 196 long itemTimeNS, 197 int iterations, 198 List<IntermediateTime> intermediates) { 199 int count = mListeners.size(); 200 for (int i = 0; i < count; i++) { 201 mListeners.get(i).performance(className, 202 itemTimeNS, 203 iterations, 204 intermediates); 205 } 206 } 207 208 public void passed(String className) { 209 mPassed++; 210 int count = mListeners.size(); 211 for (int i = 0; i < count; i++) { 212 mListeners.get(i).passed(className); 213 } 214 } 215 216 public void failed(String className, Throwable exception) { 217 mFailed++; 218 int count = mListeners.size(); 219 for (int i = 0; i < count; i++) { 220 mListeners.get(i).failed(className, exception); 221 } 222 } 223 224 public int passedCount() { 225 return mPassed; 226 } 227 228 public int failedCount() { 229 return mFailed; 230 } 231 232 public void run(String[] classes) { 233 for (String cl : classes) { 234 run(cl); 235 } 236 } 237 238 public void setInternalIterations(int count) { 239 mInternalIterations = count; 240 } 241 242 public void startTiming(boolean realTime) { 243 if (realTime) { 244 mStartTime = System.currentTimeMillis(); 245 } else { 246 mStartTime = SystemClock.currentThreadTimeMillis(); 247 } 248 } 249 250 public void addIntermediate(String name) { 251 addIntermediate(name, (System.currentTimeMillis() - mStartTime) * 1000000); 252 } 253 254 public void addIntermediate(String name, long timeInNS) { 255 mIntermediates.add(new IntermediateTime(name, timeInNS)); 256 } 257 258 public void finishTiming(boolean realTime) { 259 if (realTime) { 260 mEndTime = System.currentTimeMillis(); 261 } else { 262 mEndTime = SystemClock.currentThreadTimeMillis(); 263 } 264 } 265 266 public void setPerformanceMode(int mode) { 267 mMode = mode; 268 } 269 270 private void missingTest(String className, Throwable e) { 271 started(className); 272 finished(className); 273 failed(className, e); 274 } 275 276 /* 277 This class determines if more suites are added to this class then adds all individual 278 test classes to a test suite for run 279 */ 280 public void run(String className) { 281 try { 282 mClassName = className; 283 Class clazz = mContext.getClassLoader().loadClass(className); 284 Method method = getChildrenMethod(clazz); 285 if (method != null) { 286 String[] children = getChildren(method); 287 run(children); 288 } else if (mRunnableClass.isAssignableFrom(clazz)) { 289 Runnable test = (Runnable) clazz.newInstance(); 290 TestCase testcase = null; 291 if (test instanceof TestCase) { 292 testcase = (TestCase) test; 293 } 294 Throwable e = null; 295 boolean didSetup = false; 296 started(className); 297 try { 298 if (testcase != null) { 299 testcase.setUp(mContext); 300 didSetup = true; 301 } 302 if (mMode == PERFORMANCE) { 303 runInPerformanceMode(test, className, false, className); 304 } else if (mMode == PROFILING) { 305 //Need a way to mark a test to be run in profiling mode or not. 306 startProfiling(); 307 test.run(); 308 finishProfiling(); 309 } else { 310 test.run(); 311 } 312 } catch (Throwable ex) { 313 e = ex; 314 } 315 if (testcase != null && didSetup) { 316 try { 317 testcase.tearDown(); 318 } catch (Throwable ex) { 319 e = ex; 320 } 321 } 322 finished(className); 323 if (e == null) { 324 passed(className); 325 } else { 326 failed(className, e); 327 } 328 } else if (mJUnitClass.isAssignableFrom(clazz)) { 329 Throwable e = null; 330 //Create a Junit Suite. 331 JunitTestSuite suite = new JunitTestSuite(); 332 Method[] methods = getAllTestMethods(clazz); 333 for (Method m : methods) { 334 junit.framework.TestCase test = (junit.framework.TestCase) clazz.newInstance(); 335 test.setName(m.getName()); 336 337 if (test instanceof AndroidTestCase) { 338 AndroidTestCase testcase = (AndroidTestCase) test; 339 try { 340 testcase.setContext(mContext); 341 testcase.setTestContext(mContext); 342 } catch (Exception ex) { 343 Log.i("TestHarness", ex.toString()); 344 } 345 } 346 suite.addTest(test); 347 } 348 if (mMode == PERFORMANCE) { 349 final int testCount = suite.testCount(); 350 351 for (int j = 0; j < testCount; j++) { 352 Test test = suite.testAt(j); 353 started(test.toString()); 354 try { 355 runInPerformanceMode(test, className, true, test.toString()); 356 } catch (Throwable ex) { 357 e = ex; 358 } 359 finished(test.toString()); 360 if (e == null) { 361 passed(test.toString()); 362 } else { 363 failed(test.toString(), e); 364 } 365 } 366 } else if (mMode == PROFILING) { 367 //Need a way to mark a test to be run in profiling mode or not. 368 startProfiling(); 369 junit.textui.TestRunner.run(suite); 370 finishProfiling(); 371 } else { 372 junit.textui.TestRunner.run(suite); 373 } 374 } else { 375 System.out.println("Test wasn't Runnable and didn't have a" 376 + " children method: " + className); 377 } 378 } catch (ClassNotFoundException e) { 379 Log.e("ClassNotFoundException for " + className, e.toString()); 380 if (isJunitTest(className)) { 381 runSingleJunitTest(className); 382 } else { 383 missingTest(className, e); 384 } 385 } catch (InstantiationException e) { 386 System.out.println("InstantiationException for " + className); 387 missingTest(className, e); 388 } catch (IllegalAccessException e) { 389 System.out.println("IllegalAccessException for " + className); 390 missingTest(className, e); 391 } 392 } 393 394 public void runInPerformanceMode(Object testCase, String className, boolean junitTest, 395 String testNameInDb) throws Exception { 396 boolean increaseIterations = true; 397 int iterations = 1; 398 long duration = 0; 399 mIntermediates = null; 400 401 mInternalIterations = 1; 402 Class clazz = mContext.getClassLoader().loadClass(className); 403 Object perftest = clazz.newInstance(); 404 405 PerformanceTestCase perftestcase = null; 406 if (perftest instanceof PerformanceTestCase) { 407 perftestcase = (PerformanceTestCase) perftest; 408 // only run the test if it is not marked as a performance only test 409 if (mMode == REGRESSION && perftestcase.isPerformanceOnly()) return; 410 } 411 412 // First force GCs, to avoid GCs happening during out 413 // test and skewing its time. 414 Runtime.getRuntime().runFinalization(); 415 Runtime.getRuntime().gc(); 416 417 if (perftestcase != null) { 418 mIntermediates = new ArrayList<IntermediateTime>(); 419 iterations = perftestcase.startPerformance(this); 420 if (iterations > 0) { 421 increaseIterations = false; 422 } else { 423 iterations = 1; 424 } 425 } 426 427 // Pause briefly to let things settle down... 428 Thread.sleep(1000); 429 do { 430 mEndTime = 0; 431 if (increaseIterations) { 432 // Test case does not implement 433 // PerformanceTestCase or returned 0 iterations, 434 // so we take care of measure the whole test time. 435 mStartTime = SystemClock.currentThreadTimeMillis(); 436 } else { 437 // Try to make it obvious if the test case 438 // doesn't call startTiming(). 439 mStartTime = 0; 440 } 441 442 if (junitTest) { 443 for (int i = 0; i < iterations; i++) { 444 junit.textui.TestRunner.run((junit.framework.Test) testCase); 445 } 446 } else { 447 Runnable test = (Runnable) testCase; 448 for (int i = 0; i < iterations; i++) { 449 test.run(); 450 } 451 } 452 453 long endTime = mEndTime; 454 if (endTime == 0) { 455 endTime = SystemClock.currentThreadTimeMillis(); 456 } 457 458 duration = endTime - mStartTime; 459 if (!increaseIterations) { 460 break; 461 } 462 if (duration <= 1) { 463 iterations *= 1000; 464 } else if (duration <= 10) { 465 iterations *= 100; 466 } else if (duration < 100) { 467 iterations *= 10; 468 } else if (duration < 1000) { 469 iterations *= (int) ((1000 / duration) + 2); 470 } else { 471 break; 472 } 473 } while (true); 474 475 if (duration != 0) { 476 iterations *= mInternalIterations; 477 performance(testNameInDb, (duration * 1000000) / iterations, 478 iterations, mIntermediates); 479 } 480 } 481 482 public void runSingleJunitTest(String className) { 483 Throwable excep = null; 484 int index = className.lastIndexOf('$'); 485 String testName = ""; 486 String originalClassName = className; 487 if (index >= 0) { 488 className = className.substring(0, index); 489 testName = originalClassName.substring(index + 1); 490 } 491 try { 492 Class clazz = mContext.getClassLoader().loadClass(className); 493 if (mJUnitClass.isAssignableFrom(clazz)) { 494 junit.framework.TestCase test = (junit.framework.TestCase) clazz.newInstance(); 495 JunitTestSuite newSuite = new JunitTestSuite(); 496 test.setName(testName); 497 498 if (test instanceof AndroidTestCase) { 499 AndroidTestCase testcase = (AndroidTestCase) test; 500 try { 501 testcase.setContext(mContext); 502 } catch (Exception ex) { 503 Log.w(TAG, "Exception encountered while trying to set the context.", ex); 504 } 505 } 506 newSuite.addTest(test); 507 508 if (mMode == PERFORMANCE) { 509 try { 510 started(test.toString()); 511 runInPerformanceMode(test, className, true, test.toString()); 512 finished(test.toString()); 513 if (excep == null) { 514 passed(test.toString()); 515 } else { 516 failed(test.toString(), excep); 517 } 518 } catch (Throwable ex) { 519 excep = ex; 520 } 521 522 } else if (mMode == PROFILING) { 523 startProfiling(); 524 junit.textui.TestRunner.run(newSuite); 525 finishProfiling(); 526 } else { 527 junit.textui.TestRunner.run(newSuite); 528 } 529 } 530 } catch (ClassNotFoundException e) { 531 Log.e("TestHarness", "No test case to run", e); 532 } catch (IllegalAccessException e) { 533 Log.e("TestHarness", "Illegal Access Exception", e); 534 } catch (InstantiationException e) { 535 Log.e("TestHarness", "Instantiation Exception", e); 536 } 537 } 538 539 public static Method getChildrenMethod(Class clazz) { 540 try { 541 return clazz.getMethod("children", (Class[]) null); 542 } catch (NoSuchMethodException e) { 543 } 544 545 return null; 546 } 547 548 public static Method getChildrenMethod(Context c, String className) { 549 try { 550 return getChildrenMethod(c.getClassLoader().loadClass(className)); 551 } catch (ClassNotFoundException e) { 552 } 553 return null; 554 } 555 556 public static String[] getChildren(Context c, String className) { 557 Method m = getChildrenMethod(c, className); 558 String[] testChildren = getTestChildren(c, className); 559 if (m == null & testChildren == null) { 560 throw new RuntimeException("couldn't get children method for " 561 + className); 562 } 563 if (m != null) { 564 String[] children = getChildren(m); 565 if (testChildren != null) { 566 String[] allChildren = new String[testChildren.length + children.length]; 567 System.arraycopy(children, 0, allChildren, 0, children.length); 568 System.arraycopy(testChildren, 0, allChildren, children.length, testChildren.length); 569 return allChildren; 570 } else { 571 return children; 572 } 573 } else { 574 if (testChildren != null) { 575 return testChildren; 576 } 577 } 578 return null; 579 } 580 581 public static String[] getChildren(Method m) { 582 try { 583 if (!Modifier.isStatic(m.getModifiers())) { 584 throw new RuntimeException("children method is not static"); 585 } 586 return (String[]) m.invoke(null, (Object[]) null); 587 } catch (IllegalAccessException e) { 588 } catch (InvocationTargetException e) { 589 } 590 return new String[0]; 591 } 592 593 public static String[] getTestChildren(Context c, String className) { 594 try { 595 Class clazz = c.getClassLoader().loadClass(className); 596 597 if (mJUnitClass.isAssignableFrom(clazz)) { 598 return getTestChildren(clazz); 599 } 600 } catch (ClassNotFoundException e) { 601 Log.e("TestHarness", "No class found", e); 602 } 603 return null; 604 } 605 606 public static String[] getTestChildren(Class clazz) { 607 Method[] methods = getAllTestMethods(clazz); 608 609 String[] onScreenTestNames = new String[methods.length]; 610 int index = 0; 611 for (Method m : methods) { 612 onScreenTestNames[index] = clazz.getName() + "$" + m.getName(); 613 index++; 614 } 615 return onScreenTestNames; 616 } 617 618 public static Method[] getAllTestMethods(Class clazz) { 619 Method[] allMethods = clazz.getDeclaredMethods(); 620 int numOfMethods = 0; 621 for (Method m : allMethods) { 622 boolean mTrue = isTestMethod(m); 623 if (mTrue) { 624 numOfMethods++; 625 } 626 } 627 int index = 0; 628 Method[] testMethods = new Method[numOfMethods]; 629 for (Method m : allMethods) { 630 boolean mTrue = isTestMethod(m); 631 if (mTrue) { 632 testMethods[index] = m; 633 index++; 634 } 635 } 636 return testMethods; 637 } 638 639 private static boolean isTestMethod(Method m) { 640 return m.getName().startsWith("test") && 641 m.getReturnType() == void.class && 642 m.getParameterTypes().length == 0; 643 } 644 645 public static int countJunitTests(Class clazz) { 646 Method[] allTestMethods = getAllTestMethods(clazz); 647 int numberofMethods = allTestMethods.length; 648 649 return numberofMethods; 650 } 651 652 public static boolean isTestSuite(Context c, String className) { 653 boolean childrenMethods = getChildrenMethod(c, className) != null; 654 655 try { 656 Class clazz = c.getClassLoader().loadClass(className); 657 if (mJUnitClass.isAssignableFrom(clazz)) { 658 int numTests = countJunitTests(clazz); 659 if (numTests > 0) 660 childrenMethods = true; 661 } 662 } catch (ClassNotFoundException e) { 663 } 664 return childrenMethods; 665 } 666 667 668 public boolean isJunitTest(String className) { 669 int index = className.lastIndexOf('$'); 670 if (index >= 0) { 671 className = className.substring(0, index); 672 } 673 try { 674 Class clazz = mContext.getClassLoader().loadClass(className); 675 if (mJUnitClass.isAssignableFrom(clazz)) { 676 return true; 677 } 678 } catch (ClassNotFoundException e) { 679 } 680 return false; 681 } 682 683 /** 684 * Returns the number of tests that will be run if you try to do this. 685 */ 686 public static int countTests(Context c, String className) { 687 try { 688 Class clazz = c.getClassLoader().loadClass(className); 689 Method method = getChildrenMethod(clazz); 690 if (method != null) { 691 692 String[] children = getChildren(method); 693 int rv = 0; 694 for (String child : children) { 695 rv += countTests(c, child); 696 } 697 return rv; 698 } else if (mRunnableClass.isAssignableFrom(clazz)) { 699 return 1; 700 } else if (mJUnitClass.isAssignableFrom(clazz)) { 701 return countJunitTests(clazz); 702 } 703 } catch (ClassNotFoundException e) { 704 return 1; // this gets the count right, because either this test 705 // is missing, and it will fail when run or it is a single Junit test to be run. 706 } 707 return 0; 708 } 709 710 /** 711 * Returns a title to display given the className of a test. 712 * <p/> 713 * <p>Currently this function just returns the portion of the 714 * class name after the last '.' 715 */ 716 public static String getTitle(String className) { 717 int indexDot = className.lastIndexOf('.'); 718 int indexDollar = className.lastIndexOf('$'); 719 int index = indexDot > indexDollar ? indexDot : indexDollar; 720 if (index >= 0) { 721 className = className.substring(index + 1); 722 } 723 return className; 724 } 725 } 726
