1 /* 2 * Copyright (C) 2009 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.coretests; 17 18 import java.lang.reflect.Field; 19 import java.lang.reflect.Method; 20 import java.lang.reflect.Modifier; 21 import java.util.Enumeration; 22 import java.util.Vector; 23 24 import junit.framework.Test; 25 import junit.framework.TestCase; 26 import junit.framework.TestFailure; 27 import junit.framework.TestResult; 28 import junit.framework.TestSuite; 29 import dalvik.annotation.AndroidOnly; 30 import dalvik.annotation.BrokenTest; 31 import dalvik.annotation.KnownFailure; 32 import dalvik.annotation.SideEffect; 33 34 /** 35 * A special TestSuite implementation that flattens the hierarchy of a given 36 * JUnit TestSuite and removes tests after executing them. This is so the core 37 * tests actually have a chance to succeed, since they do consume quite some 38 * memory and many tests do not (have a chance to) cleanup properly after 39 * themselves. The class also implements our filtering mechanism for tests, so 40 * it becomes easy to only include or exclude tests based on their annotations 41 * (like, say, execute all Android-only tests that are not known to be broken). 42 */ 43 public class CoreTestSuite implements Test { 44 45 /** 46 * Include all normal tests in the suite. 47 */ 48 public static final int RUN_NORMAL_TESTS = 1; 49 50 /** 51 * Include all broken tests in the suite. 52 */ 53 public static final int RUN_BROKEN_TESTS = 2; 54 55 /** 56 * Include all known failures in the suite. 57 */ 58 public static final int RUN_KNOWN_FAILURES = 4; 59 60 /** 61 * Include all Android-only tests in the suite. 62 */ 63 public static final int RUN_ANDROID_ONLY = 8; 64 65 /** 66 * Include side-effective tests in the suite. 67 */ 68 public static final int RUN_SIDE_EFFECTS = 16; 69 70 /** 71 * Include all tests in the suite. 72 */ 73 public static final int RUN_ALL_TESTS = 74 RUN_NORMAL_TESTS | RUN_BROKEN_TESTS | 75 RUN_KNOWN_FAILURES | RUN_SIDE_EFFECTS | RUN_ANDROID_ONLY; 76 77 /** 78 * Special treatment for known failures: they are expected to fail, so we 79 * throw an Exception if they succeed and accept them failing. 80 */ 81 public static final int INVERT_KNOWN_FAILURES = 32; 82 83 /** 84 * Run each test in its own VM. 85 */ 86 public static final int ISOLATE_ALL = 64; 87 88 /** 89 * Run no test in its own VM. 90 */ 91 public static final int ISOLATE_NONE = 128; 92 93 /** 94 * Be verbose. 95 */ 96 public static final int VERBOSE = 256; 97 98 public static final int REVERSE = 512; 99 100 public static final int DRY_RUN = 1024; 101 102 private final String name; 103 104 /** 105 * The total number of tests in the original suite. 106 */ 107 protected int fTotalCount; 108 109 /** 110 * The number of Android-only tests in the original suite. 111 */ 112 protected int fAndroidOnlyCount; 113 114 /** 115 * The number of broken tests in the original suite. 116 */ 117 protected int fBrokenCount; 118 119 /** 120 * The number of known failures in the original suite. 121 */ 122 protected int fKnownFailureCount; 123 124 /** 125 * The number of side-effective tests in the original suite. 126 */ 127 protected int fSideEffectCount; 128 129 /** 130 * The number of normal (non-annotated) tests in the original suite. 131 */ 132 protected int fNormalCount; 133 134 /** 135 * The number of ignored tests, that is, the number of tests that were 136 * excluded from this suite due to their annotations. 137 */ 138 protected int fIgnoredCount; 139 140 /** 141 * Contains the actual test cases in a reverse-ordered, flat list. 142 */ 143 private Vector<Test> fTests = new Vector<Test>(); 144 145 private TestCase fVictim; 146 147 private int fStep; 148 149 private int fFlags; 150 151 /** 152 * Creates a new CoreTestSuite for the given ordinary JUnit Test (which may 153 * be a TestCase or TestSuite). The CoreTestSuite will be a flattened and 154 * potentially filtered subset of the original JUnit Test. The flags 155 * determine the way we filter. 156 */ 157 public CoreTestSuite(Test suite, int flags, int step, TestCase victim) { 158 super(); 159 160 name = suite.toString(); 161 fStep = step; 162 addAndFlatten(suite, flags); 163 fVictim = victim; 164 fFlags = flags; 165 } 166 167 /** 168 * Adds the given ordinary JUnit Test (which may be a TestCase or TestSuite) 169 * to this CoreTestSuite. Note we are storing the tests in reverse order, 170 * so it's easier to remove a finished test from the end of the list. 171 */ 172 private void addAndFlatten(Test test, int flags) { 173 if (test instanceof TestSuite) { 174 TestSuite suite = (TestSuite)test; 175 176 if ((flags & REVERSE) != 0) { 177 for (int i = suite.testCount() - 1; i >= 0; i--) { 178 addAndFlatten(suite.testAt(i), flags); 179 } 180 } else { 181 for (int i = 0; i < suite.testCount(); i++) { 182 addAndFlatten(suite.testAt(i), flags); 183 } 184 } 185 } else if (test instanceof TestCase) { 186 TestCase testCase = (TestCase)test; 187 boolean ignoreMe = false; 188 189 boolean isAndroidOnly = hasAnnotation(testCase, AndroidOnly.class); 190 boolean isBrokenTest = hasAnnotation(testCase, BrokenTest.class); 191 boolean isKnownFailure = hasAnnotation(testCase, KnownFailure.class); 192 boolean isSideEffect = hasAnnotation(testCase, SideEffect.class); 193 boolean isNormalTest = 194 !(isAndroidOnly || isBrokenTest || isKnownFailure || 195 isSideEffect); 196 197 if (isAndroidOnly) { 198 fAndroidOnlyCount++; 199 } 200 201 if (isBrokenTest) { 202 fBrokenCount++; 203 } 204 205 if (isKnownFailure) { 206 fKnownFailureCount++; 207 } 208 209 if (isNormalTest) { 210 fNormalCount++; 211 } 212 213 if (isSideEffect) { 214 fSideEffectCount++; 215 } 216 217 if ((flags & RUN_ANDROID_ONLY) == 0 && isAndroidOnly) { 218 ignoreMe = true; 219 } 220 221 if ((flags & RUN_BROKEN_TESTS) == 0 && isBrokenTest) { 222 ignoreMe = true; 223 } 224 225 if ((flags & RUN_KNOWN_FAILURES) == 0 && isKnownFailure) { 226 ignoreMe = true; 227 } 228 229 if (((flags & RUN_NORMAL_TESTS) == 0) && isNormalTest) { 230 ignoreMe = true; 231 } 232 233 if (((flags & RUN_SIDE_EFFECTS) == 0) && isSideEffect) { 234 ignoreMe = true; 235 } 236 237 this.fTotalCount++; 238 239 if (!ignoreMe) { 240 fTests.add(test); 241 } else { 242 this.fIgnoredCount++; 243 } 244 } else { 245 System.out.println("Warning: Don't know how to handle " + 246 test.getClass().getName() + " " + test.toString()); 247 } 248 } 249 250 /** 251 * Checks whether the given TestCase class has the given annotation. 252 */ 253 @SuppressWarnings("unchecked") 254 private boolean hasAnnotation(TestCase test, Class clazz) { 255 try { 256 Method method = test.getClass().getMethod(test.getName()); 257 return method.getAnnotation(clazz) != null; 258 } catch (Exception e) { 259 // Ignore 260 } 261 262 return false; 263 } 264 265 /** 266 * Runs the tests and collects their result in a TestResult. 267 */ 268 public void run(TestResult result) { 269 // Run tests 270 int i = 0; 271 272 while (fTests.size() != 0 && !result.shouldStop()) { 273 TestCase test = (TestCase)fTests.elementAt(i); 274 275 Thread.currentThread().setContextClassLoader( 276 test.getClass().getClassLoader()); 277 278 test.run(result); 279 280 /* 281 if (fVictim != null) { 282 TestResult dummy = fVictim.run(); 283 284 if (dummy.failureCount() != 0) { 285 result.addError(fTests.elementAt(i), new RuntimeException( 286 "Probable side effect", 287 ((TestFailure)dummy.failures().nextElement()). 288 thrownException())); 289 } else if (dummy.errorCount() != 0) { 290 result.addError(fTests.elementAt(i), new RuntimeException( 291 "Probable side effect", 292 ((TestFailure)dummy.errors().nextElement()). 293 thrownException())); 294 } 295 } 296 */ 297 298 fTests.remove(i); 299 300 if (fTests.size() != 0) { 301 i = (i + fStep - 1) % fTests.size(); 302 } 303 304 } 305 306 // Forward overall stats to TestResult, so ResultPrinter can see it. 307 if (result instanceof CoreTestResult) { 308 ((CoreTestResult)result).updateStats( 309 fTotalCount, fAndroidOnlyCount, fBrokenCount, 310 fKnownFailureCount, fNormalCount, fIgnoredCount, 311 fSideEffectCount); 312 } 313 } 314 315 /** 316 * Nulls all reference fields in the given test object. This method helps 317 * us with those test classes that don't have an explicit tearDown() 318 * method. Normally the garbage collector should take care of everything, 319 * but it can't hurt to support it a bit. 320 */ 321 private void cleanup(TestCase test) { 322 Field[] fields = test.getClass().getDeclaredFields(); 323 for (int i = 0; i < fields.length; i++) { 324 Field f = fields[i]; 325 if (!f.getType().isPrimitive() && 326 (f.getModifiers() & Modifier.STATIC) == 0) { 327 try { 328 f.setAccessible(true); 329 f.set(test, null); 330 } catch (Exception ex) { 331 // Nothing we can do about it. 332 } 333 } 334 } 335 } 336 337 /** 338 * Returns the tests as an enumeration. Note this is empty once the tests 339 * have been executed. 340 */ 341 @SuppressWarnings("unchecked") 342 public Enumeration tests() { 343 return fTests.elements(); 344 } 345 346 /** 347 * Returns the number of tests in the suite. Note this is zero once the 348 * tests have been executed. 349 */ 350 public int countTestCases() { 351 return fTests.size(); 352 } 353 354 @Override public String toString() { 355 return name; 356 } 357 } 358