1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.cts.core.runner.support; 18 19 import android.util.Log; 20 21 import org.junit.runner.Description; 22 import org.junit.runner.Runner; 23 import org.junit.runner.manipulation.Filter; 24 import org.junit.runner.manipulation.Filterable; 25 import org.junit.runner.manipulation.NoTestsRemainException; 26 import org.junit.runner.notification.Failure; 27 import org.junit.runner.notification.RunNotifier; 28 29 import java.lang.reflect.Method; 30 import java.lang.reflect.Modifier; 31 import java.util.HashSet; 32 import java.util.Map; 33 34 /** 35 * A {@link Runner} that can TestNG tests. 36 * 37 * <p>Implementation note: Avoid extending ParentRunner since that also has 38 * logic to handle BeforeClass/AfterClass and other junit-specific functionality 39 * that would be invalid for TestNG.</p> 40 */ 41 class TestNgRunner extends Runner implements Filterable { 42 43 private static final boolean DEBUG = false; 44 45 private Description mDescription; 46 /** Class name for debugging. */ 47 private String mClassName; 48 /** Don't include the same method names twice. */ 49 private HashSet<String> mMethodSet = new HashSet<>(); 50 51 /** 52 * @param testClass the test class to run 53 */ 54 TestNgRunner(Class<?> testClass) { 55 mDescription = generateTestNgDescription(testClass); 56 mClassName = testClass.getName(); 57 } 58 59 // Runner implementation 60 @Override 61 public Description getDescription() { 62 return mDescription; 63 } 64 65 // Runner implementation 66 @Override 67 public int testCount() { 68 if (!descriptionHasChildren(getDescription())) { // Avoid NPE when description is null. 69 return 0; 70 } 71 72 // We always follow a flat Parent->Leaf hierarchy, so no recursion necessary. 73 return getDescription().testCount(); 74 } 75 76 // Filterable implementation 77 @Override 78 public void filter(Filter filter) throws NoTestsRemainException { 79 mDescription = filterDescription(mDescription, filter); 80 81 if (!descriptionHasChildren(getDescription())) { // Avoid NPE when description is null. 82 if (DEBUG) { 83 Log.d("TestNgRunner", 84 "Filtering has removed all tests :( for class " + mClassName); 85 } 86 throw new NoTestsRemainException(); 87 } 88 89 if (DEBUG) { 90 Log.d("TestNgRunner", 91 "Filtering has retained " + testCount() + " tests for class " + mClassName); 92 } 93 } 94 95 // Filterable implementation 96 @Override 97 public void run(RunNotifier notifier) { 98 if (!descriptionHasChildren(getDescription())) { // Avoid NPE when description is null. 99 // Nothing to do. 100 return; 101 } 102 103 for (Description child : getDescription().getChildren()) { 104 String className = child.getClassName(); 105 String methodName = child.getMethodName(); 106 107 Class<?> klass; 108 try { 109 klass = Class.forName(className, false, Thread.currentThread().getContextClassLoader()); 110 } catch (ClassNotFoundException e) { 111 throw new AssertionError(e); 112 } 113 114 notifier.fireTestStarted(child); 115 116 // Avoid looking at all the methods by just using the string method name. 117 SingleTestNgTestExecutor.Result result = SingleTestNgTestExecutor.execute(klass, methodName); 118 if (result.hasFailure()) { 119 // TODO: get the error messages from testng somehow. 120 notifier.fireTestFailure(new Failure(child, extractException(result.getFailures()))); 121 } 122 123 notifier.fireTestFinished(child); 124 // TODO: Check @Test(enabled=false) and invoke #fireTestIgnored instead. 125 } 126 } 127 128 private Throwable extractException(Map<String, Throwable> failures) { 129 if (failures.isEmpty()) { 130 return new AssertionError(); 131 } 132 if (failures.size() == 1) { 133 return failures.values().iterator().next(); 134 } 135 136 StringBuilder errorMessage = new StringBuilder("========== Multiple Failures =========="); 137 for (Map.Entry<String, Throwable> failureEntry : failures.entrySet()) { 138 errorMessage.append("\n\n=== "). append(failureEntry.getKey()).append(" ===\n"); 139 Throwable throwable = failureEntry.getValue(); 140 errorMessage 141 .append(throwable.getClass()).append(": ") 142 .append(throwable.getMessage()); 143 for (StackTraceElement e : throwable.getStackTrace()) { 144 if (e.getClassName().equals(getClass().getName())) { 145 break; 146 } 147 errorMessage.append("\n at ").append(e); 148 } 149 } 150 errorMessage.append("\n=======================================\n\n"); 151 return new AssertionError(errorMessage.toString()); 152 } 153 154 155 /** 156 * Recursively (preorder traversal) apply the filter to all the descriptions. 157 * 158 * @return null if the filter rejects the whole tree. 159 */ 160 private static Description filterDescription(Description desc, Filter filter) { 161 if (!filter.shouldRun(desc)) { // XX: Does the filter itself do the recursion? 162 return null; 163 } 164 165 Description newDesc = desc.childlessCopy(); 166 167 // Return leafs. 168 if (!descriptionHasChildren(desc)) { 169 return newDesc; 170 } 171 172 // Filter all subtrees, only copying them if the filter accepts them. 173 for (Description child : desc.getChildren()) { 174 Description filteredChild = filterDescription(child, filter); 175 176 if (filteredChild != null) { 177 newDesc.addChild(filteredChild); 178 } 179 } 180 181 return newDesc; 182 } 183 184 private Description generateTestNgDescription(Class<?> cls) { 185 // Add the overall class description as the parent. 186 Description parent = Description.createSuiteDescription(cls); 187 188 if (DEBUG) { 189 Log.d("TestNgRunner", "Generating TestNg Description for class " + cls.getName()); 190 } 191 192 // Add each test method as a child. 193 for (Method m : cls.getDeclaredMethods()) { 194 195 // Filter to only 'public void' signatures. 196 if ((m.getModifiers() & Modifier.PUBLIC) == 0) { 197 continue; 198 } 199 200 if (!m.getReturnType().equals(Void.TYPE)) { 201 continue; 202 } 203 204 // Note that TestNG methods may actually have parameters 205 // (e.g. with @DataProvider) which TestNG will populate itself. 206 207 // Add [Class, MethodName] as a Description leaf node. 208 String name = m.getName(); 209 210 if (!mMethodSet.add(name)) { 211 // Overloaded methods have the same name, don't add them twice. 212 if (DEBUG) { 213 Log.d("TestNgRunner", "Already added child " + cls.getName() + "#" + name); 214 } 215 continue; 216 } 217 218 Description child = Description.createTestDescription(cls, name); 219 220 parent.addChild(child); 221 222 if (DEBUG) { 223 Log.d("TestNgRunner", "Add child " + cls.getName() + "#" + name); 224 } 225 } 226 227 return parent; 228 } 229 230 private static boolean descriptionHasChildren(Description desc) { 231 // Note: Although "desc.isTest()" is equivalent to "!desc.getChildren().isEmpty()" 232 // we add the pre-requisite 2 extra null checks to avoid throwing NPEs. 233 return desc != null && desc.getChildren() != null && !desc.getChildren().isEmpty(); 234 } 235 } 236