1 /* 2 * Copyright (C) 2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype.suite; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertTrue; 20 21 import com.android.ddmlib.testrunner.TestResult.TestStatus; 22 import com.android.tradefed.config.IConfiguration; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.DeviceUnresponsiveException; 25 import com.android.tradefed.device.metric.IMetricCollector; 26 import com.android.tradefed.invoker.InvocationContext; 27 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 28 import com.android.tradefed.testtype.IRemoteTest; 29 import com.android.tradefed.testtype.ITestFilterReceiver; 30 import com.android.tradefed.result.CollectingTestListener; 31 import com.android.tradefed.result.FileSystemLogSaver; 32 import com.android.tradefed.result.ITestInvocationListener; 33 import com.android.tradefed.result.TestDescription; 34 import com.android.tradefed.result.TestResult; 35 import com.android.tradefed.result.TestRunResult; 36 37 import org.junit.Test; 38 import org.junit.runner.RunWith; 39 import org.junit.runners.JUnit4; 40 import org.mockito.Mockito; 41 42 import java.util.ArrayList; 43 import java.util.Collections; 44 import java.util.HashMap; 45 import java.util.List; 46 import java.util.Map; 47 import java.util.Set; 48 49 /** Unit tests for {@link com.android.tradefed.testtype.suite.GranularRetriableTestWrapper}. */ 50 @RunWith(JUnit4.class) 51 public class GranularRetriableTestWrapperTest { 52 53 private class BasicFakeTest implements IRemoteTest { 54 55 protected ArrayList<TestDescription> mTestCases; 56 private Map<TestDescription, Boolean> mShouldFail; 57 58 public BasicFakeTest() { 59 mTestCases = new ArrayList<>(); 60 TestDescription defaultTestCase = new TestDescription("ClassFoo", "TestFoo"); 61 mTestCases.add(defaultTestCase); 62 mShouldFail = new HashMap<TestDescription, Boolean>(); 63 mShouldFail.put(defaultTestCase, false); 64 } 65 66 public BasicFakeTest(ArrayList<TestDescription> testCases) { 67 mTestCases = testCases; 68 mShouldFail = new HashMap<TestDescription, Boolean>(); 69 for (TestDescription testCase : testCases) { 70 mShouldFail.put(testCase, false); 71 } 72 } 73 74 public void setFailedTestCase(TestDescription testCase) { 75 mShouldFail.put(testCase, true); 76 } 77 78 @Override 79 public void run(ITestInvocationListener listener) throws DeviceUnresponsiveException { 80 listener.testRunStarted("test run", mTestCases.size()); 81 for (TestDescription td : mTestCases) { 82 listener.testStarted(td); 83 if (mShouldFail.get(td)) { 84 listener.testFailed(td, String.format("Fake failure %s", td.toString())); 85 } 86 listener.testEnded(td, new HashMap<String, Metric>()); 87 } 88 listener.testRunEnded(0, new HashMap<String, Metric>()); 89 } 90 } 91 92 private class FakeTest extends BasicFakeTest implements ITestFilterReceiver { 93 94 public FakeTest(ArrayList<TestDescription> testCases) { 95 super(testCases); 96 } 97 98 public FakeTest() { 99 super(); 100 } 101 102 @Override 103 public void addIncludeFilter(String filter) { 104 String[] descriptionStr = filter.split("#"); 105 mTestCases = new ArrayList<>(); 106 mTestCases.add(new TestDescription(descriptionStr[0], descriptionStr[1])); 107 } 108 109 @Override 110 public void addAllIncludeFilters(Set<String> filters) {} 111 112 @Override 113 public void addExcludeFilter(String filters) {} 114 115 @Override 116 public void addAllExcludeFilters(Set<String> filters) {} 117 } 118 119 public GranularRetriableTestWrapper createGranularTestWrapper( 120 IRemoteTest test, int maxRunCount) { 121 GranularRetriableTestWrapper granularTestWrapper = 122 new GranularRetriableTestWrapper(test, null, null, maxRunCount); 123 granularTestWrapper.setModuleId("test module"); 124 granularTestWrapper.setMarkTestsSkipped(false); 125 granularTestWrapper.setMetricCollectors(new ArrayList<IMetricCollector>()); 126 // Setup InvocationContext. 127 granularTestWrapper.setInvocationContext(new InvocationContext()); 128 // Setup logsaver. 129 granularTestWrapper.setLogSaver(new FileSystemLogSaver()); 130 IConfiguration mockModuleConfiguration = Mockito.mock(IConfiguration.class); 131 granularTestWrapper.setModuleConfig(mockModuleConfiguration); 132 return granularTestWrapper; 133 } 134 135 /** 136 * Test the intra module "run" triggers IRemoteTest run method with a dedicated ModuleListener. 137 */ 138 @Test 139 public void testIntraModuleRun_pass() throws Exception { 140 GranularRetriableTestWrapper granularTestWrapper = 141 createGranularTestWrapper(new FakeTest(), 1); 142 assertEquals(0, granularTestWrapper.getTestRunResultCollector().size()); 143 granularTestWrapper.intraModuleRun(); 144 assertEquals(1, granularTestWrapper.getTestRunResultCollector().size()); 145 Set<TestDescription> completedTests = 146 granularTestWrapper.getFinalTestRunResult().getCompletedTests(); 147 assertEquals(1, completedTests.size()); 148 assertEquals("ClassFoo#TestFoo", completedTests.toArray()[0].toString()); 149 } 150 151 /** 152 * Test that the intra module "run" method catches DeviceNotAvailableException and raises it 153 * after record the tests. 154 */ 155 @Test(expected = DeviceNotAvailableException.class) 156 public void testIntraModuleRun_catchDeviceNotAvailableException() throws Exception { 157 IRemoteTest mockTest = Mockito.mock(IRemoteTest.class); 158 Mockito.doThrow(new DeviceNotAvailableException("fake message", "serial")) 159 .when(mockTest) 160 .run(Mockito.any(ITestInvocationListener.class)); 161 GranularRetriableTestWrapper granularTestWrapper = createGranularTestWrapper(mockTest, 1); 162 // Verify. 163 granularTestWrapper.intraModuleRun(); 164 } 165 166 /** 167 * Test that the intra module "run" method catches DeviceUnresponsiveException and doesn't raise 168 * it again. 169 */ 170 @Test 171 public void testIntraModuleRun_catchDeviceUnresponsiveException() throws Exception { 172 FakeTest test = 173 new FakeTest() { 174 @Override 175 public void run(ITestInvocationListener listener) 176 throws DeviceUnresponsiveException { 177 listener.testRunStarted("test run", 1); 178 throw new DeviceUnresponsiveException("fake message", "serial"); 179 } 180 }; 181 GranularRetriableTestWrapper granularTestWrapper = createGranularTestWrapper(test, 1); 182 granularTestWrapper.intraModuleRun(); 183 TestRunResult finalResult = granularTestWrapper.getTestRunResultCollector().get(0); 184 assertTrue(finalResult.isRunFailure()); 185 } 186 187 /** 188 * Test that the "run" method has built-in retry logic and each run has an individual 189 * ModuleListener and TestRunResult. 190 */ 191 @Test 192 public void testRun_withMultipleRun() throws Exception { 193 ArrayList<TestDescription> testCases = new ArrayList<>(); 194 TestDescription fakeTestCase = new TestDescription("Class", "Test"); 195 testCases.add(fakeTestCase); 196 FakeTest test = new FakeTest(testCases); 197 test.setFailedTestCase(fakeTestCase); 198 199 int maxRunCount = 5; 200 GranularRetriableTestWrapper granularTestWrapper = 201 createGranularTestWrapper(test, maxRunCount); 202 granularTestWrapper.run(new CollectingTestListener()); 203 // Verify the test has run 5 times. 204 assertEquals(maxRunCount, granularTestWrapper.getTestRunResultCollector().size()); 205 Map<TestDescription, TestResult> testResults = 206 granularTestWrapper.getFinalTestRunResult().getTestResults(); 207 testResults.containsKey(fakeTestCase); 208 // Verify the final TestRunResult is a merged value of every retried TestRunResults. 209 assertEquals(TestStatus.FAILURE, testResults.get(fakeTestCase).getStatus()); 210 String stacktrace = 211 String.join("\n", Collections.nCopies(maxRunCount, "Fake failure Class#Test")); 212 assertEquals(stacktrace, testResults.get(fakeTestCase).getStackTrace()); 213 } 214 215 /** 216 * Test that the "run" method only retry failed test cases, and merge the multiple test results 217 * to a single TestRunResult. 218 */ 219 @Test 220 public void testRun_retryOnFailedTestCaseOnly() throws Exception { 221 ArrayList<TestDescription> testCases = new ArrayList<>(); 222 TestDescription fakeTestCase1 = new TestDescription("Class1", "Test1"); 223 TestDescription fakeTestCase2 = new TestDescription("Class2", "Test2"); 224 testCases.add(fakeTestCase1); 225 testCases.add(fakeTestCase2); 226 FakeTest test = new FakeTest(testCases); 227 // Only the first testcase is failed and retried. 228 test.setFailedTestCase(fakeTestCase1); 229 // Run each testcases (if failed) max to 3 times. 230 int maxRunCount = 3; 231 GranularRetriableTestWrapper granularTestWrapper = 232 createGranularTestWrapper(test, maxRunCount); 233 granularTestWrapper.run(new CollectingTestListener()); 234 235 TestRunResult finalResult = granularTestWrapper.getFinalTestRunResult(); 236 assertEquals( 237 TestStatus.FAILURE, finalResult.getTestResults().get(fakeTestCase1).getStatus()); 238 assertEquals( 239 TestStatus.PASSED, finalResult.getTestResults().get(fakeTestCase2).getStatus()); 240 // Verify the test has 3 TestRunResults, indicating it runs 3 times. And only failed test 241 // case is in the retried TestRunResult. 242 assertEquals(maxRunCount, granularTestWrapper.getTestRunResultCollector().size()); 243 List<TestRunResult> resultCollector = granularTestWrapper.getTestRunResultCollector(); 244 TestRunResult latestRunResult = resultCollector.get(resultCollector.size() - 1); 245 assertEquals(1, latestRunResult.getNumTests()); 246 latestRunResult.getTestResults().containsKey(fakeTestCase1); 247 } 248 249 /** 250 * Test that if IRemoteTest doesn't implement ITestFilterReceiver, the "run" method will retry 251 * all test cases. 252 */ 253 @Test 254 public void testRun_retryAllTestCasesIfNotSupportTestFilterReceiver() throws Exception { 255 ArrayList<TestDescription> testCases = new ArrayList<>(); 256 TestDescription fakeTestCase1 = new TestDescription("Class1", "Test1"); 257 TestDescription fakeTestCase2 = new TestDescription("Class2", "Test2"); 258 testCases.add(fakeTestCase1); 259 testCases.add(fakeTestCase2); 260 BasicFakeTest test = new BasicFakeTest(testCases); 261 // Only the first testcase is failed. 262 test.setFailedTestCase(fakeTestCase1); 263 // Run each testcases (if has failure) max to 3 times. 264 int maxRunCount = 3; 265 GranularRetriableTestWrapper granularTestWrapper = 266 createGranularTestWrapper(test, maxRunCount); 267 granularTestWrapper.run(new CollectingTestListener()); 268 // Verify the test has 3 TestRunResults, indicating it runs 3 times. And all test cases 269 // are retried. 270 assertEquals(maxRunCount, granularTestWrapper.getTestRunResultCollector().size()); 271 List<TestRunResult> resultCollector = granularTestWrapper.getTestRunResultCollector(); 272 for (TestRunResult runResult : resultCollector) { 273 assertEquals(2, runResult.getNumTests()); 274 assertEquals( 275 TestStatus.FAILURE, runResult.getTestResults().get(fakeTestCase1).getStatus()); 276 assertEquals( 277 TestStatus.PASSED, runResult.getTestResults().get(fakeTestCase2).getStatus()); 278 } 279 } 280 } 281
