1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result; 17 18 import com.android.ddmlib.testrunner.TestIdentifier; 19 import com.android.ddmlib.testrunner.TestResult.TestStatus; 20 import com.android.ddmlib.testrunner.TestRunResult; 21 import com.android.tradefed.build.BuildInfo; 22 import com.android.tradefed.invoker.IInvocationContext; 23 import com.android.tradefed.invoker.InvocationContext; 24 25 import junit.framework.TestCase; 26 27 import java.util.Collections; 28 import java.util.HashMap; 29 import java.util.Iterator; 30 import java.util.Map; 31 32 /** 33 * Unit tests for {@link CollectingTestListener}. 34 */ 35 public class CollectingTestListenerTest extends TestCase { 36 37 private static final String METRIC_VALUE = "value"; 38 private static final String TEST_KEY = "key"; 39 private static final String METRIC_VALUE2 = "value2"; 40 private static final String RUN_KEY = "key2"; 41 42 43 private CollectingTestListener mCollectingTestListener; 44 45 /** 46 * {@inheritDoc} 47 */ 48 @Override 49 protected void setUp() throws Exception { 50 super.setUp(); 51 mCollectingTestListener = new CollectingTestListener(); 52 IInvocationContext context = new InvocationContext(); 53 context.addDeviceBuildInfo("fakeDevice", new BuildInfo()); 54 mCollectingTestListener.invocationStarted(context); 55 } 56 57 /** 58 * Test the listener under a single normal test run. 59 */ 60 public void testSingleRun() { 61 final TestIdentifier test = injectTestRun("run", "testFoo", METRIC_VALUE); 62 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 63 assertTrue(runResult.isRunComplete()); 64 assertFalse(runResult.isRunFailure()); 65 assertEquals(1, mCollectingTestListener.getNumTotalTests()); 66 assertEquals(TestStatus.PASSED, 67 runResult.getTestResults().get(test).getStatus()); 68 assertTrue(runResult.getTestResults().get(test).getStartTime() > 0); 69 assertTrue(runResult.getTestResults().get(test).getEndTime() >= 70 runResult.getTestResults().get(test).getStartTime()); 71 } 72 73 /** 74 * Test the listener where test run has failed. 75 */ 76 @SuppressWarnings("unchecked") 77 public void testRunFailed() { 78 mCollectingTestListener.testRunStarted("foo", 1); 79 mCollectingTestListener.testRunFailed("error"); 80 mCollectingTestListener.testRunEnded(0, Collections.EMPTY_MAP); 81 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 82 assertTrue(runResult.isRunComplete()); 83 assertTrue(runResult.isRunFailure()); 84 assertEquals("error", runResult.getRunFailureMessage()); 85 } 86 87 /** 88 * Test the listener when invocation is composed of two test runs. 89 */ 90 public void testTwoRuns() { 91 final TestIdentifier test1 = injectTestRun("run1", "testFoo1", METRIC_VALUE); 92 final TestIdentifier test2 = injectTestRun("run2", "testFoo2", METRIC_VALUE2); 93 assertEquals(2, mCollectingTestListener.getNumTotalTests()); 94 assertEquals(2, mCollectingTestListener.getNumTestsInState(TestStatus.PASSED)); 95 assertEquals(2, mCollectingTestListener.getRunResults().size()); 96 Iterator<TestRunResult> runIter = mCollectingTestListener.getRunResults().iterator(); 97 final TestRunResult runResult1 = runIter.next(); 98 final TestRunResult runResult2 = runIter.next(); 99 100 assertEquals("run1", runResult1.getName()); 101 assertEquals("run2", runResult2.getName()); 102 assertEquals(TestStatus.PASSED, 103 runResult1.getTestResults().get(test1).getStatus()); 104 assertEquals(TestStatus.PASSED, 105 runResult2.getTestResults().get(test2).getStatus()); 106 assertEquals(METRIC_VALUE, 107 runResult1.getRunMetrics().get(RUN_KEY)); 108 assertEquals(METRIC_VALUE, 109 runResult1.getTestResults().get(test1).getMetrics().get(TEST_KEY)); 110 assertEquals(METRIC_VALUE2, 111 runResult2.getTestResults().get(test2).getMetrics().get(TEST_KEY)); 112 } 113 114 /** 115 * Test the listener when invocation is composed of a re-executed test run. 116 */ 117 public void testReRun() { 118 final TestIdentifier test1 = injectTestRun("run", "testFoo1", METRIC_VALUE); 119 final TestIdentifier test2 = injectTestRun("run", "testFoo2", METRIC_VALUE2); 120 assertEquals(2, mCollectingTestListener.getNumTotalTests()); 121 assertEquals(2, mCollectingTestListener.getNumTestsInState(TestStatus.PASSED)); 122 assertEquals(1, mCollectingTestListener.getRunResults().size()); 123 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 124 assertEquals(2, runResult.getNumTestsInState(TestStatus.PASSED)); 125 assertTrue(runResult.getCompletedTests().contains(test1)); 126 assertTrue(runResult.getCompletedTests().contains(test2)); 127 } 128 129 /** 130 * Test the listener when invocation is composed of a re-executed test run, containing the same 131 * tests 132 */ 133 public void testReRun_overlap() { 134 injectTestRun("run", "testFoo1", METRIC_VALUE); 135 injectTestRun("run", "testFoo1", METRIC_VALUE2, true); 136 assertEquals(1, mCollectingTestListener.getNumTotalTests()); 137 assertEquals(0, mCollectingTestListener.getNumTestsInState(TestStatus.PASSED)); 138 assertEquals(1, mCollectingTestListener.getNumTestsInState(TestStatus.FAILURE)); 139 assertEquals(1, mCollectingTestListener.getRunResults().size()); 140 TestRunResult runResult = mCollectingTestListener.getCurrentRunResults(); 141 assertEquals(0, runResult.getNumTestsInState(TestStatus.PASSED)); 142 assertEquals(1, runResult.getNumTestsInState(TestStatus.FAILURE)); 143 assertEquals(1, runResult.getNumTests()); 144 } 145 146 /** 147 * Test run with incomplete tests 148 */ 149 @SuppressWarnings("unchecked") 150 public void testSingleRun_incomplete() { 151 mCollectingTestListener.testRunStarted("run", 1); 152 mCollectingTestListener.testStarted(new TestIdentifier("FooTest", "incomplete")); 153 mCollectingTestListener.testRunEnded(0, Collections.EMPTY_MAP); 154 assertEquals(1, mCollectingTestListener.getNumTestsInState(TestStatus.INCOMPLETE)); 155 } 156 157 /** 158 * Test aggregating of metrics with long values 159 */ 160 public void testRunEnded_aggregateLongMetrics() { 161 mCollectingTestListener.setIsAggregrateMetrics(true); 162 injectTestRun("run", "testFoo1", "1"); 163 injectTestRun("run", "testFoo1", "1"); 164 assertEquals("2", mCollectingTestListener.getCurrentRunResults().getRunMetrics().get( 165 RUN_KEY)); 166 } 167 168 /** 169 * Test aggregating of metrics with double values 170 */ 171 public void testRunEnded_aggregateDoubleMetrics() { 172 mCollectingTestListener.setIsAggregrateMetrics(true); 173 injectTestRun("run", "testFoo1", "1.1"); 174 injectTestRun("run", "testFoo1", "1.1"); 175 assertEquals("2.2", mCollectingTestListener.getCurrentRunResults().getRunMetrics().get( 176 RUN_KEY)); 177 } 178 179 /** 180 * Test aggregating of metrics with different data types 181 */ 182 public void testRunEnded_aggregateMixedMetrics() { 183 mCollectingTestListener.setIsAggregrateMetrics(true); 184 injectTestRun("run", "testFoo1", "1"); 185 injectTestRun("run", "testFoo1", "1.1"); 186 mCollectingTestListener.invocationEnded(0); 187 assertEquals("2.1", mCollectingTestListener.getCurrentRunResults().getRunMetrics().get( 188 RUN_KEY)); 189 } 190 191 /** 192 * Test aggregating of metrics when new metric isn't a number 193 */ 194 public void testRunEnded_aggregateNewStringMetrics() { 195 mCollectingTestListener.setIsAggregrateMetrics(true); 196 injectTestRun("run", "testFoo1", "1"); 197 injectTestRun("run", "testFoo1", "bar"); 198 mCollectingTestListener.invocationEnded(0); 199 assertEquals("bar", mCollectingTestListener.getCurrentRunResults().getRunMetrics().get( 200 RUN_KEY)); 201 } 202 203 /** 204 * Test aggregating of metrics when existing metric isn't a number 205 */ 206 public void testRunEnded_aggregateExistingStringMetrics() { 207 mCollectingTestListener.setIsAggregrateMetrics(true); 208 injectTestRun("run", "testFoo1", "bar"); 209 injectTestRun("run", "testFoo1", "1"); 210 mCollectingTestListener.invocationEnded(0); 211 assertEquals("1", mCollectingTestListener.getCurrentRunResults().getRunMetrics().get( 212 RUN_KEY)); 213 } 214 215 public void testGetNumTestsInState() { 216 injectTestRun("run", "testFoo1", "1"); 217 injectTestRun("run", "testFoo2", "1"); 218 int testsPassed = mCollectingTestListener.getNumTestsInState(TestStatus.PASSED); 219 assertEquals(2, testsPassed); 220 injectTestRun("run", "testFoo3", "1"); 221 testsPassed = mCollectingTestListener.getNumTestsInState(TestStatus.PASSED); 222 assertEquals(3, testsPassed); 223 } 224 225 public void testGetNumTotalTests() { 226 injectTestRun("run", "testFoo1", "1"); 227 injectTestRun("run", "testFoo2", "1"); 228 int total = mCollectingTestListener.getNumTotalTests(); 229 assertEquals(2, total); 230 injectTestRun("run", "testFoo3", "1", true); 231 total = mCollectingTestListener.getNumTotalTests(); 232 assertEquals(3, total); 233 } 234 235 /** 236 * Injects a single test run with 1 passed test into the {@link CollectingTestListener} under 237 * test 238 * @return the {@link TestIdentifier} of added test 239 */ 240 private TestIdentifier injectTestRun(String runName, String testName, String metricValue) { 241 return injectTestRun(runName, testName, metricValue, false); 242 } 243 244 /** 245 * Injects a single test run with 1 test into the {@link CollectingTestListener} under 246 * test. 247 * @return the {@link TestIdentifier} of added test 248 */ 249 private TestIdentifier injectTestRun(String runName, String testName, String metricValue, 250 boolean failtest) { 251 Map<String, String> runMetrics = new HashMap<String, String>(1); 252 runMetrics.put(RUN_KEY, metricValue); 253 Map<String, String> testMetrics = new HashMap<String, String>(1); 254 testMetrics.put(TEST_KEY, metricValue); 255 256 mCollectingTestListener.testRunStarted(runName, 1); 257 final TestIdentifier test = new TestIdentifier("FooTest", testName); 258 mCollectingTestListener.testStarted(test); 259 if (failtest) { 260 mCollectingTestListener.testFailed(test, "trace"); 261 } 262 mCollectingTestListener.testEnded(test, testMetrics); 263 mCollectingTestListener.testRunEnded(0, runMetrics); 264 return test; 265 } 266 } 267
