1 /* 2 * Copyright (C) 2011 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.result; 17 18 import com.android.ddmlib.testrunner.TestIdentifier; 19 import com.android.tradefed.device.ITestDevice; 20 import com.android.tradefed.result.BugreportCollector.Filter; 21 import com.android.tradefed.result.BugreportCollector.Freq; 22 import com.android.tradefed.result.BugreportCollector.Noun; 23 import com.android.tradefed.result.BugreportCollector.Predicate; 24 import com.android.tradefed.result.BugreportCollector.Relation; 25 import com.android.tradefed.result.BugreportCollector.SubPredicate; 26 27 import junit.framework.TestCase; 28 29 import java.io.ByteArrayInputStream; 30 import java.io.InputStream; 31 import java.util.HashMap; 32 import java.util.Map; 33 34 import org.easymock.Capture; 35 import org.easymock.EasyMock; 36 37 /** Unit tests for {@link BugreportCollector} */ 38 public class BugreportCollectorTest extends TestCase { 39 private BugreportCollector mCollector = null; 40 private ITestDevice mMockDevice = null; 41 private ITestInvocationListener mMockListener = null; 42 private InputStreamSource mBugreportISS = null; 43 private Capture<Map<String, String>> mTestCapture = new Capture<>(); 44 private Capture<Map<String, String>> mRunCapture = new Capture<>(); 45 46 private static final String TEST_KEY = "key"; 47 private static final String RUN_KEY = "key2"; 48 private static final String BUGREPORT_STRING = "This is a bugreport\nYeah!\n"; 49 private static final String STACK_TRACE = "boo-hoo"; 50 51 private static class BugreportISS implements InputStreamSource { 52 @Override 53 public InputStream createInputStream() { 54 return new ByteArrayInputStream(BUGREPORT_STRING.getBytes()); 55 } 56 57 @Override 58 public void close() { 59 // ignore 60 } 61 62 @Override 63 public long size() { 64 return BUGREPORT_STRING.getBytes().length; 65 } 66 } 67 68 @Override 69 public void setUp() throws Exception { 70 super.setUp(); 71 mMockDevice = EasyMock.createStrictMock(ITestDevice.class); 72 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 73 74 mBugreportISS = new BugreportISS(); 75 76 EasyMock.expect(mMockDevice.getBugreport()).andStubReturn(mBugreportISS); 77 mCollector = new BugreportCollector(mMockListener, mMockDevice); 78 79 mTestCapture = new Capture<>(); 80 mRunCapture = new Capture<>(); 81 } 82 83 public void testCreatePredicate() throws Exception { 84 Predicate foo = new Predicate(Relation.AFTER, Freq.EACH, Noun.TESTCASE); 85 assertEquals("AFTER_EACH_TESTCASE", foo.toString()); 86 } 87 88 public void testPredicateEquals() throws Exception { 89 Predicate foo = new Predicate(Relation.AFTER, Freq.EACH, Noun.TESTCASE); 90 Predicate bar = new Predicate(Relation.AFTER, Freq.EACH, Noun.TESTCASE); 91 Predicate baz = new Predicate(Relation.AFTER, Freq.EACH, Noun.INVOCATION); 92 93 assertTrue(foo.equals(bar)); 94 assertTrue(bar.equals(foo)); 95 assertFalse(foo.equals(baz)); 96 assertFalse(baz.equals(bar)); 97 } 98 99 public void testPredicatePartialMatch() throws Exception { 100 Predicate shortP = new Predicate(Relation.AFTER, Freq.EACH, Noun.INVOCATION); 101 Predicate longP = new Predicate(Relation.AFTER, Freq.EACH, Noun.INVOCATION, 102 Filter.WITH_ANY, Noun.TESTCASE); 103 assertTrue(longP.partialMatch(shortP)); 104 assertTrue(shortP.partialMatch(longP)); 105 } 106 107 public void testPredicateFullMatch() throws Exception { 108 Predicate shortP = new Predicate(Relation.AFTER, Freq.EACH, Noun.INVOCATION); 109 Predicate longP = new Predicate(Relation.AFTER, Freq.EACH, Noun.INVOCATION, 110 Filter.WITH_ANY, Noun.TESTCASE); 111 Predicate longP2 = new Predicate(Relation.AFTER, Freq.EACH, Noun.INVOCATION, 112 Filter.WITH_ANY, Noun.TESTCASE); 113 assertFalse(longP.fullMatch(shortP)); 114 assertFalse(shortP.fullMatch(longP)); 115 116 assertTrue(longP.fullMatch(longP2)); 117 assertTrue(longP2.fullMatch(longP)); 118 } 119 120 /** 121 * A test to verify that invalid predicates are rejected 122 */ 123 public void testInvalidPredicate() throws Exception { 124 SubPredicate[][] predicates = new SubPredicate[][] { 125 // AT_START_OF (Freq) FAILED_(Noun) 126 {Relation.AT_START_OF, Freq.EACH, Noun.FAILED_TESTCASE}, 127 {Relation.AT_START_OF, Freq.EACH, Noun.FAILED_TESTRUN}, 128 {Relation.AT_START_OF, Freq.EACH, Noun.FAILED_INVOCATION}, 129 {Relation.AT_START_OF, Freq.FIRST, Noun.FAILED_TESTCASE}, 130 {Relation.AT_START_OF, Freq.FIRST, Noun.FAILED_TESTRUN}, 131 {Relation.AT_START_OF, Freq.FIRST, Noun.FAILED_INVOCATION}, 132 // (Relation) FIRST [FAILED_]INVOCATION 133 {Relation.AT_START_OF, Freq.FIRST, Noun.INVOCATION}, 134 {Relation.AT_START_OF, Freq.FIRST, Noun.FAILED_INVOCATION}, 135 {Relation.AFTER, Freq.FIRST, Noun.INVOCATION}, 136 {Relation.AFTER, Freq.FIRST, Noun.FAILED_INVOCATION}, 137 }; 138 139 for (SubPredicate[] pred : predicates) { 140 try { 141 assertEquals(3, pred.length); 142 new Predicate((Relation)pred[0], (Freq)pred[1], (Noun)pred[2]); 143 fail(String.format( 144 "Expected IllegalArgumentException for invalid predicate [%s %s %s]", 145 pred[0], pred[1], pred[2])); 146 } catch (IllegalArgumentException e) { 147 // expected 148 // FIXME: validate message 149 } 150 } 151 } 152 153 /** 154 * Make sure that BugreportCollector passes events through to its child listener 155 */ 156 public void testPassThrough() throws Exception { 157 setListenerTestRunExpectations(mMockListener, "runName", "testName"); 158 replayMocks(); 159 injectTestRun("runName", "testName", "value"); 160 verifyMocks(); 161 assertEquals("value", mTestCapture.getValue().get("key")); 162 assertEquals("value", mRunCapture.getValue().get("key2")); 163 } 164 165 public void testTestFailed() throws Exception { 166 Predicate pred = new Predicate(Relation.AFTER, Freq.EACH, Noun.FAILED_TESTCASE); 167 mCollector.addPredicate(pred); 168 mMockDevice.waitForDeviceOnline(EasyMock.anyLong()); 169 EasyMock.expectLastCall().times(2); 170 setListenerTestRunExpectations(mMockListener, "runName1", "testName1", true /*failed*/); 171 mMockListener.testLog(EasyMock.contains("bug-FAILED-FooTest__testName1."), 172 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 173 setListenerTestRunExpectations(mMockListener, "runName2", "testName2", true /*failed*/); 174 mMockListener.testLog(EasyMock.contains("bug-FAILED-FooTest__testName2."), 175 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 176 replayMocks(); 177 injectTestRun("runName1", "testName1", "value", true /*failed*/); 178 injectTestRun("runName2", "testName2", "value", true /*failed*/); 179 verifyMocks(); 180 assertEquals("value", mTestCapture.getValue().get("key")); 181 assertEquals("value", mRunCapture.getValue().get("key2")); 182 } 183 184 public void testTestEnded() throws Exception { 185 Predicate pred = new Predicate(Relation.AFTER, Freq.EACH, Noun.TESTCASE); 186 mCollector.addPredicate(pred); 187 mMockDevice.waitForDeviceOnline(EasyMock.anyLong()); 188 EasyMock.expectLastCall().times(2); 189 setListenerTestRunExpectations(mMockListener, "runName1", "testName1"); 190 mMockListener.testLog(EasyMock.contains("bug-FooTest__testName1."), 191 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 192 setListenerTestRunExpectations(mMockListener, "runName2", "testName2"); 193 mMockListener.testLog(EasyMock.contains("bug-FooTest__testName2."), 194 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 195 replayMocks(); 196 injectTestRun("runName1", "testName1", "value"); 197 injectTestRun("runName2", "testName2", "value"); 198 verifyMocks(); 199 assertEquals("value", mTestCapture.getValue().get("key")); 200 assertEquals("value", mRunCapture.getValue().get("key2")); 201 } 202 203 public void testWaitForDevice() throws Exception { 204 Predicate pred = new Predicate(Relation.AFTER, Freq.EACH, Noun.TESTCASE); 205 mCollector.addPredicate(pred); 206 mCollector.setDeviceWaitTime(1); 207 208 mMockDevice.waitForDeviceOnline(1000); 209 EasyMock.expectLastCall().times(2); // Once per ending test method 210 setListenerTestRunExpectations(mMockListener, "runName1", "testName1"); 211 mMockListener.testLog(EasyMock.contains("bug-FooTest__testName1."), 212 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 213 setListenerTestRunExpectations(mMockListener, "runName2", "testName2"); 214 mMockListener.testLog(EasyMock.contains("bug-FooTest__testName2."), 215 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 216 replayMocks(); 217 injectTestRun("runName1", "testName1", "value"); 218 injectTestRun("runName2", "testName2", "value"); 219 verifyMocks(); 220 assertEquals("value", mTestCapture.getValue().get("key")); 221 assertEquals("value", mRunCapture.getValue().get("key2")); 222 } 223 224 public void testTestEnded_firstCase() throws Exception { 225 Predicate pred = new Predicate(Relation.AFTER, Freq.FIRST, Noun.TESTCASE); 226 mCollector.addPredicate(pred); 227 mMockDevice.waitForDeviceOnline(EasyMock.anyLong()); 228 EasyMock.expectLastCall().times(2); 229 setListenerTestRunExpectations(mMockListener, "runName1", "testName1"); 230 mMockListener.testLog(EasyMock.contains("bug-FooTest__testName1."), 231 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 232 setListenerTestRunExpectations(mMockListener, "runName2", "testName2"); 233 mMockListener.testLog(EasyMock.contains("bug-FooTest__testName2."), 234 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 235 replayMocks(); 236 injectTestRun("runName1", "testName1", "value"); 237 injectTestRun("runName2", "testName2", "value"); 238 verifyMocks(); 239 assertEquals("value", mTestCapture.getValue().get("key")); 240 assertEquals("value", mRunCapture.getValue().get("key2")); 241 } 242 243 public void testTestEnded_firstRun() throws Exception { 244 Predicate pred = new Predicate(Relation.AFTER, Freq.FIRST, Noun.TESTRUN); 245 mCollector.addPredicate(pred); 246 mMockDevice.waitForDeviceOnline(EasyMock.anyLong()); 247 // Note: only one testLog 248 setListenerTestRunExpectations(mMockListener, "runName", "testName"); 249 mMockListener.testLog(EasyMock.contains(pred.toString()), 250 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 251 setListenerTestRunExpectations(mMockListener, "runName2", "testName2"); 252 replayMocks(); 253 injectTestRun("runName", "testName", "value"); 254 injectTestRun("runName2", "testName2", "value"); 255 verifyMocks(); 256 assertEquals("value", mTestCapture.getValue().get("key")); 257 assertEquals("value", mRunCapture.getValue().get("key2")); 258 } 259 260 public void testTestRunEnded() throws Exception { 261 Predicate pred = new Predicate(Relation.AFTER, Freq.EACH, Noun.TESTRUN); 262 mCollector.addPredicate(pred); 263 mMockDevice.waitForDeviceOnline(EasyMock.anyLong()); 264 setListenerTestRunExpectations(mMockListener, "runName", "testName"); 265 mMockListener.testLog(EasyMock.contains(pred.toString()), 266 EasyMock.eq(LogDataType.BUGREPORT), EasyMock.eq(mBugreportISS)); 267 replayMocks(); 268 injectTestRun("runName", "testName", "value"); 269 verifyMocks(); 270 assertEquals("value", mTestCapture.getValue().get("key")); 271 assertEquals("value", mRunCapture.getValue().get("key2")); 272 } 273 274 public void testDescriptiveName() throws Exception { 275 final String normalName = "AT_START_OF_FIRST_TESTCASE"; 276 final String descName = "custom_descriptive_name"; 277 mMockDevice.waitForDeviceOnline(EasyMock.anyLong()); 278 EasyMock.expectLastCall().times(2); 279 mMockListener.testLog(EasyMock.contains(normalName), EasyMock.eq(LogDataType.BUGREPORT), 280 EasyMock.eq(mBugreportISS)); 281 mMockListener.testLog(EasyMock.contains(descName), EasyMock.eq(LogDataType.BUGREPORT), 282 EasyMock.eq(mBugreportISS)); 283 replayMocks(); 284 mCollector.grabBugreport(normalName); 285 mCollector.setDescriptiveName(descName); 286 mCollector.grabBugreport(normalName); 287 verifyMocks(); 288 } 289 290 /** 291 * Injects a single test run with 1 passed test into the {@link CollectingTestListener} under 292 * test 293 * @return the {@link TestIdentifier} of added test 294 */ 295 private TestIdentifier injectTestRun(String runName, String testName, String metricValue) { 296 return injectTestRun(runName, testName, metricValue, false); 297 } 298 299 /** 300 * Injects a single test run with 1 passed test into the {@link CollectingTestListener} under 301 * test 302 * @return the {@link TestIdentifier} of added test 303 */ 304 private TestIdentifier injectTestRun(String runName, String testName, String metricValue, 305 boolean shouldFail) { 306 Map<String, String> runMetrics = new HashMap<String, String>(1); 307 runMetrics.put(RUN_KEY, metricValue); 308 Map<String, String> testMetrics = new HashMap<String, String>(1); 309 testMetrics.put(TEST_KEY, metricValue); 310 311 mCollector.testRunStarted(runName, 1); 312 final TestIdentifier test = new TestIdentifier("FooTest", testName); 313 mCollector.testStarted(test); 314 if (shouldFail) { 315 mCollector.testFailed(test, STACK_TRACE); 316 } 317 mCollector.testEnded(test, testMetrics); 318 mCollector.testRunEnded(0, runMetrics); 319 return test; 320 } 321 322 private void setListenerTestRunExpectations( 323 ITestInvocationListener listener, String runName, String testName) { 324 setListenerTestRunExpectations(listener, runName, testName, false); 325 } 326 327 @SuppressWarnings("unchecked") 328 private void setListenerTestRunExpectations( 329 ITestInvocationListener listener, String runName, String testName, boolean shouldFail) { 330 listener.testRunStarted(EasyMock.eq(runName), EasyMock.eq(1)); 331 final TestIdentifier test = new TestIdentifier("FooTest", testName); 332 listener.testStarted(EasyMock.eq(test)); 333 if (shouldFail) { 334 listener.testFailed(EasyMock.eq(test), EasyMock.eq(STACK_TRACE)); 335 } 336 listener.testEnded(EasyMock.eq(test), EasyMock.capture(mTestCapture)); 337 listener.testRunEnded(EasyMock.anyInt(), EasyMock.capture(mRunCapture)); 338 } 339 340 /** 341 * Convenience method to replay all mocks 342 */ 343 private void replayMocks() { 344 EasyMock.replay(mMockDevice, mMockListener); 345 } 346 347 /** 348 * Convenience method to verify all mocks 349 */ 350 private void verifyMocks() { 351 EasyMock.verify(mMockDevice, mMockListener); 352 } 353 } 354 355
