1 /* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import static org.junit.Assert.assertFalse; 19 import static org.junit.Assert.fail; 20 21 import com.android.tradefed.device.DeviceNotAvailableException; 22 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 23 import com.android.tradefed.result.ITestInvocationListener; 24 import com.android.tradefed.result.TestDescription; 25 import com.android.tradefed.util.proto.TfMetricProtoUtil; 26 27 import junit.framework.TestCase; 28 29 import org.easymock.Capture; 30 import org.easymock.EasyMock; 31 import org.junit.Test; 32 import org.junit.runner.RunWith; 33 import org.junit.runners.JUnit4; 34 35 import java.lang.annotation.Retention; 36 import java.lang.annotation.RetentionPolicy; 37 import java.util.HashMap; 38 import java.util.List; 39 import java.util.Map; 40 41 /** Unit tests for {@link DeviceTestCase}. */ 42 @RunWith(JUnit4.class) 43 public class DeviceTestCaseTest { 44 45 public static class MockTest extends DeviceTestCase { 46 47 public void test1() { 48 // test adding a metric during the test. 49 addTestMetric("test", "value"); 50 } 51 public void test2() {} 52 } 53 54 @MyAnnotation1 55 public static class MockAnnotatedTest extends DeviceTestCase { 56 57 @MyAnnotation1 58 public void test1() {} 59 @MyAnnotation2 60 public void test2() {} 61 } 62 63 /** A test class that illustrate duplicate names but from only one real test. */ 64 public static class DuplicateTest extends DeviceTestCase { 65 66 public void test1() { 67 test1(""); 68 } 69 70 private void test1(String arg) { 71 assertTrue(arg.isEmpty()); 72 } 73 74 public void test2() {} 75 } 76 77 /** 78 * Simple Annotation class for testing 79 */ 80 @Retention(RetentionPolicy.RUNTIME) 81 public @interface MyAnnotation1 { 82 } 83 84 /** 85 * Simple Annotation class for testing 86 */ 87 @Retention(RetentionPolicy.RUNTIME) 88 public @interface MyAnnotation2 { 89 } 90 91 public static class MockAbortTest extends DeviceTestCase { 92 93 private static final String EXCEP_MSG = "failed"; 94 private static final String FAKE_SERIAL = "fakeserial"; 95 96 public void test1() throws DeviceNotAvailableException { 97 throw new DeviceNotAvailableException(EXCEP_MSG, FAKE_SERIAL); 98 } 99 } 100 101 /** Verify that calling run on a DeviceTestCase will run all test methods. */ 102 @Test 103 public void testRun_suite() throws Exception { 104 MockTest test = new MockTest(); 105 106 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 107 listener.testRunStarted(MockTest.class.getName(), 2); 108 final TestDescription test1 = new TestDescription(MockTest.class.getName(), "test1"); 109 final TestDescription test2 = new TestDescription(MockTest.class.getName(), "test2"); 110 listener.testStarted(test1); 111 Map<String, String> metrics = new HashMap<>(); 112 metrics.put("test", "value"); 113 listener.testEnded(test1, TfMetricProtoUtil.upgradeConvert(metrics)); 114 listener.testStarted(test2); 115 listener.testEnded(test2, new HashMap<String, Metric>()); 116 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 117 EasyMock.replay(listener); 118 119 test.run(listener); 120 EasyMock.verify(listener); 121 } 122 123 /** 124 * Verify that calling run on a {@link DeviceTestCase} will only run methods included by 125 * filtering. 126 */ 127 @Test 128 public void testRun_includeFilter() throws Exception { 129 MockTest test = new MockTest(); 130 test.addIncludeFilter("com.android.tradefed.testtype.DeviceTestCaseTest$MockTest#test1"); 131 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 132 listener.testRunStarted(MockTest.class.getName(), 1); 133 final TestDescription test1 = new TestDescription(MockTest.class.getName(), "test1"); 134 listener.testStarted(test1); 135 Map<String, String> metrics = new HashMap<>(); 136 metrics.put("test", "value"); 137 listener.testEnded(test1, TfMetricProtoUtil.upgradeConvert(metrics)); 138 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 139 EasyMock.replay(listener); 140 141 test.run(listener); 142 EasyMock.verify(listener); 143 } 144 145 /** 146 * Verify that calling run on a {@link DeviceTestCase} will not run methods excluded by 147 * filtering. 148 */ 149 @Test 150 public void testRun_excludeFilter() throws Exception { 151 MockTest test = new MockTest(); 152 test.addExcludeFilter("com.android.tradefed.testtype.DeviceTestCaseTest$MockTest#test1"); 153 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 154 listener.testRunStarted(MockTest.class.getName(), 1); 155 final TestDescription test2 = new TestDescription(MockTest.class.getName(), "test2"); 156 listener.testStarted(test2); 157 listener.testEnded(test2, new HashMap<String, Metric>()); 158 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 159 EasyMock.replay(listener); 160 161 test.run(listener); 162 EasyMock.verify(listener); 163 } 164 165 /** 166 * Verify that calling run on a {@link DeviceTestCase} only runs AnnotatedElements included by 167 * filtering. 168 */ 169 @Test 170 public void testRun_includeAnnotationFiltering() throws Exception { 171 MockAnnotatedTest test = new MockAnnotatedTest(); 172 test.addIncludeAnnotation( 173 "com.android.tradefed.testtype.DeviceTestCaseTest$MyAnnotation1"); 174 test.addExcludeAnnotation("com.android.tradefed.testtype.DeviceTestCaseTest$MyAnnotation2"); 175 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 176 listener.testRunStarted(MockAnnotatedTest.class.getName(), 1); 177 final TestDescription test1 = 178 new TestDescription(MockAnnotatedTest.class.getName(), "test1"); 179 listener.testStarted(test1); 180 listener.testEnded(test1, new HashMap<String, Metric>()); 181 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 182 EasyMock.replay(listener); 183 184 test.run(listener); 185 EasyMock.verify(listener); 186 } 187 188 /** Verify that we properly carry the annotations of the methods. */ 189 @Test 190 public void testRun_checkAnnotation() throws Exception { 191 MockAnnotatedTest test = new MockAnnotatedTest(); 192 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 193 listener.testRunStarted(MockAnnotatedTest.class.getName(), 2); 194 Capture<TestDescription> capture = new Capture<>(); 195 listener.testStarted(EasyMock.capture(capture)); 196 listener.testEnded( 197 EasyMock.capture(capture), (HashMap<String, Metric>) EasyMock.anyObject()); 198 listener.testStarted(EasyMock.capture(capture)); 199 listener.testEnded( 200 EasyMock.capture(capture), (HashMap<String, Metric>) EasyMock.anyObject()); 201 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 202 EasyMock.replay(listener); 203 204 test.run(listener); 205 EasyMock.verify(listener); 206 207 List<TestDescription> descriptions = capture.getValues(); 208 // Ensure we properly capture the annotations for both methods. 209 for (TestDescription desc : descriptions) { 210 assertFalse(desc.getAnnotations().isEmpty()); 211 } 212 } 213 214 /** 215 * Verify that calling run on a {@link DeviceTestCase} does not run AnnotatedElements excluded 216 * by filtering. 217 */ 218 @Test 219 public void testRun_excludeAnnotationFiltering() throws Exception { 220 MockAnnotatedTest test = new MockAnnotatedTest(); 221 test.addExcludeAnnotation( 222 "com.android.tradefed.testtype.DeviceTestCaseTest$MyAnnotation2"); 223 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 224 listener.testRunStarted(MockAnnotatedTest.class.getName(), 1); 225 final TestDescription test1 = 226 new TestDescription(MockAnnotatedTest.class.getName(), "test1"); 227 listener.testStarted(test1); 228 listener.testEnded(test1, new HashMap<String, Metric>()); 229 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 230 EasyMock.replay(listener); 231 232 test.run(listener); 233 EasyMock.verify(listener); 234 } 235 236 /** Regression test to verify a single test can still be run. */ 237 @Test 238 public void testRun_singleTest() throws DeviceNotAvailableException { 239 MockTest test = new MockTest(); 240 test.setName("test1"); 241 242 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 243 listener.testRunStarted(MockTest.class.getName(), 1); 244 final TestDescription test1 = new TestDescription(MockTest.class.getName(), "test1"); 245 listener.testStarted(test1); 246 Map<String, String> metrics = new HashMap<>(); 247 metrics.put("test", "value"); 248 listener.testEnded(test1, TfMetricProtoUtil.upgradeConvert(metrics)); 249 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 250 EasyMock.replay(listener); 251 252 test.run(listener); 253 EasyMock.verify(listener); 254 } 255 256 /** Verify that a device not available exception is thrown up. */ 257 @Test 258 public void testRun_deviceNotAvail() { 259 MockAbortTest test = new MockAbortTest(); 260 // create a mock ITestInvocationListener, because results are easier to verify 261 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 262 263 final TestDescription test1 = new TestDescription(MockAbortTest.class.getName(), "test1"); 264 listener.testRunStarted(MockAbortTest.class.getName(), 1); 265 listener.testStarted(test1); 266 listener.testFailed(EasyMock.eq(test1), 267 EasyMock.contains(MockAbortTest.EXCEP_MSG)); 268 listener.testEnded(test1, new HashMap<String, Metric>()); 269 listener.testRunFailed(EasyMock.contains(MockAbortTest.EXCEP_MSG)); 270 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 271 EasyMock.replay(listener); 272 try { 273 test.run(listener); 274 fail("DeviceNotAvailableException not thrown"); 275 } catch (DeviceNotAvailableException e) { 276 // expected 277 } 278 EasyMock.verify(listener); 279 } 280 281 /** 282 * Test success case for {@link DeviceTestCase#run(ITestInvocationListener)} in collector mode, 283 * where test to run is a {@link TestCase} 284 */ 285 @Test 286 public void testRun_testcaseCollectMode() throws Exception { 287 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 288 MockTest test = new MockTest(); 289 test.setCollectTestsOnly(true); 290 listener.testRunStarted((String)EasyMock.anyObject(), EasyMock.eq(2)); 291 listener.testStarted(EasyMock.anyObject()); 292 listener.testEnded(EasyMock.anyObject(), (HashMap<String, Metric>) EasyMock.anyObject()); 293 listener.testStarted(EasyMock.anyObject()); 294 listener.testEnded(EasyMock.anyObject(), (HashMap<String, Metric>) EasyMock.anyObject()); 295 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 296 EasyMock.replay(listener); 297 test.run(listener); 298 EasyMock.verify(listener); 299 } 300 301 /** 302 * Test success case for {@link DeviceTestCase#run(ITestInvocationListener)} in collector mode, 303 * where test to run is a {@link TestCase} 304 */ 305 @Test 306 public void testRun_testcaseCollectMode_singleMethod() throws Exception { 307 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 308 MockTest test = new MockTest(); 309 test.setName("test1"); 310 test.setCollectTestsOnly(true); 311 listener.testRunStarted((String)EasyMock.anyObject(), EasyMock.eq(1)); 312 listener.testStarted(EasyMock.anyObject()); 313 listener.testEnded(EasyMock.anyObject(), (HashMap<String, Metric>) EasyMock.anyObject()); 314 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 315 EasyMock.replay(listener); 316 test.run(listener); 317 EasyMock.verify(listener); 318 } 319 320 /** 321 * Test that when a test class has some private method with a method name we properly ignore it 322 * and only consider the actual real method that can execute in the filtering. 323 */ 324 @Test 325 public void testRun_duplicateName() throws Exception { 326 DuplicateTest test = new DuplicateTest(); 327 ITestInvocationListener listener = EasyMock.createMock(ITestInvocationListener.class); 328 329 listener.testRunStarted(DuplicateTest.class.getName(), 2); 330 final TestDescription test1 = new TestDescription(DuplicateTest.class.getName(), "test1"); 331 final TestDescription test2 = new TestDescription(DuplicateTest.class.getName(), "test2"); 332 listener.testStarted(test1); 333 listener.testEnded(test1, new HashMap<String, Metric>()); 334 listener.testStarted(test2); 335 listener.testEnded(test2, new HashMap<String, Metric>()); 336 listener.testRunEnded(EasyMock.anyLong(), (HashMap<String, Metric>) EasyMock.anyObject()); 337 EasyMock.replay(listener); 338 339 test.run(listener); 340 EasyMock.verify(listener); 341 } 342 } 343
