1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.device.collectors; 17 18 import android.app.Instrumentation; 19 import android.device.collectors.annotations.MetricOption; 20 import android.device.collectors.annotations.OptionClass; 21 import android.device.collectors.util.SendToInstrumentation; 22 import android.os.Bundle; 23 import android.support.test.runner.AndroidJUnit4; 24 25 import org.junit.Before; 26 import org.junit.Test; 27 import org.junit.runner.Description; 28 import org.junit.runner.Result; 29 import org.junit.runner.RunWith; 30 import org.mockito.ArgumentCaptor; 31 import org.mockito.Mockito; 32 33 import java.lang.annotation.Annotation; 34 import java.util.List; 35 36 import static org.junit.Assert.assertEquals; 37 import static org.junit.Assert.assertFalse; 38 import static org.junit.Assert.assertTrue; 39 40 /** 41 * Android Unit Tests for {@link BaseMetricListener}. 42 */ 43 @RunWith(AndroidJUnit4.class) 44 public class BaseMetricListenerInstrumentedTest { 45 46 private static final String RUN_START_KEY = "run_start_key"; 47 private static final String RUN_END_KEY = "run_end_key"; 48 private static final String RUN_START_VALUE = "run_start_value"; 49 private static final String RUN_END_VALUE = "run_end_value"; 50 private static final String TEST_START_KEY = "test_start_key"; 51 private static final String TEST_END_KEY = "test_end_key"; 52 private static final String TEST_START_VALUE = "test_start_value"; 53 private static final String TEST_END_VALUE = "test_end_value"; 54 private BaseMetricListener mListener; 55 private Instrumentation mMockInstrumentation; 56 57 @Before 58 public void setUp() { 59 mMockInstrumentation = Mockito.mock(Instrumentation.class); 60 mListener = createWithArgs(null); 61 mListener.setInstrumentation(mMockInstrumentation); 62 } 63 64 private BaseMetricListener createWithArgs(Bundle args) { 65 if (args == null) { 66 args = new Bundle(); 67 } 68 return new BaseMetricListener(args) { 69 @Override 70 public void onTestStart(DataRecord testData, Description description) { 71 // In this test check that a new DataRecord is passed to testStart each time. 72 assertFalse(testData.hasMetrics()); 73 testData.addStringMetric(TEST_START_KEY, TEST_START_VALUE 74 + description.getMethodName()); 75 } 76 77 @Override 78 public void onTestEnd(DataRecord testData, Description description) { 79 testData.addStringMetric(TEST_END_KEY, TEST_END_VALUE 80 + description.getMethodName()); 81 } 82 83 @Override 84 public void onTestRunStart(DataRecord runData, Description description) { 85 assertFalse(runData.hasMetrics()); 86 runData.addStringMetric(RUN_START_KEY, RUN_START_VALUE); 87 } 88 89 @Override 90 public void onTestRunEnd(DataRecord runData, Result result) { 91 runData.addStringMetric(RUN_END_KEY, RUN_END_VALUE); 92 } 93 }; 94 } 95 96 /** 97 * When metrics are logged during a test, expect them to be added to the bundle. 98 */ 99 @MetricOption(group = "testGroup,testGroup1") 100 @Test 101 public void testReportMetrics() throws Exception { 102 Description runDescription = Description.createSuiteDescription("run"); 103 mListener.testRunStarted(runDescription); 104 Description testDescription = Description.createTestDescription("class", "method"); 105 mListener.testStarted(testDescription); 106 mListener.testFinished(testDescription); 107 mListener.testRunFinished(new Result()); 108 // AJUR runner is then gonna call instrumentationRunFinished 109 Bundle resultBundle = new Bundle(); 110 mListener.instrumentationRunFinished(System.out, resultBundle, new Result()); 111 112 // Check that the in progress status contains the metrics. 113 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 114 Mockito.verify(mMockInstrumentation) 115 .sendStatus(Mockito.eq( 116 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 117 List<Bundle> capturedBundle = capture.getAllValues(); 118 assertEquals(1, capturedBundle.size()); 119 Bundle check = capturedBundle.get(0); 120 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 121 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 122 assertEquals(2, check.size()); 123 124 // Check that final bundle contains run results 125 assertEquals(RUN_START_VALUE, resultBundle.getString(RUN_START_KEY)); 126 assertEquals(RUN_END_VALUE, resultBundle.getString(RUN_END_KEY)); 127 assertEquals(2, resultBundle.size()); 128 } 129 130 /** 131 * Test that only included group are running collection. 132 */ 133 @MetricOption(group = "testGroup") 134 @Test 135 public void testReportMetrics_withIncludeFilters() throws Exception { 136 Bundle args = new Bundle(); 137 args.putString(BaseMetricListener.INCLUDE_FILTER_GROUP_KEY, "group1,group2"); 138 mListener = createWithArgs(args); 139 mListener.setInstrumentation(mMockInstrumentation); 140 141 Description runDescription = Description.createSuiteDescription("run"); 142 mListener.testRunStarted(runDescription); 143 Description testDescription = Description.createTestDescription("class", "method", 144 new TestAnnotation("group1")); 145 mListener.testStarted(testDescription); 146 mListener.testFinished(testDescription); 147 // A second test case that will not be included 148 Description testDescription2 = Description.createTestDescription("class", "method2", 149 new TestAnnotation("group3")); 150 mListener.testStarted(testDescription2); 151 mListener.testFinished(testDescription2); 152 // A third test case that will be included 153 Description testDescription3 = Description.createTestDescription("class", "method3", 154 new TestAnnotation("group2")); 155 mListener.testStarted(testDescription3); 156 mListener.testFinished(testDescription3); 157 158 // Check that the in progress status contains the metrics. 159 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 160 Mockito.verify(mMockInstrumentation, Mockito.times(2)) 161 .sendStatus(Mockito.eq( 162 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 163 // Check that only method2 did not generate any metrics since it was filtered. 164 List<Bundle> capturedBundle = capture.getAllValues(); 165 assertEquals(2, capturedBundle.size()); 166 Bundle check = capturedBundle.get(0); 167 assertEquals(TEST_START_VALUE + "method", check.getString(TEST_START_KEY)); 168 assertEquals(TEST_END_VALUE + "method", check.getString(TEST_END_KEY)); 169 assertEquals(2, check.size()); 170 // Got a call from the method3 171 Bundle check2 = capturedBundle.get(1); 172 assertEquals(TEST_START_VALUE + "method3", check2.getString(TEST_START_KEY)); 173 assertEquals(TEST_END_VALUE + "method3", check2.getString(TEST_END_KEY)); 174 assertEquals(2, check2.size()); 175 } 176 177 /** 178 * Test that only included group are running collection, even if some method have multiple 179 * groups, if any of its group is included, the method is included. 180 */ 181 @MetricOption(group = "testGroup") 182 @Test 183 public void testReportMetrics_withIncludeFilters_multiGroup() throws Exception { 184 Bundle args = new Bundle(); 185 args.putString(BaseMetricListener.INCLUDE_FILTER_GROUP_KEY, "group4"); 186 mListener = createWithArgs(args); 187 mListener.setInstrumentation(mMockInstrumentation); 188 189 Description runDescription = Description.createSuiteDescription("run"); 190 mListener.testRunStarted(runDescription); 191 Description testDescription = Description.createTestDescription("class", "method", 192 new TestAnnotation("group1")); 193 mListener.testStarted(testDescription); 194 mListener.testFinished(testDescription); 195 // A second test case that will not be included 196 Description testDescription2 = Description.createTestDescription("class", "method2", 197 new TestAnnotation("group3,group4")); 198 mListener.testStarted(testDescription2); 199 mListener.testFinished(testDescription2); 200 // A third test case that will be included 201 Description testDescription3 = Description.createTestDescription("class", "method3", 202 new TestAnnotation("group2")); 203 mListener.testStarted(testDescription3); 204 mListener.testFinished(testDescription3); 205 206 // Check that the in progress status contains the metrics. 207 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 208 Mockito.verify(mMockInstrumentation, Mockito.times(1)) 209 .sendStatus(Mockito.eq( 210 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 211 // Check that only method2 did generate metrics since it was included. 212 List<Bundle> capturedBundle = capture.getAllValues(); 213 assertEquals(1, capturedBundle.size()); 214 Bundle check = capturedBundle.get(0); 215 // Got call from method2 216 assertEquals(TEST_START_VALUE + "method2", check.getString(TEST_START_KEY)); 217 assertEquals(TEST_END_VALUE + "method2", check.getString(TEST_END_KEY)); 218 assertEquals(2, check.size()); 219 } 220 221 /** 222 * Test that only not excluded group are running collection. 223 */ 224 @MetricOption(group = "testGroup") 225 @Test 226 public void testReportMetrics_withExcludeFilters() throws Exception { 227 Bundle args = new Bundle(); 228 args.putString(BaseMetricListener.EXCLUDE_FILTER_GROUP_KEY, "group1,group2"); 229 mListener = createWithArgs(args); 230 mListener.setInstrumentation(mMockInstrumentation); 231 232 Description runDescription = Description.createSuiteDescription("run"); 233 mListener.testRunStarted(runDescription); 234 // A first test case that will not be included 235 Description testDescription = Description.createTestDescription("class", "method", 236 new TestAnnotation("group1")); 237 mListener.testStarted(testDescription); 238 mListener.testFinished(testDescription); 239 // A second test case that will run 240 Description testDescription2 = Description.createTestDescription("class", "method2", 241 new TestAnnotation("group3")); 242 mListener.testStarted(testDescription2); 243 mListener.testFinished(testDescription2); 244 // A third test case that will not be included 245 Description testDescription3 = Description.createTestDescription("class", "method3", 246 new TestAnnotation("group2")); 247 mListener.testStarted(testDescription3); 248 mListener.testFinished(testDescription3); 249 250 // Check that the in progress status contains the metrics. 251 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 252 Mockito.verify(mMockInstrumentation, Mockito.times(1)) 253 .sendStatus(Mockito.eq( 254 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 255 // Check that only method2 generates some metrics 256 List<Bundle> capturedBundle = capture.getAllValues(); 257 assertEquals(1, capturedBundle.size()); 258 Bundle check = capturedBundle.get(0); 259 assertEquals(TEST_START_VALUE + "method2", check.getString(TEST_START_KEY)); 260 assertEquals(TEST_END_VALUE + "method2", check.getString(TEST_END_KEY)); 261 assertEquals(2, check.size()); 262 } 263 264 /** 265 * Test that when both filters are present, exclude filters have priority. 266 */ 267 @MetricOption(group = "testGroup") 268 @Test 269 public void testReportMetrics_withBothFilters() throws Exception { 270 Bundle args = new Bundle(); 271 args.putString(BaseMetricListener.EXCLUDE_FILTER_GROUP_KEY, "group1,group2"); 272 args.putString(BaseMetricListener.INCLUDE_FILTER_GROUP_KEY, "group2,group3"); 273 mListener = createWithArgs(args); 274 mListener.setInstrumentation(mMockInstrumentation); 275 276 Description runDescription = Description.createSuiteDescription("run"); 277 mListener.testRunStarted(runDescription); 278 // A first test case that will not be included 279 Description testDescription = Description.createTestDescription("class", "method", 280 new TestAnnotation("group1")); 281 mListener.testStarted(testDescription); 282 mListener.testFinished(testDescription); 283 // A second test case that will run 284 Description testDescription2 = Description.createTestDescription("class", "method2", 285 new TestAnnotation("group3")); 286 mListener.testStarted(testDescription2); 287 mListener.testFinished(testDescription2); 288 // A third test case that will not be included 289 Description testDescription3 = Description.createTestDescription("class", "method3", 290 new TestAnnotation("group2")); 291 mListener.testStarted(testDescription3); 292 mListener.testFinished(testDescription3); 293 294 // Check that the in progress status contains the metrics. 295 ArgumentCaptor<Bundle> capture = ArgumentCaptor.forClass(Bundle.class); 296 Mockito.verify(mMockInstrumentation, Mockito.times(1)) 297 .sendStatus(Mockito.eq( 298 SendToInstrumentation.INST_STATUS_IN_PROGRESS), capture.capture()); 299 // Check that only method2 generates some metrics 300 List<Bundle> capturedBundle = capture.getAllValues(); 301 assertEquals(1, capturedBundle.size()); 302 Bundle check = capturedBundle.get(0); 303 assertEquals(TEST_START_VALUE + "method2", check.getString(TEST_START_KEY)); 304 assertEquals(TEST_END_VALUE + "method2", check.getString(TEST_END_KEY)); 305 assertEquals(2, check.size()); 306 } 307 308 /** 309 * Test annotation that allows to instantiate {@link MetricOption} for testing purpose. 310 */ 311 public static class TestAnnotation implements MetricOption { 312 private String mGroup; 313 314 public TestAnnotation(String group) { 315 mGroup = group; 316 } 317 318 @Override 319 public String group() { 320 return mGroup; 321 } 322 323 @Override 324 public Class<? extends Annotation> annotationType() { 325 return MetricOption.class; 326 } 327 } 328 329 /** 330 * Test listener with an {@link OptionClass} specified to be used for arguments testing. 331 */ 332 @OptionClass(alias = "test-alias-class") 333 public static class TestListener extends BaseMetricListener { 334 335 public TestListener(Bundle b) { 336 super(b); 337 } 338 } 339 340 /** 341 * Test when the listener does not have an {@link OptionClass} specified, option with alias 342 * are filtered. 343 */ 344 @MetricOption(group = "testGroup") 345 @Test 346 public void testArgsAlias_noOptionClass() throws Exception { 347 Bundle args = new Bundle(); 348 args.putString("optionalias:optionname", "optionvalue"); 349 args.putString("noalias", "noaliasvalue"); 350 mListener = createWithArgs(args); 351 mListener.setInstrumentation(mMockInstrumentation); 352 Description runDescription = Description.createSuiteDescription("run"); 353 mListener.testRunStarted(runDescription); 354 Bundle filteredArgs = mListener.getArgsBundle(); 355 assertFalse(filteredArgs.containsKey("optionalias:optionname")); 356 assertFalse(filteredArgs.containsKey("optionname")); 357 assertTrue(filteredArgs.containsKey("noalias")); 358 } 359 360 /** 361 * Test when a listener does have an {@link OptionClass} specified, in that case only options 362 * matching the alias or with no alias are preserved. 363 */ 364 @MetricOption(group = "testGroup") 365 @Test 366 public void testArgsAlias_optionClass() throws Exception { 367 Bundle args = new Bundle(); 368 args.putString("test-alias-class:optionname", "optionvalue"); 369 args.putString("noalias", "noaliasvalue"); 370 args.putString("anotheralias:optionname2", "value"); 371 TestListener listener = new TestListener(args); 372 listener.setInstrumentation(mMockInstrumentation); 373 Description runDescription = Description.createSuiteDescription("run"); 374 listener.testRunStarted(runDescription); 375 Bundle filteredArgs = listener.getArgsBundle(); 376 assertTrue(filteredArgs.containsKey("noalias")); 377 assertTrue(filteredArgs.containsKey("optionname")); 378 379 assertFalse(filteredArgs.containsKey("test-alias-class:optionname")); 380 assertFalse(filteredArgs.containsKey("anotheralias:optionname2")); 381 assertFalse(filteredArgs.containsKey("optionname2")); 382 } 383 } 384
