1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.collectors; 17 18 import static org.junit.Assert.assertEquals; 19 import static org.junit.Assert.assertFalse; 20 import static org.junit.Assert.assertTrue; 21 import static org.mockito.Mockito.doReturn; 22 import static org.mockito.Mockito.mock; 23 24 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 25 import com.android.tradefed.invoker.IInvocationContext; 26 import com.android.tradefed.result.CollectingTestListener; 27 import com.android.tradefed.result.TestDescription; 28 import com.android.tradefed.result.TestResult; 29 import com.android.tradefed.result.TestRunResult; 30 import com.android.tradefed.testtype.AndroidJUnitTest; 31 import com.android.tradefed.testtype.DeviceJUnit4ClassRunner; 32 import com.android.tradefed.testtype.junit4.BaseHostJUnit4Test; 33 34 import org.junit.Before; 35 import org.junit.Test; 36 import org.junit.runner.RunWith; 37 38 import java.util.Arrays; 39 import java.util.Collection; 40 import java.util.Map.Entry; 41 42 /** 43 * Host side tests for the core device collectors, this ensure that we are able to use the 44 * collectors in a similar way as the infra. 45 * 46 * Command: 47 * mm CollectorHostsideLibTest CollectorDeviceLibTest -j16 48 * tradefed.sh run commandAndExit template/local_min --template:map test=CollectorHostsideLibTest 49 */ 50 @RunWith(DeviceJUnit4ClassRunner.class) 51 public class DeviceCollectorsTest extends BaseHostJUnit4Test { 52 private static final String TEST_APK = "CollectorDeviceLibTest.apk"; 53 private static final String PACKAGE_NAME = "android.device.collectors"; 54 private static final String AJUR_RUNNER = "android.support.test.runner.AndroidJUnitRunner"; 55 56 private static final String STUB_BASE_COLLECTOR = 57 "android.device.collectors.StubTestMetricListener"; 58 private static final String SCHEDULED_COLLECTOR = 59 "android.device.collectors.StubScheduledRunMetricListener"; 60 61 private RemoteAndroidTestRunner mTestRunner; 62 private IInvocationContext mContext; 63 64 @Before 65 public void setUp() throws Exception { 66 installPackage(TEST_APK); 67 assertTrue(isPackageInstalled(PACKAGE_NAME)); 68 mTestRunner = 69 new RemoteAndroidTestRunner(PACKAGE_NAME, AJUR_RUNNER, getDevice().getIDevice()); 70 // Set the new runListener order to ensure test cases can show their metrics. 71 mTestRunner.addInstrumentationArg(AndroidJUnitTest.NEW_RUN_LISTENER_ORDER_KEY, "true"); 72 mContext = mock(IInvocationContext.class); 73 doReturn(Arrays.asList(getDevice())).when(mContext).getDevices(); 74 doReturn(Arrays.asList(getBuild())).when(mContext).getBuildInfos(); 75 } 76 77 /** 78 * Test that our base metric listener can output metrics. 79 */ 80 @Test 81 public void testBaseListenerRuns() throws Exception { 82 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 83 CollectingTestListener listener = new CollectingTestListener(); 84 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 85 Collection<TestRunResult> results = listener.getRunResults(); 86 assertEquals(1, results.size()); 87 TestRunResult result = results.iterator().next(); 88 assertFalse(result.isRunFailure()); 89 assertFalse(result.hasFailedTests()); 90 // Ensure the listener added a metric at test run start and end. 91 assertTrue(result.getRunMetrics().containsKey("run_start")); 92 assertTrue(result.getRunMetrics().containsKey("run_end")); 93 // Check that each test case has results with the metrics associated. 94 for (TestResult res : result.getTestResults().values()) { 95 assertTrue(res.getMetrics().containsKey("test_start")); 96 assertTrue(res.getMetrics().containsKey("test_end")); 97 } 98 } 99 100 /** 101 * Test that our base metric listener can filter metrics to run only against some groups tagged 102 * with an annotation. All annotation of BaseMetricListenerInstrumentedTest are annotated with 103 * the group 'testGroup'. 104 */ 105 @Test 106 public void testBaseListenerRuns_withExcludeFilters() throws Exception { 107 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 108 mTestRunner.addInstrumentationArg("exclude-filter-group", "testGroup"); 109 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 110 CollectingTestListener listener = new CollectingTestListener(); 111 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 112 Collection<TestRunResult> results = listener.getRunResults(); 113 assertEquals(1, results.size()); 114 TestRunResult result = results.iterator().next(); 115 assertFalse(result.isRunFailure()); 116 assertFalse(result.hasFailedTests()); 117 // Ensure the listener added a metric at test run start and end. 118 assertTrue(result.getRunMetrics().containsKey("run_start")); 119 assertTrue(result.getRunMetrics().containsKey("run_end")); 120 // We did run some tests 121 assertTrue(!result.getTestResults().isEmpty()); 122 // After filtering none of the test case should contain any of the metrics since it was 123 // filtered. 124 for (TestResult testCaseResult : result.getTestResults().values()) { 125 assertFalse(testCaseResult.getMetrics().containsKey("test_start")); 126 assertFalse(testCaseResult.getMetrics().containsKey("test_fail")); 127 assertFalse(testCaseResult.getMetrics().containsKey("test_end")); 128 } 129 } 130 131 /** 132 * Test that if an include and exclude filters are provided, the exclude filters has priority. 133 */ 134 @Test 135 public void testBaseListenerRuns_withIncludeAndExcludeFilters() throws Exception { 136 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 137 mTestRunner.addInstrumentationArg("include-filter-group", "testGroup"); 138 mTestRunner.addInstrumentationArg("exclude-filter-group", "testGroup"); 139 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 140 CollectingTestListener listener = new CollectingTestListener(); 141 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 142 Collection<TestRunResult> results = listener.getRunResults(); 143 assertEquals(1, results.size()); 144 TestRunResult result = results.iterator().next(); 145 assertFalse(result.isRunFailure()); 146 assertFalse(result.hasFailedTests()); 147 // Ensure the listener added a metric at test run start and end. 148 assertTrue(result.getRunMetrics().containsKey("run_start")); 149 assertTrue(result.getRunMetrics().containsKey("run_end")); 150 // We did run some tests 151 assertTrue(!result.getTestResults().isEmpty()); 152 // After filtering none of the test case should contain any of the metrics since it was 153 // filtered. Exclusion has priority over inclusion. 154 for (TestResult testCaseResult : result.getTestResults().values()) { 155 assertFalse(testCaseResult.getMetrics().containsKey("test_start")); 156 assertFalse(testCaseResult.getMetrics().containsKey("test_fail")); 157 assertFalse(testCaseResult.getMetrics().containsKey("test_end")); 158 } 159 } 160 161 /** 162 * Test that if an include filter is provided, only method part of the included group will 163 * run the collection. 164 */ 165 @Test 166 public void testBaseListenerRuns_withIncludeFilters() throws Exception { 167 mTestRunner.addInstrumentationArg("listener", STUB_BASE_COLLECTOR); 168 mTestRunner.addInstrumentationArg("include-filter-group", "testGroup1"); 169 mTestRunner.setClassName("android.device.collectors.BaseMetricListenerInstrumentedTest"); 170 CollectingTestListener listener = new CollectingTestListener(); 171 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 172 Collection<TestRunResult> results = listener.getRunResults(); 173 assertEquals(1, results.size()); 174 TestRunResult result = results.iterator().next(); 175 assertFalse(result.isRunFailure()); 176 assertFalse(result.hasFailedTests()); 177 // Ensure the listener added a metric at test run start and end. 178 assertTrue(result.getRunMetrics().containsKey("run_start")); 179 assertTrue(result.getRunMetrics().containsKey("run_end")); 180 // We did run some tests 181 assertTrue(!result.getTestResults().isEmpty()); 182 // After filtering none of the test case should contain any of the metrics since it was 183 // filtered. Exclusion has priority over inclusion. 184 for (Entry<TestDescription, TestResult> testResult : result.getTestResults().entrySet()) { 185 // testReportMetrics method is the only one annotated with 'testGroup1' it should be the 186 // only one collecting metrics. 187 if ("testReportMetrics".equals(testResult.getKey().getTestName())) { 188 assertTrue(testResult.getValue().getMetrics().containsKey("test_start")); 189 assertTrue(testResult.getValue().getMetrics().containsKey("test_end")); 190 } else { 191 assertFalse(testResult.getValue().getMetrics().containsKey("test_start")); 192 assertFalse(testResult.getValue().getMetrics().containsKey("test_fail")); 193 assertFalse(testResult.getValue().getMetrics().containsKey("test_end")); 194 } 195 } 196 } 197 198 /** 199 * Test that our base scheduled listener can output metrics periodically. 200 */ 201 @Test 202 public void testScheduledListenerRuns() throws Exception { 203 mTestRunner.addInstrumentationArg("listener", SCHEDULED_COLLECTOR); 204 mTestRunner.addInstrumentationArg("interval", "100"); 205 CollectingTestListener listener = new CollectingTestListener(); 206 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 207 Collection<TestRunResult> results = listener.getRunResults(); 208 assertEquals(1, results.size()); 209 TestRunResult result = results.iterator().next(); 210 assertFalse(result.isRunFailure()); 211 assertFalse(result.hasFailedTests()); 212 // There is time during the test to output at least a handful of periodic metrics. 213 assertTrue(result.getRunMetrics().containsKey("collect0")); 214 assertTrue(result.getRunMetrics().containsKey("collect1")); 215 assertTrue(result.getRunMetrics().containsKey("collect2")); 216 } 217 218 /** 219 * Test that our base scheduled listener can use its default period to run when the interval 220 * given is not valid. 221 */ 222 @Test 223 public void testScheduledListenerRuns_defaultValue() throws Exception { 224 mTestRunner.addInstrumentationArg("listener", SCHEDULED_COLLECTOR); 225 // Invalid interval will results in the default period to be used. 226 mTestRunner.addInstrumentationArg("interval", "-100"); 227 CollectingTestListener listener = new CollectingTestListener(); 228 assertTrue(getDevice().runInstrumentationTests(mTestRunner, listener)); 229 Collection<TestRunResult> results = listener.getRunResults(); 230 assertEquals(1, results.size()); 231 TestRunResult result = results.iterator().next(); 232 assertFalse(result.isRunFailure()); 233 assertFalse(result.hasFailedTests()); 234 // The default interval value is one minute so it will only have time to run once. 235 assertEquals(1, result.getRunMetrics().size()); 236 assertTrue(result.getRunMetrics().containsKey("collect0")); 237 } 238 } 239
