1 /* 2 * Copyright (C) 2016 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import static com.google.common.truth.Truth.assertThat; 19 import static org.mockito.AdditionalMatchers.gt; 20 import static org.mockito.Mockito.any; 21 import static org.mockito.Mockito.anyCollection; 22 import static org.mockito.Mockito.anyInt; 23 import static org.mockito.Mockito.anyString; 24 import static org.mockito.Mockito.doAnswer; 25 import static org.mockito.Mockito.doReturn; 26 import static org.mockito.Mockito.eq; 27 import static org.mockito.Mockito.mock; 28 import static org.mockito.Mockito.verify; 29 30 import com.android.ddmlib.IDevice; 31 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 32 import com.android.ddmlib.testrunner.ITestRunListener; 33 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 34 import com.android.ddmlib.testrunner.TestIdentifier; 35 import com.android.ddmlib.testrunner.TestRunResult; 36 import com.android.tradefed.build.BuildInfo; 37 import com.android.tradefed.build.IBuildInfo; 38 import com.android.tradefed.device.DeviceNotAvailableException; 39 import com.android.tradefed.device.ITestDevice; 40 import com.android.tradefed.log.ITestLogger; 41 import com.android.tradefed.result.InputStreamSource; 42 import com.android.tradefed.result.ITestInvocationListener; 43 import com.android.tradefed.result.LogDataType; 44 import com.android.tradefed.util.ICompressionStrategy; 45 import com.android.tradefed.util.ListInstrumentationParser; 46 import com.android.tradefed.util.ListInstrumentationParser.InstrumentationTarget; 47 48 import com.google.common.collect.ImmutableList; 49 import com.google.common.collect.ImmutableMap; 50 import com.google.common.collect.Lists; 51 import com.google.protobuf.ByteString; 52 53 import org.junit.Before; 54 import org.junit.Rule; 55 import org.junit.Test; 56 import org.junit.rules.TemporaryFolder; 57 import org.junit.runner.RunWith; 58 import org.junit.runners.JUnit4; 59 import org.mockito.Mock; 60 import org.mockito.Mockito; 61 import org.mockito.MockitoAnnotations; 62 import org.mockito.invocation.InvocationOnMock; 63 import org.mockito.stubbing.Answer; 64 65 import java.io.File; 66 import java.io.FileInputStream; 67 import java.io.FileOutputStream; 68 import java.io.InputStream; 69 import java.io.IOException; 70 import java.util.ArrayList; 71 import java.util.Collection; 72 import java.util.HashMap; 73 import java.util.List; 74 import java.util.Map; 75 76 /** Unit tests for {@link CodeCoverageTestBase}. */ 77 @RunWith(JUnit4.class) 78 public class CodeCoverageTestBaseTest { 79 80 private static final long TEST_RUN_TIME = 1000; 81 private static final String COVERAGE_PATH = "/data/user/0/%s/files/coverage.ec"; 82 83 private static final String PACKAGE_NAME1 = "com.example.foo.test"; 84 private static final String PACKAGE_NAME2 = "com.example.bar.test"; 85 private static final String PACKAGE_NAME3 = "com.example.baz.test"; 86 87 private static final String RUNNER_NAME1 = "android.support.test.runner.AndroidJUnitRunner"; 88 private static final String RUNNER_NAME2 = "android.test.InstrumentationTestRunner"; 89 private static final String RUNNER_NAME3 = "com.example.custom.Runner"; 90 91 private static final TestIdentifier FOO_TEST1 = new TestIdentifier(".FooTest", "test1"); 92 private static final TestIdentifier FOO_TEST2 = new TestIdentifier(".FooTest", "test2"); 93 private static final TestIdentifier FOO_TEST3 = new TestIdentifier(".FooTest", "test3"); 94 private static final List<TestIdentifier> FOO_TESTS = 95 ImmutableList.of(FOO_TEST1, FOO_TEST2, FOO_TEST3); 96 97 private static final TestIdentifier BAR_TEST1 = new TestIdentifier(".BarTest", "test1"); 98 private static final TestIdentifier BAR_TEST2 = new TestIdentifier(".BarTest", "test2"); 99 private static final List<TestIdentifier> BAR_TESTS = ImmutableList.of(BAR_TEST1, BAR_TEST2); 100 101 private static final TestIdentifier BAZ_TEST1 = new TestIdentifier(".BazTest", "test1"); 102 private static final TestIdentifier BAZ_TEST2 = new TestIdentifier(".BazTest", "test2"); 103 private static final List<TestIdentifier> BAZ_TESTS = ImmutableList.of(BAZ_TEST1, BAZ_TEST2); 104 105 private static final ByteString FAKE_REPORT_CONTENTS = 106 ByteString.copyFromUtf8("Mi estas kovrado raporto"); 107 108 private static final ByteString FAKE_MEASUREMENT1 = 109 ByteString.copyFromUtf8("Mi estas kovrado mezurado"); 110 private static final ByteString FAKE_MEASUREMENT2 = 111 ByteString.copyFromUtf8("Mi estas ankau kovrado mezurado"); 112 private static final ByteString FAKE_MEASUREMENT3 = 113 ByteString.copyFromUtf8("Mi estas ankorau alia priraportado mezurado"); 114 115 private static final IBuildInfo BUILD_INFO = new BuildInfo("123456", "device-userdebug"); 116 117 @Rule public TemporaryFolder mFolder = new TemporaryFolder(); 118 119 // Mocks 120 @Mock ITestDevice mDevice; 121 122 @Mock ITestInvocationListener mListener; 123 124 @Mock ListInstrumentationParser mInstrumentationParser; 125 126 // Fake test data 127 @Mock TestDataRegistry<List<TestIdentifier>> mTests; 128 129 @Mock TestDataRegistry<ByteString> mMeasurements; 130 131 interface TestDataRegistry<T> { 132 T get(String packageName, String runnerName, int shardIndex, int numShards); 133 } 134 135 /** Object under test */ 136 CodeCoverageTestStub mCoverageTest; 137 138 @Before 139 public void setUp() throws DeviceNotAvailableException { 140 MockitoAnnotations.initMocks(this); 141 doAnswer(CALL_RUNNER) 142 .when(mDevice) 143 .runInstrumentationTests( 144 any(IRemoteAndroidTestRunner.class), any(ITestRunListener.class)); 145 mCoverageTest = new CodeCoverageTestStub(); 146 } 147 148 static enum FakeReportFormat implements CodeCoverageReportFormat { 149 CSV(LogDataType.JACOCO_CSV), 150 XML(LogDataType.JACOCO_XML), 151 HTML(LogDataType.HTML); 152 153 private LogDataType mLogDataType; 154 155 private FakeReportFormat(LogDataType logDataType) { 156 mLogDataType = logDataType; 157 } 158 159 @Override 160 public LogDataType getLogDataType() { return mLogDataType; } 161 } 162 163 /** A subclass of {@link CodeCoverageTest} with certain methods stubbed out for testing. */ 164 private class CodeCoverageTestStub extends CodeCoverageTestBase<FakeReportFormat> { 165 166 // Captured data 167 private ImmutableList.Builder<ByteString> mCapturedMeasurements = 168 new ImmutableList.Builder<>(); 169 170 public CodeCoverageTestStub() { 171 setDevice(mDevice); 172 } 173 174 @Override 175 public IBuildInfo getBuild() { return BUILD_INFO; } 176 177 @Override 178 ICompressionStrategy getCompressionStrategy() { 179 return mock(ICompressionStrategy.class); 180 } 181 182 ImmutableList<ByteString> getMeasurements() { 183 return mCapturedMeasurements.build(); 184 } 185 186 @Override 187 protected File generateCoverageReport( 188 Collection<File> measurementFiles, FakeReportFormat format) throws IOException { 189 // Capture the measurements for verification later 190 for (File measurementFile : measurementFiles) { 191 try (FileInputStream inputStream = new FileInputStream(measurementFile)) { 192 mCapturedMeasurements.add(ByteString.readFrom(inputStream)); 193 } 194 } 195 196 // Write the fake report 197 File ret = mFolder.newFile(); 198 FAKE_REPORT_CONTENTS.writeTo(new FileOutputStream(ret)); 199 return ret; 200 } 201 202 @Override 203 protected List<FakeReportFormat> getReportFormat() { 204 return ImmutableList.of(FakeReportFormat.HTML); 205 } 206 207 @Override 208 InstrumentationTest internalCreateTest() { 209 return new InstrumentationTest() { 210 @Override 211 IRemoteAndroidTestRunner createRemoteAndroidTestRunner( 212 String packageName, String runnerName, IDevice device) { 213 return new FakeTestRunner(packageName, runnerName); 214 } 215 }; 216 } 217 218 @Override 219 IRemoteAndroidTestRunner internalCreateTestRunner(String packageName, String runnerName) { 220 return new FakeTestRunner(packageName, runnerName); 221 } 222 223 @Override 224 ListInstrumentationParser internalCreateListInstrumentationParser() { 225 return mInstrumentationParser; 226 } 227 } 228 229 private static final class LogCaptorAnswer implements Answer<Void> { 230 private ByteString mValue; 231 232 @Override 233 public Void answer(InvocationOnMock invocation) throws Throwable { 234 Object[] args = invocation.getArguments(); 235 InputStream reportStream = ((InputStreamSource) args[2]).createInputStream(); 236 mValue = ByteString.readFrom(reportStream); 237 return null; 238 } 239 240 ByteString getValue() { 241 return mValue; 242 } 243 } 244 245 @Test 246 public void testRun() throws DeviceNotAvailableException, IOException { 247 // Prepare some test data 248 doReturn(ImmutableList.of(new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""))) 249 .when(mInstrumentationParser) 250 .getInstrumentationTargets(); 251 252 // Mocking boilerplate 253 doReturn(FOO_TESTS) 254 .when(mTests) 255 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 256 doReturn(FAKE_MEASUREMENT1) 257 .when(mMeasurements) 258 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 259 260 // Validate the report when it gets log, since the file will get cleaned up later 261 LogCaptorAnswer logCaptor = new LogCaptorAnswer(); 262 doAnswer(logCaptor) 263 .when(mListener) 264 .testLog( 265 eq("coverage"), 266 eq(FakeReportFormat.HTML.getLogDataType()), 267 any(InputStreamSource.class)); 268 269 // Run the test 270 mCoverageTest.run(mListener); 271 272 // Verify that the measurements were collected and the report was logged 273 assertThat(mCoverageTest.getMeasurements()).containsExactly(FAKE_MEASUREMENT1); 274 assertThat(logCaptor.getValue()).isEqualTo(FAKE_REPORT_CONTENTS); 275 } 276 277 @Test 278 public void testRun_multipleInstrumentationTargets() 279 throws DeviceNotAvailableException, IOException { 280 // Prepare some test data 281 ImmutableList<InstrumentationTarget> targets = 282 ImmutableList.of( 283 new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""), 284 new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME1, ""), 285 new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, "")); 286 doReturn(targets).when(mInstrumentationParser).getInstrumentationTargets(); 287 288 // Mocking boilerplate 289 doReturn(FOO_TESTS) 290 .when(mTests) 291 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 292 doReturn(FAKE_MEASUREMENT1) 293 .when(mMeasurements) 294 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 295 296 doReturn(BAR_TESTS) 297 .when(mTests) 298 .get(eq(PACKAGE_NAME2), eq(RUNNER_NAME1), anyInt(), anyInt()); 299 doReturn(FAKE_MEASUREMENT2) 300 .when(mMeasurements) 301 .get(eq(PACKAGE_NAME2), eq(RUNNER_NAME1), anyInt(), anyInt()); 302 303 doReturn(BAZ_TESTS) 304 .when(mTests) 305 .get(eq(PACKAGE_NAME3), eq(RUNNER_NAME1), anyInt(), anyInt()); 306 doReturn(FAKE_MEASUREMENT3) 307 .when(mMeasurements) 308 .get(eq(PACKAGE_NAME3), eq(RUNNER_NAME1), anyInt(), anyInt()); 309 310 // Run the test 311 mCoverageTest.run(mListener); 312 313 // Verify that all targets were run by checking that we recieved measurements from each 314 assertThat(mCoverageTest.getMeasurements()) 315 .containsExactly(FAKE_MEASUREMENT1, FAKE_MEASUREMENT2, FAKE_MEASUREMENT3); 316 } 317 318 @Test 319 public void testRun_multipleShards() throws DeviceNotAvailableException { 320 // Prepare some test data 321 doReturn(ImmutableList.of(new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""))) 322 .when(mInstrumentationParser) 323 .getInstrumentationTargets(); 324 List<List<TestIdentifier>> shards = Lists.partition(FOO_TESTS, 1); 325 326 // Indicate that the test should be split into 3 shards 327 doReturn(FOO_TESTS).when(mTests).get(PACKAGE_NAME1, RUNNER_NAME1, 0, 1); 328 doReturn(FOO_TESTS.subList(0, FOO_TESTS.size() / 2)) 329 .when(mTests) 330 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(0), eq(2)); 331 mCoverageTest.setMaxTestsPerChunk(1); 332 333 // Return subsets of FOO_TESTS when running shards 334 doReturn(shards.get(0)).when(mTests).get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(0), gt(1)); 335 doReturn(FAKE_MEASUREMENT1) 336 .when(mMeasurements) 337 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(0), gt(1)); 338 339 doReturn(shards.get(1)).when(mTests).get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(1), gt(1)); 340 doReturn(FAKE_MEASUREMENT2) 341 .when(mMeasurements) 342 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(1), gt(1)); 343 344 doReturn(shards.get(2)).when(mTests).get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(2), gt(1)); 345 doReturn(FAKE_MEASUREMENT3) 346 .when(mMeasurements) 347 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(2), gt(1)); 348 349 // Run the test 350 mCoverageTest.run(mListener); 351 352 // Verify that all shards were run by checking that we recieved measurements from each 353 assertThat(mCoverageTest.getMeasurements()) 354 .containsExactly(FAKE_MEASUREMENT1, FAKE_MEASUREMENT2, FAKE_MEASUREMENT3); 355 } 356 357 @Test 358 public void testRun_rerunIndividualTests_failedRun() throws DeviceNotAvailableException { 359 // Prepare some test data 360 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 361 doReturn(ImmutableList.of(target)).when(mInstrumentationParser).getInstrumentationTargets(); 362 doReturn(FOO_TESTS) 363 .when(mTests) 364 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 365 366 TestRunResult success = mock(TestRunResult.class); 367 doReturn(false).when(success).isRunFailure(); 368 TestRunResult failure = mock(TestRunResult.class); 369 doReturn(true).when(failure).isRunFailure(); 370 371 // Mocking boilerplate 372 ITestInvocationListener mockListener = mock(ITestInvocationListener.class); 373 CodeCoverageTestStub coverageTest = Mockito.spy(new CodeCoverageTestStub()); 374 doReturn(FOO_TESTS).when(mTests).get(PACKAGE_NAME1, RUNNER_NAME1, 0, 1); 375 376 doReturn(failure).when(coverageTest).runTest(eq(target), eq(0), eq(1), 377 any(ITestInvocationListener.class)); 378 doReturn(success).when(coverageTest).runTest(eq(target), eq(FOO_TEST1), 379 any(ITestInvocationListener.class)); 380 doReturn(failure).when(coverageTest).runTest(eq(target), eq(FOO_TEST2), 381 any(ITestInvocationListener.class)); 382 doReturn(success).when(coverageTest).runTest(eq(target), eq(FOO_TEST3), 383 any(ITestInvocationListener.class)); 384 385 // Run the test 386 coverageTest.run(mockListener); 387 388 // Verify that individual tests are rerun 389 verify(coverageTest).runTest(eq(target), eq(0), eq(1), any(ITestInvocationListener.class)); 390 verify(coverageTest).runTest(eq(target), eq(FOO_TEST1), any(ITestInvocationListener.class)); 391 verify(coverageTest).runTest(eq(target), eq(FOO_TEST2), any(ITestInvocationListener.class)); 392 verify(coverageTest).runTest(eq(target), eq(FOO_TEST3), any(ITestInvocationListener.class)); 393 } 394 395 @Test 396 public void testRun_rerunIndividualTests_missingCoverageFile() 397 throws DeviceNotAvailableException { 398 // Prepare some test data 399 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 400 doReturn(ImmutableList.of(target)).when(mInstrumentationParser).getInstrumentationTargets(); 401 doReturn(FOO_TESTS) 402 .when(mTests) 403 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 404 405 TestRunResult success = mock(TestRunResult.class); 406 doReturn(false).when(success).isRunFailure(); 407 408 // Mocking boilerplate 409 ITestInvocationListener mockListener = mock(ITestInvocationListener.class); 410 CodeCoverageTestStub coverageTest = Mockito.spy(new CodeCoverageTestStub()); 411 doReturn(FOO_TESTS).when(mTests).get(PACKAGE_NAME1, RUNNER_NAME1, 0, 1); 412 413 doReturn(success) 414 .when(coverageTest) 415 .runTest(any(InstrumentationTest.class), any(ITestInvocationListener.class)); 416 ITestDevice mockDevice = coverageTest.getDevice(); 417 doReturn(false).doReturn(true).when(mockDevice).doesFileExist(anyString()); 418 419 // Run the test 420 coverageTest.run(mockListener); 421 422 // Verify that individual tests are rerun 423 verify(coverageTest).runTest(eq(target), eq(0), eq(1), any(ITestInvocationListener.class)); 424 verify(coverageTest).runTest(eq(target), eq(FOO_TEST1), any(ITestInvocationListener.class)); 425 verify(coverageTest).runTest(eq(target), eq(FOO_TEST2), any(ITestInvocationListener.class)); 426 verify(coverageTest).runTest(eq(target), eq(FOO_TEST3), any(ITestInvocationListener.class)); 427 } 428 429 @Test 430 public void testRun_multipleFormats() throws DeviceNotAvailableException, IOException { 431 // Prepare some test data 432 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 433 doReturn(ImmutableList.of(target)).when(mInstrumentationParser).getInstrumentationTargets(); 434 doReturn(FOO_TESTS) 435 .when(mTests) 436 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 437 doReturn(FAKE_MEASUREMENT1) 438 .when(mMeasurements) 439 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 440 441 File fakeHtmlReport = new File("/some/fake/xml/report/"); 442 File fakeXmlReport = new File("/some/fake/xml/report.xml"); 443 444 // Mocking boilerplate 445 ITestInvocationListener mockListener = mock(ITestInvocationListener.class); 446 CodeCoverageTestStub coverageTest = Mockito.spy(new CodeCoverageTestStub()); 447 doReturn(FOO_TESTS).when(mTests).get(PACKAGE_NAME1, RUNNER_NAME1, 0, 1); 448 doReturn(ImmutableList.of(FakeReportFormat.XML, FakeReportFormat.HTML)) 449 .when(coverageTest) 450 .getReportFormat(); 451 doReturn(fakeHtmlReport) 452 .when(coverageTest) 453 .generateCoverageReport(anyCollection(), eq(FakeReportFormat.HTML)); 454 doReturn(fakeXmlReport) 455 .when(coverageTest) 456 .generateCoverageReport(anyCollection(), eq(FakeReportFormat.XML)); 457 458 // Run the test 459 coverageTest.run(mockListener); 460 461 // Verify that the test was run, and that the reports were logged 462 verify(coverageTest) 463 .runTest(any(InstrumentationTest.class), any(ITestInvocationListener.class)); 464 verify(coverageTest).generateCoverageReport(anyCollection(), eq(FakeReportFormat.HTML)); 465 verify(coverageTest).doLogReport(anyString(), eq(FakeReportFormat.HTML.getLogDataType()), 466 eq(fakeHtmlReport), any(ITestLogger.class)); 467 verify(coverageTest).generateCoverageReport(anyCollection(), eq(FakeReportFormat.XML)); 468 verify(coverageTest).doLogReport(anyString(), eq(FakeReportFormat.XML.getLogDataType()), 469 eq(fakeXmlReport), any(ITestLogger.class)); 470 } 471 472 @Test 473 public void testGetInstrumentationTargets() throws DeviceNotAvailableException { 474 // Prepare some test data 475 InstrumentationTarget target1 = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 476 InstrumentationTarget target2 = new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME2, ""); 477 InstrumentationTarget target3 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, ""); 478 InstrumentationTarget target4 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME3, ""); 479 480 // Set up mocks 481 doReturn(ImmutableList.of(target1, target2, target3, target4)) 482 .when(mInstrumentationParser) 483 .getInstrumentationTargets(); 484 485 // Get the instrumentation targets 486 Collection<InstrumentationTarget> targets = mCoverageTest.getInstrumentationTargets(); 487 488 // Verify that all of the instrumentation targets were found 489 assertThat(targets).containsExactly(target1, target2, target3, target4); 490 } 491 492 @Test 493 public void testGetInstrumentationTargets_packageFilterSingleFilterSingleResult() 494 throws DeviceNotAvailableException { 495 // Prepare some test data 496 InstrumentationTarget target1 = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 497 InstrumentationTarget target2 = new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME2, ""); 498 InstrumentationTarget target3 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, ""); 499 InstrumentationTarget target4 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME3, ""); 500 501 // Set up mocks 502 doReturn(ImmutableList.of(target1, target2, target3, target4)) 503 .when(mInstrumentationParser) 504 .getInstrumentationTargets(); 505 mCoverageTest.setPackageFilter(ImmutableList.of(PACKAGE_NAME1)); 506 507 // Get the instrumentation targets 508 Collection<InstrumentationTarget> targets = mCoverageTest.getInstrumentationTargets(); 509 510 // Verify that only the PACKAGE_NAME1 target was returned 511 assertThat(targets).containsExactly(target1); 512 } 513 514 @Test 515 public void testGetInstrumentationTargets_packageFilterSingleFilterMultipleResults() 516 throws DeviceNotAvailableException { 517 // Prepare some test data 518 InstrumentationTarget target1 = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 519 InstrumentationTarget target2 = new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME2, ""); 520 InstrumentationTarget target3 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, ""); 521 InstrumentationTarget target4 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME3, ""); 522 523 // Set up mocks 524 doReturn(ImmutableList.of(target1, target2, target3, target4)) 525 .when(mInstrumentationParser) 526 .getInstrumentationTargets(); 527 mCoverageTest.setPackageFilter(ImmutableList.of(PACKAGE_NAME3)); 528 529 // Get the instrumentation targets 530 Collection<InstrumentationTarget> targets = mCoverageTest.getInstrumentationTargets(); 531 532 // Verify that both PACKAGE_NAME3 targets were returned 533 assertThat(targets).containsExactly(target3, target4); 534 } 535 536 @Test 537 public void testGetInstrumentationTargets_packageFilterMultipleFilters() 538 throws DeviceNotAvailableException { 539 // Prepare some test data 540 InstrumentationTarget target1 = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 541 InstrumentationTarget target2 = new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME2, ""); 542 InstrumentationTarget target3 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, ""); 543 InstrumentationTarget target4 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME3, ""); 544 545 // Set up mocks 546 doReturn(ImmutableList.of(target1, target2, target3, target4)) 547 .when(mInstrumentationParser) 548 .getInstrumentationTargets(); 549 mCoverageTest.setPackageFilter(ImmutableList.of(PACKAGE_NAME1, PACKAGE_NAME2)); 550 551 // Get the instrumentation targets 552 Collection<InstrumentationTarget> targets = mCoverageTest.getInstrumentationTargets(); 553 554 // Verify that the PACKAGE_NAME1 and PACKAGE_NAME2 targets were returned 555 assertThat(targets).containsExactly(target1, target2); 556 } 557 558 @Test 559 public void testGetInstrumentationTargets_runnerFilterSingleFilterSingleResult() 560 throws DeviceNotAvailableException { 561 // Prepare some test data 562 InstrumentationTarget target1 = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 563 InstrumentationTarget target2 = new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME2, ""); 564 InstrumentationTarget target3 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, ""); 565 InstrumentationTarget target4 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME3, ""); 566 567 // Set up mocks 568 doReturn(ImmutableList.of(target1, target2, target3, target4)) 569 .when(mInstrumentationParser) 570 .getInstrumentationTargets(); 571 mCoverageTest.setRunnerFilter(ImmutableList.of(RUNNER_NAME2)); 572 573 // Get the instrumentation targets 574 Collection<InstrumentationTarget> targets = mCoverageTest.getInstrumentationTargets(); 575 576 // Verify that only the RUNNER_NAME2 target was returned 577 assertThat(targets).containsExactly(target2); 578 } 579 580 @Test 581 public void testGetInstrumentationTargets_runnerFilterSingleFilterMultipleResults() 582 throws DeviceNotAvailableException { 583 // Prepare some test data 584 InstrumentationTarget target1 = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 585 InstrumentationTarget target2 = new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME2, ""); 586 InstrumentationTarget target3 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, ""); 587 InstrumentationTarget target4 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME3, ""); 588 589 // Set up mocks 590 doReturn(ImmutableList.of(target1, target2, target3, target4)) 591 .when(mInstrumentationParser) 592 .getInstrumentationTargets(); 593 mCoverageTest.setRunnerFilter(ImmutableList.of(RUNNER_NAME1)); 594 595 // Get the instrumentation targets 596 Collection<InstrumentationTarget> targets = mCoverageTest.getInstrumentationTargets(); 597 598 // Verify that both RUNNER_NAME1 targets were returned 599 assertThat(targets).containsExactly(target1, target3); 600 } 601 602 @Test 603 public void testGetInstrumentationTargets_runnerFilterMultipleFilters() 604 throws DeviceNotAvailableException { 605 // Prepare some test data 606 InstrumentationTarget target1 = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 607 InstrumentationTarget target2 = new InstrumentationTarget(PACKAGE_NAME2, RUNNER_NAME2, ""); 608 InstrumentationTarget target3 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME1, ""); 609 InstrumentationTarget target4 = new InstrumentationTarget(PACKAGE_NAME3, RUNNER_NAME3, ""); 610 611 // Set up mocks 612 doReturn(ImmutableList.of(target1, target2, target3, target4)) 613 .when(mInstrumentationParser) 614 .getInstrumentationTargets(); 615 mCoverageTest.setRunnerFilter(ImmutableList.of(RUNNER_NAME2, RUNNER_NAME3)); 616 617 // Get the instrumentation targets 618 Collection<InstrumentationTarget> targets = mCoverageTest.getInstrumentationTargets(); 619 620 // Verify that the RUNNER_NAME2 and RUNNER_NAME3 targets were returned 621 assertThat(targets).containsExactly(target2, target4); 622 } 623 624 @Test 625 public void testDoesRunnerSupportSharding_true() throws DeviceNotAvailableException { 626 // Prepare some test data 627 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 628 629 // Set up mocks. Return fewer tests when sharding is enabled. 630 doReturn(ImmutableList.of(target)).when(mInstrumentationParser).getInstrumentationTargets(); 631 doReturn(FOO_TESTS).when(mTests).get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(0), eq(1)); 632 doReturn(Lists.partition(FOO_TESTS, 2).get(0)) 633 .when(mTests) 634 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(0), gt(1)); 635 636 // Verify that the method returns true 637 assertThat(mCoverageTest.doesRunnerSupportSharding(target)).isTrue(); 638 } 639 640 @Test 641 public void testDoesRunnerSupportSharding_false() throws DeviceNotAvailableException { 642 // Prepare some test data 643 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 644 645 // Set up mocks. Return the same number of tests for any number of shards. 646 doReturn(ImmutableList.of(target)).when(mInstrumentationParser).getInstrumentationTargets(); 647 doReturn(FOO_TESTS) 648 .when(mTests) 649 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 650 651 // Verify that the method returns false 652 assertThat(mCoverageTest.doesRunnerSupportSharding(target)).isFalse(); 653 } 654 655 @Test 656 public void testGetNumberOfShards() throws DeviceNotAvailableException { 657 // Prepare some test data 658 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 659 List<TestIdentifier> tests = new ArrayList<>(); 660 for (int i = 0; i < 10; i++) { 661 tests.add(new TestIdentifier(PACKAGE_NAME1, String.format("test%d", i))); 662 } 663 664 // Set up mocks 665 doReturn(tests).when(mTests).get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 666 mCoverageTest.setMaxTestsPerChunk(1); 667 668 // Verify that each test will run in a separate shard 669 assertThat(mCoverageTest.getNumberOfShards(target)).isEqualTo(tests.size()); 670 } 671 672 @Test 673 public void testGetNumberOfShards_allTestsInSingleShard() throws DeviceNotAvailableException { 674 // Prepare some test data 675 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 676 List<TestIdentifier> tests = new ArrayList<>(); 677 for (int i = 0; i < 10; i++) { 678 tests.add(new TestIdentifier(PACKAGE_NAME1, String.format("test%d", i))); 679 } 680 681 // Set up mocks 682 doReturn(tests).when(mTests).get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 683 mCoverageTest.setMaxTestsPerChunk(10); 684 685 // Verify that all tests will run in a single shard 686 assertThat(mCoverageTest.getNumberOfShards(target)).isEqualTo(1); 687 } 688 689 @Test 690 public void testCollectTests() throws DeviceNotAvailableException { 691 // Prepare some test data 692 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 693 List<TestIdentifier> tests = new ArrayList<>(); 694 for (int i = 0; i < 10; i++) { 695 tests.add(new TestIdentifier(PACKAGE_NAME1, String.format("test%d", i))); 696 } 697 698 // Set up mocks 699 doReturn(tests).when(mTests).get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), anyInt(), anyInt()); 700 701 // Collect the tests 702 Collection<TestIdentifier> collectedTests = mCoverageTest.collectTests(target, 0, 1); 703 704 // Verify that all of the tests were returned 705 assertThat(collectedTests).containsExactlyElementsIn(tests); 706 } 707 708 @Test 709 public void testCollectTests_withShards() throws DeviceNotAvailableException { 710 // Prepare some test data 711 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 712 int numShards = 3; 713 List<TestIdentifier> tests = new ArrayList<>(); 714 for (int i = 0; i < 10; i++) { 715 tests.add(new TestIdentifier(PACKAGE_NAME1, String.format("test%d", i))); 716 } 717 718 // Set up mocks 719 mCoverageTest.setMaxTestsPerChunk((int) Math.ceil((double) tests.size() / numShards)); 720 doReturn(tests).when(mTests).get(PACKAGE_NAME1, RUNNER_NAME1, 0, 1); 721 doReturn(tests.subList(0, tests.size() / 2)) 722 .when(mTests) 723 .get(eq(PACKAGE_NAME1), eq(RUNNER_NAME1), eq(0), eq(2)); 724 725 List<List<TestIdentifier>> shards = 726 Lists.partition(tests, (int) Math.ceil((double) tests.size() / numShards)); 727 int currentIndex = 0; 728 for (List<TestIdentifier> shard : shards) { 729 doReturn(shards.get(currentIndex)) 730 .when(mTests) 731 .get(PACKAGE_NAME1, RUNNER_NAME1, currentIndex, shards.size()); 732 currentIndex++; 733 } 734 735 // Collect the tests in shards 736 ArrayList<TestIdentifier> allCollectedTests = new ArrayList<>(); 737 for (int shardIndex = 0; shardIndex < numShards; shardIndex++) { 738 // Verify that each shard contains some tests 739 Collection<TestIdentifier> collectedTests = 740 mCoverageTest.collectTests(target, shardIndex, numShards); 741 assertThat(collectedTests).containsExactlyElementsIn(shards.get(shardIndex)); 742 743 allCollectedTests.addAll(collectedTests); 744 } 745 // Verify that all of the tests were returned in the end 746 assertThat(allCollectedTests).containsExactlyElementsIn(tests); 747 } 748 749 @Test 750 public void testCreateTestRunner() throws DeviceNotAvailableException { 751 // Prepare some test data 752 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 753 754 // Create a test runner 755 IRemoteAndroidTestRunner runner = mCoverageTest.createTestRunner(target, 0, 1); 756 757 // Verify that the runner has the correct values 758 assertThat(runner.getPackageName()).isEqualTo(PACKAGE_NAME1); 759 assertThat(runner.getRunnerName()).isEqualTo(RUNNER_NAME1); 760 } 761 762 @Test 763 public void testCreateTestRunner_withArgs() throws DeviceNotAvailableException { 764 // Prepare some test data 765 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 766 Map<String, String> args = ImmutableMap.of("arg1", "value1", "arg2", "value2"); 767 768 // Set up mocks 769 mCoverageTest.setInstrumentationArgs(args); 770 771 // Create a test runner 772 FakeTestRunner runner = (FakeTestRunner) mCoverageTest.createTestRunner(target, 0, 1); 773 774 // Verify that the addInstrumentationArg(..) method was called with each argument 775 assertThat(runner.getArgs()).containsExactlyEntriesIn(args); 776 } 777 778 @Test 779 public void testCreateTestRunner_withShards() throws DeviceNotAvailableException { 780 // Prepare some test data 781 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 782 int shardIndex = 3; 783 int numShards = 5; 784 785 // Create a test runner 786 FakeTestRunner runner = 787 (FakeTestRunner) mCoverageTest.createTestRunner(target, shardIndex, numShards); 788 789 // Verify that the addInstrumentationArg(..) method was called to configure the shards 790 assertThat(runner.getArgs()).containsEntry("shardIndex", Integer.toString(shardIndex)); 791 assertThat(runner.getArgs()).containsEntry("numShards", Integer.toString(numShards)); 792 } 793 794 @Test 795 public void testCreateCoverageTest() throws DeviceNotAvailableException { 796 // Prepare some test data 797 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 798 799 // Create a CodeCoverageTest instance 800 InstrumentationTest test = mCoverageTest.createTest(target); 801 802 // Verify that the test has the correct values 803 assertThat(test.getPackageName()).isEqualTo(PACKAGE_NAME1); 804 assertThat(test.getRunnerName()).isEqualTo(RUNNER_NAME1); 805 assertThat(test.getInstrumentationArg("coverage")).isEqualTo("true"); 806 } 807 808 @Test 809 public void testCreateCoverageTest_withArgs() throws DeviceNotAvailableException { 810 // Prepare some test data 811 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 812 Map<String, String> args = ImmutableMap.of("arg1", "value1", "arg2", "value2"); 813 814 // Set up mocks 815 mCoverageTest.setInstrumentationArgs(args); 816 817 // Create a CodeCoverageTest instance 818 InstrumentationTest test = mCoverageTest.createTest(target, 0, 3); 819 820 // Verify that the test has the correct values 821 for (Map.Entry<String, String> entry : args.entrySet()) { 822 assertThat(test.getInstrumentationArg(entry.getKey())).isEqualTo(entry.getValue()); 823 } 824 } 825 826 @Test 827 public void testCreateCoverageTest_withShards() throws DeviceNotAvailableException { 828 // Prepare some test data 829 InstrumentationTarget target = new InstrumentationTarget(PACKAGE_NAME1, RUNNER_NAME1, ""); 830 int shardIndex = 3; 831 int numShards = 5; 832 833 // Create a CodeCoverageTest instance 834 InstrumentationTest test = mCoverageTest.createTest(target, shardIndex, numShards); 835 836 // Verify that the addInstrumentationArg(..) method was called to configure the shards 837 assertThat(test.getInstrumentationArg("shardIndex")) 838 .isEqualTo(Integer.toString(shardIndex)); 839 assertThat(test.getInstrumentationArg("numShards")).isEqualTo(Integer.toString(numShards)); 840 assertThat(test.getInstrumentationArg("coverage")).isEqualTo("true"); 841 } 842 843 private static final Answer<Void> CALL_RUNNER = 844 invocation -> { 845 Object[] args = invocation.getArguments(); 846 ((IRemoteAndroidTestRunner) args[0]).run((ITestRunListener) args[1]); 847 return null; 848 }; 849 850 /** 851 * A fake {@link IRemoteAndroidTestRunner} which simulates a test run by notifying the {@link 852 * ITestRunListener}s but does not actually run anything. 853 */ 854 private class FakeTestRunner extends RemoteAndroidTestRunner { 855 private Map<String, String> mArgs = new HashMap<>(); 856 857 FakeTestRunner(String packageName, String runnerName) { 858 super(packageName, runnerName, null); 859 } 860 861 @Override 862 public void addInstrumentationArg(String name, String value) { 863 super.addInstrumentationArg(name, value); 864 mArgs.put(name, value); 865 } 866 867 Map<String, String> getArgs() { 868 return mArgs; 869 } 870 871 @Override 872 public void run(Collection<ITestRunListener> listeners) { 873 int shardIndex = Integer.parseInt(getArgs().getOrDefault("shardIndex", "0")); 874 int numShards = Integer.parseInt(getArgs().getOrDefault("numShards", "1")); 875 List<TestIdentifier> tests = 876 mTests.get(getPackageName(), getRunnerName(), shardIndex, numShards); 877 878 // Start the test run 879 listeners.stream().forEach(l -> l.testRunStarted(getPackageName(), tests.size())); 880 881 // Run each of the tests 882 for (TestIdentifier test : tests) { 883 listeners.stream().forEach(l -> l.testStarted(test)); 884 listeners.stream().forEach(l -> l.testEnded(test, ImmutableMap.of())); 885 } 886 887 // Mock out the coverage measurement if necessary 888 Map<String, String> metrics = new HashMap<>(); 889 if (getArgs().getOrDefault("coverage", "false").equals("true")) { 890 String devicePath = String.format(COVERAGE_PATH, getPackageName()); 891 ByteString measurement = 892 mMeasurements.get(getPackageName(), getRunnerName(), shardIndex, numShards); 893 mockDeviceFile(devicePath, measurement); 894 metrics.put(CodeCoverageTest.COVERAGE_REMOTE_FILE_LABEL, devicePath); 895 } 896 897 // End the test run 898 listeners.stream().forEach(l -> l.testRunEnded(TEST_RUN_TIME, metrics)); 899 } 900 } 901 902 private void mockDeviceFile(String devicePath, ByteString contents) { 903 Answer<File> pullFile = 904 unused -> { 905 File ret = mFolder.newFile(); 906 contents.writeTo(new FileOutputStream(ret)); 907 return ret; 908 }; 909 try { 910 doReturn(true).when(mDevice).doesFileExist(devicePath); 911 doAnswer(pullFile).when(mDevice).pullFile(devicePath); 912 } catch (DeviceNotAvailableException impossible) { 913 // Mocks won't actually throw. 914 throw new AssertionError(impossible); 915 } 916 } 917 } 918
