1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.tradefed.testtype; 18 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 21 import com.android.ddmlib.testrunner.ITestRunListener; 22 import com.android.ddmlib.testrunner.TestIdentifier; 23 import com.android.tradefed.config.ConfigurationException; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.ITestDevice; 26 import com.android.tradefed.result.ITestInvocationListener; 27 28 import junit.framework.TestCase; 29 30 import org.easymock.EasyMock; 31 import org.easymock.IAnswer; 32 33 import java.io.BufferedReader; 34 import java.io.File; 35 import java.io.FileReader; 36 import java.io.IOException; 37 import java.util.ArrayList; 38 import java.util.Collection; 39 import java.util.Collections; 40 41 /** 42 * Unit tests for {@link InstrumentationFileTest}. 43 */ 44 public class InstrumentationFileTestTest extends TestCase { 45 46 private static final String TEST_PACKAGE_VALUE = "com.foo"; 47 48 /** The {@link InstrumentationFileTest} under test, with all dependencies mocked out */ 49 private InstrumentationFileTest mInstrumentationFileTest; 50 51 private ITestDevice mMockTestDevice; 52 private ITestInvocationListener mMockListener; 53 private InstrumentationTest mMockITest; 54 55 private File mTestFile; 56 57 @Override 58 protected void setUp() throws Exception { 59 super.setUp(); 60 61 mTestFile = null; 62 63 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 64 mMockTestDevice = EasyMock.createMock(ITestDevice.class); 65 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 66 67 EasyMock.expect(mMockTestDevice.getIDevice()).andStubReturn(mockIDevice); 68 EasyMock.expect(mMockTestDevice.getSerialNumber()).andStubReturn("serial"); 69 70 // mock out InstrumentationTest that will be used to create InstrumentationFileTest 71 mMockITest = new InstrumentationTest() { 72 @Override 73 protected String queryRunnerName() { 74 return "runner"; 75 } 76 }; 77 mMockITest.setDevice(mMockTestDevice); 78 mMockITest.setPackageName(TEST_PACKAGE_VALUE); 79 } 80 81 /** 82 * Test normal run scenario with a single test. 83 */ 84 @SuppressWarnings("unchecked") 85 public void testRun_singleSuccessfulTest() throws DeviceNotAvailableException, 86 ConfigurationException { 87 final Collection<TestIdentifier> testsList = new ArrayList<>(1); 88 final TestIdentifier test = new TestIdentifier("ClassFoo", "methodBar"); 89 testsList.add(test); 90 91 // verify the mock listener is passed through to the runner 92 RunTestAnswer runTestResponse = new RunTestAnswer() { 93 @Override 94 public Boolean answer(IRemoteAndroidTestRunner runner, 95 ITestRunListener listener) { 96 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 97 listener.testStarted(test); 98 listener.testEnded(test, Collections.EMPTY_MAP); 99 listener.testRunEnded(0, Collections.EMPTY_MAP); 100 return true; 101 } 102 }; 103 setRunTestExpectations(runTestResponse); 104 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, true, -1) { 105 @Override 106 InstrumentationTest createInstrumentationTest() { 107 return mMockITest; 108 } 109 @Override 110 boolean pushFileToTestDevice(File file, String destinationPath) 111 throws DeviceNotAvailableException { 112 // simulate successful push and store created file 113 mTestFile = file; 114 // verify that the content of the testFile contains all expected tests 115 verifyTestFile(testsList); 116 return true; 117 } 118 @Override 119 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 120 //ignore 121 } 122 }; 123 124 // mock successful test run lifecycle 125 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 1); 126 mMockListener.testStarted(EasyMock.eq(test), EasyMock.anyLong()); 127 mMockListener.testEnded( 128 EasyMock.eq(test), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 129 mMockListener.testRunEnded(0, Collections.EMPTY_MAP); 130 131 EasyMock.replay(mMockListener, mMockTestDevice); 132 mInstrumentationFileTest.run(mMockListener); 133 assertEquals(mMockTestDevice, mMockITest.getDevice()); 134 } 135 136 /** 137 * Test re-run scenario when 1 out of 3 tests fails to complete but is successful after re-run 138 */ 139 @SuppressWarnings("unchecked") 140 public void testRun_reRunOneFailedToCompleteTest() 141 throws DeviceNotAvailableException, ConfigurationException { 142 final Collection<TestIdentifier> testsList = new ArrayList<>(1); 143 final TestIdentifier test1 = new TestIdentifier("ClassFoo1", "methodBar1"); 144 final TestIdentifier test2 = new TestIdentifier("ClassFoo2", "methodBar2"); 145 final TestIdentifier test3 = new TestIdentifier("ClassFoo3", "methodBar3"); 146 testsList.add(test1); 147 testsList.add(test2); 148 testsList.add(test3); 149 150 // verify the test1 is completed and test2 was started but never finished 151 RunTestAnswer firstRunAnswer = new RunTestAnswer() { 152 @Override 153 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 154 // first test started and ended successfully 155 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 156 listener.testStarted(test1); 157 listener.testEnded(test1, Collections.EMPTY_MAP); 158 listener.testRunEnded(1, Collections.EMPTY_MAP); 159 // second test started but never finished 160 listener.testStarted(test2); 161 return true; 162 } 163 }; 164 setRunTestExpectations(firstRunAnswer); 165 166 // now expect second run to rerun remaining test3 and test2 167 RunTestAnswer secondRunAnswer = new RunTestAnswer() { 168 @Override 169 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 170 // third test started and ended successfully 171 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 172 listener.testStarted(test3); 173 listener.testEnded(test3, Collections.EMPTY_MAP); 174 listener.testRunEnded(1, Collections.EMPTY_MAP); 175 // second test is rerun but completed successfully this time 176 listener.testStarted(test2); 177 listener.testEnded(test2, Collections.EMPTY_MAP); 178 listener.testRunEnded(1, Collections.EMPTY_MAP); 179 return true; 180 } 181 }; 182 setRunTestExpectations(secondRunAnswer); 183 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, true, -1) { 184 @Override 185 InstrumentationTest createInstrumentationTest() { 186 return mMockITest; 187 } 188 @Override 189 boolean pushFileToTestDevice(File file, String destinationPath) 190 throws DeviceNotAvailableException { 191 // simulate successful push and store created file 192 mTestFile = file; 193 // verify that the content of the testFile contains all expected tests 194 verifyTestFile(testsList); 195 return true; 196 } 197 @Override 198 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 199 //ignore 200 } 201 }; 202 203 // First run: 204 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 205 // expect test1 to start and finish successfully 206 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 207 mMockListener.testEnded( 208 EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 209 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 210 // expect test2 to start but never finish 211 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 212 // Second run: 213 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 214 // expect test3 to start and finish successfully 215 mMockListener.testStarted(EasyMock.eq(test3), EasyMock.anyLong()); 216 mMockListener.testEnded( 217 EasyMock.eq(test3), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 218 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 219 // expect to rerun test2 successfully 220 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 221 mMockListener.testEnded( 222 EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 223 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 224 225 EasyMock.replay(mMockListener, mMockTestDevice); 226 mInstrumentationFileTest.run(mMockListener); 227 assertEquals(mMockTestDevice, mMockITest.getDevice()); 228 } 229 230 /** 231 * Test re-run scenario when 2 remaining tests fail to complete and need to be run serially 232 */ 233 @SuppressWarnings("unchecked") 234 public void testRun_serialReRunOfTwoFailedToCompleteTests() 235 throws DeviceNotAvailableException, ConfigurationException { 236 final Collection<TestIdentifier> testsList = new ArrayList<>(1); 237 final TestIdentifier test1 = new TestIdentifier("ClassFoo1", "methodBar1"); 238 final TestIdentifier test2 = new TestIdentifier("ClassFoo2", "methodBar2"); 239 testsList.add(test1); 240 testsList.add(test2); 241 242 // verify the test1 and test2 started but never completed 243 RunTestAnswer firstRunAnswer = new RunTestAnswer() { 244 @Override 245 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 246 // first and second tests started but never completed 247 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 248 listener.testStarted(test1); 249 listener.testStarted(test2); 250 // verify that the content of the testFile contains all expected tests 251 verifyTestFile(testsList); 252 return true; 253 } 254 }; 255 setRunTestExpectations(firstRunAnswer); 256 257 // verify successful serial execution of test1 258 RunTestAnswer firstSerialRunAnswer = new RunTestAnswer() { 259 @Override 260 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 261 // first test started and ended successfully in serial mode 262 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 263 listener.testStarted(test1); 264 listener.testEnded(test1, Collections.EMPTY_MAP); 265 listener.testRunEnded(1, Collections.EMPTY_MAP); 266 return true; 267 } 268 }; 269 setRunTestExpectations(firstSerialRunAnswer); 270 271 // verify successful serial execution of test2 272 RunTestAnswer secdondSerialRunAnswer = new RunTestAnswer() { 273 @Override 274 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 275 // Second test started and ended successfully in serial mode 276 listener.testRunStarted(TEST_PACKAGE_VALUE, 1); 277 listener.testStarted(test2); 278 listener.testEnded(test2, Collections.EMPTY_MAP); 279 listener.testRunEnded(1, Collections.EMPTY_MAP); 280 return true; 281 } 282 }; 283 setRunTestExpectations(secdondSerialRunAnswer); 284 285 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, true, -1) { 286 @Override 287 InstrumentationTest createInstrumentationTest() { 288 return mMockITest; 289 } 290 @Override 291 boolean pushFileToTestDevice(File file, String destinationPath) 292 throws DeviceNotAvailableException { 293 // simulate successful push and store created file 294 mTestFile = file; 295 return true; 296 } 297 @Override 298 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 299 //ignore 300 } 301 }; 302 303 // First run: 304 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 305 // expect test1 and test 2 to start but never finish 306 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 307 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 308 309 // re-run test1 and test2 serially 310 // first serial re-run: 311 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 1); 312 // expect test1 to start and finish successfully 313 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 314 mMockListener.testEnded( 315 EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 316 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 317 // first serial re-run: 318 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 1); 319 // expect test2 to start and finish successfully 320 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 321 mMockListener.testEnded( 322 EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 323 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 324 325 EasyMock.replay(mMockListener, mMockTestDevice); 326 mInstrumentationFileTest.run(mMockListener); 327 assertEquals(mMockTestDevice, mMockITest.getDevice()); 328 // test file is expected to be null since we defaulted to serial test execution 329 assertEquals(null, mMockITest.getTestFilePathOnDevice()); 330 } 331 332 /** 333 * Test no serial re-run tests fail to complete. 334 */ 335 @SuppressWarnings("unchecked") 336 public void testRun_noSerialReRun() 337 throws DeviceNotAvailableException, ConfigurationException { 338 final Collection<TestIdentifier> testsList = new ArrayList<>(1); 339 final TestIdentifier test1 = new TestIdentifier("ClassFoo1", "methodBar1"); 340 final TestIdentifier test2 = new TestIdentifier("ClassFoo2", "methodBar2"); 341 testsList.add(test1); 342 testsList.add(test2); 343 344 // verify the test1 and test2 started but never completed 345 RunTestAnswer firstRunAnswer = new RunTestAnswer() { 346 @Override 347 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 348 // first and second tests started but never completed 349 listener.testRunStarted(TEST_PACKAGE_VALUE, 2); 350 listener.testStarted(test1); 351 listener.testStarted(test2); 352 // verify that the content of the testFile contains all expected tests 353 verifyTestFile(testsList); 354 return true; 355 } 356 }; 357 setRunTestExpectations(firstRunAnswer); 358 359 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, false, -1) { 360 @Override 361 InstrumentationTest createInstrumentationTest() { 362 return mMockITest; 363 } 364 @Override 365 boolean pushFileToTestDevice(File file, String destinationPath) 366 throws DeviceNotAvailableException { 367 // simulate successful push and store created file 368 mTestFile = file; 369 return true; 370 } 371 @Override 372 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 373 //ignore 374 } 375 }; 376 377 // First run: 378 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 2); 379 // expect test1 and test 2 to start but never finish 380 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 381 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 382 383 EasyMock.replay(mMockListener, mMockTestDevice); 384 mInstrumentationFileTest.run(mMockListener); 385 assertEquals(mMockTestDevice, mMockITest.getDevice()); 386 } 387 388 /** 389 * Test attempting times exceed max attempts. 390 */ 391 @SuppressWarnings("unchecked") 392 public void testRun_exceedMaxAttempts() 393 throws DeviceNotAvailableException, ConfigurationException { 394 final ArrayList<TestIdentifier> testsList = new ArrayList<>(1); 395 final TestIdentifier test1 = new TestIdentifier("ClassFoo1", "methodBar1"); 396 final TestIdentifier test2 = new TestIdentifier("ClassFoo2", "methodBar2"); 397 final TestIdentifier test3 = new TestIdentifier("ClassFoo3", "methodBar3"); 398 final TestIdentifier test4 = new TestIdentifier("ClassFoo4", "methodBar4"); 399 final TestIdentifier test5 = new TestIdentifier("ClassFoo5", "methodBar5"); 400 final TestIdentifier test6 = new TestIdentifier("ClassFoo6", "methodBar6"); 401 402 testsList.add(test1); 403 testsList.add(test2); 404 testsList.add(test3); 405 testsList.add(test4); 406 testsList.add(test5); 407 testsList.add(test6); 408 409 final ArrayList<TestIdentifier> expectedTestsList = new ArrayList<>(testsList); 410 411 // test1 fininshed, test2 started but not finished. 412 RunTestAnswer firstRunAnswer = new RunTestAnswer() { 413 @Override 414 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 415 listener.testRunStarted(TEST_PACKAGE_VALUE, 6); 416 // first test started and ended successfully 417 listener.testStarted(test1); 418 listener.testEnded(test1, Collections.EMPTY_MAP); 419 listener.testRunEnded(1, Collections.EMPTY_MAP); 420 // second test started but never finished 421 listener.testStarted(test2); 422 // verify that the content of the testFile contains all expected tests 423 verifyTestFile(expectedTestsList); 424 return true; 425 } 426 }; 427 setRunTestExpectations(firstRunAnswer); 428 429 // test2 finished, test3 started but not finished. 430 RunTestAnswer secondRunAnswer = new RunTestAnswer() { 431 @Override 432 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 433 // test2 started and ended successfully 434 listener.testRunStarted(TEST_PACKAGE_VALUE, 5); 435 listener.testStarted(test2); 436 listener.testEnded(test2, Collections.EMPTY_MAP); 437 listener.testRunEnded(1, Collections.EMPTY_MAP); 438 // test3 started but never finished 439 listener.testStarted(test3); 440 // verify that the content of the testFile contains all expected tests 441 verifyTestFile(expectedTestsList.subList(1, expectedTestsList.size())); 442 return true; 443 } 444 }; 445 setRunTestExpectations(secondRunAnswer); 446 447 // test3 finished, test4 started but not finished. 448 RunTestAnswer thirdRunAnswer = new RunTestAnswer() { 449 @Override 450 public Boolean answer(IRemoteAndroidTestRunner runner, ITestRunListener listener) { 451 // test3 started and ended successfully 452 listener.testRunStarted(TEST_PACKAGE_VALUE, 4); 453 listener.testStarted(test3); 454 listener.testEnded(test3, Collections.EMPTY_MAP); 455 listener.testRunEnded(1, Collections.EMPTY_MAP); 456 // test4 started but never finished 457 listener.testStarted(test4); 458 // verify that the content of the testFile contains all expected tests 459 verifyTestFile(expectedTestsList.subList(2, expectedTestsList.size())); 460 return true; 461 } 462 }; 463 setRunTestExpectations(thirdRunAnswer); 464 465 mInstrumentationFileTest = new InstrumentationFileTest(mMockITest, testsList, false, 3) { 466 @Override 467 InstrumentationTest createInstrumentationTest() { 468 return mMockITest; 469 } 470 @Override 471 boolean pushFileToTestDevice(File file, String destinationPath) 472 throws DeviceNotAvailableException { 473 // simulate successful push and store created file 474 mTestFile = file; 475 return true; 476 } 477 @Override 478 void deleteTestFileFromDevice(String pathToFile) throws DeviceNotAvailableException { 479 //ignore 480 } 481 }; 482 483 // First run: 484 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 6); 485 mMockListener.testStarted(EasyMock.eq(test1), EasyMock.anyLong()); 486 mMockListener.testEnded( 487 EasyMock.eq(test1), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 488 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 489 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 490 491 // Second run: 492 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 5); 493 mMockListener.testStarted(EasyMock.eq(test2), EasyMock.anyLong()); 494 mMockListener.testEnded( 495 EasyMock.eq(test2), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 496 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 497 mMockListener.testStarted(EasyMock.eq(test3), EasyMock.anyLong()); 498 499 // Third run: 500 mMockListener.testRunStarted(TEST_PACKAGE_VALUE, 4); 501 mMockListener.testStarted(EasyMock.eq(test3), EasyMock.anyLong()); 502 mMockListener.testEnded( 503 EasyMock.eq(test3), EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); 504 mMockListener.testRunEnded(1, Collections.EMPTY_MAP); 505 mMockListener.testStarted(EasyMock.eq(test4), EasyMock.anyLong()); 506 507 // MAX_ATTEMPTS is 3, so there will be no forth run. 508 509 EasyMock.replay(mMockListener, mMockTestDevice); 510 mInstrumentationFileTest.run(mMockListener); 511 assertEquals(mMockTestDevice, mMockITest.getDevice()); 512 } 513 514 /** 515 * Helper class that verifies tetFile's content match the expected list of test to be run 516 * @param testsList list of test to be executed 517 */ 518 private void verifyTestFile(Collection<TestIdentifier> testsList) { 519 // fail if the file was never created 520 assertNotNull(mTestFile); 521 522 try (BufferedReader br = new BufferedReader(new FileReader(mTestFile))) { 523 String line; 524 while ((line = br.readLine()) != null) { 525 String[] str = line.split("#"); 526 TestIdentifier test = new TestIdentifier(str[0], str[1]); 527 assertTrue(String.format( 528 "Test with class name: %s and method name: %s does not exists", 529 test.getClassName(), test.getTestName()), testsList.contains(test)); 530 } 531 } catch (IOException e) { 532 // fail if the file is corrupt in any way 533 fail("failed reading test file"); 534 } 535 } 536 537 /** 538 * Helper class for providing an EasyMock {@link IAnswer} to a 539 * {@link ITestDevice#runInstrumentationTests(IRemoteAndroidTestRunner, Collection)} call. 540 */ 541 private static abstract class RunTestAnswer implements IAnswer<Boolean> { 542 @Override 543 public Boolean answer() throws Throwable { 544 Object[] args = EasyMock.getCurrentArguments(); 545 return answer((IRemoteAndroidTestRunner) args[0], (ITestRunListener) args[1]); 546 } 547 548 public abstract Boolean answer(IRemoteAndroidTestRunner runner, 549 ITestRunListener listener) throws DeviceNotAvailableException; 550 } 551 552 private void setRunTestExpectations(RunTestAnswer runTestResponse) 553 throws DeviceNotAvailableException { 554 555 EasyMock.expect(mMockTestDevice 556 .runInstrumentationTests((IRemoteAndroidTestRunner) EasyMock.anyObject(), 557 (ITestRunListener) EasyMock.anyObject())).andAnswer(runTestResponse); 558 } 559 } 560
