1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.ddmlib.IDevice; 19 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 20 import com.android.ddmlib.testrunner.TestIdentifier; 21 import com.android.tradefed.config.OptionSetter; 22 import com.android.tradefed.device.DeviceNotAvailableException; 23 import com.android.tradefed.device.ITestDevice; 24 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 25 import com.android.tradefed.result.ITestInvocationListener; 26 import com.android.tradefed.result.ITestLifeCycleReceiver; 27 import com.android.tradefed.util.FileUtil; 28 29 import junit.framework.TestCase; 30 31 import org.easymock.EasyMock; 32 33 import java.io.File; 34 import java.util.HashMap; 35 import java.util.List; 36 import java.util.concurrent.TimeUnit; 37 38 /** 39 * Unit tests for {@link AndroidJUnitTest} 40 */ 41 public class AndroidJUnitTestTest extends TestCase { 42 43 private static final int TEST_TIMEOUT = 0; 44 private static final long SHELL_TIMEOUT = 0; 45 private static final String TEST_PACKAGE_VALUE = "com.foo"; 46 private static final TestIdentifier TEST1 = new TestIdentifier("Test", "test1"); 47 private static final TestIdentifier TEST2 = new TestIdentifier("Test", "test2"); 48 49 /** The {@link AndroidJUnitTest} under test, with all dependencies mocked out */ 50 private AndroidJUnitTest mAndroidJUnitTest; 51 52 // The mock objects. 53 private IDevice mMockIDevice; 54 private ITestDevice mMockTestDevice; 55 private IRemoteAndroidTestRunner mMockRemoteRunner; 56 private ITestInvocationListener mMockListener; 57 58 @Override 59 protected void setUp() throws Exception { 60 super.setUp(); 61 62 mMockIDevice = EasyMock.createMock(IDevice.class); 63 mMockTestDevice = EasyMock.createMock(ITestDevice.class); 64 EasyMock.expect(mMockTestDevice.getIDevice()).andStubReturn(mMockIDevice); 65 EasyMock.expect(mMockTestDevice.getSerialNumber()).andStubReturn("serial"); 66 mMockRemoteRunner = EasyMock.createMock(IRemoteAndroidTestRunner.class); 67 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 68 69 mAndroidJUnitTest = new AndroidJUnitTest() { 70 @Override 71 IRemoteAndroidTestRunner createRemoteAndroidTestRunner(String packageName, 72 String runnerName, IDevice device) { 73 return mMockRemoteRunner; 74 } 75 }; 76 mAndroidJUnitTest.setPackageName(TEST_PACKAGE_VALUE); 77 mAndroidJUnitTest.setDevice(mMockTestDevice); 78 // default to no rerun, for simplicity 79 mAndroidJUnitTest.setRerunMode(false); 80 // default to no timeout for simplicity 81 mAndroidJUnitTest.setTestTimeout(TEST_TIMEOUT); 82 mAndroidJUnitTest.setShellTimeout(SHELL_TIMEOUT); 83 mMockRemoteRunner.setMaxTimeToOutputResponse(SHELL_TIMEOUT, TimeUnit.MILLISECONDS); 84 mMockRemoteRunner.setMaxTimeout(0L, TimeUnit.MILLISECONDS); 85 mMockRemoteRunner.addInstrumentationArg(InstrumentationTest.TEST_TIMEOUT_INST_ARGS_KEY, 86 Long.toString(SHELL_TIMEOUT)); 87 mMockRemoteRunner.addInstrumentationArg( 88 AndroidJUnitTest.NEW_RUN_LISTENER_ORDER_KEY, "true"); 89 } 90 91 /** 92 * Test list of tests to run is filtered by include filters. 93 */ 94 public void testRun_includeFilterClass() throws Exception { 95 // expect this call 96 mMockRemoteRunner.addInstrumentationArg("class", TEST1.toString()); 97 setRunTestExpectations(); 98 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 99 mAndroidJUnitTest.addIncludeFilter(TEST1.toString()); 100 mAndroidJUnitTest.run(mMockListener); 101 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 102 } 103 104 /** 105 * Test list of tests to run is filtered by exclude filters. 106 */ 107 public void testRun_excludeFilterClass() throws Exception { 108 // expect this call 109 mMockRemoteRunner.addInstrumentationArg("notClass", TEST1.toString()); 110 setRunTestExpectations(); 111 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 112 mAndroidJUnitTest.addExcludeFilter(TEST1.toString()); 113 mAndroidJUnitTest.run(mMockListener); 114 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 115 } 116 117 /** 118 * Test list of tests to run is filtered by include and exclude filters. 119 */ 120 public void testRun_includeAndExcludeFilterClass() throws Exception { 121 // expect this call 122 mMockRemoteRunner.addInstrumentationArg("class", TEST1.getClassName()); 123 mMockRemoteRunner.addInstrumentationArg("notClass", TEST2.toString()); 124 setRunTestExpectations(); 125 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 126 mAndroidJUnitTest.addIncludeFilter(TEST1.getClassName()); 127 mAndroidJUnitTest.addExcludeFilter(TEST2.toString()); 128 mAndroidJUnitTest.run(mMockListener); 129 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 130 } 131 132 /** 133 * Test list of tests to run is filtered by include filters. 134 */ 135 public void testRun_includeFilterPackage() throws Exception { 136 // expect this call 137 mMockRemoteRunner.addInstrumentationArg("package", "com.android.test"); 138 setRunTestExpectations(); 139 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 140 mAndroidJUnitTest.addIncludeFilter("com.android.test"); 141 mAndroidJUnitTest.run(mMockListener); 142 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 143 } 144 145 /** 146 * Test list of tests to run is filtered by exclude filters. 147 */ 148 public void testRun_excludeFilterPackage() throws Exception { 149 // expect this call 150 mMockRemoteRunner.addInstrumentationArg("notPackage", "com.android.not"); 151 setRunTestExpectations(); 152 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 153 mAndroidJUnitTest.addExcludeFilter("com.android.not"); 154 mAndroidJUnitTest.run(mMockListener); 155 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 156 } 157 158 /** 159 * Test list of tests to run is filtered by include and exclude filters. 160 */ 161 public void testRun_includeAndExcludeFilterPackage() throws Exception { 162 // expect this call 163 mMockRemoteRunner.addInstrumentationArg("package", "com.android.test"); 164 mMockRemoteRunner.addInstrumentationArg("notPackage", "com.android.not"); 165 setRunTestExpectations(); 166 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 167 mAndroidJUnitTest.addIncludeFilter("com.android.test"); 168 mAndroidJUnitTest.addExcludeFilter("com.android.not"); 169 mAndroidJUnitTest.run(mMockListener); 170 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 171 } 172 173 /** 174 * Test list of tests to run is filtered by include and exclude filters. 175 */ 176 public void testRun_includeAndExcludeFilters() throws Exception { 177 // expect this call 178 mMockRemoteRunner.addInstrumentationArg("class", TEST1.getClassName()); 179 mMockRemoteRunner.addInstrumentationArg("notClass", TEST2.toString()); 180 mMockRemoteRunner.addInstrumentationArg("package", "com.android.test"); 181 mMockRemoteRunner.addInstrumentationArg("notPackage", "com.android.not"); 182 setRunTestExpectations(); 183 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 184 mAndroidJUnitTest.addIncludeFilter(TEST1.getClassName()); 185 mAndroidJUnitTest.addExcludeFilter(TEST2.toString()); 186 mAndroidJUnitTest.addIncludeFilter("com.android.test"); 187 mAndroidJUnitTest.addExcludeFilter("com.android.not"); 188 mAndroidJUnitTest.run(mMockListener); 189 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 190 } 191 192 /** 193 * Test list of tests to run is filtered by include file. 194 */ 195 public void testRun_includeFile() throws Exception { 196 mMockRemoteRunner.addInstrumentationArg( 197 EasyMock.eq("testFile"), EasyMock.<String>anyObject()); 198 setRunTestExpectations(); 199 EasyMock.expect(mMockTestDevice.pushFile( 200 EasyMock.<File>anyObject(), EasyMock.<String>anyObject())).andReturn(Boolean.TRUE); 201 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 202 .andReturn("") 203 .times(2); 204 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 205 206 File tmpFile = FileUtil.createTempFile("testFile", ".txt"); 207 try { 208 mAndroidJUnitTest.setIncludeTestFile(tmpFile); 209 mAndroidJUnitTest.run(mMockListener); 210 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 211 } finally { 212 FileUtil.deleteFile(tmpFile); 213 } 214 215 } 216 217 /** 218 * Test list of tests to run is filtered by exclude file. 219 */ 220 public void testRun_excludeFile() throws Exception { 221 mMockRemoteRunner.addInstrumentationArg( 222 EasyMock.eq("notTestFile"), EasyMock.<String>anyObject()); 223 setRunTestExpectations(); 224 EasyMock.expect(mMockTestDevice.pushFile( 225 EasyMock.<File>anyObject(), EasyMock.<String>anyObject())).andReturn(Boolean.TRUE); 226 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 227 .andReturn("") 228 .times(2); 229 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 230 231 File tmpFile = FileUtil.createTempFile("notTestFile", ".txt"); 232 try { 233 mAndroidJUnitTest.setExcludeTestFile(tmpFile); 234 mAndroidJUnitTest.run(mMockListener); 235 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 236 } finally { 237 FileUtil.deleteFile(tmpFile); 238 } 239 240 } 241 242 /** 243 * Test list of tests to run is filtered by include file, does not override existing filters. 244 */ 245 public void testRun_testFileAndFilters() throws Exception { 246 mMockRemoteRunner.addInstrumentationArg( 247 EasyMock.eq("testFile"), EasyMock.<String>anyObject()); 248 mMockRemoteRunner.addInstrumentationArg( 249 EasyMock.eq("notTestFile"), EasyMock.<String>anyObject()); 250 mMockRemoteRunner.addInstrumentationArg("class", TEST1.getClassName()); 251 mMockRemoteRunner.addInstrumentationArg("notClass", TEST2.toString()); 252 setRunTestExpectations(); 253 EasyMock.expect(mMockTestDevice.pushFile(EasyMock.<File>anyObject(), 254 EasyMock.<String>anyObject())).andReturn(Boolean.TRUE).times(2); 255 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 256 .andReturn("") 257 .times(4); 258 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 259 260 File tmpFileInclude = FileUtil.createTempFile("includeFile", ".txt"); 261 File tmpFileExclude = FileUtil.createTempFile("excludeFile", ".txt"); 262 try { 263 mAndroidJUnitTest.addIncludeFilter(TEST1.getClassName()); 264 mAndroidJUnitTest.addExcludeFilter(TEST2.toString()); 265 mAndroidJUnitTest.setIncludeTestFile(tmpFileInclude); 266 mAndroidJUnitTest.setExcludeTestFile(tmpFileExclude); 267 mAndroidJUnitTest.run(mMockListener); 268 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 269 } finally { 270 FileUtil.deleteFile(tmpFileInclude); 271 FileUtil.deleteFile(tmpFileExclude); 272 } 273 } 274 275 /** 276 * Test that when pushing the filters fails, we have a test run failure since we were not able 277 * to run anything. 278 */ 279 public void testRun_testFileAndFilters_fails() throws Exception { 280 mMockRemoteRunner = EasyMock.createMock(IRemoteAndroidTestRunner.class); 281 EasyMock.expect( 282 mMockTestDevice.pushFile( 283 EasyMock.<File>anyObject(), EasyMock.<String>anyObject())) 284 .andThrow(new DeviceNotAvailableException("failed to push", "device1")); 285 286 mMockListener.testRunStarted(EasyMock.anyObject(), EasyMock.eq(0)); 287 mMockListener.testRunFailed("failed to push"); 288 mMockListener.testRunEnded(0, new HashMap<String, Metric>()); 289 290 EasyMock.replay(mMockRemoteRunner, mMockTestDevice, mMockListener); 291 File tmpFileInclude = FileUtil.createTempFile("includeFile", ".txt"); 292 File tmpFileExclude = FileUtil.createTempFile("excludeFile", ".txt"); 293 try { 294 mAndroidJUnitTest.addIncludeFilter(TEST1.getClassName()); 295 mAndroidJUnitTest.addExcludeFilter(TEST2.toString()); 296 mAndroidJUnitTest.setIncludeTestFile(tmpFileInclude); 297 mAndroidJUnitTest.setExcludeTestFile(tmpFileExclude); 298 mAndroidJUnitTest.run(mMockListener); 299 fail("Should have thrown an exception."); 300 } catch (DeviceNotAvailableException expected) { 301 //expected 302 } finally { 303 FileUtil.deleteFile(tmpFileInclude); 304 FileUtil.deleteFile(tmpFileExclude); 305 } 306 EasyMock.verify(mMockRemoteRunner, mMockTestDevice, mMockListener); 307 } 308 309 /** 310 * Test that setting option for "test-file-filter" works as intended 311 */ 312 public void testRun_setTestFileOptions() throws Exception { 313 mMockRemoteRunner.addInstrumentationArg( 314 EasyMock.eq("testFile"), EasyMock.<String>anyObject()); 315 mMockRemoteRunner.addInstrumentationArg( 316 EasyMock.eq("notTestFile"), EasyMock.<String>anyObject()); 317 setRunTestExpectations(); 318 EasyMock.expect( 319 mMockTestDevice.pushFile( 320 EasyMock.<File>anyObject(), EasyMock.<String>anyObject())) 321 .andReturn(Boolean.TRUE) 322 .times(2); 323 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 324 .andReturn("") 325 .times(4); 326 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 327 328 File tmpFileInclude = FileUtil.createTempFile("includeFile", ".txt"); 329 File tmpFileExclude = FileUtil.createTempFile("excludeFile", ".txt"); 330 try { 331 OptionSetter setter = new OptionSetter(mAndroidJUnitTest); 332 setter.setOptionValue("test-file-include-filter", tmpFileInclude.getAbsolutePath()); 333 setter.setOptionValue("test-file-exclude-filter", tmpFileExclude.getAbsolutePath()); 334 mAndroidJUnitTest.run(mMockListener); 335 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 336 } finally { 337 FileUtil.deleteFile(tmpFileInclude); 338 FileUtil.deleteFile(tmpFileExclude); 339 } 340 341 } 342 343 private void setRunTestExpectations() throws DeviceNotAvailableException { 344 EasyMock.expect( 345 mMockTestDevice.runInstrumentationTests( 346 EasyMock.eq(mMockRemoteRunner), 347 (ITestLifeCycleReceiver) EasyMock.anyObject())) 348 .andReturn(Boolean.TRUE); 349 } 350 351 /** 352 * Test isClassOrMethod returns true for <package>.<class> and <package>.<class>#<method> but 353 * not for <package>. 354 */ 355 public void testIsClassOrMethod() throws Exception { 356 assertFalse("String was just package", mAndroidJUnitTest.isClassOrMethod("android.test")); 357 assertTrue("String was class", mAndroidJUnitTest.isClassOrMethod("android.test.Foo")); 358 assertTrue("String was method", mAndroidJUnitTest.isClassOrMethod("android.test.Foo#bar")); 359 } 360 361 /** 362 * Test that {@link AndroidJUnitTest#split()} returns null if the runner is not shardable. 363 */ 364 public void testSplit_notShardable() { 365 mAndroidJUnitTest.setRunnerName("fake.runner.not.shardable"); 366 assertNull(mAndroidJUnitTest.split()); 367 } 368 369 /** 370 * Test that {@link AndroidJUnitTest#split()} returns null if no shards have been requested. 371 */ 372 public void testSplit_noShardRequested() { 373 assertEquals(AndroidJUnitTest.AJUR, mAndroidJUnitTest.getRunnerName()); 374 assertNull(mAndroidJUnitTest.split()); 375 } 376 377 /** Test that {@link AndroidJUnitTest#split(int)} returns 3 shards when requested to do so. */ 378 public void testSplit_threeShards() throws Exception { 379 mAndroidJUnitTest = new AndroidJUnitTest(); 380 assertEquals(AndroidJUnitTest.AJUR, mAndroidJUnitTest.getRunnerName()); 381 OptionSetter setter = new OptionSetter(mAndroidJUnitTest); 382 setter.setOptionValue("runtime-hint", "60s"); 383 List<IRemoteTest> res = (List<IRemoteTest>) mAndroidJUnitTest.split(3); 384 assertNotNull(res); 385 assertEquals(3, res.size()); 386 // Third of the execution time on each shard. 387 assertEquals(20000L, ((AndroidJUnitTest)res.get(0)).getRuntimeHint()); 388 assertEquals(20000L, ((AndroidJUnitTest)res.get(1)).getRuntimeHint()); 389 assertEquals(20000L, ((AndroidJUnitTest)res.get(2)).getRuntimeHint()); 390 // Make sure shards cannot be re-sharded 391 assertNull(((AndroidJUnitTest) res.get(0)).split(2)); 392 assertNull(((AndroidJUnitTest) res.get(0)).split()); 393 } 394 395 /** 396 * Test that {@link AndroidJUnitTest#split(int)} can only split up to the ajur-max-shard option. 397 */ 398 public void testSplit_maxShard() throws Exception { 399 mAndroidJUnitTest = new AndroidJUnitTest(); 400 assertEquals(AndroidJUnitTest.AJUR, mAndroidJUnitTest.getRunnerName()); 401 OptionSetter setter = new OptionSetter(mAndroidJUnitTest); 402 setter.setOptionValue("runtime-hint", "60s"); 403 setter.setOptionValue("ajur-max-shard", "2"); 404 List<IRemoteTest> res = (List<IRemoteTest>) mAndroidJUnitTest.split(3); 405 assertNotNull(res); 406 assertEquals(2, res.size()); 407 // Third of the execution time on each shard. 408 assertEquals(30000L, ((AndroidJUnitTest) res.get(0)).getRuntimeHint()); 409 assertEquals(30000L, ((AndroidJUnitTest) res.get(1)).getRuntimeHint()); 410 // Make sure shards cannot be re-sharded 411 assertNull(((AndroidJUnitTest) res.get(0)).split(2)); 412 assertNull(((AndroidJUnitTest) res.get(0)).split()); 413 } 414 } 415
