1 /* 2 * Copyright (C) 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.android.tradefed.testtype; 17 18 import com.android.ddmlib.IDevice; 19 import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; 20 import com.android.ddmlib.testrunner.ITestRunListener; 21 import com.android.ddmlib.testrunner.TestIdentifier; 22 import com.android.tradefed.config.OptionSetter; 23 import com.android.tradefed.device.DeviceNotAvailableException; 24 import com.android.tradefed.device.ITestDevice; 25 import com.android.tradefed.result.ITestInvocationListener; 26 import com.android.tradefed.util.FileUtil; 27 28 import junit.framework.TestCase; 29 30 import java.io.File; 31 import java.util.List; 32 import java.util.concurrent.TimeUnit; 33 34 import org.easymock.EasyMock; 35 36 /** 37 * Unit tests for {@link AndroidJUnitTest} 38 */ 39 public class AndroidJUnitTestTest extends TestCase { 40 41 private static final int TEST_TIMEOUT = 0; 42 private static final long SHELL_TIMEOUT = 0; 43 private static final String TEST_PACKAGE_VALUE = "com.foo"; 44 private static final TestIdentifier TEST1 = new TestIdentifier("Test", "test1"); 45 private static final TestIdentifier TEST2 = new TestIdentifier("Test", "test2"); 46 47 /** The {@link AndroidJUnitTest} under test, with all dependencies mocked out */ 48 private AndroidJUnitTest mAndroidJUnitTest; 49 50 // The mock objects. 51 private IDevice mMockIDevice; 52 private ITestDevice mMockTestDevice; 53 private IRemoteAndroidTestRunner mMockRemoteRunner; 54 private ITestInvocationListener mMockListener; 55 56 @Override 57 protected void setUp() throws Exception { 58 super.setUp(); 59 60 mMockIDevice = EasyMock.createMock(IDevice.class); 61 mMockTestDevice = EasyMock.createMock(ITestDevice.class); 62 EasyMock.expect(mMockTestDevice.getIDevice()).andStubReturn(mMockIDevice); 63 EasyMock.expect(mMockTestDevice.getSerialNumber()).andStubReturn("serial"); 64 mMockRemoteRunner = EasyMock.createMock(IRemoteAndroidTestRunner.class); 65 mMockListener = EasyMock.createMock(ITestInvocationListener.class); 66 67 mAndroidJUnitTest = new AndroidJUnitTest() { 68 @Override 69 IRemoteAndroidTestRunner createRemoteAndroidTestRunner(String packageName, 70 String runnerName, IDevice device) { 71 return mMockRemoteRunner; 72 } 73 }; 74 mAndroidJUnitTest.setPackageName(TEST_PACKAGE_VALUE); 75 mAndroidJUnitTest.setDevice(mMockTestDevice); 76 // default to no rerun, for simplicity 77 mAndroidJUnitTest.setRerunMode(false); 78 // default to no timeout for simplicity 79 mAndroidJUnitTest.setTestTimeout(TEST_TIMEOUT); 80 mAndroidJUnitTest.setShellTimeout(SHELL_TIMEOUT); 81 mMockRemoteRunner.setMaxTimeToOutputResponse(SHELL_TIMEOUT, TimeUnit.MILLISECONDS); 82 mMockRemoteRunner.addInstrumentationArg(InstrumentationTest.TEST_TIMEOUT_INST_ARGS_KEY, 83 Long.toString(SHELL_TIMEOUT)); 84 } 85 86 /** 87 * Test list of tests to run is filtered by include filters. 88 */ 89 public void testRun_includeFilterClass() throws Exception { 90 // expect this call 91 mMockRemoteRunner.addInstrumentationArg("class", TEST1.toString()); 92 setRunTestExpectations(); 93 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 94 mAndroidJUnitTest.addIncludeFilter(TEST1.toString()); 95 mAndroidJUnitTest.run(mMockListener); 96 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 97 } 98 99 /** 100 * Test list of tests to run is filtered by exclude filters. 101 */ 102 public void testRun_excludeFilterClass() throws Exception { 103 // expect this call 104 mMockRemoteRunner.addInstrumentationArg("notClass", TEST1.toString()); 105 setRunTestExpectations(); 106 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 107 mAndroidJUnitTest.addExcludeFilter(TEST1.toString()); 108 mAndroidJUnitTest.run(mMockListener); 109 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 110 } 111 112 /** 113 * Test list of tests to run is filtered by include and exclude filters. 114 */ 115 public void testRun_includeAndExcludeFilterClass() throws Exception { 116 // expect this call 117 mMockRemoteRunner.addInstrumentationArg("class", TEST1.getClassName()); 118 mMockRemoteRunner.addInstrumentationArg("notClass", TEST2.toString()); 119 setRunTestExpectations(); 120 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 121 mAndroidJUnitTest.addIncludeFilter(TEST1.getClassName()); 122 mAndroidJUnitTest.addExcludeFilter(TEST2.toString()); 123 mAndroidJUnitTest.run(mMockListener); 124 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 125 } 126 127 /** 128 * Test list of tests to run is filtered by include filters. 129 */ 130 public void testRun_includeFilterPackage() throws Exception { 131 // expect this call 132 mMockRemoteRunner.addInstrumentationArg("package", "com.android.test"); 133 setRunTestExpectations(); 134 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 135 mAndroidJUnitTest.addIncludeFilter("com.android.test"); 136 mAndroidJUnitTest.run(mMockListener); 137 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 138 } 139 140 /** 141 * Test list of tests to run is filtered by exclude filters. 142 */ 143 public void testRun_excludeFilterPackage() throws Exception { 144 // expect this call 145 mMockRemoteRunner.addInstrumentationArg("notPackage", "com.android.not"); 146 setRunTestExpectations(); 147 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 148 mAndroidJUnitTest.addExcludeFilter("com.android.not"); 149 mAndroidJUnitTest.run(mMockListener); 150 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 151 } 152 153 /** 154 * Test list of tests to run is filtered by include and exclude filters. 155 */ 156 public void testRun_includeAndExcludeFilterPackage() throws Exception { 157 // expect this call 158 mMockRemoteRunner.addInstrumentationArg("package", "com.android.test"); 159 mMockRemoteRunner.addInstrumentationArg("notPackage", "com.android.not"); 160 setRunTestExpectations(); 161 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 162 mAndroidJUnitTest.addIncludeFilter("com.android.test"); 163 mAndroidJUnitTest.addExcludeFilter("com.android.not"); 164 mAndroidJUnitTest.run(mMockListener); 165 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 166 } 167 168 /** 169 * Test list of tests to run is filtered by include and exclude filters. 170 */ 171 public void testRun_includeAndExcludeFilters() throws Exception { 172 // expect this call 173 mMockRemoteRunner.addInstrumentationArg("class", TEST1.getClassName()); 174 mMockRemoteRunner.addInstrumentationArg("notClass", TEST2.toString()); 175 mMockRemoteRunner.addInstrumentationArg("package", "com.android.test"); 176 mMockRemoteRunner.addInstrumentationArg("notPackage", "com.android.not"); 177 setRunTestExpectations(); 178 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 179 mAndroidJUnitTest.addIncludeFilter(TEST1.getClassName()); 180 mAndroidJUnitTest.addExcludeFilter(TEST2.toString()); 181 mAndroidJUnitTest.addIncludeFilter("com.android.test"); 182 mAndroidJUnitTest.addExcludeFilter("com.android.not"); 183 mAndroidJUnitTest.run(mMockListener); 184 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 185 } 186 187 /** 188 * Test list of tests to run is filtered by include file. 189 */ 190 public void testRun_includeFile() throws Exception { 191 mMockRemoteRunner.addInstrumentationArg( 192 EasyMock.eq("testFile"), EasyMock.<String>anyObject()); 193 setRunTestExpectations(); 194 EasyMock.expect(mMockTestDevice.pushFile( 195 EasyMock.<File>anyObject(), EasyMock.<String>anyObject())).andReturn(Boolean.TRUE); 196 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 197 .andReturn(""); 198 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 199 200 File tmpFile = FileUtil.createTempFile("testFile", ".txt"); 201 try { 202 mAndroidJUnitTest.setIncludeTestFile(tmpFile); 203 mAndroidJUnitTest.run(mMockListener); 204 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 205 } finally { 206 FileUtil.deleteFile(tmpFile); 207 } 208 209 } 210 211 /** 212 * Test list of tests to run is filtered by exclude file. 213 */ 214 public void testRun_excludeFile() throws Exception { 215 mMockRemoteRunner.addInstrumentationArg( 216 EasyMock.eq("notTestFile"), EasyMock.<String>anyObject()); 217 setRunTestExpectations(); 218 EasyMock.expect(mMockTestDevice.pushFile( 219 EasyMock.<File>anyObject(), EasyMock.<String>anyObject())).andReturn(Boolean.TRUE); 220 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 221 .andReturn(""); 222 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 223 224 File tmpFile = FileUtil.createTempFile("notTestFile", ".txt"); 225 try { 226 mAndroidJUnitTest.setExcludeTestFile(tmpFile); 227 mAndroidJUnitTest.run(mMockListener); 228 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 229 } finally { 230 FileUtil.deleteFile(tmpFile); 231 } 232 233 } 234 235 /** 236 * Test list of tests to run is filtered by include file, does not override existing filters. 237 */ 238 public void testRun_testFileAndFilters() throws Exception { 239 mMockRemoteRunner.addInstrumentationArg( 240 EasyMock.eq("testFile"), EasyMock.<String>anyObject()); 241 mMockRemoteRunner.addInstrumentationArg( 242 EasyMock.eq("notTestFile"), EasyMock.<String>anyObject()); 243 mMockRemoteRunner.addInstrumentationArg("class", TEST1.getClassName()); 244 mMockRemoteRunner.addInstrumentationArg("notClass", TEST2.toString()); 245 setRunTestExpectations(); 246 EasyMock.expect(mMockTestDevice.pushFile(EasyMock.<File>anyObject(), 247 EasyMock.<String>anyObject())).andReturn(Boolean.TRUE).times(2); 248 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 249 .andReturn("") 250 .times(2); 251 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 252 253 File tmpFileInclude = FileUtil.createTempFile("includeFile", ".txt"); 254 File tmpFileExclude = FileUtil.createTempFile("excludeFile", ".txt"); 255 try { 256 mAndroidJUnitTest.addIncludeFilter(TEST1.getClassName()); 257 mAndroidJUnitTest.addExcludeFilter(TEST2.toString()); 258 mAndroidJUnitTest.setIncludeTestFile(tmpFileInclude); 259 mAndroidJUnitTest.setExcludeTestFile(tmpFileExclude); 260 mAndroidJUnitTest.run(mMockListener); 261 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 262 } finally { 263 FileUtil.deleteFile(tmpFileInclude); 264 FileUtil.deleteFile(tmpFileExclude); 265 } 266 } 267 268 /** 269 * Test that setting option for "test-file-filter" works as intended 270 */ 271 public void testRun_setTestFileOptions() throws Exception { 272 mMockRemoteRunner.addInstrumentationArg( 273 EasyMock.eq("testFile"), EasyMock.<String>anyObject()); 274 mMockRemoteRunner.addInstrumentationArg( 275 EasyMock.eq("notTestFile"), EasyMock.<String>anyObject()); 276 setRunTestExpectations(); 277 EasyMock.expect( 278 mMockTestDevice.pushFile( 279 EasyMock.<File>anyObject(), EasyMock.<String>anyObject())) 280 .andReturn(Boolean.TRUE) 281 .times(2); 282 EasyMock.expect(mMockTestDevice.executeShellCommand(EasyMock.<String>anyObject())) 283 .andReturn("") 284 .times(2); 285 EasyMock.replay(mMockRemoteRunner, mMockTestDevice); 286 287 File tmpFileInclude = FileUtil.createTempFile("includeFile", ".txt"); 288 File tmpFileExclude = FileUtil.createTempFile("excludeFile", ".txt"); 289 try { 290 OptionSetter setter = new OptionSetter(mAndroidJUnitTest); 291 setter.setOptionValue("test-file-include-filter", tmpFileInclude.getAbsolutePath()); 292 setter.setOptionValue("test-file-exclude-filter", tmpFileExclude.getAbsolutePath()); 293 mAndroidJUnitTest.run(mMockListener); 294 EasyMock.verify(mMockRemoteRunner, mMockTestDevice); 295 } finally { 296 FileUtil.deleteFile(tmpFileInclude); 297 FileUtil.deleteFile(tmpFileExclude); 298 } 299 300 } 301 302 private void setRunTestExpectations() throws DeviceNotAvailableException { 303 EasyMock.expect(mMockTestDevice.runInstrumentationTests(EasyMock.eq(mMockRemoteRunner), 304 (ITestRunListener)EasyMock.anyObject())).andReturn(Boolean.TRUE); 305 } 306 307 /** 308 * Test isClassOrMethod returns true for <package>.<class> and <package>.<class>#<method> but 309 * not for <package>. 310 */ 311 public void testIsClassOrMethod() throws Exception { 312 assertFalse("String was just package", mAndroidJUnitTest.isClassOrMethod("android.test")); 313 assertTrue("String was class", mAndroidJUnitTest.isClassOrMethod("android.test.Foo")); 314 assertTrue("String was method", mAndroidJUnitTest.isClassOrMethod("android.test.Foo#bar")); 315 } 316 317 /** 318 * Test that {@link AndroidJUnitTest#split()} returns null if the runner is not shardable. 319 */ 320 public void testSplit_notShardable() { 321 mAndroidJUnitTest.setRunnerName("fake.runner.not.shardable"); 322 assertNull(mAndroidJUnitTest.split()); 323 } 324 325 /** 326 * Test that {@link AndroidJUnitTest#split()} returns null if no shards have been requested. 327 */ 328 public void testSplit_noShardRequested() { 329 assertEquals(AndroidJUnitTest.AJUR, mAndroidJUnitTest.getRunnerName()); 330 assertNull(mAndroidJUnitTest.split()); 331 } 332 333 /** 334 * Test that {@link AndroidJUnitTest#split()} returns 3 shards when requested to do so. 335 */ 336 public void testSplit_threeShards() throws Exception { 337 mAndroidJUnitTest = new AndroidJUnitTest(); 338 assertEquals(AndroidJUnitTest.AJUR, mAndroidJUnitTest.getRunnerName()); 339 OptionSetter setter = new OptionSetter(mAndroidJUnitTest); 340 setter.setOptionValue("runtime-hint", "60s"); 341 List<IRemoteTest> res = (List<IRemoteTest>) mAndroidJUnitTest.split(3); 342 assertNotNull(res); 343 assertEquals(3, res.size()); 344 // Third of the execution time on each shard. 345 assertEquals(20000L, ((AndroidJUnitTest)res.get(0)).getRuntimeHint()); 346 assertEquals(20000L, ((AndroidJUnitTest)res.get(1)).getRuntimeHint()); 347 assertEquals(20000L, ((AndroidJUnitTest)res.get(2)).getRuntimeHint()); 348 // Make sure shards cannot be re-sharded 349 assertNull(((AndroidJUnitTest) res.get(0)).split(2)); 350 assertNull(((AndroidJUnitTest) res.get(0)).split()); 351 } 352 } 353
