1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.drawelements.deqp.runner; 17 18 import static org.junit.Assert.assertSame; 19 import static org.junit.Assert.assertTrue; 20 21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 22 import com.android.ddmlib.IDevice; 23 import com.android.ddmlib.IShellOutputReceiver; 24 import com.android.tradefed.build.IFolderBuildInfo; 25 import com.android.tradefed.config.ConfigurationException; 26 import com.android.tradefed.config.OptionSetter; 27 import com.android.tradefed.device.DeviceNotAvailableException; 28 import com.android.tradefed.device.IManagedTestDevice; 29 import com.android.tradefed.device.ITestDevice; 30 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 31 import com.android.tradefed.result.ITestInvocationListener; 32 import com.android.tradefed.result.TestDescription; 33 import com.android.tradefed.testtype.Abi; 34 import com.android.tradefed.testtype.IAbi; 35 import com.android.tradefed.testtype.IRemoteTest; 36 import com.android.tradefed.testtype.IRuntimeHintProvider; 37 import com.android.tradefed.testtype.NativeCodeCoverageListener; 38 import com.android.tradefed.util.AbiUtils; 39 import com.android.tradefed.util.FileUtil; 40 import com.android.tradefed.util.IRunUtil; 41 import com.android.tradefed.util.RunInterruptedException; 42 43 import junit.framework.TestCase; 44 45 import org.easymock.EasyMock; 46 import org.easymock.IAnswer; 47 import org.easymock.IMocksControl; 48 49 import java.io.File; 50 import java.io.FileNotFoundException; 51 import java.io.FileWriter; 52 import java.io.IOException; 53 import java.io.PrintWriter; 54 import java.io.StringReader; 55 import java.io.StringWriter; 56 import java.util.ArrayList; 57 import java.util.Collection; 58 import java.util.HashMap; 59 import java.util.HashSet; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.Set; 63 import java.util.concurrent.TimeUnit; 64 65 /** 66 * Unit tests for {@link DeqpTestRunner}. 67 */ 68 public class DeqpTestRunnerTest extends TestCase { 69 private static final String NAME = "dEQP-GLES3"; 70 private static final IAbi ABI = new Abi("armeabi-v7a", "32"); 71 private static final String APP_DIR = "/sdcard/"; 72 private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; 73 private static final String LOG_FILE_NAME = "TestLog.qpa"; 74 private static final String INSTRUMENTATION_NAME = 75 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation"; 76 private static final String QUERY_INSTRUMENTATION_NAME = 77 "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation"; 78 private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk"; 79 private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp"; 80 private static final String ONLY_LANDSCAPE_FEATURES = 81 "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE; 82 private static final String ALL_FEATURES = 83 ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT; 84 private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS; 85 86 static { 87 DEFAULT_INSTANCE_ARGS = new ArrayList<>(1); 88 DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>()); 89 DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8"); 90 DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified"); 91 DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window"); 92 } 93 94 private File mTestsDir = null; 95 96 public static class BuildHelperMock extends CompatibilityBuildHelper { 97 private File mTestsDir = null; 98 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { 99 super(buildInfo); 100 mTestsDir = testsDir; 101 } 102 @Override 103 public File getTestsDir() throws FileNotFoundException { 104 return mTestsDir; 105 } 106 } 107 108 109 /** 110 * {@inheritDoc} 111 */ 112 @Override 113 protected void setUp() throws Exception { 114 super.setUp(); 115 mTestsDir = FileUtil.createTempDir("deqp-test-cases"); 116 } 117 118 /** 119 * {@inheritDoc} 120 */ 121 @Override 122 protected void tearDown() throws Exception { 123 FileUtil.recursiveDelete(mTestsDir); 124 super.tearDown(); 125 } 126 127 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 128 int minorVersion, 129 Collection<TestDescription> tests, 130 File testsDir) throws ConfigurationException { 131 StringWriter testlist = new StringWriter(); 132 for (TestDescription test : tests) { 133 testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 134 } 135 return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir); 136 } 137 138 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { 139 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); 140 EasyMock.replay(mockIFolderBuildInfo); 141 return new BuildHelperMock(mockIFolderBuildInfo, testsDir); 142 } 143 144 private static DeqpTestRunner buildGlesTestRunner(int majorVersion, 145 int minorVersion, 146 String testlist, 147 File testsDir) throws ConfigurationException { 148 DeqpTestRunner runner = new DeqpTestRunner(); 149 OptionSetter setter = new OptionSetter(runner); 150 151 String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion) 152 + (minorVersion > 0 ? Integer.toString(minorVersion) : ""); 153 154 setter.setOptionValue("deqp-package", deqpPackage); 155 setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8"); 156 setter.setOptionValue("deqp-caselist-file", "dummyfile.txt"); 157 setter.setOptionValue("deqp-screen-rotation", "unspecified"); 158 setter.setOptionValue("deqp-surface-type", "window"); 159 160 runner.setCaselistReader(new StringReader(testlist)); 161 runner.setAbi(ABI); 162 runner.setBuildHelper(getMockBuildHelper(testsDir)); 163 164 return runner; 165 } 166 167 private static String getTestId(DeqpTestRunner runner) { 168 return AbiUtils.createId(ABI.getName(), runner.getPackageName()); 169 } 170 171 /** 172 * Test version of OpenGL ES. 173 */ 174 private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception { 175 final TestDescription testId = new TestDescription("dEQP-GLES" 176 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 177 + ".info", "version"); 178 179 final String testPath = "dEQP-GLES" 180 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 181 +".info.version"; 182 183 final String testTrie = "{dEQP-GLES" 184 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion) 185 + "{info{version}}}"; 186 187 final String resultCode = "Pass"; 188 189 /* MultiLineReceiver expects "\r\n" line ending. */ 190 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 191 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 192 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 193 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 194 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 195 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 196 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 197 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 198 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 199 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 200 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 201 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 202 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 203 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 204 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 205 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 206 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 207 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 208 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 209 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 210 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 211 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 212 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 213 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 214 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 215 + "INSTRUMENTATION_CODE: 0\r\n"; 216 217 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 218 ITestInvocationListener mockListener 219 = EasyMock.createStrictMock(ITestInvocationListener.class); 220 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 221 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 222 tests.add(testId); 223 224 DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir); 225 226 int version = (majorVersion << 16) | minorVersion; 227 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 228 .andReturn(Integer.toString(version)).atLeastOnce(); 229 230 if (majorVersion > requiredMajorVersion 231 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) { 232 233 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 234 andReturn("").once(); 235 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 236 andReturn("").once(); 237 238 expectRenderConfigQuery(mockDevice, requiredMajorVersion, 239 requiredMinorVersion); 240 241 String commandLine = String.format( 242 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 243 + "--deqp-screen-rotation=unspecified " 244 + "--deqp-surface-type=window " 245 + "--deqp-log-images=disable " 246 + "--deqp-watchdog=enable", 247 APP_DIR + CASE_LIST_FILE_NAME); 248 249 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 250 output); 251 } 252 253 mockListener.testRunStarted(getTestId(deqpTest), 1); 254 EasyMock.expectLastCall().once(); 255 256 mockListener.testStarted(EasyMock.eq(testId)); 257 EasyMock.expectLastCall().once(); 258 259 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 260 EasyMock.expectLastCall().once(); 261 262 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 263 EasyMock.expectLastCall().once(); 264 265 EasyMock.replay(mockDevice, mockIDevice); 266 EasyMock.replay(mockListener); 267 268 deqpTest.setDevice(mockDevice); 269 deqpTest.run(mockListener); 270 271 EasyMock.verify(mockListener); 272 EasyMock.verify(mockDevice, mockIDevice); 273 } 274 275 private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion, 276 int minorVersion) throws Exception { 277 expectRenderConfigQuery(mockDevice, 278 String.format("--deqp-gl-config-name=rgba8888d24s8 " 279 + "--deqp-screen-rotation=unspecified " 280 + "--deqp-surface-type=window " 281 + "--deqp-gl-major-version=%d " 282 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion)); 283 } 284 285 private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine) 286 throws Exception { 287 expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes"); 288 } 289 290 private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine, 291 String output) throws Exception { 292 final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n" 293 + "INSTRUMENTATION_CODE: 0\r\n"; 294 final String command = String.format( 295 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine " 296 + "\"%s\" %s", 297 AbiUtils.createAbiFlag(ABI.getName()), commandLine, 298 QUERY_INSTRUMENTATION_NAME); 299 300 mockDevice.executeShellCommand(EasyMock.eq(command), 301 EasyMock.<IShellOutputReceiver>notNull()); 302 303 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 304 @Override 305 public Object answer() { 306 IShellOutputReceiver receiver 307 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 308 309 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length()); 310 receiver.flush(); 311 312 return null; 313 } 314 }); 315 } 316 317 /** 318 * Test that result code produces correctly pass or fail. 319 */ 320 private void testResultCode(final String resultCode, boolean pass) throws Exception { 321 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 322 final String testPath = "dEQP-GLES3.info.version"; 323 final String testTrie = "{dEQP-GLES3{info{version}}}"; 324 325 /* MultiLineReceiver expects "\r\n" line ending. */ 326 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 327 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 328 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 329 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 330 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 331 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 332 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 333 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 334 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 335 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 336 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 337 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 338 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 339 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 340 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 341 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 342 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 343 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 344 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 345 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 346 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 347 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 348 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 349 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 350 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 351 + "INSTRUMENTATION_CODE: 0\r\n"; 352 353 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 354 ITestInvocationListener mockListener 355 = EasyMock.createStrictMock(ITestInvocationListener.class); 356 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 357 358 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 359 tests.add(testId); 360 361 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 362 363 int version = 3 << 16; 364 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 365 .andReturn(Integer.toString(version)).atLeastOnce(); 366 367 expectRenderConfigQuery(mockDevice, 3, 0); 368 369 String commandLine = String.format( 370 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 371 + "--deqp-screen-rotation=unspecified " 372 + "--deqp-surface-type=window " 373 + "--deqp-log-images=disable " 374 + "--deqp-watchdog=enable", 375 APP_DIR + CASE_LIST_FILE_NAME); 376 377 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 378 379 mockListener.testRunStarted(getTestId(deqpTest), 1); 380 EasyMock.expectLastCall().once(); 381 382 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 383 andReturn("").once(); 384 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 385 andReturn("").once(); 386 387 mockListener.testStarted(EasyMock.eq(testId)); 388 EasyMock.expectLastCall().once(); 389 390 if (!pass) { 391 mockListener.testFailed(testId, 392 "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 393 + resultCode + ": Detail" + resultCode); 394 395 EasyMock.expectLastCall().once(); 396 } 397 398 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 399 EasyMock.expectLastCall().once(); 400 401 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 402 EasyMock.expectLastCall().once(); 403 404 EasyMock.replay(mockDevice, mockIDevice); 405 EasyMock.replay(mockListener); 406 407 deqpTest.setDevice(mockDevice); 408 deqpTest.run(mockListener); 409 410 EasyMock.verify(mockListener); 411 EasyMock.verify(mockDevice, mockIDevice); 412 } 413 414 /** 415 * Test running multiple test cases. 416 */ 417 public void testRun_multipleTests() throws Exception { 418 /* MultiLineReceiver expects "\r\n" line ending. */ 419 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 420 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 421 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 422 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 423 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 424 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 425 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 426 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 427 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 428 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 429 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 430 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 431 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 432 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 433 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 434 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 435 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 436 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 437 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 438 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 439 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 440 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 441 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 442 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 443 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 444 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 445 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 446 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 447 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 448 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 449 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 450 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 451 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 452 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 453 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 454 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 455 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 456 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 457 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 458 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 459 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 460 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 461 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 462 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 463 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 464 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 465 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 466 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 467 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 468 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 469 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 470 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 471 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 472 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 473 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 474 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 475 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 476 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 477 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 478 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 479 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 480 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 481 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 482 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 483 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 484 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 485 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 486 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 487 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 488 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 489 + "INSTRUMENTATION_CODE: 0\r\n"; 490 491 final TestDescription[] testIds = { 492 new TestDescription("dEQP-GLES3.info", "vendor"), 493 new TestDescription("dEQP-GLES3.info", "renderer"), 494 new TestDescription("dEQP-GLES3.info", "version"), 495 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 496 new TestDescription("dEQP-GLES3.info", "extensions"), 497 new TestDescription("dEQP-GLES3.info", "render_target") 498 }; 499 500 final String[] testPaths = { 501 "dEQP-GLES3.info.vendor", 502 "dEQP-GLES3.info.renderer", 503 "dEQP-GLES3.info.version", 504 "dEQP-GLES3.info.shading_language_version", 505 "dEQP-GLES3.info.extensions", 506 "dEQP-GLES3.info.render_target" 507 }; 508 509 final String testTrie 510 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 511 512 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 513 ITestInvocationListener mockListener 514 = EasyMock.createStrictMock(ITestInvocationListener.class); 515 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 516 517 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 518 519 for (TestDescription id : testIds) { 520 tests.add(id); 521 } 522 523 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 524 525 int version = 3 << 16; 526 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 527 .andReturn(Integer.toString(version)).atLeastOnce(); 528 529 expectRenderConfigQuery(mockDevice, 3, 0); 530 531 String commandLine = String.format( 532 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 533 + "--deqp-screen-rotation=unspecified " 534 + "--deqp-surface-type=window " 535 + "--deqp-log-images=disable " 536 + "--deqp-watchdog=enable", 537 APP_DIR + CASE_LIST_FILE_NAME); 538 539 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 540 541 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 542 EasyMock.expectLastCall().once(); 543 544 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 545 andReturn("").once(); 546 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 547 andReturn("").once(); 548 549 for (int i = 0; i < testPaths.length; i++) { 550 mockListener.testStarted(EasyMock.eq(testIds[i])); 551 EasyMock.expectLastCall().once(); 552 553 mockListener.testEnded(EasyMock.eq(testIds[i]), 554 EasyMock.<HashMap<String, Metric>>notNull()); 555 556 EasyMock.expectLastCall().once(); 557 } 558 559 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 560 EasyMock.expectLastCall().once(); 561 562 EasyMock.replay(mockDevice, mockIDevice); 563 EasyMock.replay(mockListener); 564 565 deqpTest.setDevice(mockDevice); 566 deqpTest.run(mockListener); 567 568 EasyMock.verify(mockListener); 569 EasyMock.verify(mockDevice, mockIDevice); 570 } 571 572 static private String buildTestProcessOutput(List<TestDescription> tests) { 573 /* MultiLineReceiver expects "\r\n" line ending. */ 574 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 575 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 576 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 577 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 578 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 579 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 580 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 581 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 582 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 583 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 584 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 585 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 586 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 587 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; 588 589 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 590 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 591 + "INSTRUMENTATION_CODE: 0\r\n"; 592 593 StringWriter output = new StringWriter(); 594 output.write(outputHeader); 595 for (TestDescription test : tests) { 596 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); 597 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); 598 output.write(test.getClassName()); 599 output.write("."); 600 output.write(test.getTestName()); 601 output.write("\r\n"); 602 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 603 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); 604 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); 605 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); 606 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 607 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); 608 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 609 } 610 output.write(outputEnd); 611 return output.toString(); 612 } 613 614 private void testFiltering(DeqpTestRunner deqpTest, 615 String expectedTrie, 616 List<TestDescription> expectedTests) throws Exception { 617 int version = 3 << 16; 618 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 619 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 620 .andReturn(Integer.toString(version)).atLeastOnce(); 621 622 boolean thereAreTests = !expectedTests.isEmpty(); 623 624 ITestInvocationListener mockListener 625 = EasyMock.createStrictMock(ITestInvocationListener.class); 626 mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size()); 627 EasyMock.expectLastCall().once(); 628 629 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 630 andReturn("").once(); 631 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 632 andReturn("").once(); 633 634 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 635 if (thereAreTests) 636 { 637 expectRenderConfigQuery(mockDevice, 3, 0); 638 639 String testOut = buildTestProcessOutput(expectedTests); 640 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 641 642 for (int i = 0; i < expectedTests.size(); i++) { 643 mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); 644 EasyMock.expectLastCall().once(); 645 646 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)), 647 EasyMock.<HashMap<String, Metric>>notNull()); 648 649 EasyMock.expectLastCall().once(); 650 } 651 } 652 653 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 654 EasyMock.expectLastCall().once(); 655 656 EasyMock.replay(mockDevice, mockIDevice); 657 EasyMock.replay(mockListener); 658 659 deqpTest.setDevice(mockDevice); 660 deqpTest.run(mockListener); 661 662 EasyMock.verify(mockListener); 663 EasyMock.verify(mockDevice, mockIDevice); 664 } 665 666 public void testRun_trivialIncludeFilter() throws Exception { 667 final TestDescription[] testIds = { 668 new TestDescription("dEQP-GLES3.missing", "no"), 669 new TestDescription("dEQP-GLES3.missing", "nope"), 670 new TestDescription("dEQP-GLES3.missing", "donotwant"), 671 new TestDescription("dEQP-GLES3.pick_me", "yes"), 672 new TestDescription("dEQP-GLES3.pick_me", "ok"), 673 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 674 }; 675 676 List<TestDescription> allTests = new ArrayList<TestDescription>(); 677 for (TestDescription id : testIds) { 678 allTests.add(id); 679 } 680 681 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 682 activeTests.add(testIds[3]); 683 activeTests.add(testIds[4]); 684 activeTests.add(testIds[5]); 685 686 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 687 688 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 689 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*"); 690 testFiltering(deqpTest, expectedTrie, activeTests); 691 } 692 693 public void testRun_trivialExcludeFilter() throws Exception { 694 final TestDescription[] testIds = { 695 new TestDescription("dEQP-GLES3.missing", "no"), 696 new TestDescription("dEQP-GLES3.missing", "nope"), 697 new TestDescription("dEQP-GLES3.missing", "donotwant"), 698 new TestDescription("dEQP-GLES3.pick_me", "yes"), 699 new TestDescription("dEQP-GLES3.pick_me", "ok"), 700 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 701 }; 702 703 List<TestDescription> allTests = new ArrayList<TestDescription>(); 704 for (TestDescription id : testIds) { 705 allTests.add(id); 706 } 707 708 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 709 activeTests.add(testIds[3]); 710 activeTests.add(testIds[4]); 711 activeTests.add(testIds[5]); 712 713 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 714 715 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 716 deqpTest.addExcludeFilter("dEQP-GLES3.missing#*"); 717 testFiltering(deqpTest, expectedTrie, activeTests); 718 } 719 720 public void testRun_includeAndExcludeFilter() throws Exception { 721 final TestDescription[] testIds = { 722 new TestDescription("dEQP-GLES3.group1", "foo"), 723 new TestDescription("dEQP-GLES3.group1", "nope"), 724 new TestDescription("dEQP-GLES3.group1", "donotwant"), 725 new TestDescription("dEQP-GLES3.group2", "foo"), 726 new TestDescription("dEQP-GLES3.group2", "yes"), 727 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 728 }; 729 730 List<TestDescription> allTests = new ArrayList<TestDescription>(); 731 for (TestDescription id : testIds) { 732 allTests.add(id); 733 } 734 735 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 736 activeTests.add(testIds[4]); 737 738 String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; 739 740 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 741 742 Set<String> includes = new HashSet<>(); 743 includes.add("dEQP-GLES3.group2#*"); 744 deqpTest.addAllIncludeFilters(includes); 745 746 Set<String> excludes = new HashSet<>(); 747 excludes.add("*foo"); 748 excludes.add("*thoushallnotpass"); 749 deqpTest.addAllExcludeFilters(excludes); 750 testFiltering(deqpTest, expectedTrie, activeTests); 751 } 752 753 public void testRun_includeAll() throws Exception { 754 final TestDescription[] testIds = { 755 new TestDescription("dEQP-GLES3.group1", "mememe"), 756 new TestDescription("dEQP-GLES3.group1", "yeah"), 757 new TestDescription("dEQP-GLES3.group1", "takeitall"), 758 new TestDescription("dEQP-GLES3.group2", "jeba"), 759 new TestDescription("dEQP-GLES3.group2", "yes"), 760 new TestDescription("dEQP-GLES3.group2", "granted"), 761 }; 762 763 List<TestDescription> allTests = new ArrayList<TestDescription>(); 764 for (TestDescription id : testIds) { 765 allTests.add(id); 766 } 767 768 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; 769 770 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 771 deqpTest.addIncludeFilter("*"); 772 testFiltering(deqpTest, expectedTrie, allTests); 773 } 774 775 public void testRun_excludeAll() throws Exception { 776 final TestDescription[] testIds = { 777 new TestDescription("dEQP-GLES3.group1", "no"), 778 new TestDescription("dEQP-GLES3.group1", "nope"), 779 new TestDescription("dEQP-GLES3.group1", "nottoday"), 780 new TestDescription("dEQP-GLES3.group2", "banned"), 781 new TestDescription("dEQP-GLES3.group2", "notrecognized"), 782 new TestDescription("dEQP-GLES3.group2", "-2"), 783 }; 784 785 List<TestDescription> allTests = new ArrayList<TestDescription>(); 786 for (TestDescription id : testIds) { 787 allTests.add(id); 788 } 789 790 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 791 deqpTest.addExcludeFilter("*"); 792 ITestInvocationListener mockListener 793 = EasyMock.createStrictMock(ITestInvocationListener.class); 794 mockListener.testRunStarted(getTestId(deqpTest), 0); 795 EasyMock.expectLastCall().once(); 796 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 797 EasyMock.expectLastCall().once(); 798 799 EasyMock.replay(mockListener); 800 deqpTest.run(mockListener); 801 EasyMock.verify(mockListener); 802 } 803 804 /** 805 * Test running a unexecutable test. 806 */ 807 public void testRun_unexecutableTests() throws Exception { 808 final String instrumentationAnswerNoExecs = 809 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 810 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 811 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 812 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 813 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 814 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 815 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 816 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 817 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 818 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 819 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 820 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 821 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 822 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 823 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 824 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 825 + "INSTRUMENTATION_CODE: 0\r\n"; 826 827 final TestDescription[] testIds = { 828 new TestDescription("dEQP-GLES3.missing", "no"), 829 new TestDescription("dEQP-GLES3.missing", "nope"), 830 new TestDescription("dEQP-GLES3.missing", "donotwant"), 831 }; 832 833 final String[] testPaths = { 834 "dEQP-GLES3.missing.no", 835 "dEQP-GLES3.missing.nope", 836 "dEQP-GLES3.missing.donotwant", 837 }; 838 839 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 840 ITestInvocationListener mockListener 841 = EasyMock.createStrictMock(ITestInvocationListener.class); 842 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 843 844 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 845 846 for (TestDescription id : testIds) { 847 tests.add(id); 848 } 849 850 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 851 852 int version = 3 << 16; 853 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 854 .andReturn(Integer.toString(version)).atLeastOnce(); 855 856 expectRenderConfigQuery(mockDevice, 3, 0); 857 858 String commandLine = String.format( 859 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 860 + "--deqp-screen-rotation=unspecified " 861 + "--deqp-surface-type=window " 862 + "--deqp-log-images=disable " 863 + "--deqp-watchdog=enable", 864 APP_DIR + CASE_LIST_FILE_NAME); 865 866 // first try 867 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 868 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 869 870 // splitting begins 871 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 872 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs); 873 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 874 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs); 875 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 876 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs); 877 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 878 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs); 879 880 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 881 EasyMock.expectLastCall().once(); 882 883 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 884 andReturn("").once(); 885 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 886 andReturn("").once(); 887 888 for (int i = 0; i < testPaths.length; i++) { 889 mockListener.testStarted(EasyMock.eq(testIds[i])); 890 EasyMock.expectLastCall().once(); 891 892 mockListener.testFailed(EasyMock.eq(testIds[i]), 893 EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n" 894 + "Abort: Test cannot be executed")); 895 EasyMock.expectLastCall().once(); 896 897 mockListener.testEnded(EasyMock.eq(testIds[i]), 898 EasyMock.<HashMap<String, Metric>>notNull()); 899 EasyMock.expectLastCall().once(); 900 } 901 902 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 903 EasyMock.expectLastCall().once(); 904 905 EasyMock.replay(mockDevice, mockIDevice); 906 EasyMock.replay(mockListener); 907 908 deqpTest.setDevice(mockDevice); 909 deqpTest.run(mockListener); 910 911 EasyMock.verify(mockListener); 912 EasyMock.verify(mockDevice, mockIDevice); 913 } 914 915 /** 916 * Test that test are left unexecuted if pm list query fails 917 */ 918 public void testRun_queryPmListFailure() 919 throws Exception { 920 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 921 922 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 923 ITestInvocationListener mockListener 924 = EasyMock.createStrictMock(ITestInvocationListener.class); 925 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 926 tests.add(testId); 927 928 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 929 OptionSetter setter = new OptionSetter(deqpTest); 930 // Note: If the rotation is the default unspecified, features are not queried at all 931 setter.setOptionValue("deqp-screen-rotation", "90"); 932 933 deqpTest.setDevice(mockDevice); 934 935 int version = 3 << 16; 936 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 937 .andReturn(Integer.toString(version)).atLeastOnce(); 938 939 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 940 .andReturn("not a valid format"); 941 942 mockListener.testRunStarted(getTestId(deqpTest), 1); 943 EasyMock.expectLastCall().once(); 944 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 945 andReturn("").once(); 946 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 947 andReturn("").once(); 948 949 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 950 EasyMock.expectLastCall().once(); 951 952 EasyMock.replay(mockDevice); 953 EasyMock.replay(mockListener); 954 deqpTest.run(mockListener); 955 EasyMock.verify(mockListener); 956 EasyMock.verify(mockDevice); 957 } 958 959 /** 960 * Test that test are left unexecuted if renderablity query fails 961 */ 962 public void testRun_queryRenderabilityFailure() 963 throws Exception { 964 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 965 966 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 967 ITestInvocationListener mockListener 968 = EasyMock.createStrictMock(ITestInvocationListener.class); 969 970 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 971 tests.add(testId); 972 973 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 974 975 deqpTest.setDevice(mockDevice); 976 977 int version = 3 << 16; 978 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 979 .andReturn(Integer.toString(version)).atLeastOnce(); 980 981 expectRenderConfigQueryAndReturn(mockDevice, 982 "--deqp-gl-config-name=rgba8888d24s8 " 983 + "--deqp-screen-rotation=unspecified " 984 + "--deqp-surface-type=window " 985 + "--deqp-gl-major-version=3 " 986 + "--deqp-gl-minor-version=0", "Maybe?"); 987 988 mockListener.testRunStarted(getTestId(deqpTest), 1); 989 EasyMock.expectLastCall().once(); 990 991 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 992 andReturn("").once(); 993 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 994 andReturn("").once(); 995 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 996 EasyMock.expectLastCall().once(); 997 998 EasyMock.replay(mockDevice); 999 EasyMock.replay(mockListener); 1000 deqpTest.run(mockListener); 1001 EasyMock.verify(mockListener); 1002 EasyMock.verify(mockDevice); 1003 } 1004 1005 /** 1006 * Test that orientation is supplied to runner correctly 1007 */ 1008 private void testOrientation(final String rotation, final String featureString) 1009 throws Exception { 1010 final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test"); 1011 final String testPath = "dEQP-GLES3.orientation.test"; 1012 final String testTrie = "{dEQP-GLES3{orientation{test}}}"; 1013 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1014 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1015 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1016 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1017 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1018 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1019 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1020 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1021 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1022 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1023 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1024 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1025 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1026 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1027 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1028 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1029 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1030 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1031 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1032 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1033 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1034 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1035 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1036 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1037 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1038 + "INSTRUMENTATION_CODE: 0\r\n"; 1039 1040 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1041 ITestInvocationListener mockListener 1042 = EasyMock.createStrictMock(ITestInvocationListener.class); 1043 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1044 1045 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1046 tests.add(testId); 1047 1048 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1049 OptionSetter setter = new OptionSetter(deqpTest); 1050 setter.setOptionValue("deqp-screen-rotation", rotation); 1051 1052 deqpTest.setDevice(mockDevice); 1053 1054 int version = 3 << 16; 1055 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1056 .andReturn(Integer.toString(version)).atLeastOnce(); 1057 1058 if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) { 1059 EasyMock.expect(mockDevice.executeShellCommand("pm list features")) 1060 .andReturn(featureString); 1061 } 1062 1063 final boolean isPortraitOrientation = 1064 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) || 1065 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT); 1066 final boolean isLandscapeOrientation = 1067 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) || 1068 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE); 1069 final boolean executable = 1070 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) || 1071 (isPortraitOrientation && 1072 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) || 1073 (isLandscapeOrientation && 1074 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE)); 1075 1076 if (executable) { 1077 expectRenderConfigQuery(mockDevice, String.format( 1078 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s " 1079 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1080 + "--deqp-gl-minor-version=0", rotation)); 1081 1082 String commandLine = String.format( 1083 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1084 + "--deqp-screen-rotation=%s " 1085 + "--deqp-surface-type=window " 1086 + "--deqp-log-images=disable " 1087 + "--deqp-watchdog=enable", 1088 APP_DIR + CASE_LIST_FILE_NAME, rotation); 1089 1090 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1091 output); 1092 } 1093 1094 mockListener.testRunStarted(getTestId(deqpTest), 1); 1095 EasyMock.expectLastCall().once(); 1096 1097 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1098 andReturn("").once(); 1099 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1100 andReturn("").once(); 1101 1102 mockListener.testStarted(EasyMock.eq(testId)); 1103 EasyMock.expectLastCall().once(); 1104 1105 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1106 EasyMock.expectLastCall().once(); 1107 1108 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1109 EasyMock.expectLastCall().once(); 1110 1111 EasyMock.replay(mockDevice, mockIDevice); 1112 EasyMock.replay(mockListener); 1113 deqpTest.run(mockListener); 1114 EasyMock.verify(mockListener); 1115 EasyMock.verify(mockDevice, mockIDevice); 1116 } 1117 1118 /** 1119 * Test OpeGL ES3 tests on device with OpenGL ES2. 1120 */ 1121 public void testRun_require30DeviceVersion20() throws Exception { 1122 testGlesVersion(3, 0, 2, 0); 1123 } 1124 1125 /** 1126 * Test OpeGL ES3.1 tests on device with OpenGL ES2. 1127 */ 1128 public void testRun_require31DeviceVersion20() throws Exception { 1129 testGlesVersion(3, 1, 2, 0); 1130 } 1131 1132 /** 1133 * Test OpeGL ES3 tests on device with OpenGL ES3. 1134 */ 1135 public void testRun_require30DeviceVersion30() throws Exception { 1136 testGlesVersion(3, 0, 3, 0); 1137 } 1138 1139 /** 1140 * Test OpeGL ES3.1 tests on device with OpenGL ES3. 1141 */ 1142 public void testRun_require31DeviceVersion30() throws Exception { 1143 testGlesVersion(3, 1, 3, 0); 1144 } 1145 1146 /** 1147 * Test OpeGL ES3 tests on device with OpenGL ES3.1. 1148 */ 1149 public void testRun_require30DeviceVersion31() throws Exception { 1150 testGlesVersion(3, 0, 3, 1); 1151 } 1152 1153 /** 1154 * Test OpeGL ES3.1 tests on device with OpenGL ES3.1. 1155 */ 1156 public void testRun_require31DeviceVersion31() throws Exception { 1157 testGlesVersion(3, 1, 3, 1); 1158 } 1159 1160 /** 1161 * Test dEQP Pass result code. 1162 */ 1163 public void testRun_resultPass() throws Exception { 1164 testResultCode("Pass", true); 1165 } 1166 1167 /** 1168 * Test dEQP Fail result code. 1169 */ 1170 public void testRun_resultFail() throws Exception { 1171 testResultCode("Fail", false); 1172 } 1173 1174 /** 1175 * Test dEQP NotSupported result code. 1176 */ 1177 public void testRun_resultNotSupported() throws Exception { 1178 testResultCode("NotSupported", true); 1179 } 1180 1181 /** 1182 * Test dEQP QualityWarning result code. 1183 */ 1184 public void testRun_resultQualityWarning() throws Exception { 1185 testResultCode("QualityWarning", true); 1186 } 1187 1188 /** 1189 * Test dEQP CompatibilityWarning result code. 1190 */ 1191 public void testRun_resultCompatibilityWarning() throws Exception { 1192 testResultCode("CompatibilityWarning", true); 1193 } 1194 1195 /** 1196 * Test dEQP ResourceError result code. 1197 */ 1198 public void testRun_resultResourceError() throws Exception { 1199 testResultCode("ResourceError", false); 1200 } 1201 1202 /** 1203 * Test dEQP InternalError result code. 1204 */ 1205 public void testRun_resultInternalError() throws Exception { 1206 testResultCode("InternalError", false); 1207 } 1208 1209 /** 1210 * Test dEQP Crash result code. 1211 */ 1212 public void testRun_resultCrash() throws Exception { 1213 testResultCode("Crash", false); 1214 } 1215 1216 /** 1217 * Test dEQP Timeout result code. 1218 */ 1219 public void testRun_resultTimeout() throws Exception { 1220 testResultCode("Timeout", false); 1221 } 1222 /** 1223 * Test dEQP Orientation 1224 */ 1225 public void testRun_orientationLandscape() throws Exception { 1226 testOrientation("90", ALL_FEATURES); 1227 } 1228 1229 /** 1230 * Test dEQP Orientation 1231 */ 1232 public void testRun_orientationPortrait() throws Exception { 1233 testOrientation("0", ALL_FEATURES); 1234 } 1235 1236 /** 1237 * Test dEQP Orientation 1238 */ 1239 public void testRun_orientationReverseLandscape() throws Exception { 1240 testOrientation("270", ALL_FEATURES); 1241 } 1242 1243 /** 1244 * Test dEQP Orientation 1245 */ 1246 public void testRun_orientationReversePortrait() throws Exception { 1247 testOrientation("180", ALL_FEATURES); 1248 } 1249 1250 /** 1251 * Test dEQP Orientation 1252 */ 1253 public void testRun_orientationUnspecified() throws Exception { 1254 testOrientation("unspecified", ALL_FEATURES); 1255 } 1256 1257 /** 1258 * Test dEQP Orientation with limited features 1259 */ 1260 public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception { 1261 testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES); 1262 } 1263 1264 /** 1265 * Test dEQP Orientation with limited features 1266 */ 1267 public void testRun_orientationLandscapeLimitedFeatures() throws Exception { 1268 testOrientation("90", ONLY_LANDSCAPE_FEATURES); 1269 } 1270 1271 /** 1272 * Test dEQP Orientation with limited features 1273 */ 1274 public void testRun_orientationPortraitLimitedFeatures() throws Exception { 1275 testOrientation("0", ONLY_LANDSCAPE_FEATURES); 1276 } 1277 1278 /** 1279 * Test dEQP unsupported pixel format 1280 */ 1281 public void testRun_unsupportedPixelFormat() throws Exception { 1282 final String pixelFormat = "rgba5658d16m4"; 1283 final TestDescription testId = new TestDescription("dEQP-GLES3.pixelformat", "test"); 1284 1285 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1286 ITestInvocationListener mockListener 1287 = EasyMock.createStrictMock(ITestInvocationListener.class); 1288 1289 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1290 tests.add(testId); 1291 1292 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1293 OptionSetter setter = new OptionSetter(deqpTest); 1294 setter.setOptionValue("deqp-gl-config-name", pixelFormat); 1295 1296 deqpTest.setDevice(mockDevice); 1297 1298 int version = 3 << 16; 1299 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1300 .andReturn(Integer.toString(version)).atLeastOnce(); 1301 1302 expectRenderConfigQueryAndReturn(mockDevice, String.format( 1303 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified " 1304 + "--deqp-surface-type=window " 1305 + "--deqp-gl-major-version=3 " 1306 + "--deqp-gl-minor-version=0", pixelFormat), "No"); 1307 1308 mockListener.testRunStarted(getTestId(deqpTest), 1); 1309 EasyMock.expectLastCall().once(); 1310 1311 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1312 andReturn("").once(); 1313 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1314 andReturn("").once(); 1315 1316 mockListener.testStarted(EasyMock.eq(testId)); 1317 EasyMock.expectLastCall().once(); 1318 1319 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 1320 EasyMock.expectLastCall().once(); 1321 1322 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1323 EasyMock.expectLastCall().once(); 1324 1325 EasyMock.replay(mockDevice); 1326 EasyMock.replay(mockListener); 1327 deqpTest.run(mockListener); 1328 EasyMock.verify(mockListener); 1329 EasyMock.verify(mockDevice); 1330 } 1331 1332 /** 1333 * Test interface to mock Tradefed device types. 1334 */ 1335 public static interface RecoverableTestDevice extends ITestDevice, IManagedTestDevice { 1336 } 1337 1338 private static enum RecoveryEvent { 1339 PROGRESS, 1340 FAIL_CONNECTION_REFUSED, 1341 FAIL_LINK_KILLED, 1342 } 1343 1344 private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events) 1345 throws DeviceNotAvailableException { 1346 for (RecoveryEvent event : events) { 1347 switch (event) { 1348 case PROGRESS: 1349 recovery.onExecutionProgressed(); 1350 break; 1351 case FAIL_CONNECTION_REFUSED: 1352 recovery.recoverConnectionRefused(); 1353 break; 1354 case FAIL_LINK_KILLED: 1355 recovery.recoverComLinkKilled(); 1356 break; 1357 } 1358 } 1359 } 1360 1361 private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) { 1362 mockSleepProvider.sleep(EasyMock.gt(0)); 1363 EasyMock.expectLastCall().once(); 1364 } 1365 1366 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, 1367 DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { 1368 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1369 andReturn("root 1234 com.drawelement.deqp").once(); 1370 1371 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1372 andReturn("").once(); 1373 1374 // Recovery checks if kill failed 1375 mockSleepProvider.sleep(EasyMock.gt(0)); 1376 EasyMock.expectLastCall().once(); 1377 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1378 andReturn("").once(); 1379 } 1380 1381 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 1382 throws DeviceNotAvailableException { 1383 mockDevice.recoverDevice(); 1384 EasyMock.expectLastCall().once(); 1385 } 1386 1387 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 1388 throws DeviceNotAvailableException { 1389 mockDevice.reboot(); 1390 EasyMock.expectLastCall().once(); 1391 } 1392 1393 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, 1394 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1395 throws DeviceNotAvailableException { 1396 switch (numConsecutiveErrors) { 1397 case 0: 1398 case 1: 1399 setRecoveryExpectationRecovery(mockDevice); 1400 return 2; 1401 case 2: 1402 setRecoveryExpectationReboot(mockDevice); 1403 return 3; 1404 default: 1405 return 4; 1406 } 1407 } 1408 1409 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, 1410 DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 1411 throws DeviceNotAvailableException { 1412 switch (numConsecutiveErrors) { 1413 case 0: 1414 setRecoveryExpectationWait(mockSleepProvider); 1415 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1416 return 1; 1417 case 1: 1418 setRecoveryExpectationRecovery(mockDevice); 1419 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 1420 return 2; 1421 case 2: 1422 setRecoveryExpectationReboot(mockDevice); 1423 return 3; 1424 default: 1425 return 4; 1426 } 1427 } 1428 1429 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, 1430 DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 1431 throws DeviceNotAvailableException { 1432 int numConsecutiveErrors = 0; 1433 for (RecoveryEvent event : events) { 1434 switch (event) { 1435 case PROGRESS: 1436 numConsecutiveErrors = 0; 1437 break; 1438 case FAIL_CONNECTION_REFUSED: 1439 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, 1440 mockSleepProvider, numConsecutiveErrors); 1441 break; 1442 case FAIL_LINK_KILLED: 1443 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, 1444 mockSleepProvider, numConsecutiveErrors); 1445 break; 1446 } 1447 } 1448 } 1449 1450 /** 1451 * Test dEQP runner recovery state machine. 1452 */ 1453 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) 1454 throws Exception { 1455 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1456 IMocksControl orderedControl = EasyMock.createStrictControl(); 1457 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1458 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); 1459 DeqpTestRunner.ISleepProvider mockSleepProvider = 1460 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1461 1462 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); 1463 1464 orderedControl.replay(); 1465 1466 recovery.setDevice(mockDevice); 1467 recovery.setSleepProvider(mockSleepProvider); 1468 try { 1469 runRecoveryWithPattern(recovery, pattern); 1470 if (!expectSuccess) { 1471 fail("Expected DeviceNotAvailableException"); 1472 } 1473 } catch (DeviceNotAvailableException ex) { 1474 if (expectSuccess) { 1475 fail("Did not expect DeviceNotAvailableException"); 1476 } 1477 } 1478 1479 orderedControl.verify(); 1480 } 1481 1482 // basic patterns 1483 1484 public void testRecovery_NoEvents() throws Exception { 1485 testRecoveryWithPattern(true); 1486 } 1487 1488 public void testRecovery_AllOk() throws Exception { 1489 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); 1490 } 1491 1492 // conn fail patterns 1493 1494 public void testRecovery_OneConnectionFailureBegin() throws Exception { 1495 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1496 RecoveryEvent.PROGRESS); 1497 } 1498 1499 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 1500 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1501 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1502 } 1503 1504 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 1505 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1506 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); 1507 } 1508 1509 public void testRecovery_OneConnectionFailureMid() throws Exception { 1510 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1511 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1512 } 1513 1514 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 1515 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1516 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1517 RecoveryEvent.PROGRESS); 1518 } 1519 1520 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 1521 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 1522 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1523 RecoveryEvent.FAIL_CONNECTION_REFUSED); 1524 } 1525 1526 // link fail patterns 1527 1528 public void testRecovery_OneLinkFailureBegin() throws Exception { 1529 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1530 RecoveryEvent.PROGRESS); 1531 } 1532 1533 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 1534 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1535 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1536 } 1537 1538 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 1539 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1540 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1541 RecoveryEvent.PROGRESS); 1542 } 1543 1544 public void testRecovery_FourLinkFailuresBegin() throws Exception { 1545 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 1546 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1547 RecoveryEvent.FAIL_LINK_KILLED); 1548 } 1549 1550 public void testRecovery_OneLinkFailureMid() throws Exception { 1551 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1552 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1553 } 1554 1555 public void testRecovery_TwoLinkFailuresMid() throws Exception { 1556 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1557 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1558 RecoveryEvent.PROGRESS); 1559 } 1560 1561 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 1562 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1563 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1564 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1565 } 1566 1567 public void testRecovery_FourLinkFailuresMid() throws Exception { 1568 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1569 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1570 RecoveryEvent.FAIL_LINK_KILLED); 1571 } 1572 1573 // mixed patterns 1574 1575 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 1576 testRecoveryWithPattern(true, 1577 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1578 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1579 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1580 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1581 RecoveryEvent.PROGRESS); 1582 } 1583 1584 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 1585 testRecoveryWithPattern(true, 1586 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1587 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, 1588 RecoveryEvent.PROGRESS); 1589 } 1590 1591 /** 1592 * Test recovery if process cannot be killed 1593 */ 1594 public void testRecovery_unkillableProcess () throws Exception { 1595 DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery(); 1596 IMocksControl orderedControl = EasyMock.createStrictControl(); 1597 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1598 DeqpTestRunner.ISleepProvider mockSleepProvider = 1599 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class); 1600 1601 // recovery attempts to kill the process after a timeout 1602 mockSleepProvider.sleep(EasyMock.gt(0)); 1603 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1604 andReturn("root 1234 com.drawelement.deqp").once(); 1605 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1606 andReturn("").once(); 1607 1608 // Recovery checks if kill failed 1609 mockSleepProvider.sleep(EasyMock.gt(0)); 1610 EasyMock.expectLastCall().once(); 1611 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1612 andReturn("root 1234 com.drawelement.deqp").once(); 1613 1614 // Recovery resets the connection 1615 mockDevice.recoverDevice(); 1616 EasyMock.expectLastCall().once(); 1617 1618 // and attempts to kill the process again 1619 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1620 andReturn("root 1234 com.drawelement.deqp").once(); 1621 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1622 andReturn("").once(); 1623 1624 // Recovery checks if kill failed 1625 mockSleepProvider.sleep(EasyMock.gt(0)); 1626 EasyMock.expectLastCall().once(); 1627 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1628 andReturn("root 1234 com.drawelement.deqp").once(); 1629 1630 // recovery reboots the device 1631 mockDevice.reboot(); 1632 EasyMock.expectLastCall().once(); 1633 1634 orderedControl.replay(); 1635 recovery.setDevice(mockDevice); 1636 recovery.setSleepProvider(mockSleepProvider); 1637 recovery.recoverComLinkKilled(); 1638 orderedControl.verify(); 1639 } 1640 1641 /** 1642 * Test external interruption before batch run. 1643 */ 1644 public void testInterrupt_killBeforeBatch() throws Exception { 1645 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1646 1647 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1648 tests.add(testId); 1649 1650 ITestInvocationListener mockListener 1651 = EasyMock.createStrictMock(ITestInvocationListener.class); 1652 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1653 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1654 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1655 1656 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1657 1658 deqpTest.setDevice(mockDevice); 1659 deqpTest.setRunUtil(mockRunUtil); 1660 1661 int version = 3 << 16; 1662 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1663 .andReturn(Integer.toString(version)).atLeastOnce(); 1664 1665 expectRenderConfigQuery(mockDevice, 1666 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1667 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1668 + "--deqp-gl-minor-version=0"); 1669 1670 mockRunUtil.sleep(0); 1671 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 1672 1673 mockListener.testRunStarted(getTestId(deqpTest), 1); 1674 EasyMock.expectLastCall().once(); 1675 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1676 andReturn("").once(); 1677 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1678 andReturn("").once(); 1679 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 1680 EasyMock.expectLastCall().once(); 1681 1682 EasyMock.replay(mockDevice, mockIDevice); 1683 EasyMock.replay(mockListener); 1684 EasyMock.replay(mockRunUtil); 1685 try { 1686 deqpTest.run(mockListener); 1687 fail("expected RunInterruptedException"); 1688 } catch (RunInterruptedException ex) { 1689 // expected 1690 } 1691 EasyMock.verify(mockRunUtil); 1692 EasyMock.verify(mockListener); 1693 EasyMock.verify(mockDevice, mockIDevice); 1694 } 1695 1696 private void runShardedTest(TestDescription[] testIds, 1697 ArrayList<ArrayList<TestDescription>> testsForShard) throws Exception { 1698 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1699 for (TestDescription id : testIds) tests.add(id); 1700 1701 DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir); 1702 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1703 1704 for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) { 1705 DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex); 1706 shard.setBuildHelper(getMockBuildHelper(mTestsDir)); 1707 1708 ArrayList<TestDescription> shardTests = testsForShard.get(shardIndex); 1709 1710 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1711 ITestInvocationListener mockListener 1712 = EasyMock.createStrictMock(ITestInvocationListener.class); 1713 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1714 int version = 3 << 16; 1715 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1716 .andReturn(Integer.toString(version)).atLeastOnce(); 1717 1718 mockListener.testRunStarted(getTestId(shard), shardTests.size()); 1719 EasyMock.expectLastCall().once(); 1720 1721 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1722 andReturn("").once(); 1723 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1724 andReturn("").once(); 1725 1726 expectRenderConfigQuery(mockDevice, 3, 0); 1727 1728 String testOut = buildTestProcessOutput(shardTests); 1729 // NOTE: This assumes that there won't be multiple batches per shard! 1730 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut); 1731 1732 for (int i = 0; i < shardTests.size(); i++) { 1733 mockListener.testStarted(EasyMock.eq(shardTests.get(i))); 1734 EasyMock.expectLastCall().once(); 1735 1736 mockListener.testEnded(EasyMock.eq(shardTests.get(i)), 1737 EasyMock.<HashMap<String, Metric>>notNull()); 1738 1739 EasyMock.expectLastCall().once(); 1740 } 1741 1742 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 1743 EasyMock.expectLastCall().once(); 1744 1745 EasyMock.replay(mockDevice, mockIDevice); 1746 EasyMock.replay(mockListener); 1747 1748 shard.setDevice(mockDevice); 1749 shard.run(mockListener); 1750 1751 EasyMock.verify(mockListener); 1752 EasyMock.verify(mockDevice, mockIDevice); 1753 } 1754 } 1755 1756 public void testSharding_smallTrivial() throws Exception { 1757 final TestDescription[] testIds = { 1758 new TestDescription("dEQP-GLES3.info", "vendor"), 1759 new TestDescription("dEQP-GLES3.info", "renderer"), 1760 new TestDescription("dEQP-GLES3.info", "version"), 1761 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 1762 new TestDescription("dEQP-GLES3.info", "extensions"), 1763 new TestDescription("dEQP-GLES3.info", "render_target") 1764 }; 1765 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1766 ArrayList<TestDescription> shardOne = new ArrayList<>(); 1767 for (int i = 0; i < testIds.length; i++) { 1768 shardOne.add(testIds[i]); 1769 } 1770 shardedTests.add(shardOne); 1771 runShardedTest(testIds, shardedTests); 1772 } 1773 1774 public void testSharding_twoShards() throws Exception { 1775 final int TEST_COUNT = 1237; 1776 final int SHARD_SIZE = 1000; 1777 1778 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 1779 for (int i = 0; i < TEST_COUNT; i++) { 1780 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 1781 } 1782 1783 ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>(); 1784 ArrayList<TestDescription> shard = new ArrayList<>(); 1785 for (int i = 0; i < testIds.size(); i++) { 1786 if (i == SHARD_SIZE) { 1787 shardedTests.add(shard); 1788 shard = new ArrayList<>(); 1789 } 1790 shard.add(testIds.get(i)); 1791 } 1792 shardedTests.add(shard); 1793 runShardedTest(testIds.toArray(new TestDescription[testIds.size()]), shardedTests); 1794 } 1795 1796 public void testSharding_empty() throws Exception { 1797 DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestDescription>(), mTestsDir); 1798 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split(); 1799 // Returns null when cannot be sharded. 1800 assertNull(shards); 1801 } 1802 1803 /** 1804 * Test external interruption in testFailed(). 1805 */ 1806 public void testInterrupt_killReportTestFailed() throws Exception { 1807 final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test"); 1808 final String testPath = "dEQP-GLES3.interrupt.test"; 1809 final String testTrie = "{dEQP-GLES3{interrupt{test}}}"; 1810 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1811 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1812 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1813 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1814 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1815 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1816 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1817 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1818 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1819 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1820 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1821 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1822 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1823 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1824 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1825 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 1826 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1827 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n" 1828 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n" 1829 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1830 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1831 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1832 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1833 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1834 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1835 + "INSTRUMENTATION_CODE: 0\r\n"; 1836 1837 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 1838 tests.add(testId); 1839 1840 ITestInvocationListener mockListener 1841 = EasyMock.createStrictMock(ITestInvocationListener.class); 1842 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 1843 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 1844 IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class); 1845 1846 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 1847 1848 deqpTest.setDevice(mockDevice); 1849 deqpTest.setRunUtil(mockRunUtil); 1850 1851 int version = 3 << 16; 1852 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 1853 .andReturn(Integer.toString(version)).atLeastOnce(); 1854 1855 expectRenderConfigQuery(mockDevice, 1856 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified " 1857 + "--deqp-surface-type=window --deqp-gl-major-version=3 " 1858 + "--deqp-gl-minor-version=0"); 1859 1860 mockRunUtil.sleep(0); 1861 EasyMock.expectLastCall().once(); 1862 1863 String commandLine = String.format( 1864 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 1865 + "--deqp-screen-rotation=unspecified " 1866 + "--deqp-surface-type=window " 1867 + "--deqp-log-images=disable " 1868 + "--deqp-watchdog=enable", 1869 APP_DIR + CASE_LIST_FILE_NAME); 1870 1871 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, 1872 output); 1873 1874 mockListener.testRunStarted(getTestId(deqpTest), 1); 1875 EasyMock.expectLastCall().once(); 1876 1877 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 1878 andReturn("").once(); 1879 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 1880 andReturn("").once(); 1881 1882 mockListener.testStarted(EasyMock.eq(testId)); 1883 EasyMock.expectLastCall().once(); 1884 1885 mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull()); 1886 EasyMock.expectLastCall().andThrow(new RunInterruptedException()); 1887 1888 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject()); 1889 EasyMock.expectLastCall().once(); 1890 EasyMock.replay(mockDevice, mockIDevice); 1891 EasyMock.replay(mockListener); 1892 EasyMock.replay(mockRunUtil); 1893 try { 1894 deqpTest.run(mockListener); 1895 fail("expected RunInterruptedException"); 1896 } catch (RunInterruptedException ex) { 1897 // expected 1898 } 1899 EasyMock.verify(mockRunUtil); 1900 EasyMock.verify(mockListener); 1901 EasyMock.verify(mockDevice, mockIDevice); 1902 } 1903 1904 public void testRuntimeHint_optionSet() throws Exception { 1905 /* MultiLineReceiver expects "\r\n" line ending. */ 1906 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 1907 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1908 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 1909 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1910 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 1911 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1912 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 1913 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1914 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 1915 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 1916 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 1917 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1918 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 1919 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1920 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1921 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n" 1922 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1923 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1924 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1925 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1926 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1927 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1928 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1929 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1930 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n" 1931 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1932 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1933 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1934 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1935 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1936 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1937 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1938 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1939 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n" 1940 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1941 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1942 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1943 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1944 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1945 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1946 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1947 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1948 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n" 1949 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1950 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1951 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1952 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1953 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1954 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1955 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1956 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1957 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n" 1958 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1959 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1960 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1961 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1962 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1963 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1964 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1965 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 1966 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n" 1967 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1968 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n" 1969 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n" 1970 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 1971 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1972 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 1973 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1974 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 1975 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 1976 + "INSTRUMENTATION_CODE: 0\r\n"; 1977 1978 final TestDescription[] testIds = { 1979 new TestDescription("dEQP-GLES3.info", "vendor"), 1980 new TestDescription("dEQP-GLES3.info", "renderer"), 1981 new TestDescription("dEQP-GLES3.info", "version"), 1982 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 1983 new TestDescription("dEQP-GLES3.info", "extensions"), 1984 new TestDescription("dEQP-GLES3.info", "render_target") 1985 }; 1986 1987 final String[] testPaths = { 1988 "dEQP-GLES3.info.vendor", 1989 "dEQP-GLES3.info.renderer", 1990 "dEQP-GLES3.info.version", 1991 "dEQP-GLES3.info.shading_language_version", 1992 "dEQP-GLES3.info.extensions", 1993 "dEQP-GLES3.info.render_target" 1994 }; 1995 1996 final String testTrie 1997 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 1998 1999 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2000 ITestInvocationListener mockListener 2001 = EasyMock.createStrictMock(ITestInvocationListener.class); 2002 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 2003 2004 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2005 2006 for (TestDescription id : testIds) { 2007 tests.add(id); 2008 } 2009 2010 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2011 OptionSetter setter = new OptionSetter(deqpTest); 2012 final long runtimeMs = 123456; 2013 setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs)); 2014 assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint()); 2015 2016 // Try running the tests as well. The unit tests do not set the hint be default, 2017 // so that case is covered. 2018 2019 int version = 3 << 16; 2020 EasyMock.expect(mockDevice.getProperty("ro.opengles.version")) 2021 .andReturn(Integer.toString(version)).atLeastOnce(); 2022 2023 expectRenderConfigQuery(mockDevice, 3, 0); 2024 2025 String commandLine = String.format( 2026 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2027 + "--deqp-screen-rotation=unspecified " 2028 + "--deqp-surface-type=window " 2029 + "--deqp-log-images=disable " 2030 + "--deqp-watchdog=enable", 2031 APP_DIR + CASE_LIST_FILE_NAME); 2032 2033 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output); 2034 2035 mockListener.testRunStarted(getTestId(deqpTest), testPaths.length); 2036 EasyMock.expectLastCall().once(); 2037 2038 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))). 2039 andReturn("").once(); 2040 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))). 2041 andReturn("").once(); 2042 2043 for (int i = 0; i < testPaths.length; i++) { 2044 mockListener.testStarted(EasyMock.eq(testIds[i])); 2045 EasyMock.expectLastCall().once(); 2046 2047 mockListener.testEnded(EasyMock.eq(testIds[i]), 2048 EasyMock.<HashMap<String, Metric>>notNull()); 2049 2050 EasyMock.expectLastCall().once(); 2051 } 2052 2053 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 2054 EasyMock.expectLastCall().once(); 2055 2056 EasyMock.replay(mockDevice, mockIDevice); 2057 EasyMock.replay(mockListener); 2058 2059 deqpTest.setDevice(mockDevice); 2060 deqpTest.run(mockListener); 2061 2062 EasyMock.verify(mockListener); 2063 EasyMock.verify(mockDevice, mockIDevice); 2064 } 2065 2066 public void testRuntimeHint_optionSetSharded() throws Exception { 2067 final int TEST_COUNT = 1237; 2068 final int SHARD_SIZE = 1000; 2069 2070 ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT); 2071 for (int i = 0; i < TEST_COUNT; i++) { 2072 testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i))); 2073 } 2074 2075 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir); 2076 OptionSetter setter = new OptionSetter(deqpTest); 2077 final long fullRuntimeMs = testIds.size()*100; 2078 setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs)); 2079 2080 ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split(); 2081 assertEquals("First shard's time not proportional to test count", 2082 (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT, 2083 ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint()); 2084 assertEquals("Second shard's time not proportional to test count", 2085 (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT, 2086 ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint()); 2087 } 2088 2089 public void testRuntimeHint_optionNotSet() throws Exception { 2090 final TestDescription[] testIds = { 2091 new TestDescription("dEQP-GLES3.info", "vendor"), 2092 new TestDescription("dEQP-GLES3.info", "renderer"), 2093 new TestDescription("dEQP-GLES3.info", "version"), 2094 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 2095 new TestDescription("dEQP-GLES3.info", "extensions"), 2096 new TestDescription("dEQP-GLES3.info", "render_target") 2097 }; 2098 Collection<TestDescription> tests = new ArrayList<TestDescription>(); 2099 2100 for (TestDescription id : testIds) { 2101 tests.add(id); 2102 } 2103 2104 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2105 2106 long runtime = deqpTest.getRuntimeHint(); 2107 assertTrue("Runtime for tests must be positive", runtime > 0); 2108 assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s 2109 } 2110 2111 2112 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2113 final String output) throws Exception { 2114 String cmd = String.format( 2115 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 " 2116 + "--deqp-screen-rotation=unspecified " 2117 + "--deqp-surface-type=window " 2118 + "--deqp-log-images=disable " 2119 + "--deqp-watchdog=enable", 2120 APP_DIR + CASE_LIST_FILE_NAME); 2121 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output); 2122 } 2123 2124 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 2125 final String testTrie, final String cmd, final String output) throws Exception { 2126 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME))) 2127 .andReturn("").once(); 2128 2129 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME))) 2130 .andReturn("").once(); 2131 2132 if (testTrie == null) { 2133 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME)); 2134 } 2135 else { 2136 mockDevice.pushString(testTrie + "\n", APP_DIR + CASE_LIST_FILE_NAME); 2137 } 2138 EasyMock.expectLastCall().andReturn(true).once(); 2139 2140 String command = String.format( 2141 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" " 2142 + "-e deqpLogData \"%s\" %s", 2143 AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, false, 2144 INSTRUMENTATION_NAME); 2145 2146 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 2147 mockIDevice.executeShellCommand(EasyMock.eq(command), 2148 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 2149 EasyMock.isA(TimeUnit.class)); 2150 2151 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 2152 @Override 2153 public Object answer() { 2154 IShellOutputReceiver receiver 2155 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 2156 2157 receiver.addOutput(output.getBytes(), 0, output.length()); 2158 receiver.flush(); 2159 2160 return null; 2161 } 2162 }); 2163 } 2164 2165 static private void writeStringsToFile(File target, Set<String> strings) throws IOException { 2166 try (PrintWriter out = new PrintWriter(new FileWriter(target))) { 2167 out.print(String.join(System.lineSeparator(), strings)); 2168 out.println(); 2169 } 2170 } 2171 2172 private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option) 2173 throws IOException, ConfigurationException { 2174 String filterFile = option + ".txt"; 2175 writeStringsToFile(new File(mTestsDir, filterFile), filters); 2176 OptionSetter setter = new OptionSetter(test); 2177 setter.setOptionValue(option, filterFile); 2178 } 2179 2180 public void testIncludeFilterFile() throws Exception { 2181 final TestDescription[] testIds = { 2182 new TestDescription("dEQP-GLES3.missing", "no"), 2183 new TestDescription("dEQP-GLES3.missing", "nope"), 2184 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2185 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2186 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2187 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2188 }; 2189 2190 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2191 for (TestDescription id : testIds) { 2192 allTests.add(id); 2193 } 2194 2195 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2196 activeTests.add(testIds[3]); 2197 activeTests.add(testIds[4]); 2198 activeTests.add(testIds[5]); 2199 2200 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2201 2202 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2203 Set<String> includes = new HashSet<>(); 2204 includes.add("dEQP-GLES3.pick_me#*"); 2205 addFilterFileForOption(deqpTest, includes, "include-filter-file"); 2206 testFiltering(deqpTest, expectedTrie, activeTests); 2207 } 2208 2209 public void testMissingIncludeFilterFile() throws Exception { 2210 final TestDescription[] testIds = { 2211 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2212 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2213 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2214 }; 2215 2216 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2217 for (TestDescription id : testIds) { 2218 allTests.add(id); 2219 } 2220 2221 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2222 2223 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2224 OptionSetter setter = new OptionSetter(deqpTest); 2225 setter.setOptionValue("include-filter-file", "not-a-file.txt"); 2226 try { 2227 testFiltering(deqpTest, expectedTrie, allTests); 2228 fail("Test execution should have aborted with exception."); 2229 } catch (RuntimeException e) { 2230 } 2231 } 2232 2233 public void testExcludeFilterFile() throws Exception { 2234 final TestDescription[] testIds = { 2235 new TestDescription("dEQP-GLES3.missing", "no"), 2236 new TestDescription("dEQP-GLES3.missing", "nope"), 2237 new TestDescription("dEQP-GLES3.missing", "donotwant"), 2238 new TestDescription("dEQP-GLES3.pick_me", "yes"), 2239 new TestDescription("dEQP-GLES3.pick_me", "ok"), 2240 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 2241 }; 2242 2243 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2244 for (TestDescription id : testIds) { 2245 allTests.add(id); 2246 } 2247 2248 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2249 activeTests.add(testIds[3]); 2250 activeTests.add(testIds[4]); 2251 activeTests.add(testIds[5]); 2252 2253 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 2254 2255 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2256 Set<String> excludes = new HashSet<>(); 2257 excludes.add("dEQP-GLES3.missing#*"); 2258 addFilterFileForOption(deqpTest, excludes, "exclude-filter-file"); 2259 testFiltering(deqpTest, expectedTrie, activeTests); 2260 } 2261 2262 public void testFilterComboWithFiles() throws Exception { 2263 final TestDescription[] testIds = { 2264 new TestDescription("dEQP-GLES3.group1", "footah"), 2265 new TestDescription("dEQP-GLES3.group1", "foo"), 2266 new TestDescription("dEQP-GLES3.group1", "nope"), 2267 new TestDescription("dEQP-GLES3.group1", "nonotwant"), 2268 new TestDescription("dEQP-GLES3.group2", "foo"), 2269 new TestDescription("dEQP-GLES3.group2", "yes"), 2270 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 2271 }; 2272 2273 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2274 for (TestDescription id : testIds) { 2275 allTests.add(id); 2276 } 2277 2278 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2279 activeTests.add(testIds[0]); 2280 activeTests.add(testIds[5]); 2281 2282 String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}"; 2283 2284 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2285 2286 Set<String> includes = new HashSet<>(); 2287 includes.add("dEQP-GLES3.group2#*"); 2288 deqpTest.addAllIncludeFilters(includes); 2289 2290 Set<String> fileIncludes = new HashSet<>(); 2291 fileIncludes.add("dEQP-GLES3.group1#no*"); 2292 fileIncludes.add("dEQP-GLES3.group1#foo*"); 2293 addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file"); 2294 2295 Set<String> fileExcludes = new HashSet<>(); 2296 fileExcludes.add("*foo"); 2297 fileExcludes.add("*thoushallnotpass"); 2298 addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file"); 2299 2300 deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*"); 2301 2302 testFiltering(deqpTest, expectedTrie, activeTests); 2303 } 2304 2305 public void testDotToHashConversionInFilters() throws Exception { 2306 final TestDescription[] testIds = { 2307 new TestDescription("dEQP-GLES3.missing", "no"), 2308 new TestDescription("dEQP-GLES3.pick_me", "donotwant"), 2309 new TestDescription("dEQP-GLES3.pick_me", "yes") 2310 }; 2311 2312 List<TestDescription> allTests = new ArrayList<TestDescription>(); 2313 for (TestDescription id : testIds) { 2314 allTests.add(id); 2315 } 2316 2317 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 2318 activeTests.add(testIds[2]); 2319 2320 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; 2321 2322 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir); 2323 deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes"); 2324 testFiltering(deqpTest, expectedTrie, activeTests); 2325 } 2326 2327 public void testCoverageDisabled_noCoverageListener() throws Exception { 2328 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 2329 2330 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2331 ITestInvocationListener mockListener = 2332 EasyMock.createStrictMock(ITestInvocationListener.class); 2333 2334 Collection<TestDescription> tests = new ArrayList<>(); 2335 tests.add(testId); 2336 2337 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2338 deqpTest.setNativeCoverage(false); 2339 2340 ITestInvocationListener listener = 2341 deqpTest.addNativeCoverageListenerIfEnabled(mockDevice, mockListener); 2342 assertSame(listener, mockListener); 2343 } 2344 2345 public void testCoverageEnabled_newCoverageListener() throws Exception { 2346 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 2347 2348 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 2349 ITestInvocationListener mockListener = 2350 EasyMock.createStrictMock(ITestInvocationListener.class); 2351 2352 Collection<TestDescription> tests = new ArrayList<>(); 2353 tests.add(testId); 2354 2355 DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir); 2356 deqpTest.setNativeCoverage(true); 2357 2358 ITestInvocationListener listener = 2359 deqpTest.addNativeCoverageListenerIfEnabled(mockDevice, mockListener); 2360 assertTrue(listener instanceof NativeCodeCoverageListener); 2361 } 2362 } 2363