Home | History | Annotate | Download | only in runner
      1 /*
      2  * Copyright (C) 2014 The Android Open Source Project
      3  *
      4  * Licensed under the Apache License, Version 2.0 (the "License");
      5  * you may not use this file except in compliance with the License.
      6  * You may obtain a copy of the License at
      7  *
      8  *      http://www.apache.org/licenses/LICENSE-2.0
      9  *
     10  * Unless required by applicable law or agreed to in writing, software
     11  * distributed under the License is distributed on an "AS IS" BASIS,
     12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     13  * See the License for the specific language governing permissions and
     14  * limitations under the License.
     15  */
     16 package com.drawelements.deqp.runner;
     17 
     18 import static org.junit.Assert.assertSame;
     19 import static org.junit.Assert.assertTrue;
     20 
     21 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper;
     22 import com.android.ddmlib.IDevice;
     23 import com.android.ddmlib.IShellOutputReceiver;
     24 import com.android.tradefed.build.IFolderBuildInfo;
     25 import com.android.tradefed.config.ConfigurationException;
     26 import com.android.tradefed.config.OptionSetter;
     27 import com.android.tradefed.device.DeviceNotAvailableException;
     28 import com.android.tradefed.device.IManagedTestDevice;
     29 import com.android.tradefed.device.ITestDevice;
     30 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
     31 import com.android.tradefed.result.ITestInvocationListener;
     32 import com.android.tradefed.result.TestDescription;
     33 import com.android.tradefed.testtype.Abi;
     34 import com.android.tradefed.testtype.IAbi;
     35 import com.android.tradefed.testtype.IRemoteTest;
     36 import com.android.tradefed.testtype.IRuntimeHintProvider;
     37 import com.android.tradefed.testtype.NativeCodeCoverageListener;
     38 import com.android.tradefed.util.AbiUtils;
     39 import com.android.tradefed.util.FileUtil;
     40 import com.android.tradefed.util.IRunUtil;
     41 import com.android.tradefed.util.RunInterruptedException;
     42 
     43 import junit.framework.TestCase;
     44 
     45 import org.easymock.EasyMock;
     46 import org.easymock.IAnswer;
     47 import org.easymock.IMocksControl;
     48 
     49 import java.io.File;
     50 import java.io.FileNotFoundException;
     51 import java.io.FileWriter;
     52 import java.io.IOException;
     53 import java.io.PrintWriter;
     54 import java.io.StringReader;
     55 import java.io.StringWriter;
     56 import java.util.ArrayList;
     57 import java.util.Collection;
     58 import java.util.HashMap;
     59 import java.util.HashSet;
     60 import java.util.List;
     61 import java.util.Map;
     62 import java.util.Set;
     63 import java.util.concurrent.TimeUnit;
     64 
     65 /**
     66  * Unit tests for {@link DeqpTestRunner}.
     67  */
     68 public class DeqpTestRunnerTest extends TestCase {
     69     private static final String NAME = "dEQP-GLES3";
     70     private static final IAbi ABI = new Abi("armeabi-v7a", "32");
     71     private static final String APP_DIR = "/sdcard/";
     72     private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt";
     73     private static final String LOG_FILE_NAME = "TestLog.qpa";
     74     private static final String INSTRUMENTATION_NAME =
     75             "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
     76     private static final String QUERY_INSTRUMENTATION_NAME =
     77             "com.drawelements.deqp/com.drawelements.deqp.platformutil.DeqpPlatformCapabilityQueryInstrumentation";
     78     private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
     79     private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
     80     private static final String ONLY_LANDSCAPE_FEATURES =
     81             "feature:"+DeqpTestRunner.FEATURE_LANDSCAPE;
     82     private static final String ALL_FEATURES =
     83             ONLY_LANDSCAPE_FEATURES + "\nfeature:"+DeqpTestRunner.FEATURE_PORTRAIT;
     84     private static List<Map<String,String>> DEFAULT_INSTANCE_ARGS;
     85 
     86     static {
     87         DEFAULT_INSTANCE_ARGS = new ArrayList<>(1);
     88         DEFAULT_INSTANCE_ARGS.add(new HashMap<String,String>());
     89         DEFAULT_INSTANCE_ARGS.iterator().next().put("glconfig", "rgba8888d24s8");
     90         DEFAULT_INSTANCE_ARGS.iterator().next().put("rotation", "unspecified");
     91         DEFAULT_INSTANCE_ARGS.iterator().next().put("surfacetype", "window");
     92     }
     93 
     94     private File mTestsDir = null;
     95 
     96     public static class BuildHelperMock extends CompatibilityBuildHelper {
     97         private File mTestsDir = null;
     98         public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) {
     99             super(buildInfo);
    100             mTestsDir = testsDir;
    101         }
    102         @Override
    103         public File getTestsDir() throws FileNotFoundException {
    104             return mTestsDir;
    105         }
    106     }
    107 
    108 
    109     /**
    110      * {@inheritDoc}
    111      */
    112     @Override
    113     protected void setUp() throws Exception {
    114         super.setUp();
    115         mTestsDir = FileUtil.createTempDir("deqp-test-cases");
    116     }
    117 
    118     /**
    119      * {@inheritDoc}
    120      */
    121     @Override
    122     protected void tearDown() throws Exception {
    123         FileUtil.recursiveDelete(mTestsDir);
    124         super.tearDown();
    125     }
    126 
    127     private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
    128                                                       int minorVersion,
    129                                                       Collection<TestDescription> tests,
    130                                                       File testsDir) throws ConfigurationException {
    131         StringWriter testlist = new StringWriter();
    132         for (TestDescription test : tests) {
    133             testlist.write(test.getClassName() + "." + test.getTestName() + "\n");
    134         }
    135         return buildGlesTestRunner(majorVersion, minorVersion, testlist.toString(), testsDir);
    136     }
    137 
    138     private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) {
    139         IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class);
    140         EasyMock.replay(mockIFolderBuildInfo);
    141         return new BuildHelperMock(mockIFolderBuildInfo, testsDir);
    142     }
    143 
    144     private static DeqpTestRunner buildGlesTestRunner(int majorVersion,
    145                                                       int minorVersion,
    146                                                       String testlist,
    147                                                       File testsDir) throws ConfigurationException {
    148         DeqpTestRunner runner = new DeqpTestRunner();
    149         OptionSetter setter = new OptionSetter(runner);
    150 
    151         String deqpPackage = "dEQP-GLES" + Integer.toString(majorVersion)
    152                 + (minorVersion > 0 ? Integer.toString(minorVersion) : "");
    153 
    154         setter.setOptionValue("deqp-package", deqpPackage);
    155         setter.setOptionValue("deqp-gl-config-name", "rgba8888d24s8");
    156         setter.setOptionValue("deqp-caselist-file", "dummyfile.txt");
    157         setter.setOptionValue("deqp-screen-rotation", "unspecified");
    158         setter.setOptionValue("deqp-surface-type", "window");
    159 
    160         runner.setCaselistReader(new StringReader(testlist));
    161         runner.setAbi(ABI);
    162         runner.setBuildHelper(getMockBuildHelper(testsDir));
    163 
    164         return runner;
    165     }
    166 
    167     private static String getTestId(DeqpTestRunner runner) {
    168         return AbiUtils.createId(ABI.getName(), runner.getPackageName());
    169     }
    170 
    171     /**
    172      * Test version of OpenGL ES.
    173      */
    174     private void testGlesVersion(int requiredMajorVersion, int requiredMinorVersion, int majorVersion, int minorVersion) throws Exception {
    175         final TestDescription testId = new TestDescription("dEQP-GLES"
    176                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
    177                 + ".info", "version");
    178 
    179         final String testPath = "dEQP-GLES"
    180                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
    181                 +".info.version";
    182 
    183         final String testTrie = "{dEQP-GLES"
    184                 + Integer.toString(requiredMajorVersion) + Integer.toString(requiredMinorVersion)
    185                 + "{info{version}}}";
    186 
    187         final String resultCode = "Pass";
    188 
    189         /* MultiLineReceiver expects "\r\n" line ending. */
    190         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
    191                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    192                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
    193                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    194                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
    195                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    196                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
    197                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    198                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
    199                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    200                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
    201                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    202                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
    203                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    204                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    205                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
    206                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    207                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
    208                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
    209                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    210                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    211                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    212                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    213                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
    214                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    215                 + "INSTRUMENTATION_CODE: 0\r\n";
    216 
    217         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
    218         ITestInvocationListener mockListener
    219                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    220         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
    221         Collection<TestDescription> tests = new ArrayList<TestDescription>();
    222         tests.add(testId);
    223 
    224         DeqpTestRunner deqpTest = buildGlesTestRunner(requiredMajorVersion, requiredMinorVersion, tests, mTestsDir);
    225 
    226         int version = (majorVersion << 16) | minorVersion;
    227         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
    228             .andReturn(Integer.toString(version)).atLeastOnce();
    229 
    230         if (majorVersion > requiredMajorVersion
    231                 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion)) {
    232 
    233             EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
    234                 andReturn("").once();
    235             EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
    236                 andReturn("").once();
    237 
    238             expectRenderConfigQuery(mockDevice, requiredMajorVersion,
    239                     requiredMinorVersion);
    240 
    241             String commandLine = String.format(
    242                     "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
    243                     + "--deqp-screen-rotation=unspecified "
    244                     + "--deqp-surface-type=window "
    245                     + "--deqp-log-images=disable "
    246                     + "--deqp-watchdog=enable",
    247                     APP_DIR + CASE_LIST_FILE_NAME);
    248 
    249             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
    250                     output);
    251         }
    252 
    253         mockListener.testRunStarted(getTestId(deqpTest), 1);
    254         EasyMock.expectLastCall().once();
    255 
    256         mockListener.testStarted(EasyMock.eq(testId));
    257         EasyMock.expectLastCall().once();
    258 
    259         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull());
    260         EasyMock.expectLastCall().once();
    261 
    262         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    263         EasyMock.expectLastCall().once();
    264 
    265         EasyMock.replay(mockDevice, mockIDevice);
    266         EasyMock.replay(mockListener);
    267 
    268         deqpTest.setDevice(mockDevice);
    269         deqpTest.run(mockListener);
    270 
    271         EasyMock.verify(mockListener);
    272         EasyMock.verify(mockDevice, mockIDevice);
    273     }
    274 
    275     private void expectRenderConfigQuery(ITestDevice mockDevice, int majorVersion,
    276             int minorVersion) throws Exception {
    277         expectRenderConfigQuery(mockDevice,
    278                 String.format("--deqp-gl-config-name=rgba8888d24s8 "
    279                 + "--deqp-screen-rotation=unspecified "
    280                 + "--deqp-surface-type=window "
    281                 + "--deqp-gl-major-version=%d "
    282                 + "--deqp-gl-minor-version=%d", majorVersion, minorVersion));
    283     }
    284 
    285     private void expectRenderConfigQuery(ITestDevice mockDevice, String commandLine)
    286             throws Exception {
    287         expectRenderConfigQueryAndReturn(mockDevice, commandLine, "Yes");
    288     }
    289 
    290     private void expectRenderConfigQueryAndReturn(ITestDevice mockDevice, String commandLine,
    291             String output) throws Exception {
    292         final String queryOutput = "INSTRUMENTATION_RESULT: Supported=" + output + "\r\n"
    293                 + "INSTRUMENTATION_CODE: 0\r\n";
    294         final String command = String.format(
    295                 "am instrument %s -w -e deqpQueryType renderConfigSupported -e deqpCmdLine "
    296                     + "\"%s\" %s",
    297                 AbiUtils.createAbiFlag(ABI.getName()), commandLine,
    298                 QUERY_INSTRUMENTATION_NAME);
    299 
    300         mockDevice.executeShellCommand(EasyMock.eq(command),
    301                 EasyMock.<IShellOutputReceiver>notNull());
    302 
    303         EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
    304             @Override
    305             public Object answer() {
    306                 IShellOutputReceiver receiver
    307                         = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
    308 
    309                 receiver.addOutput(queryOutput.getBytes(), 0, queryOutput.length());
    310                 receiver.flush();
    311 
    312                 return null;
    313             }
    314         });
    315     }
    316 
    317     /**
    318      * Test that result code produces correctly pass or fail.
    319      */
    320     private void testResultCode(final String resultCode, boolean pass) throws Exception {
    321         final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version");
    322         final String testPath = "dEQP-GLES3.info.version";
    323         final String testTrie = "{dEQP-GLES3{info{version}}}";
    324 
    325         /* MultiLineReceiver expects "\r\n" line ending. */
    326         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
    327                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    328                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
    329                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    330                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
    331                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    332                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
    333                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    334                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
    335                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    336                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
    337                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    338                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
    339                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    340                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    341                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
    342                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    343                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n"
    344                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n"
    345                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    346                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    347                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    348                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    349                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
    350                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    351                 + "INSTRUMENTATION_CODE: 0\r\n";
    352 
    353         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
    354         ITestInvocationListener mockListener
    355                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    356         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
    357 
    358         Collection<TestDescription> tests = new ArrayList<TestDescription>();
    359         tests.add(testId);
    360 
    361         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
    362 
    363         int version = 3 << 16;
    364         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
    365                 .andReturn(Integer.toString(version)).atLeastOnce();
    366 
    367         expectRenderConfigQuery(mockDevice, 3, 0);
    368 
    369         String commandLine = String.format(
    370                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
    371                 + "--deqp-screen-rotation=unspecified "
    372                 + "--deqp-surface-type=window "
    373                 + "--deqp-log-images=disable "
    374                 + "--deqp-watchdog=enable",
    375                 APP_DIR + CASE_LIST_FILE_NAME);
    376 
    377         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
    378 
    379         mockListener.testRunStarted(getTestId(deqpTest), 1);
    380         EasyMock.expectLastCall().once();
    381 
    382         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
    383             andReturn("").once();
    384         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
    385             andReturn("").once();
    386 
    387         mockListener.testStarted(EasyMock.eq(testId));
    388         EasyMock.expectLastCall().once();
    389 
    390         if (!pass) {
    391             mockListener.testFailed(testId,
    392                     "=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
    393                     + resultCode + ": Detail" + resultCode);
    394 
    395             EasyMock.expectLastCall().once();
    396         }
    397 
    398         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull());
    399         EasyMock.expectLastCall().once();
    400 
    401         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    402         EasyMock.expectLastCall().once();
    403 
    404         EasyMock.replay(mockDevice, mockIDevice);
    405         EasyMock.replay(mockListener);
    406 
    407         deqpTest.setDevice(mockDevice);
    408         deqpTest.run(mockListener);
    409 
    410         EasyMock.verify(mockListener);
    411         EasyMock.verify(mockDevice, mockIDevice);
    412     }
    413 
    414     /**
    415      * Test running multiple test cases.
    416      */
    417     public void testRun_multipleTests() throws Exception {
    418         /* MultiLineReceiver expects "\r\n" line ending. */
    419         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
    420                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    421                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
    422                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    423                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
    424                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    425                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
    426                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    427                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
    428                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    429                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
    430                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    431                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
    432                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    433                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    434                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
    435                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    436                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
    437                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
    438                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    439                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    440                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    441                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    442                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    443                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
    444                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    445                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
    446                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
    447                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    448                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    449                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    450                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    451                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    452                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
    453                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    454                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
    455                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
    456                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    457                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    458                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    459                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    460                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    461                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
    462                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    463                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
    464                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
    465                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    466                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    467                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    468                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    469                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    470                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
    471                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    472                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
    473                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
    474                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    475                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    476                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    477                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    478                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
    479                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
    480                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    481                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
    482                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
    483                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
    484                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    485                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
    486                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    487                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
    488                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    489                 + "INSTRUMENTATION_CODE: 0\r\n";
    490 
    491         final TestDescription[] testIds = {
    492                 new TestDescription("dEQP-GLES3.info", "vendor"),
    493                 new TestDescription("dEQP-GLES3.info", "renderer"),
    494                 new TestDescription("dEQP-GLES3.info", "version"),
    495                 new TestDescription("dEQP-GLES3.info", "shading_language_version"),
    496                 new TestDescription("dEQP-GLES3.info", "extensions"),
    497                 new TestDescription("dEQP-GLES3.info", "render_target")
    498         };
    499 
    500         final String[] testPaths = {
    501                 "dEQP-GLES3.info.vendor",
    502                 "dEQP-GLES3.info.renderer",
    503                 "dEQP-GLES3.info.version",
    504                 "dEQP-GLES3.info.shading_language_version",
    505                 "dEQP-GLES3.info.extensions",
    506                 "dEQP-GLES3.info.render_target"
    507         };
    508 
    509         final String testTrie
    510                 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
    511 
    512         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
    513         ITestInvocationListener mockListener
    514                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    515         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
    516 
    517         Collection<TestDescription> tests = new ArrayList<TestDescription>();
    518 
    519         for (TestDescription id : testIds) {
    520             tests.add(id);
    521         }
    522 
    523         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
    524 
    525         int version = 3 << 16;
    526         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
    527                 .andReturn(Integer.toString(version)).atLeastOnce();
    528 
    529         expectRenderConfigQuery(mockDevice, 3, 0);
    530 
    531         String commandLine = String.format(
    532                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
    533                 + "--deqp-screen-rotation=unspecified "
    534                 + "--deqp-surface-type=window "
    535                 + "--deqp-log-images=disable "
    536                 + "--deqp-watchdog=enable",
    537                 APP_DIR + CASE_LIST_FILE_NAME);
    538 
    539         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
    540 
    541         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
    542         EasyMock.expectLastCall().once();
    543 
    544         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
    545             andReturn("").once();
    546         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
    547             andReturn("").once();
    548 
    549         for (int i = 0; i < testPaths.length; i++) {
    550             mockListener.testStarted(EasyMock.eq(testIds[i]));
    551             EasyMock.expectLastCall().once();
    552 
    553             mockListener.testEnded(EasyMock.eq(testIds[i]),
    554                     EasyMock.<HashMap<String, Metric>>notNull());
    555 
    556             EasyMock.expectLastCall().once();
    557         }
    558 
    559         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    560         EasyMock.expectLastCall().once();
    561 
    562         EasyMock.replay(mockDevice, mockIDevice);
    563         EasyMock.replay(mockListener);
    564 
    565         deqpTest.setDevice(mockDevice);
    566         deqpTest.run(mockListener);
    567 
    568         EasyMock.verify(mockListener);
    569         EasyMock.verify(mockDevice, mockIDevice);
    570     }
    571 
    572     static private String buildTestProcessOutput(List<TestDescription> tests) {
    573         /* MultiLineReceiver expects "\r\n" line ending. */
    574         final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
    575                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    576                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
    577                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    578                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
    579                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    580                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
    581                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    582                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
    583                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    584                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
    585                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    586                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
    587                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n";
    588 
    589         final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
    590                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    591                 + "INSTRUMENTATION_CODE: 0\r\n";
    592 
    593         StringWriter output = new StringWriter();
    594         output.write(outputHeader);
    595         for (TestDescription test : tests) {
    596             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n");
    597             output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=");
    598             output.write(test.getClassName());
    599             output.write(".");
    600             output.write(test.getTestName());
    601             output.write("\r\n");
    602             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
    603             output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n");
    604             output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n");
    605             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n");
    606             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
    607             output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n");
    608             output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n");
    609         }
    610         output.write(outputEnd);
    611         return output.toString();
    612     }
    613 
    614     private void testFiltering(DeqpTestRunner deqpTest,
    615                                String expectedTrie,
    616                                List<TestDescription> expectedTests) throws Exception {
    617         int version = 3 << 16;
    618         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
    619         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
    620                 .andReturn(Integer.toString(version)).atLeastOnce();
    621 
    622         boolean thereAreTests = !expectedTests.isEmpty();
    623 
    624         ITestInvocationListener mockListener
    625                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    626         mockListener.testRunStarted(getTestId(deqpTest), expectedTests.size());
    627         EasyMock.expectLastCall().once();
    628 
    629         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
    630             andReturn("").once();
    631         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
    632             andReturn("").once();
    633 
    634         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
    635         if (thereAreTests)
    636         {
    637             expectRenderConfigQuery(mockDevice, 3, 0);
    638 
    639             String testOut = buildTestProcessOutput(expectedTests);
    640             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
    641 
    642             for (int i = 0; i < expectedTests.size(); i++) {
    643                 mockListener.testStarted(EasyMock.eq(expectedTests.get(i)));
    644                 EasyMock.expectLastCall().once();
    645 
    646                 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)),
    647                                        EasyMock.<HashMap<String, Metric>>notNull());
    648 
    649                 EasyMock.expectLastCall().once();
    650             }
    651         }
    652 
    653         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    654         EasyMock.expectLastCall().once();
    655 
    656         EasyMock.replay(mockDevice, mockIDevice);
    657         EasyMock.replay(mockListener);
    658 
    659         deqpTest.setDevice(mockDevice);
    660         deqpTest.run(mockListener);
    661 
    662         EasyMock.verify(mockListener);
    663         EasyMock.verify(mockDevice, mockIDevice);
    664     }
    665 
    666     public void testRun_trivialIncludeFilter() throws Exception {
    667         final TestDescription[] testIds = {
    668                 new TestDescription("dEQP-GLES3.missing", "no"),
    669                 new TestDescription("dEQP-GLES3.missing", "nope"),
    670                 new TestDescription("dEQP-GLES3.missing", "donotwant"),
    671                 new TestDescription("dEQP-GLES3.pick_me", "yes"),
    672                 new TestDescription("dEQP-GLES3.pick_me", "ok"),
    673                 new TestDescription("dEQP-GLES3.pick_me", "accepted"),
    674         };
    675 
    676         List<TestDescription> allTests = new ArrayList<TestDescription>();
    677         for (TestDescription id : testIds) {
    678             allTests.add(id);
    679         }
    680 
    681         List<TestDescription> activeTests = new ArrayList<TestDescription>();
    682         activeTests.add(testIds[3]);
    683         activeTests.add(testIds[4]);
    684         activeTests.add(testIds[5]);
    685 
    686         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
    687 
    688         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
    689         deqpTest.addIncludeFilter("dEQP-GLES3.pick_me#*");
    690         testFiltering(deqpTest, expectedTrie, activeTests);
    691     }
    692 
    693     public void testRun_trivialExcludeFilter() throws Exception {
    694         final TestDescription[] testIds = {
    695                 new TestDescription("dEQP-GLES3.missing", "no"),
    696                 new TestDescription("dEQP-GLES3.missing", "nope"),
    697                 new TestDescription("dEQP-GLES3.missing", "donotwant"),
    698                 new TestDescription("dEQP-GLES3.pick_me", "yes"),
    699                 new TestDescription("dEQP-GLES3.pick_me", "ok"),
    700                 new TestDescription("dEQP-GLES3.pick_me", "accepted"),
    701         };
    702 
    703         List<TestDescription> allTests = new ArrayList<TestDescription>();
    704         for (TestDescription id : testIds) {
    705             allTests.add(id);
    706         }
    707 
    708         List<TestDescription> activeTests = new ArrayList<TestDescription>();
    709         activeTests.add(testIds[3]);
    710         activeTests.add(testIds[4]);
    711         activeTests.add(testIds[5]);
    712 
    713         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
    714 
    715         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
    716         deqpTest.addExcludeFilter("dEQP-GLES3.missing#*");
    717         testFiltering(deqpTest, expectedTrie, activeTests);
    718     }
    719 
    720     public void testRun_includeAndExcludeFilter() throws Exception {
    721         final TestDescription[] testIds = {
    722                 new TestDescription("dEQP-GLES3.group1", "foo"),
    723                 new TestDescription("dEQP-GLES3.group1", "nope"),
    724                 new TestDescription("dEQP-GLES3.group1", "donotwant"),
    725                 new TestDescription("dEQP-GLES3.group2", "foo"),
    726                 new TestDescription("dEQP-GLES3.group2", "yes"),
    727                 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"),
    728         };
    729 
    730         List<TestDescription> allTests = new ArrayList<TestDescription>();
    731         for (TestDescription id : testIds) {
    732             allTests.add(id);
    733         }
    734 
    735         List<TestDescription> activeTests = new ArrayList<TestDescription>();
    736         activeTests.add(testIds[4]);
    737 
    738         String expectedTrie = "{dEQP-GLES3{group2{yes}}}";
    739 
    740         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
    741 
    742         Set<String> includes = new HashSet<>();
    743         includes.add("dEQP-GLES3.group2#*");
    744         deqpTest.addAllIncludeFilters(includes);
    745 
    746         Set<String> excludes = new HashSet<>();
    747         excludes.add("*foo");
    748         excludes.add("*thoushallnotpass");
    749         deqpTest.addAllExcludeFilters(excludes);
    750         testFiltering(deqpTest, expectedTrie, activeTests);
    751     }
    752 
    753     public void testRun_includeAll() throws Exception {
    754         final TestDescription[] testIds = {
    755                 new TestDescription("dEQP-GLES3.group1", "mememe"),
    756                 new TestDescription("dEQP-GLES3.group1", "yeah"),
    757                 new TestDescription("dEQP-GLES3.group1", "takeitall"),
    758                 new TestDescription("dEQP-GLES3.group2", "jeba"),
    759                 new TestDescription("dEQP-GLES3.group2", "yes"),
    760                 new TestDescription("dEQP-GLES3.group2", "granted"),
    761         };
    762 
    763         List<TestDescription> allTests = new ArrayList<TestDescription>();
    764         for (TestDescription id : testIds) {
    765             allTests.add(id);
    766         }
    767 
    768         String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}";
    769 
    770         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
    771         deqpTest.addIncludeFilter("*");
    772         testFiltering(deqpTest, expectedTrie, allTests);
    773     }
    774 
    775     public void testRun_excludeAll() throws Exception {
    776         final TestDescription[] testIds = {
    777                 new TestDescription("dEQP-GLES3.group1", "no"),
    778                 new TestDescription("dEQP-GLES3.group1", "nope"),
    779                 new TestDescription("dEQP-GLES3.group1", "nottoday"),
    780                 new TestDescription("dEQP-GLES3.group2", "banned"),
    781                 new TestDescription("dEQP-GLES3.group2", "notrecognized"),
    782                 new TestDescription("dEQP-GLES3.group2", "-2"),
    783         };
    784 
    785         List<TestDescription> allTests = new ArrayList<TestDescription>();
    786         for (TestDescription id : testIds) {
    787             allTests.add(id);
    788         }
    789 
    790         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
    791         deqpTest.addExcludeFilter("*");
    792         ITestInvocationListener mockListener
    793                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    794         mockListener.testRunStarted(getTestId(deqpTest), 0);
    795         EasyMock.expectLastCall().once();
    796         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    797         EasyMock.expectLastCall().once();
    798 
    799         EasyMock.replay(mockListener);
    800         deqpTest.run(mockListener);
    801         EasyMock.verify(mockListener);
    802     }
    803 
    804     /**
    805      * Test running a unexecutable test.
    806      */
    807     public void testRun_unexecutableTests() throws Exception {
    808         final String instrumentationAnswerNoExecs =
    809                 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
    810                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    811                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
    812                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    813                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
    814                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    815                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
    816                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    817                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
    818                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
    819                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
    820                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    821                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
    822                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    823                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
    824                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
    825                 + "INSTRUMENTATION_CODE: 0\r\n";
    826 
    827         final TestDescription[] testIds = {
    828                 new TestDescription("dEQP-GLES3.missing", "no"),
    829                 new TestDescription("dEQP-GLES3.missing", "nope"),
    830                 new TestDescription("dEQP-GLES3.missing", "donotwant"),
    831         };
    832 
    833         final String[] testPaths = {
    834                 "dEQP-GLES3.missing.no",
    835                 "dEQP-GLES3.missing.nope",
    836                 "dEQP-GLES3.missing.donotwant",
    837         };
    838 
    839         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
    840         ITestInvocationListener mockListener
    841                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    842         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
    843 
    844         Collection<TestDescription> tests = new ArrayList<TestDescription>();
    845 
    846         for (TestDescription id : testIds) {
    847             tests.add(id);
    848         }
    849 
    850         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
    851 
    852         int version = 3 << 16;
    853         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
    854                 .andReturn(Integer.toString(version)).atLeastOnce();
    855 
    856         expectRenderConfigQuery(mockDevice, 3, 0);
    857 
    858         String commandLine = String.format(
    859                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
    860                 + "--deqp-screen-rotation=unspecified "
    861                 + "--deqp-surface-type=window "
    862                 + "--deqp-log-images=disable "
    863                 + "--deqp-watchdog=enable",
    864                 APP_DIR + CASE_LIST_FILE_NAME);
    865 
    866         // first try
    867         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
    868                 "{dEQP-GLES3{missing{no,nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
    869 
    870         // splitting begins
    871         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
    872                 "{dEQP-GLES3{missing{no}}}", commandLine, instrumentationAnswerNoExecs);
    873         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
    874                 "{dEQP-GLES3{missing{nope,donotwant}}}", commandLine, instrumentationAnswerNoExecs);
    875         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
    876                 "{dEQP-GLES3{missing{nope}}}", commandLine, instrumentationAnswerNoExecs);
    877         runInstrumentationLineAndAnswer(mockDevice, mockIDevice,
    878                 "{dEQP-GLES3{missing{donotwant}}}", commandLine, instrumentationAnswerNoExecs);
    879 
    880         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
    881         EasyMock.expectLastCall().once();
    882 
    883         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
    884             andReturn("").once();
    885         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
    886             andReturn("").once();
    887 
    888         for (int i = 0; i < testPaths.length; i++) {
    889             mockListener.testStarted(EasyMock.eq(testIds[i]));
    890             EasyMock.expectLastCall().once();
    891 
    892             mockListener.testFailed(EasyMock.eq(testIds[i]),
    893                     EasyMock.eq("=== with config {glformat=rgba8888d24s8,rotation=unspecified,surfacetype=window,required=false} ===\n"
    894                     + "Abort: Test cannot be executed"));
    895             EasyMock.expectLastCall().once();
    896 
    897             mockListener.testEnded(EasyMock.eq(testIds[i]),
    898                     EasyMock.<HashMap<String, Metric>>notNull());
    899             EasyMock.expectLastCall().once();
    900         }
    901 
    902         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    903         EasyMock.expectLastCall().once();
    904 
    905         EasyMock.replay(mockDevice, mockIDevice);
    906         EasyMock.replay(mockListener);
    907 
    908         deqpTest.setDevice(mockDevice);
    909         deqpTest.run(mockListener);
    910 
    911         EasyMock.verify(mockListener);
    912         EasyMock.verify(mockDevice, mockIDevice);
    913     }
    914 
    915     /**
    916      * Test that test are left unexecuted if pm list query fails
    917      */
    918     public void testRun_queryPmListFailure()
    919             throws Exception {
    920         final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test");
    921 
    922         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
    923         ITestInvocationListener mockListener
    924                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    925         Collection<TestDescription> tests = new ArrayList<TestDescription>();
    926         tests.add(testId);
    927 
    928         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
    929         OptionSetter setter = new OptionSetter(deqpTest);
    930         // Note: If the rotation is the default unspecified, features are not queried at all
    931         setter.setOptionValue("deqp-screen-rotation", "90");
    932 
    933         deqpTest.setDevice(mockDevice);
    934 
    935         int version = 3 << 16;
    936         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
    937                 .andReturn(Integer.toString(version)).atLeastOnce();
    938 
    939         EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
    940                 .andReturn("not a valid format");
    941 
    942         mockListener.testRunStarted(getTestId(deqpTest), 1);
    943         EasyMock.expectLastCall().once();
    944         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
    945             andReturn("").once();
    946         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
    947             andReturn("").once();
    948 
    949         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    950         EasyMock.expectLastCall().once();
    951 
    952         EasyMock.replay(mockDevice);
    953         EasyMock.replay(mockListener);
    954         deqpTest.run(mockListener);
    955         EasyMock.verify(mockListener);
    956         EasyMock.verify(mockDevice);
    957     }
    958 
    959     /**
    960      * Test that test are left unexecuted if renderablity query fails
    961      */
    962     public void testRun_queryRenderabilityFailure()
    963             throws Exception {
    964         final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test");
    965 
    966         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
    967         ITestInvocationListener mockListener
    968                 = EasyMock.createStrictMock(ITestInvocationListener.class);
    969 
    970         Collection<TestDescription> tests = new ArrayList<TestDescription>();
    971         tests.add(testId);
    972 
    973         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
    974 
    975         deqpTest.setDevice(mockDevice);
    976 
    977         int version = 3 << 16;
    978         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
    979                 .andReturn(Integer.toString(version)).atLeastOnce();
    980 
    981         expectRenderConfigQueryAndReturn(mockDevice,
    982                 "--deqp-gl-config-name=rgba8888d24s8 "
    983                 + "--deqp-screen-rotation=unspecified "
    984                 + "--deqp-surface-type=window "
    985                 + "--deqp-gl-major-version=3 "
    986                 + "--deqp-gl-minor-version=0", "Maybe?");
    987 
    988         mockListener.testRunStarted(getTestId(deqpTest), 1);
    989         EasyMock.expectLastCall().once();
    990 
    991         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
    992             andReturn("").once();
    993         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
    994             andReturn("").once();
    995         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
    996         EasyMock.expectLastCall().once();
    997 
    998         EasyMock.replay(mockDevice);
    999         EasyMock.replay(mockListener);
   1000         deqpTest.run(mockListener);
   1001         EasyMock.verify(mockListener);
   1002         EasyMock.verify(mockDevice);
   1003     }
   1004 
   1005     /**
   1006      * Test that orientation is supplied to runner correctly
   1007      */
   1008     private void testOrientation(final String rotation, final String featureString)
   1009             throws Exception {
   1010         final TestDescription testId = new TestDescription("dEQP-GLES3.orientation", "test");
   1011         final String testPath = "dEQP-GLES3.orientation.test";
   1012         final String testTrie = "{dEQP-GLES3{orientation{test}}}";
   1013         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
   1014                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1015                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
   1016                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1017                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
   1018                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1019                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
   1020                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1021                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
   1022                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1023                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
   1024                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1025                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
   1026                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1027                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1028                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
   1029                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1030                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
   1031                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
   1032                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1033                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1034                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1035                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1036                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
   1037                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1038                 + "INSTRUMENTATION_CODE: 0\r\n";
   1039 
   1040         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   1041         ITestInvocationListener mockListener
   1042                 = EasyMock.createStrictMock(ITestInvocationListener.class);
   1043         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
   1044 
   1045         Collection<TestDescription> tests = new ArrayList<TestDescription>();
   1046         tests.add(testId);
   1047 
   1048         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   1049         OptionSetter setter = new OptionSetter(deqpTest);
   1050         setter.setOptionValue("deqp-screen-rotation", rotation);
   1051 
   1052         deqpTest.setDevice(mockDevice);
   1053 
   1054         int version = 3 << 16;
   1055         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
   1056                 .andReturn(Integer.toString(version)).atLeastOnce();
   1057 
   1058         if (!rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED)) {
   1059             EasyMock.expect(mockDevice.executeShellCommand("pm list features"))
   1060                     .andReturn(featureString);
   1061         }
   1062 
   1063         final boolean isPortraitOrientation =
   1064                 rotation.equals(BatchRunConfiguration.ROTATION_PORTRAIT) ||
   1065                 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_PORTRAIT);
   1066         final boolean isLandscapeOrientation =
   1067                 rotation.equals(BatchRunConfiguration.ROTATION_LANDSCAPE) ||
   1068                 rotation.equals(BatchRunConfiguration.ROTATION_REVERSE_LANDSCAPE);
   1069         final boolean executable =
   1070                 rotation.equals(BatchRunConfiguration.ROTATION_UNSPECIFIED) ||
   1071                 (isPortraitOrientation &&
   1072                 featureString.contains(DeqpTestRunner.FEATURE_PORTRAIT)) ||
   1073                 (isLandscapeOrientation &&
   1074                 featureString.contains(DeqpTestRunner.FEATURE_LANDSCAPE));
   1075 
   1076         if (executable) {
   1077             expectRenderConfigQuery(mockDevice, String.format(
   1078                     "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=%s "
   1079                     + "--deqp-surface-type=window --deqp-gl-major-version=3 "
   1080                     + "--deqp-gl-minor-version=0", rotation));
   1081 
   1082             String commandLine = String.format(
   1083                     "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
   1084                     + "--deqp-screen-rotation=%s "
   1085                     + "--deqp-surface-type=window "
   1086                     + "--deqp-log-images=disable "
   1087                     + "--deqp-watchdog=enable",
   1088                     APP_DIR + CASE_LIST_FILE_NAME, rotation);
   1089 
   1090             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
   1091                     output);
   1092         }
   1093 
   1094         mockListener.testRunStarted(getTestId(deqpTest), 1);
   1095         EasyMock.expectLastCall().once();
   1096 
   1097         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
   1098             andReturn("").once();
   1099         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
   1100             andReturn("").once();
   1101 
   1102         mockListener.testStarted(EasyMock.eq(testId));
   1103         EasyMock.expectLastCall().once();
   1104 
   1105         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull());
   1106         EasyMock.expectLastCall().once();
   1107 
   1108         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
   1109         EasyMock.expectLastCall().once();
   1110 
   1111         EasyMock.replay(mockDevice, mockIDevice);
   1112         EasyMock.replay(mockListener);
   1113         deqpTest.run(mockListener);
   1114         EasyMock.verify(mockListener);
   1115         EasyMock.verify(mockDevice, mockIDevice);
   1116     }
   1117 
   1118     /**
   1119      * Test OpeGL ES3 tests on device with OpenGL ES2.
   1120      */
   1121     public void testRun_require30DeviceVersion20() throws Exception {
   1122         testGlesVersion(3, 0, 2, 0);
   1123     }
   1124 
   1125     /**
   1126      * Test OpeGL ES3.1 tests on device with OpenGL ES2.
   1127      */
   1128     public void testRun_require31DeviceVersion20() throws Exception {
   1129         testGlesVersion(3, 1, 2, 0);
   1130     }
   1131 
   1132     /**
   1133      * Test OpeGL ES3 tests on device with OpenGL ES3.
   1134      */
   1135     public void testRun_require30DeviceVersion30() throws Exception {
   1136         testGlesVersion(3, 0, 3, 0);
   1137     }
   1138 
   1139     /**
   1140      * Test OpeGL ES3.1 tests on device with OpenGL ES3.
   1141      */
   1142     public void testRun_require31DeviceVersion30() throws Exception {
   1143         testGlesVersion(3, 1, 3, 0);
   1144     }
   1145 
   1146     /**
   1147      * Test OpeGL ES3 tests on device with OpenGL ES3.1.
   1148      */
   1149     public void testRun_require30DeviceVersion31() throws Exception {
   1150         testGlesVersion(3, 0, 3, 1);
   1151     }
   1152 
   1153     /**
   1154      * Test OpeGL ES3.1 tests on device with OpenGL ES3.1.
   1155      */
   1156     public void testRun_require31DeviceVersion31() throws Exception {
   1157         testGlesVersion(3, 1, 3, 1);
   1158     }
   1159 
   1160     /**
   1161      * Test dEQP Pass result code.
   1162      */
   1163     public void testRun_resultPass() throws Exception {
   1164         testResultCode("Pass", true);
   1165     }
   1166 
   1167     /**
   1168      * Test dEQP Fail result code.
   1169      */
   1170     public void testRun_resultFail() throws Exception {
   1171         testResultCode("Fail", false);
   1172     }
   1173 
   1174     /**
   1175      * Test dEQP NotSupported result code.
   1176      */
   1177     public void testRun_resultNotSupported() throws Exception {
   1178         testResultCode("NotSupported", true);
   1179     }
   1180 
   1181     /**
   1182      * Test dEQP QualityWarning result code.
   1183      */
   1184     public void testRun_resultQualityWarning() throws Exception {
   1185         testResultCode("QualityWarning", true);
   1186     }
   1187 
   1188     /**
   1189      * Test dEQP CompatibilityWarning result code.
   1190      */
   1191     public void testRun_resultCompatibilityWarning() throws Exception {
   1192         testResultCode("CompatibilityWarning", true);
   1193     }
   1194 
   1195     /**
   1196      * Test dEQP ResourceError result code.
   1197      */
   1198     public void testRun_resultResourceError() throws Exception {
   1199         testResultCode("ResourceError", false);
   1200     }
   1201 
   1202     /**
   1203      * Test dEQP InternalError result code.
   1204      */
   1205     public void testRun_resultInternalError() throws Exception {
   1206         testResultCode("InternalError", false);
   1207     }
   1208 
   1209     /**
   1210      * Test dEQP Crash result code.
   1211      */
   1212     public void testRun_resultCrash() throws Exception {
   1213         testResultCode("Crash", false);
   1214     }
   1215 
   1216     /**
   1217      * Test dEQP Timeout result code.
   1218      */
   1219     public void testRun_resultTimeout() throws Exception {
   1220         testResultCode("Timeout", false);
   1221     }
   1222     /**
   1223      * Test dEQP Orientation
   1224      */
   1225     public void testRun_orientationLandscape() throws Exception {
   1226         testOrientation("90", ALL_FEATURES);
   1227     }
   1228 
   1229     /**
   1230      * Test dEQP Orientation
   1231      */
   1232     public void testRun_orientationPortrait() throws Exception {
   1233         testOrientation("0", ALL_FEATURES);
   1234     }
   1235 
   1236     /**
   1237      * Test dEQP Orientation
   1238      */
   1239     public void testRun_orientationReverseLandscape() throws Exception {
   1240         testOrientation("270", ALL_FEATURES);
   1241     }
   1242 
   1243     /**
   1244      * Test dEQP Orientation
   1245      */
   1246     public void testRun_orientationReversePortrait() throws Exception {
   1247         testOrientation("180", ALL_FEATURES);
   1248     }
   1249 
   1250     /**
   1251      * Test dEQP Orientation
   1252      */
   1253     public void testRun_orientationUnspecified() throws Exception {
   1254         testOrientation("unspecified", ALL_FEATURES);
   1255     }
   1256 
   1257     /**
   1258      * Test dEQP Orientation with limited features
   1259      */
   1260     public void testRun_orientationUnspecifiedLimitedFeatures() throws Exception {
   1261         testOrientation("unspecified", ONLY_LANDSCAPE_FEATURES);
   1262     }
   1263 
   1264     /**
   1265      * Test dEQP Orientation with limited features
   1266      */
   1267     public void testRun_orientationLandscapeLimitedFeatures() throws Exception {
   1268         testOrientation("90", ONLY_LANDSCAPE_FEATURES);
   1269     }
   1270 
   1271     /**
   1272      * Test dEQP Orientation with limited features
   1273      */
   1274     public void testRun_orientationPortraitLimitedFeatures() throws Exception {
   1275         testOrientation("0", ONLY_LANDSCAPE_FEATURES);
   1276     }
   1277 
   1278     /**
   1279      * Test dEQP unsupported pixel format
   1280      */
   1281     public void testRun_unsupportedPixelFormat() throws Exception {
   1282         final String pixelFormat = "rgba5658d16m4";
   1283         final TestDescription testId = new TestDescription("dEQP-GLES3.pixelformat", "test");
   1284 
   1285         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   1286         ITestInvocationListener mockListener
   1287                 = EasyMock.createStrictMock(ITestInvocationListener.class);
   1288 
   1289         Collection<TestDescription> tests = new ArrayList<TestDescription>();
   1290         tests.add(testId);
   1291 
   1292         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   1293         OptionSetter setter = new OptionSetter(deqpTest);
   1294         setter.setOptionValue("deqp-gl-config-name", pixelFormat);
   1295 
   1296         deqpTest.setDevice(mockDevice);
   1297 
   1298         int version = 3 << 16;
   1299         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
   1300                 .andReturn(Integer.toString(version)).atLeastOnce();
   1301 
   1302         expectRenderConfigQueryAndReturn(mockDevice, String.format(
   1303                 "--deqp-gl-config-name=%s --deqp-screen-rotation=unspecified "
   1304                 + "--deqp-surface-type=window "
   1305                 + "--deqp-gl-major-version=3 "
   1306                 + "--deqp-gl-minor-version=0", pixelFormat), "No");
   1307 
   1308         mockListener.testRunStarted(getTestId(deqpTest), 1);
   1309         EasyMock.expectLastCall().once();
   1310 
   1311         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
   1312             andReturn("").once();
   1313         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
   1314             andReturn("").once();
   1315 
   1316         mockListener.testStarted(EasyMock.eq(testId));
   1317         EasyMock.expectLastCall().once();
   1318 
   1319         mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull());
   1320         EasyMock.expectLastCall().once();
   1321 
   1322         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
   1323         EasyMock.expectLastCall().once();
   1324 
   1325         EasyMock.replay(mockDevice);
   1326         EasyMock.replay(mockListener);
   1327         deqpTest.run(mockListener);
   1328         EasyMock.verify(mockListener);
   1329         EasyMock.verify(mockDevice);
   1330     }
   1331 
   1332     /**
   1333      * Test interface to mock Tradefed device types.
   1334      */
   1335     public static interface RecoverableTestDevice extends ITestDevice, IManagedTestDevice {
   1336     }
   1337 
   1338     private static enum RecoveryEvent {
   1339         PROGRESS,
   1340         FAIL_CONNECTION_REFUSED,
   1341         FAIL_LINK_KILLED,
   1342     }
   1343 
   1344     private void runRecoveryWithPattern(DeqpTestRunner.Recovery recovery, RecoveryEvent[] events)
   1345             throws DeviceNotAvailableException {
   1346         for (RecoveryEvent event : events) {
   1347             switch (event) {
   1348                 case PROGRESS:
   1349                     recovery.onExecutionProgressed();
   1350                     break;
   1351                 case FAIL_CONNECTION_REFUSED:
   1352                     recovery.recoverConnectionRefused();
   1353                     break;
   1354                 case FAIL_LINK_KILLED:
   1355                     recovery.recoverComLinkKilled();
   1356                     break;
   1357             }
   1358         }
   1359     }
   1360 
   1361     private void setRecoveryExpectationWait(DeqpTestRunner.ISleepProvider mockSleepProvider) {
   1362         mockSleepProvider.sleep(EasyMock.gt(0));
   1363         EasyMock.expectLastCall().once();
   1364     }
   1365 
   1366     private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice,
   1367             DeqpTestRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException {
   1368         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
   1369                 andReturn("root 1234 com.drawelement.deqp").once();
   1370 
   1371         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
   1372                 andReturn("").once();
   1373 
   1374         // Recovery checks if kill failed
   1375         mockSleepProvider.sleep(EasyMock.gt(0));
   1376         EasyMock.expectLastCall().once();
   1377         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
   1378                 andReturn("").once();
   1379     }
   1380 
   1381     private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)
   1382             throws DeviceNotAvailableException {
   1383         mockDevice.recoverDevice();
   1384         EasyMock.expectLastCall().once();
   1385     }
   1386 
   1387     private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)
   1388             throws DeviceNotAvailableException {
   1389         mockDevice.reboot();
   1390         EasyMock.expectLastCall().once();
   1391     }
   1392 
   1393     private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice,
   1394             DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
   1395             throws DeviceNotAvailableException {
   1396         switch (numConsecutiveErrors) {
   1397             case 0:
   1398             case 1:
   1399                 setRecoveryExpectationRecovery(mockDevice);
   1400                 return 2;
   1401             case 2:
   1402                 setRecoveryExpectationReboot(mockDevice);
   1403                 return 3;
   1404             default:
   1405                 return 4;
   1406         }
   1407     }
   1408 
   1409     private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice,
   1410             DeqpTestRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)
   1411             throws DeviceNotAvailableException {
   1412         switch (numConsecutiveErrors) {
   1413             case 0:
   1414                 setRecoveryExpectationWait(mockSleepProvider);
   1415                 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
   1416                 return 1;
   1417             case 1:
   1418                 setRecoveryExpectationRecovery(mockDevice);
   1419                 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider);
   1420                 return 2;
   1421             case 2:
   1422                 setRecoveryExpectationReboot(mockDevice);
   1423                 return 3;
   1424             default:
   1425                 return 4;
   1426         }
   1427     }
   1428 
   1429     private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice,
   1430             DeqpTestRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)
   1431             throws DeviceNotAvailableException {
   1432         int numConsecutiveErrors = 0;
   1433         for (RecoveryEvent event : events) {
   1434             switch (event) {
   1435                 case PROGRESS:
   1436                     numConsecutiveErrors = 0;
   1437                     break;
   1438                 case FAIL_CONNECTION_REFUSED:
   1439                     numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice,
   1440                             mockSleepProvider, numConsecutiveErrors);
   1441                     break;
   1442                 case FAIL_LINK_KILLED:
   1443                     numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice,
   1444                             mockSleepProvider, numConsecutiveErrors);
   1445                     break;
   1446             }
   1447         }
   1448     }
   1449 
   1450     /**
   1451      * Test dEQP runner recovery state machine.
   1452      */
   1453     private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)
   1454             throws Exception {
   1455         DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
   1456         IMocksControl orderedControl = EasyMock.createStrictControl();
   1457         RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
   1458         EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL");
   1459         DeqpTestRunner.ISleepProvider mockSleepProvider =
   1460                 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
   1461 
   1462         setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern);
   1463 
   1464         orderedControl.replay();
   1465 
   1466         recovery.setDevice(mockDevice);
   1467         recovery.setSleepProvider(mockSleepProvider);
   1468         try {
   1469             runRecoveryWithPattern(recovery, pattern);
   1470             if (!expectSuccess) {
   1471                 fail("Expected DeviceNotAvailableException");
   1472             }
   1473         } catch (DeviceNotAvailableException ex) {
   1474             if (expectSuccess) {
   1475                 fail("Did not expect DeviceNotAvailableException");
   1476             }
   1477         }
   1478 
   1479         orderedControl.verify();
   1480     }
   1481 
   1482     // basic patterns
   1483 
   1484     public void testRecovery_NoEvents() throws Exception {
   1485         testRecoveryWithPattern(true);
   1486     }
   1487 
   1488     public void testRecovery_AllOk() throws Exception {
   1489         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS);
   1490     }
   1491 
   1492     // conn fail patterns
   1493 
   1494     public void testRecovery_OneConnectionFailureBegin() throws Exception {
   1495         testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
   1496                 RecoveryEvent.PROGRESS);
   1497     }
   1498 
   1499     public void testRecovery_TwoConnectionFailuresBegin() throws Exception {
   1500         testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED,
   1501                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
   1502     }
   1503 
   1504     public void testRecovery_ThreeConnectionFailuresBegin() throws Exception {
   1505         testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED,
   1506                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED);
   1507     }
   1508 
   1509     public void testRecovery_OneConnectionFailureMid() throws Exception {
   1510         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
   1511                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS);
   1512     }
   1513 
   1514     public void testRecovery_TwoConnectionFailuresMid() throws Exception {
   1515         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
   1516                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
   1517                 RecoveryEvent.PROGRESS);
   1518     }
   1519 
   1520     public void testRecovery_ThreeConnectionFailuresMid() throws Exception {
   1521         testRecoveryWithPattern(false, RecoveryEvent.PROGRESS,
   1522                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED,
   1523                 RecoveryEvent.FAIL_CONNECTION_REFUSED);
   1524     }
   1525 
   1526     // link fail patterns
   1527 
   1528     public void testRecovery_OneLinkFailureBegin() throws Exception {
   1529         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
   1530                 RecoveryEvent.PROGRESS);
   1531     }
   1532 
   1533     public void testRecovery_TwoLinkFailuresBegin() throws Exception {
   1534         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
   1535                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
   1536     }
   1537 
   1538     public void testRecovery_ThreeLinkFailuresBegin() throws Exception {
   1539         testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED,
   1540                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
   1541                 RecoveryEvent.PROGRESS);
   1542     }
   1543 
   1544     public void testRecovery_FourLinkFailuresBegin() throws Exception {
   1545         testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED,
   1546                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
   1547                 RecoveryEvent.FAIL_LINK_KILLED);
   1548     }
   1549 
   1550     public void testRecovery_OneLinkFailureMid() throws Exception {
   1551         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
   1552                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
   1553     }
   1554 
   1555     public void testRecovery_TwoLinkFailuresMid() throws Exception {
   1556         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
   1557                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
   1558                 RecoveryEvent.PROGRESS);
   1559     }
   1560 
   1561     public void testRecovery_ThreeLinkFailuresMid() throws Exception {
   1562         testRecoveryWithPattern(true, RecoveryEvent.PROGRESS,
   1563                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
   1564                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS);
   1565     }
   1566 
   1567     public void testRecovery_FourLinkFailuresMid() throws Exception {
   1568         testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
   1569                 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED,
   1570                 RecoveryEvent.FAIL_LINK_KILLED);
   1571     }
   1572 
   1573     // mixed patterns
   1574 
   1575     public void testRecovery_MixedFailuresProgressBetween() throws Exception {
   1576         testRecoveryWithPattern(true,
   1577                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
   1578                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
   1579                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
   1580                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED,
   1581                 RecoveryEvent.PROGRESS);
   1582     }
   1583 
   1584     public void testRecovery_MixedFailuresNoProgressBetween() throws Exception {
   1585         testRecoveryWithPattern(true,
   1586                 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED,
   1587                 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED,
   1588                 RecoveryEvent.PROGRESS);
   1589     }
   1590 
   1591     /**
   1592      * Test recovery if process cannot be killed
   1593      */
   1594     public void testRecovery_unkillableProcess () throws Exception {
   1595         DeqpTestRunner.Recovery recovery = new DeqpTestRunner.Recovery();
   1596         IMocksControl orderedControl = EasyMock.createStrictControl();
   1597         RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class);
   1598         DeqpTestRunner.ISleepProvider mockSleepProvider =
   1599                 orderedControl.createMock(DeqpTestRunner.ISleepProvider.class);
   1600 
   1601         // recovery attempts to kill the process after a timeout
   1602         mockSleepProvider.sleep(EasyMock.gt(0));
   1603         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
   1604                 andReturn("root 1234 com.drawelement.deqp").once();
   1605         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
   1606                 andReturn("").once();
   1607 
   1608         // Recovery checks if kill failed
   1609         mockSleepProvider.sleep(EasyMock.gt(0));
   1610         EasyMock.expectLastCall().once();
   1611         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
   1612                 andReturn("root 1234 com.drawelement.deqp").once();
   1613 
   1614         // Recovery resets the connection
   1615         mockDevice.recoverDevice();
   1616         EasyMock.expectLastCall().once();
   1617 
   1618         // and attempts to kill the process again
   1619         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
   1620                 andReturn("root 1234 com.drawelement.deqp").once();
   1621         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))).
   1622                 andReturn("").once();
   1623 
   1624         // Recovery checks if kill failed
   1625         mockSleepProvider.sleep(EasyMock.gt(0));
   1626         EasyMock.expectLastCall().once();
   1627         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))).
   1628                 andReturn("root 1234 com.drawelement.deqp").once();
   1629 
   1630         // recovery reboots the device
   1631         mockDevice.reboot();
   1632         EasyMock.expectLastCall().once();
   1633 
   1634         orderedControl.replay();
   1635         recovery.setDevice(mockDevice);
   1636         recovery.setSleepProvider(mockSleepProvider);
   1637         recovery.recoverComLinkKilled();
   1638         orderedControl.verify();
   1639     }
   1640 
   1641     /**
   1642      * Test external interruption before batch run.
   1643      */
   1644     public void testInterrupt_killBeforeBatch() throws Exception {
   1645         final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test");
   1646 
   1647         Collection<TestDescription> tests = new ArrayList<TestDescription>();
   1648         tests.add(testId);
   1649 
   1650         ITestInvocationListener mockListener
   1651                 = EasyMock.createStrictMock(ITestInvocationListener.class);
   1652         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   1653         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
   1654         IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
   1655 
   1656         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   1657 
   1658         deqpTest.setDevice(mockDevice);
   1659         deqpTest.setRunUtil(mockRunUtil);
   1660 
   1661         int version = 3 << 16;
   1662         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
   1663                 .andReturn(Integer.toString(version)).atLeastOnce();
   1664 
   1665         expectRenderConfigQuery(mockDevice,
   1666                 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
   1667                 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
   1668                 + "--deqp-gl-minor-version=0");
   1669 
   1670         mockRunUtil.sleep(0);
   1671         EasyMock.expectLastCall().andThrow(new RunInterruptedException());
   1672 
   1673         mockListener.testRunStarted(getTestId(deqpTest), 1);
   1674         EasyMock.expectLastCall().once();
   1675         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
   1676             andReturn("").once();
   1677         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
   1678             andReturn("").once();
   1679         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
   1680         EasyMock.expectLastCall().once();
   1681 
   1682         EasyMock.replay(mockDevice, mockIDevice);
   1683         EasyMock.replay(mockListener);
   1684         EasyMock.replay(mockRunUtil);
   1685         try {
   1686             deqpTest.run(mockListener);
   1687             fail("expected RunInterruptedException");
   1688         } catch (RunInterruptedException ex) {
   1689             // expected
   1690         }
   1691         EasyMock.verify(mockRunUtil);
   1692         EasyMock.verify(mockListener);
   1693         EasyMock.verify(mockDevice, mockIDevice);
   1694     }
   1695 
   1696     private void runShardedTest(TestDescription[] testIds,
   1697             ArrayList<ArrayList<TestDescription>> testsForShard) throws Exception {
   1698         Collection<TestDescription> tests = new ArrayList<TestDescription>();
   1699         for (TestDescription id : testIds) tests.add(id);
   1700 
   1701         DeqpTestRunner runner = buildGlesTestRunner(3, 0, tests, mTestsDir);
   1702         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
   1703 
   1704         for (int shardIndex = 0; shardIndex < shards.size(); shardIndex++) {
   1705             DeqpTestRunner shard = (DeqpTestRunner)shards.get(shardIndex);
   1706             shard.setBuildHelper(getMockBuildHelper(mTestsDir));
   1707 
   1708             ArrayList<TestDescription> shardTests = testsForShard.get(shardIndex);
   1709 
   1710             ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   1711             ITestInvocationListener mockListener
   1712                     = EasyMock.createStrictMock(ITestInvocationListener.class);
   1713             IDevice mockIDevice = EasyMock.createMock(IDevice.class);
   1714             int version = 3 << 16;
   1715             EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
   1716                     .andReturn(Integer.toString(version)).atLeastOnce();
   1717 
   1718             mockListener.testRunStarted(getTestId(shard), shardTests.size());
   1719             EasyMock.expectLastCall().once();
   1720 
   1721             EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
   1722                 andReturn("").once();
   1723             EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
   1724                 andReturn("").once();
   1725 
   1726             expectRenderConfigQuery(mockDevice, 3, 0);
   1727 
   1728             String testOut = buildTestProcessOutput(shardTests);
   1729             // NOTE: This assumes that there won't be multiple batches per shard!
   1730             runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testOut);
   1731 
   1732             for (int i = 0; i < shardTests.size(); i++) {
   1733                 mockListener.testStarted(EasyMock.eq(shardTests.get(i)));
   1734                 EasyMock.expectLastCall().once();
   1735 
   1736                 mockListener.testEnded(EasyMock.eq(shardTests.get(i)),
   1737                                        EasyMock.<HashMap<String, Metric>>notNull());
   1738 
   1739                 EasyMock.expectLastCall().once();
   1740             }
   1741 
   1742             mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
   1743             EasyMock.expectLastCall().once();
   1744 
   1745             EasyMock.replay(mockDevice, mockIDevice);
   1746             EasyMock.replay(mockListener);
   1747 
   1748             shard.setDevice(mockDevice);
   1749             shard.run(mockListener);
   1750 
   1751             EasyMock.verify(mockListener);
   1752             EasyMock.verify(mockDevice, mockIDevice);
   1753         }
   1754     }
   1755 
   1756     public void testSharding_smallTrivial() throws Exception {
   1757         final TestDescription[] testIds = {
   1758                 new TestDescription("dEQP-GLES3.info", "vendor"),
   1759                 new TestDescription("dEQP-GLES3.info", "renderer"),
   1760                 new TestDescription("dEQP-GLES3.info", "version"),
   1761                 new TestDescription("dEQP-GLES3.info", "shading_language_version"),
   1762                 new TestDescription("dEQP-GLES3.info", "extensions"),
   1763                 new TestDescription("dEQP-GLES3.info", "render_target")
   1764         };
   1765         ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>();
   1766         ArrayList<TestDescription> shardOne = new ArrayList<>();
   1767         for (int i = 0; i < testIds.length; i++) {
   1768             shardOne.add(testIds[i]);
   1769         }
   1770         shardedTests.add(shardOne);
   1771         runShardedTest(testIds, shardedTests);
   1772     }
   1773 
   1774     public void testSharding_twoShards() throws Exception {
   1775         final int TEST_COUNT = 1237;
   1776         final int SHARD_SIZE = 1000;
   1777 
   1778         ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT);
   1779         for (int i = 0; i < TEST_COUNT; i++) {
   1780             testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i)));
   1781         }
   1782 
   1783         ArrayList<ArrayList<TestDescription>> shardedTests = new ArrayList<>();
   1784         ArrayList<TestDescription> shard = new ArrayList<>();
   1785         for (int i = 0; i < testIds.size(); i++) {
   1786             if (i == SHARD_SIZE) {
   1787                 shardedTests.add(shard);
   1788                 shard = new ArrayList<>();
   1789             }
   1790             shard.add(testIds.get(i));
   1791         }
   1792         shardedTests.add(shard);
   1793         runShardedTest(testIds.toArray(new TestDescription[testIds.size()]), shardedTests);
   1794     }
   1795 
   1796     public void testSharding_empty() throws Exception {
   1797         DeqpTestRunner runner = buildGlesTestRunner(3, 0, new ArrayList<TestDescription>(), mTestsDir);
   1798         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)runner.split();
   1799         // Returns null when cannot be sharded.
   1800         assertNull(shards);
   1801     }
   1802 
   1803     /**
   1804      * Test external interruption in testFailed().
   1805      */
   1806     public void testInterrupt_killReportTestFailed() throws Exception {
   1807         final TestDescription testId = new TestDescription("dEQP-GLES3.interrupt", "test");
   1808         final String testPath = "dEQP-GLES3.interrupt.test";
   1809         final String testTrie = "{dEQP-GLES3{interrupt{test}}}";
   1810         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
   1811                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1812                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
   1813                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1814                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
   1815                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1816                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
   1817                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1818                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
   1819                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1820                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
   1821                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1822                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
   1823                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1824                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1825                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n"
   1826                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1827                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Fail\r\n"
   1828                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Fail\r\n"
   1829                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1830                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1831                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1832                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1833                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
   1834                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1835                 + "INSTRUMENTATION_CODE: 0\r\n";
   1836 
   1837         Collection<TestDescription> tests = new ArrayList<TestDescription>();
   1838         tests.add(testId);
   1839 
   1840         ITestInvocationListener mockListener
   1841                 = EasyMock.createStrictMock(ITestInvocationListener.class);
   1842         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   1843         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
   1844         IRunUtil mockRunUtil = EasyMock.createMock(IRunUtil.class);
   1845 
   1846         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   1847 
   1848         deqpTest.setDevice(mockDevice);
   1849         deqpTest.setRunUtil(mockRunUtil);
   1850 
   1851         int version = 3 << 16;
   1852         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
   1853                 .andReturn(Integer.toString(version)).atLeastOnce();
   1854 
   1855         expectRenderConfigQuery(mockDevice,
   1856                 "--deqp-gl-config-name=rgba8888d24s8 --deqp-screen-rotation=unspecified "
   1857                 + "--deqp-surface-type=window --deqp-gl-major-version=3 "
   1858                 + "--deqp-gl-minor-version=0");
   1859 
   1860         mockRunUtil.sleep(0);
   1861         EasyMock.expectLastCall().once();
   1862 
   1863         String commandLine = String.format(
   1864                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
   1865                 + "--deqp-screen-rotation=unspecified "
   1866                 + "--deqp-surface-type=window "
   1867                 + "--deqp-log-images=disable "
   1868                 + "--deqp-watchdog=enable",
   1869                 APP_DIR + CASE_LIST_FILE_NAME);
   1870 
   1871         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine,
   1872                 output);
   1873 
   1874         mockListener.testRunStarted(getTestId(deqpTest), 1);
   1875         EasyMock.expectLastCall().once();
   1876 
   1877         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
   1878             andReturn("").once();
   1879         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
   1880             andReturn("").once();
   1881 
   1882         mockListener.testStarted(EasyMock.eq(testId));
   1883         EasyMock.expectLastCall().once();
   1884 
   1885         mockListener.testFailed(EasyMock.eq(testId), EasyMock.<String>notNull());
   1886         EasyMock.expectLastCall().andThrow(new RunInterruptedException());
   1887 
   1888         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>anyObject());
   1889         EasyMock.expectLastCall().once();
   1890         EasyMock.replay(mockDevice, mockIDevice);
   1891         EasyMock.replay(mockListener);
   1892         EasyMock.replay(mockRunUtil);
   1893         try {
   1894             deqpTest.run(mockListener);
   1895             fail("expected RunInterruptedException");
   1896         } catch (RunInterruptedException ex) {
   1897             // expected
   1898         }
   1899         EasyMock.verify(mockRunUtil);
   1900         EasyMock.verify(mockListener);
   1901         EasyMock.verify(mockDevice, mockIDevice);
   1902     }
   1903 
   1904     public void testRuntimeHint_optionSet() throws Exception {
   1905         /* MultiLineReceiver expects "\r\n" line ending. */
   1906         final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n"
   1907                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1908                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n"
   1909                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1910                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n"
   1911                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1912                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n"
   1913                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1914                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n"
   1915                 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n"
   1916                 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n"
   1917                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1918                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n"
   1919                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1920                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1921                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.vendor\r\n"
   1922                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1923                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
   1924                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
   1925                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1926                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1927                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1928                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1929                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1930                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.renderer\r\n"
   1931                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1932                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
   1933                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
   1934                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1935                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1936                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1937                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1938                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1939                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.version\r\n"
   1940                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1941                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
   1942                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
   1943                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1944                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1945                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1946                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1947                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1948                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.shading_language_version\r\n"
   1949                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1950                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
   1951                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
   1952                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1953                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1954                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1955                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1956                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1957                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.extensions\r\n"
   1958                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1959                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
   1960                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
   1961                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1962                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1963                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1964                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1965                 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"
   1966                 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=dEQP-GLES3.info.render_target\r\n"
   1967                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1968                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"
   1969                 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"
   1970                 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"
   1971                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1972                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"
   1973                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1974                 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n"
   1975                 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"
   1976                 + "INSTRUMENTATION_CODE: 0\r\n";
   1977 
   1978         final TestDescription[] testIds = {
   1979                 new TestDescription("dEQP-GLES3.info", "vendor"),
   1980                 new TestDescription("dEQP-GLES3.info", "renderer"),
   1981                 new TestDescription("dEQP-GLES3.info", "version"),
   1982                 new TestDescription("dEQP-GLES3.info", "shading_language_version"),
   1983                 new TestDescription("dEQP-GLES3.info", "extensions"),
   1984                 new TestDescription("dEQP-GLES3.info", "render_target")
   1985         };
   1986 
   1987         final String[] testPaths = {
   1988                 "dEQP-GLES3.info.vendor",
   1989                 "dEQP-GLES3.info.renderer",
   1990                 "dEQP-GLES3.info.version",
   1991                 "dEQP-GLES3.info.shading_language_version",
   1992                 "dEQP-GLES3.info.extensions",
   1993                 "dEQP-GLES3.info.render_target"
   1994         };
   1995 
   1996         final String testTrie
   1997                 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}";
   1998 
   1999         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   2000         ITestInvocationListener mockListener
   2001                 = EasyMock.createStrictMock(ITestInvocationListener.class);
   2002         IDevice mockIDevice = EasyMock.createMock(IDevice.class);
   2003 
   2004         Collection<TestDescription> tests = new ArrayList<TestDescription>();
   2005 
   2006         for (TestDescription id : testIds) {
   2007             tests.add(id);
   2008         }
   2009 
   2010         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   2011         OptionSetter setter = new OptionSetter(deqpTest);
   2012         final long runtimeMs = 123456;
   2013         setter.setOptionValue("runtime-hint", String.valueOf(runtimeMs));
   2014         assertEquals("Wrong expected runtime - option not passed cleanly", runtimeMs, deqpTest.getRuntimeHint());
   2015 
   2016         // Try running the tests as well. The unit tests do not set the hint be default,
   2017         // so that case is covered.
   2018 
   2019         int version = 3 << 16;
   2020         EasyMock.expect(mockDevice.getProperty("ro.opengles.version"))
   2021                 .andReturn(Integer.toString(version)).atLeastOnce();
   2022 
   2023         expectRenderConfigQuery(mockDevice, 3, 0);
   2024 
   2025         String commandLine = String.format(
   2026                 "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
   2027                 + "--deqp-screen-rotation=unspecified "
   2028                 + "--deqp-surface-type=window "
   2029                 + "--deqp-log-images=disable "
   2030                 + "--deqp-watchdog=enable",
   2031                 APP_DIR + CASE_LIST_FILE_NAME);
   2032 
   2033         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, commandLine, output);
   2034 
   2035         mockListener.testRunStarted(getTestId(deqpTest), testPaths.length);
   2036         EasyMock.expectLastCall().once();
   2037 
   2038         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_pkgs \"\""))).
   2039             andReturn("").once();
   2040         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings put global angle_gl_driver_selection_values \"\""))).
   2041             andReturn("").once();
   2042 
   2043         for (int i = 0; i < testPaths.length; i++) {
   2044             mockListener.testStarted(EasyMock.eq(testIds[i]));
   2045             EasyMock.expectLastCall().once();
   2046 
   2047             mockListener.testEnded(EasyMock.eq(testIds[i]),
   2048                     EasyMock.<HashMap<String, Metric>>notNull());
   2049 
   2050             EasyMock.expectLastCall().once();
   2051         }
   2052 
   2053         mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull());
   2054         EasyMock.expectLastCall().once();
   2055 
   2056         EasyMock.replay(mockDevice, mockIDevice);
   2057         EasyMock.replay(mockListener);
   2058 
   2059         deqpTest.setDevice(mockDevice);
   2060         deqpTest.run(mockListener);
   2061 
   2062         EasyMock.verify(mockListener);
   2063         EasyMock.verify(mockDevice, mockIDevice);
   2064     }
   2065 
   2066     public void testRuntimeHint_optionSetSharded() throws Exception {
   2067         final int TEST_COUNT = 1237;
   2068         final int SHARD_SIZE = 1000;
   2069 
   2070         ArrayList<TestDescription> testIds = new ArrayList<>(TEST_COUNT);
   2071         for (int i = 0; i < TEST_COUNT; i++) {
   2072             testIds.add(new TestDescription("dEQP-GLES3.funny.group", String.valueOf(i)));
   2073         }
   2074 
   2075         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, testIds, mTestsDir);
   2076         OptionSetter setter = new OptionSetter(deqpTest);
   2077         final long fullRuntimeMs = testIds.size()*100;
   2078         setter.setOptionValue("runtime-hint", String.valueOf(fullRuntimeMs));
   2079 
   2080         ArrayList<IRemoteTest> shards = (ArrayList<IRemoteTest>)deqpTest.split();
   2081         assertEquals("First shard's time not proportional to test count",
   2082                  (fullRuntimeMs*SHARD_SIZE)/TEST_COUNT,
   2083                  ((IRuntimeHintProvider)shards.get(0)).getRuntimeHint());
   2084         assertEquals("Second shard's time not proportional to test count",
   2085                  (fullRuntimeMs*(TEST_COUNT-SHARD_SIZE))/TEST_COUNT,
   2086                  ((IRuntimeHintProvider)shards.get(1)).getRuntimeHint());
   2087     }
   2088 
   2089     public void testRuntimeHint_optionNotSet() throws Exception {
   2090         final TestDescription[] testIds = {
   2091                 new TestDescription("dEQP-GLES3.info", "vendor"),
   2092                 new TestDescription("dEQP-GLES3.info", "renderer"),
   2093                 new TestDescription("dEQP-GLES3.info", "version"),
   2094                 new TestDescription("dEQP-GLES3.info", "shading_language_version"),
   2095                 new TestDescription("dEQP-GLES3.info", "extensions"),
   2096                 new TestDescription("dEQP-GLES3.info", "render_target")
   2097         };
   2098         Collection<TestDescription> tests = new ArrayList<TestDescription>();
   2099 
   2100         for (TestDescription id : testIds) {
   2101             tests.add(id);
   2102         }
   2103 
   2104         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   2105 
   2106         long runtime = deqpTest.getRuntimeHint();
   2107         assertTrue("Runtime for tests must be positive", runtime > 0);
   2108         assertTrue("Runtime for tests must be reasonable", runtime < (1000 * 10)); // Must be done in 10s
   2109     }
   2110 
   2111 
   2112     private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
   2113             final String output) throws Exception {
   2114         String cmd = String.format(
   2115             "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8 "
   2116             + "--deqp-screen-rotation=unspecified "
   2117             + "--deqp-surface-type=window "
   2118             + "--deqp-log-images=disable "
   2119             + "--deqp-watchdog=enable",
   2120             APP_DIR + CASE_LIST_FILE_NAME);
   2121         runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, cmd, output);
   2122     }
   2123 
   2124     private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice,
   2125             final String testTrie, final String cmd, final String output) throws Exception {
   2126         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME)))
   2127                 .andReturn("").once();
   2128 
   2129         EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME)))
   2130                 .andReturn("").once();
   2131 
   2132         if (testTrie == null) {
   2133             mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME));
   2134         }
   2135         else {
   2136             mockDevice.pushString(testTrie + "\n", APP_DIR + CASE_LIST_FILE_NAME);
   2137         }
   2138         EasyMock.expectLastCall().andReturn(true).once();
   2139 
   2140         String command = String.format(
   2141                 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \"%s\" "
   2142                     + "-e deqpLogData \"%s\" %s",
   2143                 AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, false,
   2144                 INSTRUMENTATION_NAME);
   2145 
   2146         EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice);
   2147         mockIDevice.executeShellCommand(EasyMock.eq(command),
   2148                 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(),
   2149                 EasyMock.isA(TimeUnit.class));
   2150 
   2151         EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {
   2152             @Override
   2153             public Object answer() {
   2154                 IShellOutputReceiver receiver
   2155                         = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1];
   2156 
   2157                 receiver.addOutput(output.getBytes(), 0, output.length());
   2158                 receiver.flush();
   2159 
   2160                 return null;
   2161             }
   2162         });
   2163     }
   2164 
   2165     static private void writeStringsToFile(File target, Set<String> strings) throws IOException {
   2166         try (PrintWriter out = new PrintWriter(new FileWriter(target))) {
   2167             out.print(String.join(System.lineSeparator(), strings));
   2168             out.println();
   2169         }
   2170     }
   2171 
   2172     private void addFilterFileForOption(DeqpTestRunner test, Set<String> filters, String option)
   2173             throws IOException, ConfigurationException {
   2174         String filterFile = option + ".txt";
   2175         writeStringsToFile(new File(mTestsDir, filterFile), filters);
   2176         OptionSetter setter = new OptionSetter(test);
   2177         setter.setOptionValue(option, filterFile);
   2178     }
   2179 
   2180     public void testIncludeFilterFile() throws Exception {
   2181         final TestDescription[] testIds = {
   2182                 new TestDescription("dEQP-GLES3.missing", "no"),
   2183                 new TestDescription("dEQP-GLES3.missing", "nope"),
   2184                 new TestDescription("dEQP-GLES3.missing", "donotwant"),
   2185                 new TestDescription("dEQP-GLES3.pick_me", "yes"),
   2186                 new TestDescription("dEQP-GLES3.pick_me", "ok"),
   2187                 new TestDescription("dEQP-GLES3.pick_me", "accepted"),
   2188         };
   2189 
   2190         List<TestDescription> allTests = new ArrayList<TestDescription>();
   2191         for (TestDescription id : testIds) {
   2192             allTests.add(id);
   2193         }
   2194 
   2195         List<TestDescription> activeTests = new ArrayList<TestDescription>();
   2196         activeTests.add(testIds[3]);
   2197         activeTests.add(testIds[4]);
   2198         activeTests.add(testIds[5]);
   2199 
   2200         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
   2201 
   2202         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
   2203         Set<String> includes = new HashSet<>();
   2204         includes.add("dEQP-GLES3.pick_me#*");
   2205         addFilterFileForOption(deqpTest, includes, "include-filter-file");
   2206         testFiltering(deqpTest, expectedTrie, activeTests);
   2207     }
   2208 
   2209     public void testMissingIncludeFilterFile() throws Exception {
   2210         final TestDescription[] testIds = {
   2211                 new TestDescription("dEQP-GLES3.pick_me", "yes"),
   2212                 new TestDescription("dEQP-GLES3.pick_me", "ok"),
   2213                 new TestDescription("dEQP-GLES3.pick_me", "accepted"),
   2214         };
   2215 
   2216         List<TestDescription> allTests = new ArrayList<TestDescription>();
   2217         for (TestDescription id : testIds) {
   2218             allTests.add(id);
   2219         }
   2220 
   2221         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
   2222 
   2223         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
   2224         OptionSetter setter = new OptionSetter(deqpTest);
   2225         setter.setOptionValue("include-filter-file", "not-a-file.txt");
   2226         try {
   2227             testFiltering(deqpTest, expectedTrie, allTests);
   2228             fail("Test execution should have aborted with exception.");
   2229         } catch (RuntimeException e) {
   2230         }
   2231     }
   2232 
   2233     public void testExcludeFilterFile() throws Exception {
   2234         final TestDescription[] testIds = {
   2235                 new TestDescription("dEQP-GLES3.missing", "no"),
   2236                 new TestDescription("dEQP-GLES3.missing", "nope"),
   2237                 new TestDescription("dEQP-GLES3.missing", "donotwant"),
   2238                 new TestDescription("dEQP-GLES3.pick_me", "yes"),
   2239                 new TestDescription("dEQP-GLES3.pick_me", "ok"),
   2240                 new TestDescription("dEQP-GLES3.pick_me", "accepted"),
   2241         };
   2242 
   2243         List<TestDescription> allTests = new ArrayList<TestDescription>();
   2244         for (TestDescription id : testIds) {
   2245             allTests.add(id);
   2246         }
   2247 
   2248         List<TestDescription> activeTests = new ArrayList<TestDescription>();
   2249         activeTests.add(testIds[3]);
   2250         activeTests.add(testIds[4]);
   2251         activeTests.add(testIds[5]);
   2252 
   2253         String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}";
   2254 
   2255         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
   2256         Set<String> excludes = new HashSet<>();
   2257         excludes.add("dEQP-GLES3.missing#*");
   2258         addFilterFileForOption(deqpTest, excludes, "exclude-filter-file");
   2259         testFiltering(deqpTest, expectedTrie, activeTests);
   2260     }
   2261 
   2262     public void testFilterComboWithFiles() throws Exception {
   2263         final TestDescription[] testIds = {
   2264                 new TestDescription("dEQP-GLES3.group1", "footah"),
   2265                 new TestDescription("dEQP-GLES3.group1", "foo"),
   2266                 new TestDescription("dEQP-GLES3.group1", "nope"),
   2267                 new TestDescription("dEQP-GLES3.group1", "nonotwant"),
   2268                 new TestDescription("dEQP-GLES3.group2", "foo"),
   2269                 new TestDescription("dEQP-GLES3.group2", "yes"),
   2270                 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"),
   2271         };
   2272 
   2273         List<TestDescription> allTests = new ArrayList<TestDescription>();
   2274         for (TestDescription id : testIds) {
   2275             allTests.add(id);
   2276         }
   2277 
   2278         List<TestDescription> activeTests = new ArrayList<TestDescription>();
   2279         activeTests.add(testIds[0]);
   2280         activeTests.add(testIds[5]);
   2281 
   2282         String expectedTrie = "{dEQP-GLES3{group1{footah}group2{yes}}}";
   2283 
   2284         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
   2285 
   2286         Set<String> includes = new HashSet<>();
   2287         includes.add("dEQP-GLES3.group2#*");
   2288         deqpTest.addAllIncludeFilters(includes);
   2289 
   2290         Set<String> fileIncludes = new HashSet<>();
   2291         fileIncludes.add("dEQP-GLES3.group1#no*");
   2292         fileIncludes.add("dEQP-GLES3.group1#foo*");
   2293         addFilterFileForOption(deqpTest, fileIncludes, "include-filter-file");
   2294 
   2295         Set<String> fileExcludes = new HashSet<>();
   2296         fileExcludes.add("*foo");
   2297         fileExcludes.add("*thoushallnotpass");
   2298         addFilterFileForOption(deqpTest, fileExcludes, "exclude-filter-file");
   2299 
   2300         deqpTest.addExcludeFilter("dEQP-GLES3.group1#no*");
   2301 
   2302         testFiltering(deqpTest, expectedTrie, activeTests);
   2303     }
   2304 
   2305     public void testDotToHashConversionInFilters() throws Exception {
   2306         final TestDescription[] testIds = {
   2307                 new TestDescription("dEQP-GLES3.missing", "no"),
   2308                 new TestDescription("dEQP-GLES3.pick_me", "donotwant"),
   2309                 new TestDescription("dEQP-GLES3.pick_me", "yes")
   2310         };
   2311 
   2312         List<TestDescription> allTests = new ArrayList<TestDescription>();
   2313         for (TestDescription id : testIds) {
   2314             allTests.add(id);
   2315         }
   2316 
   2317         List<TestDescription> activeTests = new ArrayList<TestDescription>();
   2318         activeTests.add(testIds[2]);
   2319 
   2320         String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}";
   2321 
   2322         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, allTests, mTestsDir);
   2323         deqpTest.addIncludeFilter("dEQP-GLES3.pick_me.yes");
   2324         testFiltering(deqpTest, expectedTrie, activeTests);
   2325     }
   2326 
   2327     public void testCoverageDisabled_noCoverageListener() throws Exception {
   2328         final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version");
   2329 
   2330         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   2331         ITestInvocationListener mockListener =
   2332             EasyMock.createStrictMock(ITestInvocationListener.class);
   2333 
   2334         Collection<TestDescription> tests = new ArrayList<>();
   2335         tests.add(testId);
   2336 
   2337         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   2338         deqpTest.setNativeCoverage(false);
   2339 
   2340         ITestInvocationListener listener =
   2341             deqpTest.addNativeCoverageListenerIfEnabled(mockDevice, mockListener);
   2342         assertSame(listener, mockListener);
   2343     }
   2344 
   2345     public void testCoverageEnabled_newCoverageListener() throws Exception {
   2346         final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version");
   2347 
   2348         ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class);
   2349         ITestInvocationListener mockListener =
   2350             EasyMock.createStrictMock(ITestInvocationListener.class);
   2351 
   2352         Collection<TestDescription> tests = new ArrayList<>();
   2353         tests.add(testId);
   2354 
   2355         DeqpTestRunner deqpTest = buildGlesTestRunner(3, 0, tests, mTestsDir);
   2356         deqpTest.setNativeCoverage(true);
   2357 
   2358         ITestInvocationListener listener =
   2359             deqpTest.addNativeCoverageListenerIfEnabled(mockDevice, mockListener);
   2360         assertTrue(listener instanceof NativeCodeCoverageListener);
   2361     }
   2362 }
   2363