Home | History | Annotate | Download | only in testtype 
      1 package com.android.cts.tradefed.testtype;
      2 
      3 import com.android.cts.tradefed.build.CtsBuildHelper;
      4 import com.android.cts.util.AbiUtils;
      5 import com.android.ddmlib.MultiLineReceiver;
      6 import com.android.ddmlib.testrunner.TestIdentifier;
      7 import com.android.tradefed.build.IBuildInfo;
      8 import com.android.tradefed.device.DeviceNotAvailableException;
      9 import com.android.tradefed.device.ITestDevice;
     10 import com.android.tradefed.log.LogUtil.CLog;
     11 import com.android.tradefed.result.ByteArrayInputStreamSource;
     12 import com.android.tradefed.result.ITestInvocationListener;
     13 import com.android.tradefed.result.LogDataType;
     14 import com.android.tradefed.testtype.IAbi;
     15 import com.android.tradefed.testtype.IBuildReceiver;
     16 import com.android.tradefed.testtype.IDeviceTest;
     17 import com.android.tradefed.testtype.IRemoteTest;
     18 
     19 import java.io.File;
     20 import java.io.FileNotFoundException;
     21 import java.util.ArrayList;
     22 import java.util.Collection;
     23 import java.util.Collections;
     24 import java.util.HashMap;
     25 import java.util.Iterator;
     26 import java.util.Map;
     27 
     28 /**
     29  * Test runner for dEQP tests
     30  *
     31  * Supports running drawElements Quality Program tests found under external/deqp.
     32  */
     33 public class DeqpTestRunner implements IBuildReceiver, IDeviceTest, IRemoteTest {
     34 
     35     private static final String DEQP_ONDEVICE_APK = "com.drawelements.deqp.apk";
     36     private static final String DEQP_ONDEVICE_PKG = "com.drawelements.deqp";
     37     private static final String INCOMPLETE_LOG_MESSAGE = "Crash: Incomplete test log";
     38 
     39     private final int TESTCASE_BATCH_LIMIT = 1000;
     40 
     41     private boolean mLogData;
     42 
     43     private ITestDevice mDevice;
     44 
     45     private final String mPackageName;
     46     private final String mName;
     47     private Collection<TestIdentifier> mTests;
     48     private IAbi mAbi;
     49     private CtsBuildHelper mCtsBuild;
     50 
     51     private TestIdentifier mCurrentTestId;
     52     private boolean mGotTestResult;
     53     private String mCurrentTestLog;
     54 
     55     private ITestInvocationListener mListener;
     56 
     57     public DeqpTestRunner(String packageName, String name, Collection<TestIdentifier> tests) {
     58         mPackageName = packageName;
     59         mName = name;
     60         mTests = tests;
     61         mLogData = false;
     62     }
     63 
     64     /**
     65      * @param abi the ABI to run the test on
     66      */
     67     public void setAbi(IAbi abi) {
     68         mAbi = abi;
     69     }
     70 
     71     /**
     72      * {@inheritDoc}
     73      */
     74     @Override
     75     public void setBuild(IBuildInfo buildInfo) {
     76         mCtsBuild = CtsBuildHelper.createBuildHelper(buildInfo);
     77     }
     78 
     79     /**
     80      * Set the CTS build container.
     81      * <p/>
     82      * Exposed so unit tests can mock the provided build.
     83      *
     84      * @param buildHelper
     85      */
     86     public void setBuildHelper(CtsBuildHelper buildHelper) {
     87         mCtsBuild = buildHelper;
     88     }
     89 
     90     /**
     91      * Enable or disable raw dEQP test log collection.
     92      */
     93     public void setCollectLogs(boolean logData) {
     94         mLogData = logData;
     95     }
     96 
     97     /**
     98      * dEQP instrumentation parser
     99      */
    100     class InstrumentationParser extends MultiLineReceiver {
    101         private DeqpTestRunner mDeqpTests;
    102 
    103         private Map<String, String> mValues;
    104         private String mCurrentName;
    105         private String mCurrentValue;
    106 
    107 
    108         public InstrumentationParser(DeqpTestRunner tests) {
    109             mDeqpTests = tests;
    110         }
    111 
    112         /**
    113          * {@inheritDoc}
    114          */
    115         @Override
    116         public void processNewLines(String[] lines) {
    117             for (String line : lines) {
    118                 if (mValues == null) mValues = new HashMap<String, String>();
    119 
    120                 if (line.startsWith("INSTRUMENTATION_STATUS_CODE: ")) {
    121                     if (mCurrentName != null) {
    122                         mValues.put(mCurrentName, mCurrentValue);
    123 
    124                         mCurrentName = null;
    125                         mCurrentValue = null;
    126                     }
    127 
    128                     mDeqpTests.handleStatus(mValues);
    129                     mValues = null;
    130                 } else if (line.startsWith("INSTRUMENTATION_STATUS: dEQP-")) {
    131                     if (mCurrentName != null) {
    132                         mValues.put(mCurrentName, mCurrentValue);
    133 
    134                         mCurrentValue = null;
    135                         mCurrentName = null;
    136                     }
    137 
    138                     String prefix = "INSTRUMENTATION_STATUS: ";
    139                     int nameBegin = prefix.length();
    140                     int nameEnd = line.indexOf('=');
    141                     int valueBegin = nameEnd + 1;
    142 
    143                     mCurrentName = line.substring(nameBegin, nameEnd);
    144                     mCurrentValue = line.substring(valueBegin);
    145                 } else if (mCurrentValue != null) {
    146                     mCurrentValue = mCurrentValue + line;
    147                 }
    148             }
    149         }
    150 
    151         /**
    152          * {@inheritDoc}
    153          */
    154         @Override
    155         public void done() {
    156             if (mCurrentName != null) {
    157                 mValues.put(mCurrentName, mCurrentValue);
    158 
    159                 mCurrentName = null;
    160                 mCurrentValue = null;
    161             }
    162 
    163             if (mValues != null) {
    164                 mDeqpTests.handleStatus(mValues);
    165                 mValues = null;
    166             }
    167         }
    168 
    169         /**
    170          * {@inheritDoc}
    171          */
    172         @Override
    173         public boolean isCancelled() {
    174             return false;
    175         }
    176     }
    177 
    178     /**
    179      * Converts dEQP testcase path to TestIdentifier.
    180      */
    181     private TestIdentifier pathToIdentifier(String testPath) {
    182         String[] components = testPath.split("\\.");
    183         String name = components[components.length - 1];
    184         String className = null;
    185 
    186         for (int i = 0; i < components.length - 1; i++) {
    187             if (className == null) {
    188                 className = components[i];
    189             } else {
    190                 className = className + "." + components[i];
    191             }
    192         }
    193 
    194         return new TestIdentifier(className, name);
    195     }
    196 
    197     /**
    198      * Handles beginning of dEQP session.
    199      */
    200     private void handleBeginSession(Map<String, String> values) {
    201         String id = AbiUtils.createId(mAbi.getName(), mPackageName);
    202         mListener.testRunStarted(id, mTests.size());
    203     }
    204 
    205     /**
    206      * Handles end of dEQP session.
    207      */
    208     private void handleEndSession(Map<String, String> values) {
    209         Map <String, String> emptyMap = Collections.emptyMap();
    210         mListener.testRunEnded(0, emptyMap);
    211     }
    212 
    213     /**
    214      * Handles beginning of dEQP testcase.
    215      */
    216     private void handleBeginTestCase(Map<String, String> values) {
    217         mCurrentTestId = pathToIdentifier(values.get("dEQP-BeginTestCase-TestCasePath"));
    218         mCurrentTestLog = "";
    219         mGotTestResult = false;
    220 
    221         mListener.testStarted(mCurrentTestId);
    222         mTests.remove(mCurrentTestId);
    223     }
    224 
    225     /**
    226      * Handles end of dEQP testcase.
    227      */
    228     private void handleEndTestCase(Map<String, String> values) {
    229         Map <String, String> emptyMap = Collections.emptyMap();
    230 
    231         if (!mGotTestResult) {
    232             mListener.testFailed(mCurrentTestId,
    233                     INCOMPLETE_LOG_MESSAGE);
    234         }
    235 
    236         if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
    237             ByteArrayInputStreamSource source
    238                     = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
    239 
    240             mListener.testLog(mCurrentTestId.getClassName() + "."
    241                     + mCurrentTestId.getTestName(), LogDataType.XML, source);
    242 
    243             source.cancel();
    244         }
    245 
    246         mListener.testEnded(mCurrentTestId, emptyMap);
    247         mCurrentTestId = null;
    248     }
    249 
    250     /**
    251      * Handles dEQP testcase result.
    252      */
    253     private void handleTestCaseResult(Map<String, String> values) {
    254         String code = values.get("dEQP-TestCaseResult-Code");
    255         String details = values.get("dEQP-TestCaseResult-Details");
    256 
    257         if (code.compareTo("Pass") == 0) {
    258             mGotTestResult = true;
    259         } else if (code.compareTo("NotSupported") == 0) {
    260             mGotTestResult = true;
    261         } else if (code.compareTo("QualityWarning") == 0) {
    262             mGotTestResult = true;
    263         } else if (code.compareTo("CompatibilityWarning") == 0) {
    264             mGotTestResult = true;
    265         } else if (code.compareTo("Fail") == 0 || code.compareTo("ResourceError") == 0
    266                 || code.compareTo("InternalError") == 0 || code.compareTo("Crash") == 0
    267                 || code.compareTo("Timeout") == 0) {
    268             mListener.testFailed(mCurrentTestId,
    269                     code + ": " + details);
    270             mGotTestResult = true;
    271         } else {
    272             mListener.testFailed(mCurrentTestId,
    273                     "Unknown result code: " + code + ": " + details);
    274             mGotTestResult = true;
    275         }
    276     }
    277 
    278     /**
    279      * Handles terminated dEQP testcase.
    280      */
    281     private void handleTestCaseTerminate(Map<String, String> values) {
    282         Map <String, String> emptyMap = Collections.emptyMap();
    283 
    284         String reason = values.get("dEQP-TerminateTestCase-Reason");
    285         mListener.testFailed(mCurrentTestId,
    286                 "Terminated: " + reason);
    287         mListener.testEnded(mCurrentTestId, emptyMap);
    288 
    289         if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
    290             ByteArrayInputStreamSource source
    291                     = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
    292 
    293             mListener.testLog(mCurrentTestId.getClassName() + "."
    294                     + mCurrentTestId.getTestName(), LogDataType.XML, source);
    295 
    296             source.cancel();
    297         }
    298 
    299         mCurrentTestId = null;
    300         mGotTestResult = true;
    301     }
    302 
    303     /**
    304      * Handles dEQP testlog data.
    305      */
    306     private void handleTestLogData(Map<String, String> values) {
    307         mCurrentTestLog = mCurrentTestLog + values.get("dEQP-TestLogData-Log");
    308     }
    309 
    310     /**
    311      * Handles new instrumentation status message.
    312      */
    313     public void handleStatus(Map<String, String> values) {
    314         String eventType = values.get("dEQP-EventType");
    315 
    316         if (eventType == null) {
    317             return;
    318         }
    319 
    320         if (eventType.compareTo("BeginSession") == 0) {
    321             handleBeginSession(values);
    322         } else if (eventType.compareTo("EndSession") == 0) {
    323             handleEndSession(values);
    324         } else if (eventType.compareTo("BeginTestCase") == 0) {
    325             handleBeginTestCase(values);
    326         } else if (eventType.compareTo("EndTestCase") == 0) {
    327             handleEndTestCase(values);
    328         } else if (eventType.compareTo("TestCaseResult") == 0) {
    329             handleTestCaseResult(values);
    330         } else if (eventType.compareTo("TerminateTestCase") == 0) {
    331             handleTestCaseTerminate(values);
    332         } else if (eventType.compareTo("TestLogData") == 0) {
    333             handleTestLogData(values);
    334         }
    335     }
    336 
    337     /**
    338      * Generates tescase trie from dEQP testcase paths. Used to define which testcases to execute.
    339      */
    340     private String generateTestCaseTrieFromPaths(Collection<String> tests) {
    341         String result = "{";
    342         boolean first = true;
    343 
    344         // Add testcases to results
    345         for (Iterator<String> iter = tests.iterator(); iter.hasNext();) {
    346             String test = iter.next();
    347             String[] components = test.split("\\.");
    348 
    349             if (components.length == 1) {
    350                 if (!first) {
    351                     result = result + ",";
    352                 }
    353                 first = false;
    354 
    355                 result += components[0];
    356                 iter.remove();
    357             }
    358         }
    359 
    360         if (!tests.isEmpty()) {
    361             HashMap<String, ArrayList<String> > testGroups = new HashMap<>();
    362 
    363             // Collect all sub testgroups
    364             for (String test : tests) {
    365                 String[] components = test.split("\\.");
    366                 ArrayList<String> testGroup = testGroups.get(components[0]);
    367 
    368                 if (testGroup == null) {
    369                     testGroup = new ArrayList<String>();
    370                     testGroups.put(components[0], testGroup);
    371                 }
    372 
    373                 testGroup.add(test.substring(components[0].length()+1));
    374             }
    375 
    376             for (String testGroup : testGroups.keySet()) {
    377                 if (!first) {
    378                     result = result + ",";
    379                 }
    380 
    381                 first = false;
    382                 result = result + testGroup
    383                         + generateTestCaseTrieFromPaths(testGroups.get(testGroup));
    384             }
    385         }
    386 
    387         return result + "}";
    388     }
    389 
    390     /**
    391      * Generates testcase trie from TestIdentifiers.
    392      */
    393     private String generateTestCaseTrie(Collection<TestIdentifier> tests) {
    394         ArrayList<String> testPaths = new ArrayList<String>();
    395 
    396         for (TestIdentifier test : tests) {
    397             testPaths.add(test.getClassName() + "." + test.getTestName());
    398 
    399             // Limit number of testcases for each run
    400             if (testPaths.size() > TESTCASE_BATCH_LIMIT)
    401                 break;
    402         }
    403 
    404         return generateTestCaseTrieFromPaths(testPaths);
    405     }
    406 
    407     /**
    408      * Executes tests on the device.
    409      */
    410     private void executeTests(ITestInvocationListener listener) throws DeviceNotAvailableException {
    411         InstrumentationParser parser = new InstrumentationParser(this);
    412         String caseListFileName = "/sdcard/dEQP-TestCaseList.txt";
    413         String logFileName = "/sdcard/TestLog.qpa";
    414         String testCases = generateTestCaseTrie(mTests);
    415 
    416         mDevice.executeShellCommand("rm " + caseListFileName);
    417         mDevice.executeShellCommand("rm " + logFileName);
    418         mDevice.pushString(testCases + "\n", caseListFileName);
    419 
    420         String instrumentationName =
    421                 "com.drawelements.deqp/com.drawelements.deqp.testercore.DeqpInstrumentation";
    422 
    423         String command = String.format(
    424                 "am instrument %s -w -e deqpLogFileName \"%s\" -e deqpCmdLine \""
    425                     + "--deqp-caselist-file=%s --deqp-gl-config-name=rgba8888d24s8\""
    426                     + " -e deqpLogData \"%s\" %s",
    427                 AbiUtils.createAbiFlag(mAbi.getName()), logFileName, caseListFileName, mLogData,
    428                 instrumentationName);
    429 
    430         mDevice.executeShellCommand(command, parser);
    431         parser.flush();
    432     }
    433 
    434     /**
    435      * Check if device supports OpenGL ES version.
    436      */
    437     static boolean isSupportedGles(ITestDevice device, int requiredMajorVersion, int requiredMinorVersion) throws DeviceNotAvailableException {
    438         String roOpenglesVersion = device.getProperty("ro.opengles.version");
    439 
    440         if (roOpenglesVersion == null)
    441             return false;
    442 
    443         int intValue = Integer.parseInt(roOpenglesVersion);
    444 
    445         int majorVersion = ((intValue & 0xffff0000) >> 16);
    446         int minorVersion = (intValue & 0xffff);
    447 
    448         return (majorVersion > requiredMajorVersion)
    449                 || (majorVersion == requiredMajorVersion && minorVersion >= requiredMinorVersion);
    450     }
    451 
    452     /**
    453      * Install dEQP OnDevice Package
    454      */
    455     private void installTestApk() throws DeviceNotAvailableException {
    456         try {
    457             File apkFile = mCtsBuild.getTestApp(DEQP_ONDEVICE_APK);
    458             String[] options = {AbiUtils.createAbiFlag(mAbi.getName())};
    459             String errorCode = getDevice().installPackage(apkFile, true, options);
    460             if (errorCode != null) {
    461                 CLog.e("Failed to install %s. Reason: %s", DEQP_ONDEVICE_APK, errorCode);
    462             }
    463         } catch (FileNotFoundException e) {
    464             CLog.e("Could not find test apk %s", DEQP_ONDEVICE_APK);
    465         }
    466     }
    467 
    468     /**
    469      * Uninstall dEQP OnDevice Package
    470      */
    471     private void uninstallTestApk() throws DeviceNotAvailableException {
    472         getDevice().uninstallPackage(DEQP_ONDEVICE_PKG);
    473     }
    474 
    475     /**
    476      * {@inheritDoc}
    477      */
    478     @Override
    479     public void run(ITestInvocationListener listener) throws DeviceNotAvailableException {
    480         mListener = listener;
    481 
    482         if ((mName.equals( "dEQP-GLES3") && isSupportedGles(mDevice, 3, 0))
    483             || (mName.equals("dEQP-GLES31") && isSupportedGles(mDevice, 3, 1))) {
    484 
    485             // Make sure there is no pre-existing package form earlier interrupted test run.
    486             uninstallTestApk();
    487             installTestApk();
    488 
    489             while (!mTests.isEmpty()) {
    490                 executeTests(listener);
    491 
    492                 // Set test to failed if it didn't receive test result
    493                 if (mCurrentTestId != null) {
    494                     Map <String, String> emptyMap = Collections.emptyMap();
    495 
    496                     if (mLogData && mCurrentTestLog != null && mCurrentTestLog.length() > 0) {
    497                         ByteArrayInputStreamSource source
    498                                 = new ByteArrayInputStreamSource(mCurrentTestLog.getBytes());
    499 
    500                         mListener.testLog(mCurrentTestId.getClassName() + "."
    501                                 + mCurrentTestId.getTestName(), LogDataType.XML, source);
    502 
    503                         source.cancel();
    504                     }
    505                     if (!mGotTestResult) {
    506                         mListener.testFailed(mCurrentTestId,
    507                             INCOMPLETE_LOG_MESSAGE);
    508                     }
    509 
    510                     mListener.testEnded(mCurrentTestId, emptyMap);
    511                     mCurrentTestId = null;
    512                     mListener.testRunEnded(0, emptyMap);
    513                 }
    514             }
    515 
    516             uninstallTestApk();
    517         } else {
    518             /* Pass all tests if OpenGL ES version is not supported */
    519             Map <String, String> emptyMap = Collections.emptyMap();
    520             String id = AbiUtils.createId(mAbi.getName(), mPackageName);
    521             mListener.testRunStarted(id, mTests.size());
    522 
    523             for (TestIdentifier test : mTests) {
    524                 CLog.d("Skipping test '%s', Opengl ES version not supported", test.toString());
    525                 mListener.testStarted(test);
    526                 mListener.testEnded(test, emptyMap);
    527             }
    528 
    529             mListener.testRunEnded(0, emptyMap);
    530         }
    531     }
    532 
    533     /**
    534      * {@inheritDoc}
    535      */
    536     @Override
    537     public void setDevice(ITestDevice device) {
    538         mDevice = device;
    539     }
    540 
    541     /**
    542      * {@inheritDoc}
    543      */
    544     @Override
    545     public ITestDevice getDevice() {
    546         return mDevice;
    547     }
    548 }
    549