1 /*------------------------------------------------------------------------- 2 * drawElements Quality Program Tester Core 3 * ---------------------------------------- 4 * 5 * Copyright 2014 The Android Open Source Project 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 * 19 *//*! 20 * \file 21 * \brief Class for executing tests. 22 *//*--------------------------------------------------------------------*/ 23 24 #include "tcuTestExecutor.hpp" 25 #include "tcuCommandLine.hpp" 26 #include "tcuPlatform.hpp" 27 #include "tcuTestLog.hpp" 28 29 #include "deInt32.h" 30 31 #include <typeinfo> 32 33 #include <stdio.h> 34 #include <stdlib.h> 35 #include <string.h> 36 37 using std::string; 38 using std::vector; 39 40 namespace tcu 41 { 42 43 TestExecutor::TestExecutor (TestContext& testCtx, const CommandLine& cmdLine) 44 : m_testCtx (testCtx) 45 , m_cmdLine (cmdLine) 46 , m_rootNode (DE_NULL) 47 , m_testCaseWrapper (DE_NULL) 48 , m_testCaseListFile (DE_NULL) 49 , m_testCaseListWriter (DE_NULL) 50 { 51 m_abortSession = false; 52 m_isInTestCase = false; 53 54 // Create the root node. 55 TestPackageRegistry* packageRegistry = TestPackageRegistry::getSingleton(); 56 vector<TestPackageRegistry::PackageInfo*> packageInfos = packageRegistry->getPackageInfos(); 57 vector<TestNode*> testPackages; 58 59 for (int i = 0; i < (int)packageInfos.size(); i++) 60 testPackages.push_back(packageInfos[i]->createFunc(testCtx)); 61 62 m_rootNode = new TestPackageRoot(testCtx, testPackages); 63 64 // Init traverse stack. 65 NodeIter iter(m_rootNode); 66 m_sessionStack.push_back(iter); 67 } 68 69 TestExecutor::~TestExecutor (void) 70 { 71 if (m_testCaseListWriter) 72 qpXmlWriter_destroy(m_testCaseListWriter); 73 74 if (m_testCaseListFile) 75 fclose(m_testCaseListFile); 76 77 delete m_rootNode; 78 } 79 80 // Test sub-case iteration. 81 void TestExecutor::enterTestPackage (TestPackage* testPackage, const char* packageName) 82 { 83 DE_ASSERT(testPackage && packageName); 84 85 // Open file/writer for case dumping. 86 const RunMode runMode = m_cmdLine.getRunMode(); 87 if (runMode == RUNMODE_DUMP_XML_CASELIST || runMode == RUNMODE_DUMP_TEXT_CASELIST) 88 { 89 const char* const ext = (runMode == RUNMODE_DUMP_XML_CASELIST) ? "xml" : "txt"; 90 const string fileName = string(packageName) + "-cases." + ext; 91 92 print("Dumping all test case names in '%s' to file '%s'..\n", packageName, fileName.c_str()); 93 TCU_CHECK(m_testCaseListFile = fopen(fileName.c_str(), "wb")); 94 95 if (runMode == RUNMODE_DUMP_XML_CASELIST) 96 { 97 TCU_CHECK(m_testCaseListWriter = qpXmlWriter_createFileWriter(m_testCaseListFile, DE_FALSE)); 98 99 qpXmlWriter_startDocument(m_testCaseListWriter); 100 qpXmlWriter_startElement(m_testCaseListWriter, "TestCaseList", 0, DE_NULL); 101 } 102 } 103 104 // Initialize package. 105 testPackage->init(); 106 107 // Store test case wrapper 108 m_testCaseWrapper = &testPackage->getTestCaseWrapper(); 109 DE_ASSERT(m_testCaseWrapper); 110 111 // Set archive. 112 m_testCtx.setCurrentArchive(testPackage->getArchive()); 113 } 114 115 void TestExecutor::leaveTestPackage (TestPackage* testPackage) 116 { 117 DE_ASSERT(testPackage); 118 119 const RunMode runMode = m_cmdLine.getRunMode(); 120 if (runMode == RUNMODE_DUMP_XML_CASELIST) 121 { 122 qpXmlWriter_endElement(m_testCaseListWriter, "TestCaseList"); 123 qpXmlWriter_endDocument(m_testCaseListWriter); 124 qpXmlWriter_destroy(m_testCaseListWriter); 125 m_testCaseListWriter = DE_NULL; 126 } 127 128 if (runMode == RUNMODE_DUMP_TEXT_CASELIST || runMode == RUNMODE_DUMP_XML_CASELIST) 129 { 130 fclose(m_testCaseListFile); 131 m_testCaseListFile = DE_NULL; 132 } 133 134 DE_ASSERT(!m_testCaseListWriter && !m_testCaseListFile); 135 136 m_testCaseWrapper = DE_NULL; 137 m_testCtx.setCurrentArchive(m_testCtx.getRootArchive()); 138 139 // Deinitialize package. 140 testPackage->deinit(); 141 } 142 143 void TestExecutor::enterGroupNode (TestCaseGroup* testGroup, const char* casePath) 144 { 145 DE_UNREF(casePath); 146 testGroup->init(); 147 } 148 149 void TestExecutor::leaveGroupNode (TestCaseGroup* testGroup) 150 { 151 testGroup->deinit(); 152 } 153 154 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType) 155 { 156 switch (nodeType) 157 { 158 case NODETYPE_SELF_VALIDATE: return QP_TEST_CASE_TYPE_SELF_VALIDATE; 159 case NODETYPE_PERFORMANCE: return QP_TEST_CASE_TYPE_PERFORMANCE; 160 case NODETYPE_CAPABILITY: return QP_TEST_CASE_TYPE_CAPABILITY; 161 case NODETYPE_ACCURACY: return QP_TEST_CASE_TYPE_ACCURACY; 162 default: 163 DE_ASSERT(DE_FALSE); 164 return QP_TEST_CASE_TYPE_LAST; 165 } 166 } 167 168 bool TestExecutor::enterTestCase (TestCase* testCase, const char* casePath) 169 { 170 const RunMode runMode = m_cmdLine.getRunMode(); 171 const qpTestCaseType caseType = nodeTypeToTestCaseType(testCase->getNodeType()); 172 173 if (runMode == RUNMODE_EXECUTE) 174 { 175 print("\nTest case '%s'..\n", casePath); 176 177 m_testCtx.getLog().startCase(casePath, caseType); 178 m_isInTestCase = true; 179 m_testCtx.setTestResult(QP_TEST_RESULT_LAST, ""); 180 181 if (!m_testCaseWrapper->initTestCase(testCase)) 182 { 183 if (m_testCtx.getTestResult() == QP_TEST_RESULT_LAST) 184 m_testCtx.setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, "Unexpected error in subcase init"); 185 return false; 186 } 187 } 188 189 return true; 190 } 191 192 void TestExecutor::leaveTestCase (TestCase* testCase) 193 { 194 const RunMode runMode = m_cmdLine.getRunMode(); 195 if (runMode == RUNMODE_EXECUTE) 196 { 197 // De-init case. 198 const bool deinitOk = m_testCaseWrapper->deinitTestCase(testCase); 199 const qpTestResult testResult = m_testCtx.getTestResult(); 200 const char* const testResultDesc = m_testCtx.getTestResultDesc(); 201 const bool terminateAfter = m_testCtx.getTerminateAfter(); 202 DE_ASSERT(testResult != QP_TEST_RESULT_LAST); 203 204 m_isInTestCase = false; 205 m_testCtx.getLog().endCase(testResult, testResultDesc); 206 207 // Update statistics. 208 print(" %s (%s)\n", qpGetTestResultName(testResult), testResultDesc); 209 210 m_result.numExecuted += 1; 211 switch (testResult) 212 { 213 case QP_TEST_RESULT_PASS: m_result.numPassed += 1; break; 214 case QP_TEST_RESULT_NOT_SUPPORTED: m_result.numNotSupported += 1; break; 215 case QP_TEST_RESULT_QUALITY_WARNING: m_result.numWarnings += 1; break; 216 case QP_TEST_RESULT_COMPATIBILITY_WARNING: m_result.numWarnings += 1; break; 217 default: m_result.numFailed += 1; break; 218 } 219 220 // terminateAfter, Resource error or any error in deinit means that execution should end 221 if (terminateAfter || !deinitOk || testResult == QP_TEST_RESULT_RESOURCE_ERROR) 222 m_abortSession = true; 223 224 // \todo [2011-02-09 pyry] Disable watchdog temporarily? 225 if (m_testCtx.getWatchDog()) 226 qpWatchDog_reset(m_testCtx.getWatchDog()); 227 } 228 } 229 230 // Return true while session should still continue, false otherwise. 231 bool TestExecutor::iterate (void) 232 { 233 try 234 { 235 while (!m_sessionStack.empty()) 236 { 237 // Get full path to node. 238 string nodePath = ""; 239 for (int ndx = 0; ndx < (int)m_sessionStack.size(); ndx++) 240 { 241 NodeIter& iter = m_sessionStack[ndx]; 242 if (ndx > 1) // ignore root package 243 nodePath += "."; 244 nodePath += iter.node->getName(); 245 } 246 247 // Handle the node. 248 NodeIter& iter = m_sessionStack[m_sessionStack.size()-1]; 249 DE_ASSERT(iter.node != DE_NULL); 250 TestNode* node = iter.node; 251 bool isLeaf = isTestNodeTypeExecutable(node->getNodeType()); 252 253 switch (iter.getState()) 254 { 255 case NodeIter::STATE_BEGIN: 256 { 257 // Return to parent if name doesn't match filter. 258 if (!(isLeaf ? m_cmdLine.checkTestCaseName(nodePath.c_str()) : m_cmdLine.checkTestGroupName(nodePath.c_str()))) 259 { 260 m_sessionStack.pop_back(); 261 break; 262 } 263 264 // Enter node. 265 bool enterOk = true; 266 switch (node->getNodeType()) 267 { 268 case NODETYPE_ROOT: /* nada */ break; 269 case NODETYPE_PACKAGE: enterTestPackage(static_cast<TestPackage*>(node), nodePath.c_str()); break; 270 case NODETYPE_GROUP: enterGroupNode(static_cast<TestCaseGroup*>(node), nodePath.c_str()); break; 271 case NODETYPE_PERFORMANCE: 272 case NODETYPE_CAPABILITY: 273 case NODETYPE_ACCURACY: /* fall-trough */ 274 case NODETYPE_SELF_VALIDATE: enterOk = enterTestCase(static_cast<TestCase*>(node), nodePath.c_str()); break; 275 default: DE_ASSERT(false); 276 } 277 278 if (m_cmdLine.getRunMode() == RUNMODE_EXECUTE) 279 { 280 if (isLeaf) 281 { 282 if (enterOk) 283 iter.setState(NodeIter::STATE_EXECUTE_TEST); 284 else 285 iter.setState(NodeIter::STATE_FINISH); 286 } 287 else 288 { 289 iter.setState(NodeIter::STATE_TRAVERSE_CHILDREN); 290 } 291 } 292 else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST) 293 { 294 if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE) 295 { 296 string caseName = iter.node->getName(); 297 string description = iter.node->getDescription(); 298 qpXmlAttribute attribs[8]; 299 int numAttribs = 0; 300 const char* caseType = DE_NULL; 301 302 switch (node->getNodeType()) 303 { 304 case NODETYPE_SELF_VALIDATE: caseType = "SelfValidate"; break; 305 case NODETYPE_CAPABILITY: caseType = "Capability"; break; 306 case NODETYPE_ACCURACY: caseType = "Accuracy"; break; 307 case NODETYPE_PERFORMANCE: caseType = "Performance"; break; 308 default: caseType = "TestGroup"; break; 309 } 310 311 attribs[numAttribs++] = qpSetStringAttrib("Name", caseName.c_str()); 312 attribs[numAttribs++] = qpSetStringAttrib("CaseType", caseType); 313 attribs[numAttribs++] = qpSetStringAttrib("Description", description.c_str()); 314 qpXmlWriter_startElement(m_testCaseListWriter, "TestCase", numAttribs, attribs); 315 } 316 317 iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN); 318 } 319 else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_TEXT_CASELIST) 320 { 321 // \note Case list file is not open until we are in test package. 322 if (isLeaf) 323 fprintf(m_testCaseListFile, "TEST: %s\n", nodePath.c_str()); 324 else if (node->getNodeType() != NODETYPE_ROOT) 325 fprintf(m_testCaseListFile, "GROUP: %s\n", nodePath.c_str()); 326 iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN); 327 } 328 329 break; 330 } 331 332 case NodeIter::STATE_EXECUTE_TEST: 333 { 334 // Touch the watchdog. 335 m_testCtx.touchWatchdog(); 336 337 // Iterate the sub-case. 338 TestCase::IterateResult iterateResult = m_testCaseWrapper->iterateTestCase(static_cast<TestCase*>(node)); 339 340 if (iterateResult == TestCase::STOP) 341 iter.setState(NodeIter::STATE_FINISH); 342 343 return true; // return after each iteration (when another iteration follows). 344 } 345 346 case NodeIter::STATE_TRAVERSE_CHILDREN: 347 { 348 int numChildren = (int)iter.children.size(); 349 if (++iter.curChildNdx < numChildren) 350 { 351 // Push child to stack. 352 TestNode* childNode = iter.children[iter.curChildNdx]; 353 m_sessionStack.push_back(NodeIter(childNode)); 354 } 355 else 356 iter.setState(NodeIter::STATE_FINISH); 357 358 break; 359 } 360 361 case NodeIter::STATE_FINISH: 362 { 363 if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST) 364 { 365 if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE) 366 qpXmlWriter_endElement(m_testCaseListWriter, "TestCase"); 367 } 368 369 // Leave node. 370 switch (node->getNodeType()) 371 { 372 case NODETYPE_ROOT: /* nada */ break; 373 case NODETYPE_PACKAGE: leaveTestPackage(static_cast<TestPackage*>(node)); break; 374 case NODETYPE_GROUP: leaveGroupNode(static_cast<TestCaseGroup*>(node)); break; 375 case NODETYPE_ACCURACY: 376 case NODETYPE_CAPABILITY: 377 case NODETYPE_PERFORMANCE: /* fall-thru */ 378 case NODETYPE_SELF_VALIDATE: leaveTestCase(static_cast<TestCase*>(node)); break; 379 default: DE_ASSERT(false); 380 } 381 382 m_sessionStack.pop_back(); 383 384 // Return if execution should abort. 385 if (m_abortSession) 386 return false; 387 388 // Otherwise continue iterating. 389 break; 390 } 391 392 default: 393 DE_ASSERT(DE_FALSE); 394 break; 395 } 396 } 397 } 398 catch (const std::exception& e) 399 { 400 print("TestExecutor::iterateSession(): Caught unhandled %s: %s\n", typeid(e).name(), e.what()); 401 throw; 402 } 403 404 m_result.isComplete = true; 405 return false; 406 } 407 408 } // tcu 409
