1 /*------------------------------------------------------------------------- 2 * drawElements Quality Program Tester Core 3 * ---------------------------------------- 4 * 5 * Copyright 2014 The Android Open Source Project 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 * 19 *//*! 20 * \file 21 * \brief Class for executing tests. 22 *//*--------------------------------------------------------------------*/ 23 24 #include "tcuTestExecutor.hpp" 25 #include "tcuCommandLine.hpp" 26 #include "tcuPlatform.hpp" 27 #include "tcuTestLog.hpp" 28 29 #include "deInt32.h" 30 31 #include <typeinfo> 32 33 #include <stdio.h> 34 #include <stdlib.h> 35 #include <string.h> 36 37 using std::string; 38 using std::vector; 39 40 namespace tcu 41 { 42 43 TestExecutor::TestExecutor (TestContext& testCtx, const CommandLine& cmdLine) 44 : m_testCtx (testCtx) 45 , m_cmdLine (cmdLine) 46 , m_rootNode (DE_NULL) 47 , m_testCaseWrapper (DE_NULL) 48 , m_testCaseListFile (DE_NULL) 49 , m_testCaseListWriter (DE_NULL) 50 { 51 m_abortSession = false; 52 m_isInTestCase = false; 53 54 // Create the root node. 55 TestPackageRegistry* packageRegistry = TestPackageRegistry::getSingleton(); 56 vector<TestPackageRegistry::PackageInfo*> packageInfos = packageRegistry->getPackageInfos(); 57 vector<TestNode*> testPackages; 58 59 for (int i = 0; i < (int)packageInfos.size(); i++) 60 testPackages.push_back(packageInfos[i]->createFunc(testCtx)); 61 62 m_rootNode = new TestPackageRoot(testCtx, testPackages); 63 64 // Init traverse stack. 65 NodeIter iter(m_rootNode); 66 m_sessionStack.push_back(iter); 67 } 68 69 TestExecutor::~TestExecutor (void) 70 { 71 if (m_testCaseListWriter) 72 qpXmlWriter_destroy(m_testCaseListWriter); 73 74 if (m_testCaseListFile) 75 fclose(m_testCaseListFile); 76 77 delete m_rootNode; 78 } 79 80 // Test sub-case iteration. 81 void TestExecutor::enterTestPackage (TestPackage* testPackage, const char* packageName) 82 { 83 DE_ASSERT(testPackage && packageName); 84 85 // Open file/writer for case dumping. 86 const RunMode runMode = m_cmdLine.getRunMode(); 87 if (runMode == RUNMODE_DUMP_XML_CASELIST || runMode == RUNMODE_DUMP_TEXT_CASELIST) 88 { 89 const char* const ext = (runMode == RUNMODE_DUMP_XML_CASELIST) ? "xml" : "txt"; 90 const string fileName = string(packageName) + "-cases." + ext; 91 92 print("Dumping all test case names in '%s' to file '%s'..\n", packageName, fileName.c_str()); 93 TCU_CHECK(m_testCaseListFile = fopen(fileName.c_str(), "wb")); 94 95 if (runMode == RUNMODE_DUMP_XML_CASELIST) 96 { 97 TCU_CHECK(m_testCaseListWriter = qpXmlWriter_createFileWriter(m_testCaseListFile, DE_FALSE)); 98 99 qpXmlWriter_startDocument(m_testCaseListWriter); 100 qpXmlWriter_startElement(m_testCaseListWriter, "TestCaseList", 0, DE_NULL); 101 } 102 } 103 104 // Initialize package. 105 testPackage->init(); 106 107 // Store test case wrapper 108 m_testCaseWrapper = &testPackage->getTestCaseWrapper(); 109 DE_ASSERT(m_testCaseWrapper); 110 111 // Set archive. 112 m_testCtx.setCurrentArchive(testPackage->getArchive()); 113 } 114 115 void TestExecutor::leaveTestPackage (TestPackage* testPackage) 116 { 117 DE_ASSERT(testPackage); 118 119 const RunMode runMode = m_cmdLine.getRunMode(); 120 if (runMode == RUNMODE_DUMP_XML_CASELIST) 121 { 122 qpXmlWriter_endElement(m_testCaseListWriter, "TestCaseList"); 123 qpXmlWriter_endDocument(m_testCaseListWriter); 124 qpXmlWriter_destroy(m_testCaseListWriter); 125 m_testCaseListWriter = DE_NULL; 126 } 127 128 if (runMode == RUNMODE_DUMP_TEXT_CASELIST || runMode == RUNMODE_DUMP_XML_CASELIST) 129 { 130 fclose(m_testCaseListFile); 131 m_testCaseListFile = DE_NULL; 132 } 133 134 DE_ASSERT(!m_testCaseListWriter && !m_testCaseListFile); 135 136 m_testCaseWrapper = DE_NULL; 137 m_testCtx.setCurrentArchive(m_testCtx.getRootArchive()); 138 139 // Deinitialize package. 140 testPackage->deinit(); 141 } 142 143 void TestExecutor::enterGroupNode (TestCaseGroup* testGroup, const char* casePath) 144 { 145 DE_UNREF(casePath); 146 testGroup->init(); 147 } 148 149 void TestExecutor::leaveGroupNode (TestCaseGroup* testGroup) 150 { 151 testGroup->deinit(); 152 } 153 154 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType) 155 { 156 switch (nodeType) 157 { 158 case NODETYPE_SELF_VALIDATE: return QP_TEST_CASE_TYPE_SELF_VALIDATE; 159 case NODETYPE_PERFORMANCE: return QP_TEST_CASE_TYPE_PERFORMANCE; 160 case NODETYPE_CAPABILITY: return QP_TEST_CASE_TYPE_CAPABILITY; 161 case NODETYPE_ACCURACY: return QP_TEST_CASE_TYPE_ACCURACY; 162 default: 163 DE_ASSERT(DE_FALSE); 164 return QP_TEST_CASE_TYPE_LAST; 165 } 166 } 167 168 bool TestExecutor::enterTestCase (TestCase* testCase, const char* casePath) 169 { 170 const RunMode runMode = m_cmdLine.getRunMode(); 171 const qpTestCaseType caseType = nodeTypeToTestCaseType(testCase->getNodeType()); 172 173 if (runMode == RUNMODE_EXECUTE) 174 { 175 print("\nTest case '%s'..\n", casePath); 176 177 m_testCtx.getLog().startCase(casePath, caseType); 178 m_isInTestCase = true; 179 m_testCtx.setTestResult(QP_TEST_RESULT_LAST, ""); 180 181 if (!m_testCaseWrapper->initTestCase(testCase)) 182 { 183 if (m_testCtx.getTestResult() == QP_TEST_RESULT_LAST) 184 m_testCtx.setTestResult(QP_TEST_RESULT_INTERNAL_ERROR, "Unexpected error in subcase init"); 185 return false; 186 } 187 } 188 189 return true; 190 } 191 192 void TestExecutor::leaveTestCase (TestCase* testCase) 193 { 194 const RunMode runMode = m_cmdLine.getRunMode(); 195 if (runMode == RUNMODE_EXECUTE) 196 { 197 // Update statistics. 198 qpTestResult testResult = m_testCtx.getTestResult(); 199 DE_ASSERT(testResult != QP_TEST_RESULT_LAST); 200 201 print(" %s (%s)\n", qpGetTestResultName(testResult), m_testCtx.getTestResultDesc()); 202 203 m_result.numExecuted += 1; 204 switch (testResult) 205 { 206 case QP_TEST_RESULT_PASS: m_result.numPassed += 1; break; 207 case QP_TEST_RESULT_NOT_SUPPORTED: m_result.numNotSupported += 1; break; 208 case QP_TEST_RESULT_QUALITY_WARNING: m_result.numWarnings += 1; break; 209 case QP_TEST_RESULT_COMPATIBILITY_WARNING: m_result.numWarnings += 1; break; 210 default: m_result.numFailed += 1; break; 211 } 212 213 // De-init case. 214 bool deinitOk = m_testCaseWrapper->deinitTestCase(testCase); 215 216 m_isInTestCase = false; 217 m_testCtx.getLog().endCase(m_testCtx.getTestResult(), m_testCtx.getTestResultDesc()); 218 219 // Resource error or any error in deinit means that execution should end 220 if (!deinitOk || testResult == QP_TEST_RESULT_RESOURCE_ERROR) 221 m_abortSession = true; 222 223 // \todo [2011-02-09 pyry] Disable watchdog temporarily? 224 if (m_testCtx.getWatchDog()) 225 qpWatchDog_reset(m_testCtx.getWatchDog()); 226 } 227 } 228 229 // Return true while session should still continue, false otherwise. 230 bool TestExecutor::iterate (void) 231 { 232 try 233 { 234 while (!m_sessionStack.empty()) 235 { 236 // Get full path to node. 237 string nodePath = ""; 238 for (int ndx = 0; ndx < (int)m_sessionStack.size(); ndx++) 239 { 240 NodeIter& iter = m_sessionStack[ndx]; 241 if (ndx > 1) // ignore root package 242 nodePath += "."; 243 nodePath += iter.node->getName(); 244 } 245 246 // Handle the node. 247 NodeIter& iter = m_sessionStack[m_sessionStack.size()-1]; 248 DE_ASSERT(iter.node != DE_NULL); 249 TestNode* node = iter.node; 250 bool isLeaf = isTestNodeTypeExecutable(node->getNodeType()); 251 252 switch (iter.getState()) 253 { 254 case NodeIter::STATE_BEGIN: 255 { 256 // Return to parent if name doesn't match filter. 257 if (!(isLeaf ? m_cmdLine.checkTestCaseName(nodePath.c_str()) : m_cmdLine.checkTestGroupName(nodePath.c_str()))) 258 { 259 m_sessionStack.pop_back(); 260 break; 261 } 262 263 // Enter node. 264 bool enterOk = true; 265 switch (node->getNodeType()) 266 { 267 case NODETYPE_ROOT: /* nada */ break; 268 case NODETYPE_PACKAGE: enterTestPackage(static_cast<TestPackage*>(node), nodePath.c_str()); break; 269 case NODETYPE_GROUP: enterGroupNode(static_cast<TestCaseGroup*>(node), nodePath.c_str()); break; 270 case NODETYPE_PERFORMANCE: 271 case NODETYPE_CAPABILITY: 272 case NODETYPE_ACCURACY: /* fall-trough */ 273 case NODETYPE_SELF_VALIDATE: enterOk = enterTestCase(static_cast<TestCase*>(node), nodePath.c_str()); break; 274 default: DE_ASSERT(false); 275 } 276 277 if (m_cmdLine.getRunMode() == RUNMODE_EXECUTE) 278 { 279 if (isLeaf) 280 { 281 if (enterOk) 282 iter.setState(NodeIter::STATE_EXECUTE_TEST); 283 else 284 iter.setState(NodeIter::STATE_FINISH); 285 } 286 else 287 { 288 iter.setState(NodeIter::STATE_TRAVERSE_CHILDREN); 289 } 290 } 291 else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST) 292 { 293 if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE) 294 { 295 string caseName = iter.node->getName(); 296 string description = iter.node->getDescription(); 297 qpXmlAttribute attribs[8]; 298 int numAttribs = 0; 299 const char* caseType = DE_NULL; 300 301 switch (node->getNodeType()) 302 { 303 case NODETYPE_SELF_VALIDATE: caseType = "SelfValidate"; break; 304 case NODETYPE_CAPABILITY: caseType = "Capability"; break; 305 case NODETYPE_ACCURACY: caseType = "Accuracy"; break; 306 case NODETYPE_PERFORMANCE: caseType = "Performance"; break; 307 default: caseType = "TestGroup"; break; 308 } 309 310 attribs[numAttribs++] = qpSetStringAttrib("Name", caseName.c_str()); 311 attribs[numAttribs++] = qpSetStringAttrib("CaseType", caseType); 312 attribs[numAttribs++] = qpSetStringAttrib("Description", description.c_str()); 313 qpXmlWriter_startElement(m_testCaseListWriter, "TestCase", numAttribs, attribs); 314 } 315 316 iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN); 317 } 318 else if (m_cmdLine.getRunMode() == RUNMODE_DUMP_TEXT_CASELIST) 319 { 320 // \note Case list file is not open until we are in test package. 321 if (isLeaf) 322 fprintf(m_testCaseListFile, "TEST: %s\n", nodePath.c_str()); 323 else if (node->getNodeType() != NODETYPE_ROOT) 324 fprintf(m_testCaseListFile, "GROUP: %s\n", nodePath.c_str()); 325 iter.setState(isLeaf ? NodeIter::STATE_FINISH : NodeIter::STATE_TRAVERSE_CHILDREN); 326 } 327 328 break; 329 } 330 331 case NodeIter::STATE_EXECUTE_TEST: 332 { 333 // Touch the watchdog. 334 m_testCtx.touchWatchdog(); 335 336 // Iterate the sub-case. 337 TestCase::IterateResult iterateResult = m_testCaseWrapper->iterateTestCase(static_cast<TestCase*>(node)); 338 339 if (iterateResult == TestCase::STOP) 340 iter.setState(NodeIter::STATE_FINISH); 341 342 return true; // return after each iteration (when another iteration follows). 343 } 344 345 case NodeIter::STATE_TRAVERSE_CHILDREN: 346 { 347 int numChildren = (int)iter.children.size(); 348 if (++iter.curChildNdx < numChildren) 349 { 350 // Push child to stack. 351 TestNode* childNode = iter.children[iter.curChildNdx]; 352 m_sessionStack.push_back(NodeIter(childNode)); 353 } 354 else 355 iter.setState(NodeIter::STATE_FINISH); 356 357 break; 358 } 359 360 case NodeIter::STATE_FINISH: 361 { 362 if (m_cmdLine.getRunMode() == RUNMODE_DUMP_XML_CASELIST) 363 { 364 if (node->getNodeType() != NODETYPE_ROOT && node->getNodeType() != NODETYPE_PACKAGE) 365 qpXmlWriter_endElement(m_testCaseListWriter, "TestCase"); 366 } 367 368 // Leave node. 369 switch (node->getNodeType()) 370 { 371 case NODETYPE_ROOT: /* nada */ break; 372 case NODETYPE_PACKAGE: leaveTestPackage(static_cast<TestPackage*>(node)); break; 373 case NODETYPE_GROUP: leaveGroupNode(static_cast<TestCaseGroup*>(node)); break; 374 case NODETYPE_ACCURACY: 375 case NODETYPE_CAPABILITY: 376 case NODETYPE_PERFORMANCE: /* fall-thru */ 377 case NODETYPE_SELF_VALIDATE: leaveTestCase(static_cast<TestCase*>(node)); break; 378 default: DE_ASSERT(false); 379 } 380 381 m_sessionStack.pop_back(); 382 383 // Return if execution should abort. 384 if (m_abortSession) 385 return false; 386 387 // Otherwise continue iterating. 388 break; 389 } 390 391 default: 392 DE_ASSERT(DE_FALSE); 393 break; 394 } 395 } 396 } 397 catch (const std::exception& e) 398 { 399 print("TestExecutor::iterateSession(): Caught unhandled %s: %s\n", typeid(e).name(), e.what()); 400 throw; 401 } 402 403 m_result.isComplete = true; 404 return false; 405 } 406 407 } // tcu 408
