Home | History | Annotate | Download | only in common 
      1 /*-------------------------------------------------------------------------
      2  * drawElements Quality Program Tester Core
      3  * ----------------------------------------
      4  *
      5  * Copyright 2014 The Android Open Source Project
      6  *
      7  * Licensed under the Apache License, Version 2.0 (the "License");
      8  * you may not use this file except in compliance with the License.
      9  * You may obtain a copy of the License at
     10  *
     11  *      http://www.apache.org/licenses/LICENSE-2.0
     12  *
     13  * Unless required by applicable law or agreed to in writing, software
     14  * distributed under the License is distributed on an "AS IS" BASIS,
     15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     16  * See the License for the specific language governing permissions and
     17  * limitations under the License.
     18  *
     19  *//*!
     20  * \file
     21  * \brief Test executor.
     22  *//*--------------------------------------------------------------------*/
     23 
     24 #include "tcuTestSessionExecutor.hpp"
     25 #include "tcuCommandLine.hpp"
     26 #include "tcuTestLog.hpp"
     27 
     28 #include "deClock.h"
     29 
     30 namespace tcu
     31 {
     32 
     33 using std::vector;
     34 
     35 static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
     36 {
     37 	switch (nodeType)
     38 	{
     39 		case NODETYPE_SELF_VALIDATE:	return QP_TEST_CASE_TYPE_SELF_VALIDATE;
     40 		case NODETYPE_PERFORMANCE:		return QP_TEST_CASE_TYPE_PERFORMANCE;
     41 		case NODETYPE_CAPABILITY:		return QP_TEST_CASE_TYPE_CAPABILITY;
     42 		case NODETYPE_ACCURACY:			return QP_TEST_CASE_TYPE_ACCURACY;
     43 		default:
     44 			DE_ASSERT(false);
     45 			return QP_TEST_CASE_TYPE_LAST;
     46 	}
     47 }
     48 
     49 TestSessionExecutor::TestSessionExecutor (TestPackageRoot& root, TestContext& testCtx)
     50 	: m_testCtx				(testCtx)
     51 	, m_inflater			(testCtx)
     52 	, m_caseListFilter		(testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()))
     53 	, m_iterator			(root, m_inflater, *m_caseListFilter)
     54 	, m_state				(STATE_TRAVERSE_HIERARCHY)
     55 	, m_abortSession		(false)
     56 	, m_isInTestCase		(false)
     57 	, m_testStartTime		(0)
     58 	, m_packageStartTime	(0)
     59 {
     60 }
     61 
     62 TestSessionExecutor::~TestSessionExecutor (void)
     63 {
     64 }
     65 
     66 bool TestSessionExecutor::iterate (void)
     67 {
     68 	while (!m_abortSession)
     69 	{
     70 		switch (m_state)
     71 		{
     72 			case STATE_TRAVERSE_HIERARCHY:
     73 			{
     74 				const TestHierarchyIterator::State	hierIterState	= m_iterator.getState();
     75 
     76 				if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
     77 					hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
     78 				{
     79 					TestNode* const		curNode		= m_iterator.getNode();
     80 					const TestNodeType	nodeType	= curNode->getNodeType();
     81 					const bool			isEnter		= hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
     82 
     83 					switch (nodeType)
     84 					{
     85 						case NODETYPE_PACKAGE:
     86 						{
     87 							TestPackage* const testPackage = static_cast<TestPackage*>(curNode);
     88 							isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
     89 							break;
     90 						}
     91 
     92 						case NODETYPE_GROUP:
     93 						{
     94 							isEnter ? enterTestGroup(m_iterator.getNodePath()) : leaveTestGroup(m_iterator.getNodePath());
     95 							break; // nada
     96 						}
     97 
     98 						case NODETYPE_SELF_VALIDATE:
     99 						case NODETYPE_PERFORMANCE:
    100 						case NODETYPE_CAPABILITY:
    101 						case NODETYPE_ACCURACY:
    102 						{
    103 							TestCase* const testCase = static_cast<TestCase*>(curNode);
    104 
    105 							if (isEnter)
    106 							{
    107 								if (enterTestCase(testCase, m_iterator.getNodePath()))
    108 									m_state = STATE_EXECUTE_TEST_CASE;
    109 								// else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
    110 							}
    111 							else
    112 								leaveTestCase(testCase);
    113 
    114 							break;
    115 						}
    116 
    117 						default:
    118 							DE_ASSERT(false);
    119 							break;
    120 					}
    121 
    122 					m_iterator.next();
    123 					break;
    124 				}
    125 				else
    126 				{
    127 					DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
    128 					m_status.isComplete = true;
    129 					return false;
    130 				}
    131 			}
    132 
    133 			case STATE_EXECUTE_TEST_CASE:
    134 			{
    135 				DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
    136 						  isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
    137 
    138 				TestCase* const					testCase	= static_cast<TestCase*>(m_iterator.getNode());
    139 				const TestCase::IterateResult	iterResult	= iterateTestCase(testCase);
    140 
    141 				if (iterResult == TestCase::STOP)
    142 					m_state = STATE_TRAVERSE_HIERARCHY;
    143 
    144 				return true;
    145 			}
    146 
    147 			default:
    148 				DE_ASSERT(false);
    149 				break;
    150 		}
    151 	}
    152 
    153 	return false;
    154 }
    155 
    156 void TestSessionExecutor::enterTestPackage (TestPackage* testPackage)
    157 {
    158 	// Create test case wrapper
    159 	DE_ASSERT(!m_caseExecutor);
    160 	m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
    161 	m_packageStartTime	= deGetMicroseconds();
    162 }
    163 
    164 void TestSessionExecutor::leaveTestPackage (TestPackage* testPackage)
    165 {
    166 	DE_UNREF(testPackage);
    167 	m_caseExecutor.clear();
    168 	m_testCtx.getLog().startTestsCasesTime();
    169 
    170 	{
    171 		const deInt64 duration = deGetMicroseconds() - m_packageStartTime;
    172 		m_packageStartTime = 0;
    173 		m_testCtx.getLog() << TestLog::Integer(testPackage->getName(), "Total tests case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
    174 	}
    175 
    176 	for(std::map<std::string, deUint64>::iterator it=m_groupsDurationTime.begin(); it != m_groupsDurationTime.end(); ++it)
    177 		m_testCtx.getLog() << TestLog::Integer(it->first, "The test group case duration in microseconds", "us", QP_KEY_TAG_TIME, it->second);
    178 
    179 	m_testCtx.getLog().endTestsCasesTime();
    180 }
    181 
    182 void TestSessionExecutor::enterTestGroup (const std::string& casePath)
    183 {
    184 	m_groupsDurationTime[casePath] = deGetMicroseconds();
    185 }
    186 
    187 void TestSessionExecutor::leaveTestGroup (const std::string& casePath)
    188 {
    189 	m_groupsDurationTime[casePath] = deGetMicroseconds() - m_groupsDurationTime[casePath];
    190 }
    191 
    192 bool TestSessionExecutor::enterTestCase (TestCase* testCase, const std::string& casePath)
    193 {
    194 	TestLog&				log			= m_testCtx.getLog();
    195 	const qpTestCaseType	caseType	= nodeTypeToTestCaseType(testCase->getNodeType());
    196 	bool					initOk		= false;
    197 
    198 	print("\nTest case '%s'..\n", casePath.c_str());
    199 
    200 	m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
    201 	m_testCtx.setTerminateAfter(false);
    202 	log.startCase(casePath.c_str(), caseType);
    203 
    204 	m_isInTestCase	= true;
    205 	m_testStartTime	= deGetMicroseconds();
    206 
    207 	try
    208 	{
    209 		m_caseExecutor->init(testCase, casePath);
    210 		initOk = true;
    211 	}
    212 	catch (const std::bad_alloc&)
    213 	{
    214 		DE_ASSERT(!initOk);
    215 		m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
    216 		m_testCtx.setTerminateAfter(true);
    217 	}
    218 	catch (const tcu::TestException& e)
    219 	{
    220 		DE_ASSERT(!initOk);
    221 		DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
    222 		m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
    223 		m_testCtx.setTerminateAfter(e.isFatal());
    224 		log << e;
    225 	}
    226 	catch (const tcu::Exception& e)
    227 	{
    228 		DE_ASSERT(!initOk);
    229 		m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
    230 		log << e;
    231 	}
    232 
    233 	DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
    234 
    235 	return initOk;
    236 }
    237 
    238 void TestSessionExecutor::leaveTestCase (TestCase* testCase)
    239 {
    240 	TestLog&	log		= m_testCtx.getLog();
    241 
    242 	// De-init case.
    243 	try
    244 	{
    245 		m_caseExecutor->deinit(testCase);
    246 	}
    247 	catch (const tcu::Exception& e)
    248 	{
    249 		log << e << TestLog::Message << "Error in test case deinit, test program will terminate." << TestLog::EndMessage;
    250 		m_testCtx.setTerminateAfter(true);
    251 	}
    252 
    253 	{
    254 		const deInt64 duration = deGetMicroseconds()-m_testStartTime;
    255 		m_testStartTime = 0;
    256 		m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
    257 	}
    258 
    259 	{
    260 		const qpTestResult	testResult		= m_testCtx.getTestResult();
    261 		const char* const	testResultDesc	= m_testCtx.getTestResultDesc();
    262 		const bool			terminateAfter	= m_testCtx.getTerminateAfter();
    263 		DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
    264 
    265 		m_isInTestCase = false;
    266 		m_testCtx.getLog().endCase(testResult, testResultDesc);
    267 
    268 		// Update statistics.
    269 		print("  %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
    270 
    271 		m_status.numExecuted += 1;
    272 		switch (testResult)
    273 		{
    274 			case QP_TEST_RESULT_PASS:					m_status.numPassed			+= 1;	break;
    275 			case QP_TEST_RESULT_NOT_SUPPORTED:			m_status.numNotSupported	+= 1;	break;
    276 			case QP_TEST_RESULT_QUALITY_WARNING:		m_status.numWarnings		+= 1;	break;
    277 			case QP_TEST_RESULT_COMPATIBILITY_WARNING:	m_status.numWarnings		+= 1;	break;
    278 			default:									m_status.numFailed			+= 1;	break;
    279 		}
    280 
    281 		// terminateAfter, Resource error or any error in deinit means that execution should end
    282 		if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR)
    283 			m_abortSession = true;
    284 	}
    285 
    286 	if (m_testCtx.getWatchDog())
    287 		qpWatchDog_reset(m_testCtx.getWatchDog());
    288 }
    289 
    290 TestCase::IterateResult TestSessionExecutor::iterateTestCase (TestCase* testCase)
    291 {
    292 	TestLog&				log				= m_testCtx.getLog();
    293 	TestCase::IterateResult	iterateResult	= TestCase::STOP;
    294 
    295 	m_testCtx.touchWatchdog();
    296 
    297 	try
    298 	{
    299 		iterateResult = m_caseExecutor->iterate(testCase);
    300 	}
    301 	catch (const std::bad_alloc&)
    302 	{
    303 		m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
    304 		m_testCtx.setTerminateAfter(true);
    305 	}
    306 	catch (const tcu::TestException& e)
    307 	{
    308 		log << e;
    309 		m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
    310 		m_testCtx.setTerminateAfter(e.isFatal());
    311 	}
    312 	catch (const tcu::Exception& e)
    313 	{
    314 		log << e;
    315 		m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
    316 	}
    317 
    318 	return iterateResult;
    319 }
    320 
    321 } // tcu
    322