Home | History | Annotate | Download | only in tools
      1 /*-------------------------------------------------------------------------
      2  * drawElements Quality Program Test Executor
      3  * ------------------------------------------
      4  *
      5  * Copyright 2014 The Android Open Source Project
      6  *
      7  * Licensed under the Apache License, Version 2.0 (the "License");
      8  * you may not use this file except in compliance with the License.
      9  * You may obtain a copy of the License at
     10  *
     11  *      http://www.apache.org/licenses/LICENSE-2.0
     12  *
     13  * Unless required by applicable law or agreed to in writing, software
     14  * distributed under the License is distributed on an "AS IS" BASIS,
     15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
     16  * See the License for the specific language governing permissions and
     17  * limitations under the License.
     18  *
     19  *//*!
     20  * \file
     21  * \brief Command line test executor.
     22  *//*--------------------------------------------------------------------*/
     23 
     24 #include "xeBatchExecutor.hpp"
     25 #include "xeTestCaseListParser.hpp"
     26 #include "xeTcpIpLink.hpp"
     27 #include "xeLocalTcpIpLink.hpp"
     28 #include "xeTestResultParser.hpp"
     29 #include "xeTestLogWriter.hpp"
     30 #include "deDirectoryIterator.hpp"
     31 #include "deCommandLine.hpp"
     32 #include "deString.h"
     33 
     34 #include <vector>
     35 #include <string>
     36 #include <cstdio>
     37 #include <cstdlib>
     38 #include <fstream>
     39 #include <memory>
     40 #include <algorithm>
     41 #include <iostream>
     42 #include <sstream>
     43 
     44 // Command line arguments.
     45 namespace opt
     46 {
     47 
     48 DE_DECLARE_COMMAND_LINE_OPT(StartServer,	std::string);
     49 DE_DECLARE_COMMAND_LINE_OPT(Host,			std::string);
     50 DE_DECLARE_COMMAND_LINE_OPT(Port,			int);
     51 DE_DECLARE_COMMAND_LINE_OPT(CaseListDir,	std::string);
     52 DE_DECLARE_COMMAND_LINE_OPT(TestSet,		std::vector<std::string>);
     53 DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet,		std::vector<std::string>);
     54 DE_DECLARE_COMMAND_LINE_OPT(ContinueFile,	std::string);
     55 DE_DECLARE_COMMAND_LINE_OPT(TestLogFile,	std::string);
     56 DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile,	std::string);
     57 DE_DECLARE_COMMAND_LINE_OPT(Summary,		bool);
     58 
     59 // TargetConfiguration
     60 DE_DECLARE_COMMAND_LINE_OPT(BinaryName,		std::string);
     61 DE_DECLARE_COMMAND_LINE_OPT(WorkingDir,		std::string);
     62 DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs,	std::string);
     63 
     64 static void parseCommaSeparatedList (const char* src, std::vector<std::string>* dst)
     65 {
     66 	std::istringstream	inStr	(src);
     67 	std::string			comp;
     68 
     69 	while (std::getline(inStr, comp, ','))
     70 		dst->push_back(comp);
     71 }
     72 
     73 void registerOptions (de::cmdline::Parser& parser)
     74 {
     75 	using de::cmdline::Option;
     76 	using de::cmdline::NamedValue;
     77 
     78 	static const NamedValue<bool> s_yesNo[] =
     79 	{
     80 		{ "yes",	true	},
     81 		{ "no",		false	}
     82 	};
     83 
     84 	parser << Option<StartServer>	("s",		"start-server",	"Start local execserver")
     85 		   << Option<Host>			("c",		"connect",		"Connect to host",							"127.0.0.1")
     86 		   << Option<Port>			("p",		"port",			"Select TCP port to use",					"50016")
     87 		   << Option<CaseListDir>	("cd",		"caselistdir",	"Path to test case XML files",				".")
     88 		   << Option<TestSet>		("t",		"testset",		"Test set",									parseCommaSeparatedList)
     89 		   << Option<ExcludeSet>	("e",		"exclude",		"Comma-separated list of exclude filters",	parseCommaSeparatedList)
     90 		   << Option<ContinueFile>	(DE_NULL,	"continue",		"Continue execution by initializing results from existing test log")
     91 		   << Option<TestLogFile>	("o",		"out",			"Output test log filename")
     92 		   << Option<InfoLogFile>	("i",		"info",			"Output info log filename")
     93 		   << Option<Summary>		(DE_NULL,	"summary",		"Print summary at the end",					s_yesNo,	"yes")
     94 		   << Option<BinaryName>	("b",		"binaryname",	"Test binary path, relative to working directory")
     95 		   << Option<WorkingDir>	("wd",		"workdir",		"Working directory for test execution")
     96 		   << Option<CmdLineArgs>	(DE_NULL,	"cmdline",		"Additional command line arguments for test binary");
     97 }
     98 
     99 } // opt
    100 
    101 using std::vector;
    102 using std::string;
    103 
    104 struct CommandLine
    105 {
    106 	CommandLine (void)
    107 		: port		(0)
    108 		, summary	(false)
    109 	{
    110 	}
    111 
    112 	xe::TargetConfiguration		targetCfg;
    113 	std::string					serverBin;
    114 	std::string					host;
    115 	int							port;
    116 	std::string					caseListDir;
    117 	std::vector<std::string>	testset;
    118 	std::vector<std::string>	exclude;
    119 	std::string					inFile;
    120 	std::string					outFile;
    121 	std::string					infoFile;
    122 	bool						summary;
    123 };
    124 
    125 static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
    126 {
    127 	de::cmdline::Parser			parser;
    128 	de::cmdline::CommandLine	opts;
    129 
    130 	XE_CHECK(argc >= 1);
    131 
    132 	opt::registerOptions(parser);
    133 
    134 	if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
    135 	{
    136 		std::cout << argv[0] << " [options]\n";
    137 		parser.help(std::cout);
    138 		return false;
    139 	}
    140 
    141 	cmdLine.serverBin				= opts.getOption<opt::StartServer>();
    142 	cmdLine.host					= opts.getOption<opt::Host>();
    143 	cmdLine.port					= opts.getOption<opt::Port>();
    144 	cmdLine.caseListDir				= opts.getOption<opt::CaseListDir>();
    145 	cmdLine.testset					= opts.getOption<opt::TestSet>();
    146 	cmdLine.exclude					= opts.getOption<opt::ExcludeSet>();
    147 	cmdLine.inFile					= opts.getOption<opt::ContinueFile>();
    148 	cmdLine.outFile					= opts.getOption<opt::TestLogFile>();
    149 	cmdLine.infoFile				= opts.getOption<opt::InfoLogFile>();
    150 	cmdLine.summary					= opts.getOption<opt::Summary>();
    151 	cmdLine.targetCfg.binaryName	= opts.getOption<opt::BinaryName>();
    152 	cmdLine.targetCfg.workingDir	= opts.getOption<opt::WorkingDir>();
    153 	cmdLine.targetCfg.cmdLineArgs	= opts.getOption<opt::CmdLineArgs>();
    154 
    155 	return true;
    156 }
    157 
    158 static bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
    159 {
    160 	int ptrnPos = 0;
    161 	int casePos = 0;
    162 
    163 	for (;;)
    164 	{
    165 		char c = casePath[casePos];
    166 		char p = pattern[ptrnPos];
    167 
    168 		if (p == '*')
    169 		{
    170 			/* Recurse to rest of positions. */
    171 			int next = casePos;
    172 			for (;;)
    173 			{
    174 				if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
    175 					return DE_TRUE;
    176 
    177 				if (casePath[next] == 0)
    178 					return DE_FALSE; /* No match found. */
    179 				else
    180 					next += 1;
    181 			}
    182 			DE_ASSERT(DE_FALSE);
    183 		}
    184 		else if (c == 0 && p == 0)
    185 			return true;
    186 		else if (c == 0)
    187 		{
    188 			/* Incomplete match is ok for test groups. */
    189 			return isTestGroup;
    190 		}
    191 		else if (c != p)
    192 			return false;
    193 
    194 		casePos += 1;
    195 		ptrnPos += 1;
    196 	}
    197 
    198 	DE_ASSERT(false);
    199 	return false;
    200 }
    201 
    202 static void readCaseList (xe::TestGroup* root, const char* filename)
    203 {
    204 	xe::TestCaseListParser	caseListParser;
    205 	std::ifstream			in				(filename, std::ios_base::binary);
    206 	deUint8					buf[1024];
    207 
    208 	XE_CHECK(in.good());
    209 
    210 	caseListParser.init(root);
    211 
    212 	for (;;)
    213 	{
    214 		in.read((char*)&buf[0], sizeof(buf));
    215 		int numRead = (int)in.gcount();
    216 
    217 		if (numRead > 0)
    218 			caseListParser.parse(&buf[0], numRead);
    219 
    220 		if (numRead < (int)sizeof(buf))
    221 			break; // EOF
    222 	}
    223 }
    224 
    225 static void readCaseLists (xe::TestRoot& root, const char* caseListDir)
    226 {
    227 	de::DirectoryIterator iter(caseListDir);
    228 
    229 	for (; iter.hasItem(); iter.next())
    230 	{
    231 		de::FilePath item = iter.getItem();
    232 
    233 		if (item.getType() == de::FilePath::TYPE_FILE)
    234 		{
    235 			std::string baseName = item.getBaseName();
    236 			if (baseName.find("-cases.xml") == baseName.length()-10)
    237 			{
    238 				std::string		packageName	= baseName.substr(0, baseName.length()-10);
    239 				xe::TestGroup*	package		= root.createGroup(packageName.c_str(), "");
    240 
    241 				readCaseList(package, item.getPath());
    242 			}
    243 		}
    244 	}
    245 }
    246 
    247 static void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
    248 {
    249 	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
    250 	{
    251 		const xe::TestNode* child		= group.getChild(childNdx);
    252 		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
    253 		const std::string	fullPath	= child->getFullPath();
    254 
    255 		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
    256 		{
    257 			if (isGroup)
    258 			{
    259 				// Recurse into group.
    260 				addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
    261 			}
    262 			else
    263 			{
    264 				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
    265 				testSet.add(child);
    266 			}
    267 		}
    268 	}
    269 }
    270 
    271 static void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
    272 {
    273 	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
    274 	{
    275 		const xe::TestNode* child		= group.getChild(childNdx);
    276 		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
    277 		const std::string	fullPath	= child->getFullPath();
    278 
    279 		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
    280 		{
    281 			if (isGroup)
    282 			{
    283 				// Recurse into group.
    284 				removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
    285 			}
    286 			else
    287 			{
    288 				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
    289 				testSet.remove(child);
    290 			}
    291 		}
    292 	}
    293 }
    294 
    295 class BatchResultHandler : public xe::TestLogHandler
    296 {
    297 public:
    298 	BatchResultHandler (xe::BatchResult* batchResult)
    299 		: m_batchResult(batchResult)
    300 	{
    301 	}
    302 
    303 	void setSessionInfo (const xe::SessionInfo& sessionInfo)
    304 	{
    305 		m_batchResult->getSessionInfo() = sessionInfo;
    306 	}
    307 
    308 	xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
    309 	{
    310 		// \todo [2012-11-01 pyry] What to do with duplicate results?
    311 		if (m_batchResult->hasTestCaseResult(casePath))
    312 			return m_batchResult->getTestCaseResult(casePath);
    313 		else
    314 			return m_batchResult->createTestCaseResult(casePath);
    315 	}
    316 
    317 	void testCaseResultUpdated (const xe::TestCaseResultPtr&)
    318 	{
    319 	}
    320 
    321 	void testCaseResultComplete (const xe::TestCaseResultPtr&)
    322 	{
    323 	}
    324 
    325 private:
    326 	xe::BatchResult* m_batchResult;
    327 };
    328 
    329 static void readLogFile (xe::BatchResult* batchResult, const char* filename)
    330 {
    331 	std::ifstream		in		(filename, std::ifstream::binary|std::ifstream::in);
    332 	BatchResultHandler	handler	(batchResult);
    333 	xe::TestLogParser	parser	(&handler);
    334 	deUint8				buf		[1024];
    335 	int					numRead	= 0;
    336 
    337 	for (;;)
    338 	{
    339 		in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
    340 		numRead = (int)in.gcount();
    341 
    342 		if (numRead <= 0)
    343 			break;
    344 
    345 		parser.parse(&buf[0], numRead);
    346 	}
    347 
    348 	in.close();
    349 }
    350 
    351 static void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
    352 {
    353 	int countByStatusCode[xe::TESTSTATUSCODE_LAST];
    354 	std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
    355 
    356 	for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
    357 	{
    358 		const xe::TestNode* node = *iter;
    359 		if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
    360 		{
    361 			const xe::TestCase*				testCase		= static_cast<const xe::TestCase*>(node);
    362 			std::string						fullPath;
    363 			xe::TestStatusCode				statusCode		= xe::TESTSTATUSCODE_PENDING;
    364 			testCase->getFullPath(fullPath);
    365 
    366 			// Parse result data if such exists.
    367 			if (batchResult.hasTestCaseResult(fullPath.c_str()))
    368 			{
    369 				xe::ConstTestCaseResultPtr	resultData	= batchResult.getTestCaseResult(fullPath.c_str());
    370 				xe::TestCaseResult			result;
    371 				xe::TestResultParser		parser;
    372 
    373 				xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
    374 				statusCode = result.statusCode;
    375 			}
    376 
    377 			countByStatusCode[statusCode] += 1;
    378 		}
    379 	}
    380 
    381 	printf("\nTest run summary:\n");
    382 	int totalCases = 0;
    383 	for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
    384 	{
    385 		if (countByStatusCode[code] > 0)
    386 			printf("  %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
    387 
    388 		totalCases += countByStatusCode[code];
    389 	}
    390 	printf("  %20s: %5d\n", "Total", totalCases);
    391 }
    392 
    393 static void writeInfoLog (const xe::InfoLog& log, const char* filename)
    394 {
    395 	std::ofstream out(filename, std::ios_base::binary);
    396 	XE_CHECK(out.good());
    397 	out.write((const char*)log.getBytes(), log.getSize());
    398 	out.close();
    399 }
    400 
    401 static xe::CommLink* createCommLink (const CommandLine& cmdLine)
    402 {
    403 	if (!cmdLine.serverBin.empty())
    404 	{
    405 		xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
    406 		try
    407 		{
    408 			link->start(cmdLine.serverBin.c_str(), DE_NULL, cmdLine.port);
    409 			return link;
    410 		}
    411 		catch (...)
    412 		{
    413 			delete link;
    414 			throw;
    415 		}
    416 	}
    417 	else
    418 	{
    419 		de::SocketAddress address;
    420 		address.setFamily(DE_SOCKETFAMILY_INET4);
    421 		address.setProtocol(DE_SOCKETPROTOCOL_TCP);
    422 		address.setHost(cmdLine.host.c_str());
    423 		address.setPort(cmdLine.port);
    424 
    425 		xe::TcpIpLink* link = new xe::TcpIpLink();
    426 		try
    427 		{
    428 			link->connect(address);
    429 			return link;
    430 		}
    431 		catch (...)
    432 		{
    433 			delete link;
    434 			throw;
    435 		}
    436 	}
    437 }
    438 
    439 static void runExecutor (const CommandLine& cmdLine)
    440 {
    441 	xe::TestRoot root;
    442 
    443 	// Read case list definitions.
    444 	readCaseLists(root, cmdLine.caseListDir.c_str());
    445 
    446 	// Build test set.
    447 	xe::TestSet testSet;
    448 
    449 	// Build test set.
    450 	for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
    451 		addMatchingCases(root, testSet, filterIter->c_str());
    452 
    453 	// Remove excluded cases.
    454 	for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
    455 		removeMatchingCases(root, testSet, filterIter->c_str());
    456 
    457 	// Initialize batch result.
    458 	xe::BatchResult	batchResult;
    459 	xe::InfoLog		infoLog;
    460 
    461 	// Read existing results from input file (if supplied).
    462 	if (!cmdLine.inFile.empty())
    463 		readLogFile(&batchResult, cmdLine.inFile.c_str());
    464 
    465 	// Initialize commLink.
    466 	std::auto_ptr<xe::CommLink> commLink(createCommLink(cmdLine));
    467 
    468 	xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
    469 	executor.run();
    470 
    471 	commLink.reset();
    472 
    473 	if (!cmdLine.outFile.empty())
    474 	{
    475 		xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
    476 		printf("Test log written to %s\n", cmdLine.outFile.c_str());
    477 	}
    478 
    479 	if (!cmdLine.infoFile.empty())
    480 	{
    481 		writeInfoLog(infoLog, cmdLine.infoFile.c_str());
    482 		printf("Info log written to %s\n", cmdLine.infoFile.c_str());
    483 	}
    484 
    485 	if (cmdLine.summary)
    486 		printBatchResultSummary(&root, testSet, batchResult);
    487 }
    488 
    489 int main (int argc, const char* const* argv)
    490 {
    491 	CommandLine cmdLine;
    492 
    493 	if (!parseCommandLine(cmdLine, argc, argv))
    494 		return -1;
    495 
    496 	try
    497 	{
    498 		runExecutor(cmdLine);
    499 	}
    500 	catch (const std::exception& e)
    501 	{
    502 		printf("%s\n", e.what());
    503 		return -1;
    504 	}
    505 
    506 	return 0;
    507 }
    508