1 /*------------------------------------------------------------------------- 2 * drawElements Quality Program Test Executor 3 * ------------------------------------------ 4 * 5 * Copyright 2014 The Android Open Source Project 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 * 19 *//*! 20 * \file 21 * \brief Command line test executor. 22 *//*--------------------------------------------------------------------*/ 23 24 #include "xeBatchExecutor.hpp" 25 #include "xeTestCaseListParser.hpp" 26 #include "xeTcpIpLink.hpp" 27 #include "xeLocalTcpIpLink.hpp" 28 #include "xeTestResultParser.hpp" 29 #include "xeTestLogWriter.hpp" 30 #include "deDirectoryIterator.hpp" 31 #include "deCommandLine.hpp" 32 #include "deString.h" 33 34 #include <vector> 35 #include <string> 36 #include <cstdio> 37 #include <cstdlib> 38 #include <fstream> 39 #include <memory> 40 #include <algorithm> 41 #include <iostream> 42 #include <sstream> 43 44 // Command line arguments. 45 namespace opt 46 { 47 48 DE_DECLARE_COMMAND_LINE_OPT(StartServer, std::string); 49 DE_DECLARE_COMMAND_LINE_OPT(Host, std::string); 50 DE_DECLARE_COMMAND_LINE_OPT(Port, int); 51 DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, std::string); 52 DE_DECLARE_COMMAND_LINE_OPT(TestSet, std::vector<std::string>); 53 DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, std::vector<std::string>); 54 DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, std::string); 55 DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, std::string); 56 DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, std::string); 57 DE_DECLARE_COMMAND_LINE_OPT(Summary, bool); 58 59 // TargetConfiguration 60 DE_DECLARE_COMMAND_LINE_OPT(BinaryName, std::string); 61 DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, std::string); 62 DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, std::string); 63 64 static void parseCommaSeparatedList (const char* src, std::vector<std::string>* dst) 65 { 66 std::istringstream inStr (src); 67 std::string comp; 68 69 while (std::getline(inStr, comp, ',')) 70 dst->push_back(comp); 71 } 72 73 void registerOptions (de::cmdline::Parser& parser) 74 { 75 using de::cmdline::Option; 76 using de::cmdline::NamedValue; 77 78 static const NamedValue<bool> s_yesNo[] = 79 { 80 { "yes", true }, 81 { "no", false } 82 }; 83 84 parser << Option<StartServer> ("s", "start-server", "Start local execserver", "") 85 << Option<Host> ("c", "connect", "Connect to host", "127.0.0.1") 86 << Option<Port> ("p", "port", "Select TCP port to use", "50016") 87 << Option<CaseListDir> ("cd", "caselistdir", "Path to test case XML files", ".") 88 << Option<TestSet> ("t", "testset", "Test set", parseCommaSeparatedList, "") 89 << Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters", parseCommaSeparatedList, "") 90 << Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log", "") 91 << Option<TestLogFile> ("o", "out", "Output test log filename", "") 92 << Option<InfoLogFile> ("i", "info", "Output info log filename", "") 93 << Option<Summary> (DE_NULL, "summary", "Print summary at the end", s_yesNo, "yes") 94 << Option<BinaryName> ("b", "binaryname", "Test binary path, relative to working directory", "") 95 << Option<WorkingDir> ("wd", "workdir", "Working directory for test execution", "") 96 << Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for test binary", ""); 97 } 98 99 } // opt 100 101 using std::vector; 102 using std::string; 103 104 struct CommandLine 105 { 106 CommandLine (void) 107 : port (0) 108 , summary (false) 109 { 110 } 111 112 xe::TargetConfiguration targetCfg; 113 std::string serverBin; 114 std::string host; 115 int port; 116 std::string caseListDir; 117 std::vector<std::string> testset; 118 std::vector<std::string> exclude; 119 std::string inFile; 120 std::string outFile; 121 std::string infoFile; 122 bool summary; 123 }; 124 125 static bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv) 126 { 127 de::cmdline::Parser parser; 128 de::cmdline::CommandLine opts; 129 130 XE_CHECK(argc >= 1); 131 132 opt::registerOptions(parser); 133 134 if (!parser.parse(argc-1, argv+1, &opts, std::cerr)) 135 { 136 std::cout << argv[0] << " [options]\n"; 137 parser.help(std::cout); 138 return false; 139 } 140 141 cmdLine.serverBin = opts.getOption<opt::StartServer>(); 142 cmdLine.host = opts.getOption<opt::Host>(); 143 cmdLine.port = opts.getOption<opt::Port>(); 144 cmdLine.caseListDir = opts.getOption<opt::CaseListDir>(); 145 cmdLine.testset = opts.getOption<opt::TestSet>(); 146 cmdLine.exclude = opts.getOption<opt::ExcludeSet>(); 147 cmdLine.inFile = opts.getOption<opt::ContinueFile>(); 148 cmdLine.outFile = opts.getOption<opt::TestLogFile>(); 149 cmdLine.infoFile = opts.getOption<opt::InfoLogFile>(); 150 cmdLine.summary = opts.getOption<opt::Summary>(); 151 cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>(); 152 cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>(); 153 cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>(); 154 155 return true; 156 } 157 158 static bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup) 159 { 160 int ptrnPos = 0; 161 int casePos = 0; 162 163 for (;;) 164 { 165 char c = casePath[casePos]; 166 char p = pattern[ptrnPos]; 167 168 if (p == '*') 169 { 170 /* Recurse to rest of positions. */ 171 int next = casePos; 172 for (;;) 173 { 174 if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup)) 175 return DE_TRUE; 176 177 if (casePath[next] == 0) 178 return DE_FALSE; /* No match found. */ 179 else 180 next += 1; 181 } 182 DE_ASSERT(DE_FALSE); 183 } 184 else if (c == 0 && p == 0) 185 return true; 186 else if (c == 0) 187 { 188 /* Incomplete match is ok for test groups. */ 189 return isTestGroup; 190 } 191 else if (c != p) 192 return false; 193 194 casePos += 1; 195 ptrnPos += 1; 196 } 197 198 DE_ASSERT(false); 199 return false; 200 } 201 202 static void readCaseList (xe::TestGroup* root, const char* filename) 203 { 204 xe::TestCaseListParser caseListParser; 205 std::ifstream in (filename, std::ios_base::binary); 206 deUint8 buf[1024]; 207 208 XE_CHECK(in.good()); 209 210 caseListParser.init(root); 211 212 for (;;) 213 { 214 in.read((char*)&buf[0], sizeof(buf)); 215 int numRead = (int)in.gcount(); 216 217 if (numRead > 0) 218 caseListParser.parse(&buf[0], numRead); 219 220 if (numRead < (int)sizeof(buf)) 221 break; // EOF 222 } 223 } 224 225 static void readCaseLists (xe::TestRoot& root, const char* caseListDir) 226 { 227 de::DirectoryIterator iter(caseListDir); 228 229 for (; iter.hasItem(); iter.next()) 230 { 231 de::FilePath item = iter.getItem(); 232 233 if (item.getType() == de::FilePath::TYPE_FILE) 234 { 235 std::string baseName = item.getBaseName(); 236 if (baseName.find("-cases.xml") == baseName.length()-10) 237 { 238 std::string packageName = baseName.substr(0, baseName.length()-10); 239 xe::TestGroup* package = root.createGroup(packageName.c_str(), ""); 240 241 readCaseList(package, item.getPath()); 242 } 243 } 244 } 245 } 246 247 static void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) 248 { 249 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) 250 { 251 const xe::TestNode* child = group.getChild(childNdx); 252 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; 253 const std::string fullPath = child->getFullPath(); 254 255 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) 256 { 257 if (isGroup) 258 { 259 // Recurse into group. 260 addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); 261 } 262 else 263 { 264 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); 265 testSet.add(child); 266 } 267 } 268 } 269 } 270 271 static void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) 272 { 273 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) 274 { 275 const xe::TestNode* child = group.getChild(childNdx); 276 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; 277 const std::string fullPath = child->getFullPath(); 278 279 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) 280 { 281 if (isGroup) 282 { 283 // Recurse into group. 284 removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); 285 } 286 else 287 { 288 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); 289 testSet.remove(child); 290 } 291 } 292 } 293 } 294 295 class BatchResultHandler : public xe::TestLogHandler 296 { 297 public: 298 BatchResultHandler (xe::BatchResult* batchResult) 299 : m_batchResult(batchResult) 300 { 301 } 302 303 void setSessionInfo (const xe::SessionInfo& sessionInfo) 304 { 305 m_batchResult->getSessionInfo() = sessionInfo; 306 } 307 308 xe::TestCaseResultPtr startTestCaseResult (const char* casePath) 309 { 310 // \todo [2012-11-01 pyry] What to do with duplicate results? 311 if (m_batchResult->hasTestCaseResult(casePath)) 312 return m_batchResult->getTestCaseResult(casePath); 313 else 314 return m_batchResult->createTestCaseResult(casePath); 315 } 316 317 void testCaseResultUpdated (const xe::TestCaseResultPtr&) 318 { 319 } 320 321 void testCaseResultComplete (const xe::TestCaseResultPtr&) 322 { 323 } 324 325 private: 326 xe::BatchResult* m_batchResult; 327 }; 328 329 static void readLogFile (xe::BatchResult* batchResult, const char* filename) 330 { 331 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in); 332 BatchResultHandler handler (batchResult); 333 xe::TestLogParser parser (&handler); 334 deUint8 buf [1024]; 335 int numRead = 0; 336 337 for (;;) 338 { 339 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf)); 340 numRead = (int)in.gcount(); 341 342 if (numRead <= 0) 343 break; 344 345 parser.parse(&buf[0], numRead); 346 } 347 348 in.close(); 349 } 350 351 static void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult) 352 { 353 int countByStatusCode[xe::TESTSTATUSCODE_LAST]; 354 std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0); 355 356 for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter) 357 { 358 const xe::TestNode* node = *iter; 359 if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node)) 360 { 361 const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node); 362 std::string fullPath; 363 xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING; 364 testCase->getFullPath(fullPath); 365 366 // Parse result data if such exists. 367 if (batchResult.hasTestCaseResult(fullPath.c_str())) 368 { 369 xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str()); 370 xe::TestCaseResult result; 371 xe::TestResultParser parser; 372 373 xe::parseTestCaseResultFromData(&parser, &result, *resultData.get()); 374 statusCode = result.statusCode; 375 } 376 377 countByStatusCode[statusCode] += 1; 378 } 379 } 380 381 printf("\nTest run summary:\n"); 382 int totalCases = 0; 383 for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++) 384 { 385 if (countByStatusCode[code] > 0) 386 printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]); 387 388 totalCases += countByStatusCode[code]; 389 } 390 printf(" %20s: %5d\n", "Total", totalCases); 391 } 392 393 static void writeInfoLog (const xe::InfoLog& log, const char* filename) 394 { 395 std::ofstream out(filename, std::ios_base::binary); 396 XE_CHECK(out.good()); 397 out.write((const char*)log.getBytes(), log.getSize()); 398 out.close(); 399 } 400 401 static xe::CommLink* createCommLink (const CommandLine& cmdLine) 402 { 403 if (!cmdLine.serverBin.empty()) 404 { 405 xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink(); 406 try 407 { 408 link->start(cmdLine.serverBin.c_str(), DE_NULL, cmdLine.port); 409 return link; 410 } 411 catch (...) 412 { 413 delete link; 414 throw; 415 } 416 } 417 else 418 { 419 de::SocketAddress address; 420 address.setFamily(DE_SOCKETFAMILY_INET4); 421 address.setProtocol(DE_SOCKETPROTOCOL_TCP); 422 address.setHost(cmdLine.host.c_str()); 423 address.setPort(cmdLine.port); 424 425 xe::TcpIpLink* link = new xe::TcpIpLink(); 426 try 427 { 428 link->connect(address); 429 return link; 430 } 431 catch (...) 432 { 433 delete link; 434 throw; 435 } 436 } 437 } 438 439 static void runExecutor (const CommandLine& cmdLine) 440 { 441 xe::TestRoot root; 442 443 // Read case list definitions. 444 readCaseLists(root, cmdLine.caseListDir.c_str()); 445 446 // Build test set. 447 xe::TestSet testSet; 448 449 // Build test set. 450 for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter) 451 addMatchingCases(root, testSet, filterIter->c_str()); 452 453 // Remove excluded cases. 454 for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter) 455 removeMatchingCases(root, testSet, filterIter->c_str()); 456 457 // Initialize batch result. 458 xe::BatchResult batchResult; 459 xe::InfoLog infoLog; 460 461 // Read existing results from input file (if supplied). 462 if (!cmdLine.inFile.empty()) 463 readLogFile(&batchResult, cmdLine.inFile.c_str()); 464 465 // Initialize commLink. 466 std::auto_ptr<xe::CommLink> commLink(createCommLink(cmdLine)); 467 468 xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog); 469 executor.run(); 470 471 commLink.reset(); 472 473 if (!cmdLine.outFile.empty()) 474 { 475 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str()); 476 printf("Test log written to %s\n", cmdLine.outFile.c_str()); 477 } 478 479 if (!cmdLine.infoFile.empty()) 480 { 481 writeInfoLog(infoLog, cmdLine.infoFile.c_str()); 482 printf("Info log written to %s\n", cmdLine.infoFile.c_str()); 483 } 484 485 if (cmdLine.summary) 486 printBatchResultSummary(&root, testSet, batchResult); 487 } 488 489 int main (int argc, const char* const* argv) 490 { 491 CommandLine cmdLine; 492 493 if (!parseCommandLine(cmdLine, argc, argv)) 494 return -1; 495 496 try 497 { 498 runExecutor(cmdLine); 499 } 500 catch (const std::exception& e) 501 { 502 printf("%s\n", e.what()); 503 return -1; 504 } 505 506 return 0; 507 } 508
