1 /*------------------------------------------------------------------------- 2 * drawElements Quality Program Test Executor 3 * ------------------------------------------ 4 * 5 * Copyright 2014 The Android Open Source Project 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 * 19 *//*! 20 * \file 21 * \brief Command line test executor. 22 *//*--------------------------------------------------------------------*/ 23 24 #include "xeBatchExecutor.hpp" 25 #include "xeLocalTcpIpLink.hpp" 26 #include "xeTcpIpLink.hpp" 27 #include "xeTestCaseListParser.hpp" 28 #include "xeTestLogWriter.hpp" 29 #include "xeTestResultParser.hpp" 30 31 #include "deCommandLine.hpp" 32 #include "deDirectoryIterator.hpp" 33 #include "deStringUtil.hpp" 34 #include "deUniquePtr.hpp" 35 36 #include "deString.h" 37 38 #include <algorithm> 39 #include <cstdio> 40 #include <cstdlib> 41 #include <fstream> 42 #include <iostream> 43 #include <memory> 44 #include <sstream> 45 #include <string> 46 #include <vector> 47 48 #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32) 49 # include <signal.h> 50 #endif 51 52 using std::vector; 53 using std::string; 54 55 namespace 56 { 57 58 // Command line arguments. 59 namespace opt 60 { 61 62 DE_DECLARE_COMMAND_LINE_OPT(StartServer, string); 63 DE_DECLARE_COMMAND_LINE_OPT(Host, string); 64 DE_DECLARE_COMMAND_LINE_OPT(Port, int); 65 DE_DECLARE_COMMAND_LINE_OPT(CaseListDir, string); 66 DE_DECLARE_COMMAND_LINE_OPT(TestSet, vector<string>); 67 DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet, vector<string>); 68 DE_DECLARE_COMMAND_LINE_OPT(ContinueFile, string); 69 DE_DECLARE_COMMAND_LINE_OPT(TestLogFile, string); 70 DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile, string); 71 DE_DECLARE_COMMAND_LINE_OPT(Summary, bool); 72 73 // TargetConfiguration 74 DE_DECLARE_COMMAND_LINE_OPT(BinaryName, string); 75 DE_DECLARE_COMMAND_LINE_OPT(WorkingDir, string); 76 DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs, string); 77 78 void parseCommaSeparatedList (const char* src, vector<string>* dst) 79 { 80 std::istringstream inStr (src); 81 string comp; 82 83 while (std::getline(inStr, comp, ',')) 84 dst->push_back(comp); 85 } 86 87 void registerOptions (de::cmdline::Parser& parser) 88 { 89 using de::cmdline::Option; 90 using de::cmdline::NamedValue; 91 92 static const NamedValue<bool> s_yesNo[] = 93 { 94 { "yes", true }, 95 { "no", false } 96 }; 97 98 parser << Option<StartServer> ("s", "start-server", "Start local execserver. Path to the execserver binary.") 99 << Option<Host> ("c", "connect", "Connect to host. Address of the execserver.") 100 << Option<Port> ("p", "port", "TCP port of the execserver.", "50016") 101 << Option<CaseListDir> ("cd", "caselistdir", "Path to the directory containing test case XML files.", ".") 102 << Option<TestSet> ("t", "testset", "Comma-separated list of include filters.", parseCommaSeparatedList) 103 << Option<ExcludeSet> ("e", "exclude", "Comma-separated list of exclude filters.", parseCommaSeparatedList, "") 104 << Option<ContinueFile> (DE_NULL, "continue", "Continue execution by initializing results from existing test log.") 105 << Option<TestLogFile> ("o", "out", "Output test log filename.", "TestLog.qpa") 106 << Option<InfoLogFile> ("i", "info", "Output info log filename.", "InfoLog.txt") 107 << Option<Summary> (DE_NULL, "summary", "Print summary after running tests.", s_yesNo, "yes") 108 << Option<BinaryName> ("b", "binaryname", "Test binary path. Relative to working directory.", "<Unused>") 109 << Option<WorkingDir> ("wd", "workdir", "Working directory for the test execution.", ".") 110 << Option<CmdLineArgs> (DE_NULL, "cmdline", "Additional command line arguments for the test binary.", ""); 111 } 112 113 } // opt 114 115 enum RunMode 116 { 117 RUNMODE_CONNECT, 118 RUNMODE_START_SERVER 119 }; 120 121 struct CommandLine 122 { 123 CommandLine (void) 124 : port (0) 125 , summary (false) 126 { 127 } 128 129 xe::TargetConfiguration targetCfg; 130 RunMode runMode; 131 string serverBinOrAddress; 132 int port; 133 string caseListDir; 134 vector<string> testset; 135 vector<string> exclude; 136 string inFile; 137 string outFile; 138 string infoFile; 139 bool summary; 140 }; 141 142 bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv) 143 { 144 de::cmdline::Parser parser; 145 de::cmdline::CommandLine opts; 146 147 XE_CHECK(argc >= 1); 148 149 opt::registerOptions(parser); 150 151 if (!parser.parse(argc-1, argv+1, &opts, std::cerr)) 152 { 153 std::cout << argv[0] << " [options]\n"; 154 parser.help(std::cout); 155 return false; 156 } 157 158 if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>()) 159 { 160 std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl; 161 return false; 162 } 163 else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>()) 164 { 165 std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl; 166 return false; 167 } 168 169 if (!opts.hasOption<opt::TestSet>()) 170 { 171 std::cout << "Invalid command line arguments. --testset not defined." << std::endl; 172 return false; 173 } 174 175 if (opts.hasOption<opt::StartServer>()) 176 { 177 cmdLine.runMode = RUNMODE_START_SERVER; 178 cmdLine.serverBinOrAddress = opts.getOption<opt::StartServer>(); 179 } 180 else 181 { 182 cmdLine.runMode = RUNMODE_CONNECT; 183 cmdLine.serverBinOrAddress = opts.getOption<opt::Host>(); 184 } 185 186 if (opts.hasOption<opt::ContinueFile>()) 187 { 188 cmdLine.inFile = opts.getOption<opt::ContinueFile>(); 189 190 if (cmdLine.inFile.empty()) 191 { 192 std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl; 193 return false; 194 } 195 } 196 197 cmdLine.port = opts.getOption<opt::Port>(); 198 cmdLine.caseListDir = opts.getOption<opt::CaseListDir>(); 199 cmdLine.testset = opts.getOption<opt::TestSet>(); 200 cmdLine.exclude = opts.getOption<opt::ExcludeSet>(); 201 cmdLine.outFile = opts.getOption<opt::TestLogFile>(); 202 cmdLine.infoFile = opts.getOption<opt::InfoLogFile>(); 203 cmdLine.summary = opts.getOption<opt::Summary>(); 204 cmdLine.targetCfg.binaryName = opts.getOption<opt::BinaryName>(); 205 cmdLine.targetCfg.workingDir = opts.getOption<opt::WorkingDir>(); 206 cmdLine.targetCfg.cmdLineArgs = opts.getOption<opt::CmdLineArgs>(); 207 208 return true; 209 } 210 211 bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup) 212 { 213 int ptrnPos = 0; 214 int casePos = 0; 215 216 for (;;) 217 { 218 char c = casePath[casePos]; 219 char p = pattern[ptrnPos]; 220 221 if (p == '*') 222 { 223 /* Recurse to rest of positions. */ 224 int next = casePos; 225 for (;;) 226 { 227 if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup)) 228 return DE_TRUE; 229 230 if (casePath[next] == 0) 231 return DE_FALSE; /* No match found. */ 232 else 233 next += 1; 234 } 235 DE_ASSERT(DE_FALSE); 236 } 237 else if (c == 0 && p == 0) 238 return true; 239 else if (c == 0) 240 { 241 /* Incomplete match is ok for test groups. */ 242 return isTestGroup; 243 } 244 else if (c != p) 245 return false; 246 247 casePos += 1; 248 ptrnPos += 1; 249 } 250 251 DE_ASSERT(false); 252 return false; 253 } 254 255 void readCaseList (xe::TestGroup* root, const char* filename) 256 { 257 xe::TestCaseListParser caseListParser; 258 std::ifstream in (filename, std::ios_base::binary); 259 deUint8 buf[1024]; 260 261 XE_CHECK(in.good()); 262 263 caseListParser.init(root); 264 265 for (;;) 266 { 267 in.read((char*)&buf[0], sizeof(buf)); 268 int numRead = (int)in.gcount(); 269 270 if (numRead > 0) 271 caseListParser.parse(&buf[0], numRead); 272 273 if (numRead < (int)sizeof(buf)) 274 break; // EOF 275 } 276 } 277 278 void readCaseLists (xe::TestRoot& root, const char* caseListDir) 279 { 280 int testCaseListCount = 0; 281 de::DirectoryIterator iter (caseListDir); 282 283 for (; iter.hasItem(); iter.next()) 284 { 285 de::FilePath item = iter.getItem(); 286 287 if (item.getType() == de::FilePath::TYPE_FILE) 288 { 289 string baseName = item.getBaseName(); 290 if (baseName.find("-cases.xml") == baseName.length()-10) 291 { 292 string packageName = baseName.substr(0, baseName.length()-10); 293 xe::TestGroup* package = root.createGroup(packageName.c_str(), ""); 294 295 readCaseList(package, item.getPath()); 296 testCaseListCount++; 297 } 298 } 299 } 300 301 if (testCaseListCount == 0) 302 throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir) + "'"); 303 } 304 305 void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) 306 { 307 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) 308 { 309 const xe::TestNode* child = group.getChild(childNdx); 310 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; 311 const string fullPath = child->getFullPath(); 312 313 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) 314 { 315 if (isGroup) 316 { 317 // Recurse into group. 318 addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); 319 } 320 else 321 { 322 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); 323 testSet.add(child); 324 } 325 } 326 } 327 } 328 329 void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter) 330 { 331 for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++) 332 { 333 const xe::TestNode* child = group.getChild(childNdx); 334 const bool isGroup = child->getNodeType() == xe::TESTNODETYPE_GROUP; 335 const string fullPath = child->getFullPath(); 336 337 if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup)) 338 { 339 if (isGroup) 340 { 341 // Recurse into group. 342 removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter); 343 } 344 else 345 { 346 DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE); 347 testSet.remove(child); 348 } 349 } 350 } 351 } 352 353 class BatchResultHandler : public xe::TestLogHandler 354 { 355 public: 356 BatchResultHandler (xe::BatchResult* batchResult) 357 : m_batchResult(batchResult) 358 { 359 } 360 361 void setSessionInfo (const xe::SessionInfo& sessionInfo) 362 { 363 m_batchResult->getSessionInfo() = sessionInfo; 364 } 365 366 xe::TestCaseResultPtr startTestCaseResult (const char* casePath) 367 { 368 // \todo [2012-11-01 pyry] What to do with duplicate results? 369 if (m_batchResult->hasTestCaseResult(casePath)) 370 return m_batchResult->getTestCaseResult(casePath); 371 else 372 return m_batchResult->createTestCaseResult(casePath); 373 } 374 375 void testCaseResultUpdated (const xe::TestCaseResultPtr&) 376 { 377 } 378 379 void testCaseResultComplete (const xe::TestCaseResultPtr&) 380 { 381 } 382 383 private: 384 xe::BatchResult* m_batchResult; 385 }; 386 387 void readLogFile (xe::BatchResult* batchResult, const char* filename) 388 { 389 std::ifstream in (filename, std::ifstream::binary|std::ifstream::in); 390 BatchResultHandler handler (batchResult); 391 xe::TestLogParser parser (&handler); 392 deUint8 buf [1024]; 393 int numRead = 0; 394 395 for (;;) 396 { 397 in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf)); 398 numRead = (int)in.gcount(); 399 400 if (numRead <= 0) 401 break; 402 403 parser.parse(&buf[0], numRead); 404 } 405 406 in.close(); 407 } 408 409 void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult) 410 { 411 int countByStatusCode[xe::TESTSTATUSCODE_LAST]; 412 std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0); 413 414 for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter) 415 { 416 const xe::TestNode* node = *iter; 417 if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node)) 418 { 419 const xe::TestCase* testCase = static_cast<const xe::TestCase*>(node); 420 string fullPath; 421 xe::TestStatusCode statusCode = xe::TESTSTATUSCODE_PENDING; 422 testCase->getFullPath(fullPath); 423 424 // Parse result data if such exists. 425 if (batchResult.hasTestCaseResult(fullPath.c_str())) 426 { 427 xe::ConstTestCaseResultPtr resultData = batchResult.getTestCaseResult(fullPath.c_str()); 428 xe::TestCaseResult result; 429 xe::TestResultParser parser; 430 431 xe::parseTestCaseResultFromData(&parser, &result, *resultData.get()); 432 statusCode = result.statusCode; 433 } 434 435 countByStatusCode[statusCode] += 1; 436 } 437 } 438 439 printf("\nTest run summary:\n"); 440 int totalCases = 0; 441 for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++) 442 { 443 if (countByStatusCode[code] > 0) 444 printf(" %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]); 445 446 totalCases += countByStatusCode[code]; 447 } 448 printf(" %20s: %5d\n", "Total", totalCases); 449 } 450 451 void writeInfoLog (const xe::InfoLog& log, const char* filename) 452 { 453 std::ofstream out(filename, std::ios_base::binary); 454 XE_CHECK(out.good()); 455 out.write((const char*)log.getBytes(), log.getSize()); 456 out.close(); 457 } 458 459 xe::CommLink* createCommLink (const CommandLine& cmdLine) 460 { 461 if (cmdLine.runMode == RUNMODE_START_SERVER) 462 { 463 xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink(); 464 try 465 { 466 link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port); 467 return link; 468 } 469 catch (...) 470 { 471 delete link; 472 throw; 473 } 474 } 475 else if (cmdLine.runMode == RUNMODE_CONNECT) 476 { 477 de::SocketAddress address; 478 479 address.setFamily(DE_SOCKETFAMILY_INET4); 480 address.setProtocol(DE_SOCKETPROTOCOL_TCP); 481 address.setHost(cmdLine.serverBinOrAddress.c_str()); 482 address.setPort(cmdLine.port); 483 484 xe::TcpIpLink* link = new xe::TcpIpLink(); 485 try 486 { 487 std::string error; 488 489 link->connect(address); 490 return link; 491 } 492 catch (const std::exception& error) 493 { 494 delete link; 495 throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what()); 496 } 497 catch (...) 498 { 499 delete link; 500 throw; 501 } 502 } 503 else 504 { 505 DE_ASSERT(false); 506 return DE_NULL; 507 } 508 } 509 510 #if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) 511 512 static xe::BatchExecutor* s_executor = DE_NULL; 513 514 void signalHandler (int, siginfo_t*, void*) 515 { 516 if (s_executor) 517 s_executor->cancel(); 518 } 519 520 void setupSignalHandler (xe::BatchExecutor* executor) 521 { 522 s_executor = executor; 523 struct sigaction sa; 524 525 sa.sa_sigaction = signalHandler; 526 sa.sa_flags = SA_SIGINFO | SA_RESTART; 527 sigfillset(&sa.sa_mask); 528 529 sigaction(SIGINT, &sa, DE_NULL); 530 } 531 532 void resetSignalHandler (void) 533 { 534 struct sigaction sa; 535 536 sa.sa_handler = SIG_DFL; 537 sa.sa_flags = SA_RESTART; 538 sigfillset(&sa.sa_mask); 539 540 sigaction(SIGINT, &sa, DE_NULL); 541 s_executor = DE_NULL; 542 } 543 544 #elif (DE_OS == DE_OS_WIN32) 545 546 static xe::BatchExecutor* s_executor = DE_NULL; 547 548 void signalHandler (int) 549 { 550 if (s_executor) 551 s_executor->cancel(); 552 } 553 554 void setupSignalHandler (xe::BatchExecutor* executor) 555 { 556 s_executor = executor; 557 signal(SIGINT, signalHandler); 558 } 559 560 void resetSignalHandler (void) 561 { 562 signal(SIGINT, SIG_DFL); 563 s_executor = DE_NULL; 564 } 565 566 #else 567 568 void setupSignalHandler (xe::BatchExecutor*) 569 { 570 } 571 572 void resetSignalHandler (void) 573 { 574 } 575 576 #endif 577 578 void runExecutor (const CommandLine& cmdLine) 579 { 580 xe::TestRoot root; 581 582 // Read case list definitions. 583 readCaseLists(root, cmdLine.caseListDir.c_str()); 584 585 // Build test set. 586 xe::TestSet testSet; 587 588 // Build test set. 589 for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter) 590 addMatchingCases(root, testSet, filterIter->c_str()); 591 592 if (testSet.empty()) 593 throw xe::Error("None of the test case lists contains tests matching any of the test sets."); 594 595 // Remove excluded cases. 596 for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter) 597 removeMatchingCases(root, testSet, filterIter->c_str()); 598 599 // Initialize batch result. 600 xe::BatchResult batchResult; 601 xe::InfoLog infoLog; 602 603 // Read existing results from input file (if supplied). 604 if (!cmdLine.inFile.empty()) 605 readLogFile(&batchResult, cmdLine.inFile.c_str()); 606 607 // Initialize commLink. 608 de::UniquePtr<xe::CommLink> commLink(createCommLink(cmdLine)); 609 610 xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog); 611 612 try 613 { 614 setupSignalHandler(&executor); 615 executor.run(); 616 resetSignalHandler(); 617 } 618 catch (...) 619 { 620 resetSignalHandler(); 621 622 if (!cmdLine.outFile.empty()) 623 { 624 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str()); 625 printf("Test log written to %s\n", cmdLine.outFile.c_str()); 626 } 627 628 if (!cmdLine.infoFile.empty()) 629 { 630 writeInfoLog(infoLog, cmdLine.infoFile.c_str()); 631 printf("Info log written to %s\n", cmdLine.infoFile.c_str()); 632 } 633 634 if (cmdLine.summary) 635 printBatchResultSummary(&root, testSet, batchResult); 636 637 throw; 638 } 639 640 if (!cmdLine.outFile.empty()) 641 { 642 xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str()); 643 printf("Test log written to %s\n", cmdLine.outFile.c_str()); 644 } 645 646 if (!cmdLine.infoFile.empty()) 647 { 648 writeInfoLog(infoLog, cmdLine.infoFile.c_str()); 649 printf("Info log written to %s\n", cmdLine.infoFile.c_str()); 650 } 651 652 if (cmdLine.summary) 653 printBatchResultSummary(&root, testSet, batchResult); 654 655 { 656 string err; 657 658 if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR) 659 throw xe::Error(err); 660 } 661 } 662 663 } // anonymous 664 665 int main (int argc, const char* const* argv) 666 { 667 CommandLine cmdLine; 668 669 if (!parseCommandLine(cmdLine, argc, argv)) 670 return -1; 671 672 try 673 { 674 runExecutor(cmdLine); 675 } 676 catch (const std::exception& e) 677 { 678 printf("%s\n", e.what()); 679 return -1; 680 } 681 682 return 0; 683 } 684