1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include <gtest/gtest.h> 18 19 #include <ctype.h> 20 #include <errno.h> 21 #include <fcntl.h> 22 #include <inttypes.h> 23 #include <limits.h> 24 #include <signal.h> 25 #include <stdarg.h> 26 #include <stdio.h> 27 #include <string.h> 28 #include <sys/wait.h> 29 #include <unistd.h> 30 31 #include <chrono> 32 #include <string> 33 #include <tuple> 34 #include <utility> 35 #include <vector> 36 37 #ifndef TEMP_FAILURE_RETRY 38 39 /* Used to retry syscalls that can return EINTR. */ 40 #define TEMP_FAILURE_RETRY(exp) \ 41 ({ \ 42 __typeof__(exp) _rc; \ 43 do { \ 44 _rc = (exp); \ 45 } while (_rc == -1 && errno == EINTR); \ 46 _rc; \ 47 }) 48 49 #endif 50 51 namespace testing { 52 namespace internal { 53 54 // Reuse of testing::internal::ColoredPrintf in gtest. 55 enum GTestColor { COLOR_DEFAULT, COLOR_RED, COLOR_GREEN, COLOR_YELLOW }; 56 57 void ColoredPrintf(GTestColor color, const char* fmt, ...); 58 59 } // namespace internal 60 } // namespace testing 61 62 using testing::internal::GTestColor; 63 using testing::internal::COLOR_DEFAULT; 64 using testing::internal::COLOR_RED; 65 using testing::internal::COLOR_GREEN; 66 using testing::internal::COLOR_YELLOW; 67 using testing::internal::ColoredPrintf; 68 69 constexpr int DEFAULT_GLOBAL_TEST_RUN_DEADLINE_MS = 90000; 70 constexpr int DEFAULT_GLOBAL_TEST_RUN_WARNLINE_MS = 2000; 71 72 // The time each test can run before killed for the reason of timeout. 73 // It takes effect only with --isolate option. 74 static int global_test_run_deadline_ms = DEFAULT_GLOBAL_TEST_RUN_DEADLINE_MS; 75 76 // The time each test can run before be warned for too much running time. 77 // It takes effect only with --isolate option. 78 static int global_test_run_warnline_ms = DEFAULT_GLOBAL_TEST_RUN_WARNLINE_MS; 79 80 // Return deadline duration for a test, in ms. 81 static int GetDeadlineInfo(const std::string& /*test_name*/) { 82 return global_test_run_deadline_ms; 83 } 84 85 // Return warnline duration for a test, in ms. 86 static int GetWarnlineInfo(const std::string& /*test_name*/) { 87 return global_test_run_warnline_ms; 88 } 89 90 static void PrintHelpInfo() { 91 printf( 92 "VTS Unit Test Options:\n" 93 " -j [JOB_COUNT] or -j[JOB_COUNT]\n" 94 " Run up to JOB_COUNT tests in parallel.\n" 95 " Use isolation mode, Run each test in a separate process.\n" 96 " If JOB_COUNT is not given, it is set to the count of available " 97 "processors.\n" 98 " --no-isolate\n" 99 " Don't use isolation mode, run all tests in a single process.\n" 100 " --deadline=[TIME_IN_MS]\n" 101 " Run each test in no longer than [TIME_IN_MS] time.\n" 102 " It takes effect only in isolation mode. Deafult deadline is 90000 " 103 "ms.\n" 104 " --warnline=[TIME_IN_MS]\n" 105 " Test running longer than [TIME_IN_MS] will be warned.\n" 106 " It takes effect only in isolation mode. Default warnline is 2000 " 107 "ms.\n" 108 " --gtest-filter=POSITIVE_PATTERNS[-NEGATIVE_PATTERNS]\n" 109 " Used as a synonym for --gtest_filter option in gtest.\n" 110 "Default vts unit test option is -j.\n" 111 "In isolation mode, you can send SIGQUIT to the parent process to show " 112 "current\n" 113 "running tests, or send SIGINT to the parent process to stop testing " 114 "and\n" 115 "clean up current running tests.\n" 116 "\n"); 117 } 118 119 enum TestResult { TEST_SUCCESS = 0, TEST_FAILED, TEST_TIMEOUT }; 120 121 class Test { 122 public: 123 Test() {} // For std::vector<Test>. 124 explicit Test(const char* name) : name_(name) {} 125 126 const std::string& GetName() const { return name_; } 127 128 void SetResult(TestResult result) { result_ = result; } 129 130 TestResult GetResult() const { return result_; } 131 132 void SetTestTime(int64_t elapsed_time_ns) { 133 elapsed_time_ns_ = elapsed_time_ns; 134 } 135 136 int64_t GetTestTime() const { return elapsed_time_ns_; } 137 138 void AppendTestOutput(const std::string& s) { output_ += s; } 139 140 const std::string& GetTestOutput() const { return output_; } 141 142 private: 143 const std::string name_; 144 TestResult result_; 145 int64_t elapsed_time_ns_; 146 std::string output_; 147 }; 148 149 class TestCase { 150 public: 151 TestCase() {} // For std::vector<TestCase>. 152 explicit TestCase(const char* name) : name_(name) {} 153 154 const std::string& GetName() const { return name_; } 155 156 void AppendTest(const char* test_name) { 157 test_list_.push_back(Test(test_name)); 158 } 159 160 size_t TestCount() const { return test_list_.size(); } 161 162 std::string GetTestName(size_t test_id) const { 163 VerifyTestId(test_id); 164 return name_ + "." + test_list_[test_id].GetName(); 165 } 166 167 Test& GetTest(size_t test_id) { 168 VerifyTestId(test_id); 169 return test_list_[test_id]; 170 } 171 172 const Test& GetTest(size_t test_id) const { 173 VerifyTestId(test_id); 174 return test_list_[test_id]; 175 } 176 177 void SetTestResult(size_t test_id, TestResult result) { 178 VerifyTestId(test_id); 179 test_list_[test_id].SetResult(result); 180 } 181 182 TestResult GetTestResult(size_t test_id) const { 183 VerifyTestId(test_id); 184 return test_list_[test_id].GetResult(); 185 } 186 187 void SetTestTime(size_t test_id, int64_t elapsed_time_ns) { 188 VerifyTestId(test_id); 189 test_list_[test_id].SetTestTime(elapsed_time_ns); 190 } 191 192 int64_t GetTestTime(size_t test_id) const { 193 VerifyTestId(test_id); 194 return test_list_[test_id].GetTestTime(); 195 } 196 197 private: 198 void VerifyTestId(size_t test_id) const { 199 if (test_id >= test_list_.size()) { 200 fprintf(stderr, "test_id %zu out of range [0, %zu)\n", test_id, 201 test_list_.size()); 202 exit(1); 203 } 204 } 205 206 private: 207 const std::string name_; 208 std::vector<Test> test_list_; 209 }; 210 211 class TestResultPrinter : public testing::EmptyTestEventListener { 212 public: 213 TestResultPrinter() : pinfo_(NULL) {} 214 virtual void OnTestStart(const testing::TestInfo& test_info) { 215 pinfo_ = &test_info; // Record test_info for use in OnTestPartResult. 216 } 217 virtual void OnTestPartResult(const testing::TestPartResult& result); 218 219 private: 220 const testing::TestInfo* pinfo_; 221 }; 222 223 // Called after an assertion failure. 224 void TestResultPrinter::OnTestPartResult( 225 const testing::TestPartResult& result) { 226 // If the test part succeeded, we don't need to do anything. 227 if (result.type() == testing::TestPartResult::kSuccess) return; 228 229 // Print failure message from the assertion (e.g. expected this and got that). 230 printf("%s:(%d) Failure in test %s.%s\n%s\n", result.file_name(), 231 result.line_number(), pinfo_->test_case_name(), pinfo_->name(), 232 result.message()); 233 fflush(stdout); 234 } 235 236 static int64_t NanoTime() { 237 std::chrono::nanoseconds duration( 238 std::chrono::steady_clock::now().time_since_epoch()); 239 return static_cast<int64_t>(duration.count()); 240 } 241 242 static bool EnumerateTests(int argc, char** argv, 243 std::vector<TestCase>& testcase_list) { 244 std::string command; 245 for (int i = 0; i < argc; ++i) { 246 command += argv[i]; 247 command += " "; 248 } 249 command += "--gtest_list_tests"; 250 FILE* fp = popen(command.c_str(), "r"); 251 if (fp == NULL) { 252 perror("popen"); 253 return false; 254 } 255 256 char buf[200]; 257 while (fgets(buf, sizeof(buf), fp) != NULL) { 258 char* p = buf; 259 260 while (*p != '\0' && isspace(*p)) { 261 ++p; 262 } 263 if (*p == '\0') continue; 264 char* start = p; 265 while (*p != '\0' && !isspace(*p)) { 266 ++p; 267 } 268 char* end = p; 269 while (*p != '\0' && isspace(*p)) { 270 ++p; 271 } 272 if (*p != '\0' && *p != '#') { 273 // This is not we want, gtest must meet with some error when parsing the 274 // arguments. 275 fprintf(stderr, "argument error, check with --help\n"); 276 return false; 277 } 278 *end = '\0'; 279 if (*(end - 1) == '.') { 280 *(end - 1) = '\0'; 281 testcase_list.push_back(TestCase(start)); 282 } else { 283 testcase_list.back().AppendTest(start); 284 } 285 } 286 int result = pclose(fp); 287 return (result != -1 && WEXITSTATUS(result) == 0); 288 } 289 290 // Part of the following *Print functions are copied from 291 // external/gtest/src/gtest.cc: 292 // PrettyUnitTestResultPrinter. The reason for copy is that 293 // PrettyUnitTestResultPrinter 294 // is defined and used in gtest.cc, which is hard to reuse. 295 static void OnTestIterationStartPrint( 296 const std::vector<TestCase>& testcase_list, size_t iteration, 297 int iteration_count) { 298 if (iteration_count != 1) { 299 printf("\nRepeating all tests (iteration %zu) . . .\n\n", iteration); 300 } 301 ColoredPrintf(COLOR_GREEN, "[==========] "); 302 303 size_t testcase_count = testcase_list.size(); 304 size_t test_count = 0; 305 for (const auto& testcase : testcase_list) { 306 test_count += testcase.TestCount(); 307 } 308 309 printf("Running %zu %s from %zu %s.\n", test_count, 310 (test_count == 1) ? "test" : "tests", testcase_count, 311 (testcase_count == 1) ? "test case" : "test cases"); 312 fflush(stdout); 313 } 314 315 // vts cts test needs gtest output format. 316 static void OnTestEndPrint(const TestCase& testcase, size_t test_id) { 317 ColoredPrintf(COLOR_GREEN, "[ RUN ] "); 318 printf("%s\n", testcase.GetTestName(test_id).c_str()); 319 320 const std::string& test_output = testcase.GetTest(test_id).GetTestOutput(); 321 printf("%s", test_output.c_str()); 322 323 TestResult result = testcase.GetTestResult(test_id); 324 if (result == TEST_SUCCESS) { 325 ColoredPrintf(COLOR_GREEN, "[ OK ] "); 326 } else { 327 ColoredPrintf(COLOR_RED, "[ FAILED ] "); 328 } 329 printf("%s", testcase.GetTestName(test_id).c_str()); 330 if (testing::GTEST_FLAG(print_time)) { 331 printf(" (%" PRId64 " ms)", testcase.GetTestTime(test_id) / 1000000); 332 } 333 printf("\n"); 334 fflush(stdout); 335 } 336 337 static void OnTestIterationEndPrint(const std::vector<TestCase>& testcase_list, 338 size_t /*iteration*/, 339 int64_t elapsed_time_ns) { 340 std::vector<std::string> fail_test_name_list; 341 std::vector<std::pair<std::string, int64_t>> timeout_test_list; 342 343 // For tests run exceed warnline but not timeout. 344 std::vector<std::tuple<std::string, int64_t, int>> slow_test_list; 345 size_t testcase_count = testcase_list.size(); 346 size_t test_count = 0; 347 size_t success_test_count = 0; 348 349 for (const auto& testcase : testcase_list) { 350 test_count += testcase.TestCount(); 351 for (size_t i = 0; i < testcase.TestCount(); ++i) { 352 TestResult result = testcase.GetTestResult(i); 353 if (result == TEST_SUCCESS) { 354 ++success_test_count; 355 } else if (result == TEST_FAILED) { 356 fail_test_name_list.push_back(testcase.GetTestName(i)); 357 } else if (result == TEST_TIMEOUT) { 358 timeout_test_list.push_back( 359 std::make_pair(testcase.GetTestName(i), testcase.GetTestTime(i))); 360 } 361 if (result != TEST_TIMEOUT && 362 testcase.GetTestTime(i) / 1000000 >= 363 GetWarnlineInfo(testcase.GetTestName(i))) { 364 slow_test_list.push_back( 365 std::make_tuple(testcase.GetTestName(i), testcase.GetTestTime(i), 366 GetWarnlineInfo(testcase.GetTestName(i)))); 367 } 368 } 369 } 370 371 ColoredPrintf(COLOR_GREEN, "[==========] "); 372 printf("%zu %s from %zu %s ran.", test_count, 373 (test_count == 1) ? "test" : "tests", testcase_count, 374 (testcase_count == 1) ? "test case" : "test cases"); 375 if (testing::GTEST_FLAG(print_time)) { 376 printf(" (%" PRId64 " ms total)", elapsed_time_ns / 1000000); 377 } 378 printf("\n"); 379 ColoredPrintf(COLOR_GREEN, "[ PASS ] "); 380 printf("%zu %s.\n", success_test_count, 381 (success_test_count == 1) ? "test" : "tests"); 382 383 // Print tests failed. 384 size_t fail_test_count = fail_test_name_list.size(); 385 if (fail_test_count > 0) { 386 ColoredPrintf(COLOR_RED, "[ FAIL ] "); 387 printf("%zu %s, listed below:\n", fail_test_count, 388 (fail_test_count == 1) ? "test" : "tests"); 389 for (const auto& name : fail_test_name_list) { 390 ColoredPrintf(COLOR_RED, "[ FAIL ] "); 391 printf("%s\n", name.c_str()); 392 } 393 } 394 395 // Print tests run timeout. 396 size_t timeout_test_count = timeout_test_list.size(); 397 if (timeout_test_count > 0) { 398 ColoredPrintf(COLOR_RED, "[ TIMEOUT ] "); 399 printf("%zu %s, listed below:\n", timeout_test_count, 400 (timeout_test_count == 1) ? "test" : "tests"); 401 for (const auto& timeout_pair : timeout_test_list) { 402 ColoredPrintf(COLOR_RED, "[ TIMEOUT ] "); 403 printf("%s (stopped at %" PRId64 " ms)\n", timeout_pair.first.c_str(), 404 timeout_pair.second / 1000000); 405 } 406 } 407 408 // Print tests run exceed warnline. 409 size_t slow_test_count = slow_test_list.size(); 410 if (slow_test_count > 0) { 411 ColoredPrintf(COLOR_YELLOW, "[ SLOW ] "); 412 printf("%zu %s, listed below:\n", slow_test_count, 413 (slow_test_count == 1) ? "test" : "tests"); 414 for (const auto& slow_tuple : slow_test_list) { 415 ColoredPrintf(COLOR_YELLOW, "[ SLOW ] "); 416 printf("%s (%" PRId64 " ms, exceed warnline %d ms)\n", 417 std::get<0>(slow_tuple).c_str(), std::get<1>(slow_tuple) / 1000000, 418 std::get<2>(slow_tuple)); 419 } 420 } 421 422 if (fail_test_count > 0) { 423 printf("\n%2zu FAILED %s\n", fail_test_count, 424 (fail_test_count == 1) ? "TEST" : "TESTS"); 425 } 426 if (timeout_test_count > 0) { 427 printf("%2zu TIMEOUT %s\n", timeout_test_count, 428 (timeout_test_count == 1) ? "TEST" : "TESTS"); 429 } 430 if (slow_test_count > 0) { 431 printf("%2zu SLOW %s\n", slow_test_count, 432 (slow_test_count == 1) ? "TEST" : "TESTS"); 433 } 434 fflush(stdout); 435 } 436 437 std::string XmlEscape(const std::string& xml) { 438 std::string escaped; 439 escaped.reserve(xml.size()); 440 441 for (auto c : xml) { 442 switch (c) { 443 case '<': 444 escaped.append("<"); 445 break; 446 case '>': 447 escaped.append(">"); 448 break; 449 case '&': 450 escaped.append("&"); 451 break; 452 case '\'': 453 escaped.append("'"); 454 break; 455 case '"': 456 escaped.append("""); 457 break; 458 default: 459 escaped.append(1, c); 460 break; 461 } 462 } 463 464 return escaped; 465 } 466 467 // Output xml file when --gtest_output is used, write this function as we can't 468 // reuse 469 // gtest.cc:XmlUnitTestResultPrinter. The reason is XmlUnitTestResultPrinter is 470 // totally 471 // defined in gtest.cc and not expose to outside. What's more, as we don't run 472 // gtest in 473 // the parent process, we don't have gtest classes which are needed by 474 // XmlUnitTestResultPrinter. 475 void OnTestIterationEndXmlPrint(const std::string& xml_output_filename, 476 const std::vector<TestCase>& testcase_list, 477 time_t epoch_iteration_start_time, 478 int64_t elapsed_time_ns) { 479 FILE* fp = fopen(xml_output_filename.c_str(), "w"); 480 if (fp == NULL) { 481 fprintf(stderr, "failed to open '%s': %s\n", xml_output_filename.c_str(), 482 strerror(errno)); 483 exit(1); 484 } 485 486 size_t total_test_count = 0; 487 size_t total_failed_count = 0; 488 std::vector<size_t> failed_count_list(testcase_list.size(), 0); 489 std::vector<int64_t> elapsed_time_list(testcase_list.size(), 0); 490 for (size_t i = 0; i < testcase_list.size(); ++i) { 491 auto& testcase = testcase_list[i]; 492 total_test_count += testcase.TestCount(); 493 for (size_t j = 0; j < testcase.TestCount(); ++j) { 494 if (testcase.GetTestResult(j) != TEST_SUCCESS) { 495 ++failed_count_list[i]; 496 } 497 elapsed_time_list[i] += testcase.GetTestTime(j); 498 } 499 total_failed_count += failed_count_list[i]; 500 } 501 502 const tm* time_struct = localtime(&epoch_iteration_start_time); 503 char timestamp[40]; 504 snprintf(timestamp, sizeof(timestamp), "%4d-%02d-%02dT%02d:%02d:%02d", 505 time_struct->tm_year + 1900, time_struct->tm_mon + 1, 506 time_struct->tm_mday, time_struct->tm_hour, time_struct->tm_min, 507 time_struct->tm_sec); 508 509 fputs("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n", fp); 510 fprintf( 511 fp, 512 "<testsuites tests=\"%zu\" failures=\"%zu\" disabled=\"0\" errors=\"0\"", 513 total_test_count, total_failed_count); 514 fprintf(fp, " timestamp=\"%s\" time=\"%.3lf\" name=\"AllTests\">\n", 515 timestamp, elapsed_time_ns / 1e9); 516 for (size_t i = 0; i < testcase_list.size(); ++i) { 517 auto& testcase = testcase_list[i]; 518 fprintf(fp, 519 " <testsuite name=\"%s\" tests=\"%zu\" failures=\"%zu\" " 520 "disabled=\"0\" errors=\"0\"", 521 testcase.GetName().c_str(), testcase.TestCount(), 522 failed_count_list[i]); 523 fprintf(fp, " time=\"%.3lf\">\n", elapsed_time_list[i] / 1e9); 524 525 for (size_t j = 0; j < testcase.TestCount(); ++j) { 526 fprintf(fp, 527 " <testcase name=\"%s\" status=\"run\" time=\"%.3lf\" " 528 "classname=\"%s\"", 529 testcase.GetTest(j).GetName().c_str(), 530 testcase.GetTestTime(j) / 1e9, testcase.GetName().c_str()); 531 if (testcase.GetTestResult(j) == TEST_SUCCESS) { 532 fputs(" />\n", fp); 533 } else { 534 fputs(">\n", fp); 535 const std::string& test_output = testcase.GetTest(j).GetTestOutput(); 536 const std::string escaped_test_output = XmlEscape(test_output); 537 fprintf(fp, " <failure message=\"%s\" type=\"\">\n", 538 escaped_test_output.c_str()); 539 fputs(" </failure>\n", fp); 540 fputs(" </testcase>\n", fp); 541 } 542 } 543 544 fputs(" </testsuite>\n", fp); 545 } 546 fputs("</testsuites>\n", fp); 547 fclose(fp); 548 } 549 550 static bool sigint_flag; 551 static bool sigquit_flag; 552 553 static void signal_handler(int sig) { 554 if (sig == SIGINT) { 555 sigint_flag = true; 556 } else if (sig == SIGQUIT) { 557 sigquit_flag = true; 558 } 559 } 560 561 static bool RegisterSignalHandler() { 562 sigint_flag = false; 563 sigquit_flag = false; 564 sig_t ret = signal(SIGINT, signal_handler); 565 if (ret != SIG_ERR) { 566 ret = signal(SIGQUIT, signal_handler); 567 } 568 if (ret == SIG_ERR) { 569 perror("RegisterSignalHandler"); 570 return false; 571 } 572 return true; 573 } 574 575 static bool UnregisterSignalHandler() { 576 sig_t ret = signal(SIGINT, SIG_DFL); 577 if (ret != SIG_ERR) { 578 ret = signal(SIGQUIT, SIG_DFL); 579 } 580 if (ret == SIG_ERR) { 581 perror("UnregisterSignalHandler"); 582 return false; 583 } 584 return true; 585 } 586 587 struct ChildProcInfo { 588 pid_t pid; 589 int64_t start_time_ns; 590 int64_t end_time_ns; 591 int64_t 592 deadline_end_time_ns; // The time when the test is thought of as timeout. 593 size_t testcase_id, test_id; 594 bool finished; 595 bool timed_out; 596 int exit_status; 597 int child_read_fd; // File descriptor to read child test failure info. 598 }; 599 600 // Forked Child process, run the single test. 601 static void ChildProcessFn(int argc, char** argv, 602 const std::string& test_name) { 603 char** new_argv = new char*[argc + 2]; 604 memcpy(new_argv, argv, sizeof(char*) * argc); 605 606 char* filter_arg = new char[test_name.size() + 20]; 607 strcpy(filter_arg, "--gtest_filter="); 608 strcat(filter_arg, test_name.c_str()); 609 new_argv[argc] = filter_arg; 610 new_argv[argc + 1] = NULL; 611 612 int new_argc = argc + 1; 613 testing::InitGoogleTest(&new_argc, new_argv); 614 int result = RUN_ALL_TESTS(); 615 exit(result); 616 } 617 618 static ChildProcInfo RunChildProcess(const std::string& test_name, 619 int testcase_id, int test_id, int argc, 620 char** argv) { 621 int pipefd[2]; 622 if (pipe(pipefd) == -1) { 623 perror("pipe in RunTestInSeparateProc"); 624 exit(1); 625 } 626 if (fcntl(pipefd[0], F_SETFL, O_NONBLOCK) == -1) { 627 perror("fcntl in RunTestInSeparateProc"); 628 exit(1); 629 } 630 pid_t pid = fork(); 631 if (pid == -1) { 632 perror("fork in RunTestInSeparateProc"); 633 exit(1); 634 } else if (pid == 0) { 635 // In child process, run a single test. 636 close(pipefd[0]); 637 close(STDOUT_FILENO); 638 close(STDERR_FILENO); 639 dup2(pipefd[1], STDOUT_FILENO); 640 dup2(pipefd[1], STDERR_FILENO); 641 642 if (!UnregisterSignalHandler()) { 643 exit(1); 644 } 645 ChildProcessFn(argc, argv, test_name); 646 // Unreachable. 647 } 648 // In parent process, initialize child process info. 649 close(pipefd[1]); 650 ChildProcInfo child_proc; 651 child_proc.child_read_fd = pipefd[0]; 652 child_proc.pid = pid; 653 child_proc.start_time_ns = NanoTime(); 654 child_proc.deadline_end_time_ns = 655 child_proc.start_time_ns + GetDeadlineInfo(test_name) * 1000000LL; 656 child_proc.testcase_id = testcase_id; 657 child_proc.test_id = test_id; 658 child_proc.finished = false; 659 return child_proc; 660 } 661 662 static void HandleSignals(std::vector<TestCase>& testcase_list, 663 std::vector<ChildProcInfo>& child_proc_list) { 664 if (sigquit_flag) { 665 sigquit_flag = false; 666 // Print current running tests. 667 printf("List of current running tests:\n"); 668 for (const auto& child_proc : child_proc_list) { 669 if (child_proc.pid != 0) { 670 std::string test_name = 671 testcase_list[child_proc.testcase_id].GetTestName( 672 child_proc.test_id); 673 int64_t current_time_ns = NanoTime(); 674 int64_t run_time_ms = 675 (current_time_ns - child_proc.start_time_ns) / 1000000; 676 printf(" %s (%" PRId64 " ms)\n", test_name.c_str(), run_time_ms); 677 } 678 } 679 } else if (sigint_flag) { 680 sigint_flag = false; 681 // Kill current running tests. 682 for (const auto& child_proc : child_proc_list) { 683 if (child_proc.pid != 0) { 684 // Send SIGKILL to ensure the child process can be killed 685 // unconditionally. 686 kill(child_proc.pid, SIGKILL); 687 } 688 } 689 // SIGINT kills the parent process as well. 690 exit(1); 691 } 692 } 693 694 static bool CheckChildProcExit(pid_t exit_pid, int exit_status, 695 std::vector<ChildProcInfo>& child_proc_list) { 696 for (size_t i = 0; i < child_proc_list.size(); ++i) { 697 if (child_proc_list[i].pid == exit_pid) { 698 child_proc_list[i].finished = true; 699 child_proc_list[i].timed_out = false; 700 child_proc_list[i].exit_status = exit_status; 701 child_proc_list[i].end_time_ns = NanoTime(); 702 return true; 703 } 704 } 705 return false; 706 } 707 708 static size_t CheckChildProcTimeout( 709 std::vector<ChildProcInfo>& child_proc_list) { 710 int64_t current_time_ns = NanoTime(); 711 size_t timeout_child_count = 0; 712 for (size_t i = 0; i < child_proc_list.size(); ++i) { 713 if (child_proc_list[i].deadline_end_time_ns <= current_time_ns) { 714 child_proc_list[i].finished = true; 715 child_proc_list[i].timed_out = true; 716 child_proc_list[i].end_time_ns = current_time_ns; 717 ++timeout_child_count; 718 } 719 } 720 return timeout_child_count; 721 } 722 723 static void ReadChildProcOutput(std::vector<TestCase>& testcase_list, 724 std::vector<ChildProcInfo>& child_proc_list) { 725 for (const auto& child_proc : child_proc_list) { 726 TestCase& testcase = testcase_list[child_proc.testcase_id]; 727 int test_id = child_proc.test_id; 728 while (true) { 729 char buf[1024]; 730 ssize_t bytes_read = TEMP_FAILURE_RETRY( 731 read(child_proc.child_read_fd, buf, sizeof(buf) - 1)); 732 if (bytes_read > 0) { 733 buf[bytes_read] = '\0'; 734 testcase.GetTest(test_id).AppendTestOutput(buf); 735 } else if (bytes_read == 0) { 736 break; // Read end. 737 } else { 738 if (errno == EAGAIN) { 739 break; 740 } 741 perror("failed to read child_read_fd"); 742 exit(1); 743 } 744 } 745 } 746 } 747 748 static void WaitChildProcs(std::vector<TestCase>& testcase_list, 749 std::vector<ChildProcInfo>& child_proc_list) { 750 size_t finished_child_count = 0; 751 while (true) { 752 int status; 753 pid_t result; 754 while ((result = TEMP_FAILURE_RETRY(waitpid(-1, &status, WNOHANG))) > 0) { 755 if (CheckChildProcExit(result, status, child_proc_list)) { 756 ++finished_child_count; 757 } 758 } 759 760 if (result == -1) { 761 if (errno == ECHILD) { 762 // This happens when we have no running child processes. 763 return; 764 } else { 765 perror("waitpid"); 766 exit(1); 767 } 768 } else if (result == 0) { 769 finished_child_count += CheckChildProcTimeout(child_proc_list); 770 } 771 772 ReadChildProcOutput(testcase_list, child_proc_list); 773 if (finished_child_count > 0) { 774 return; 775 } 776 777 HandleSignals(testcase_list, child_proc_list); 778 779 // sleep 1 ms to avoid busy looping. 780 timespec sleep_time; 781 sleep_time.tv_sec = 0; 782 sleep_time.tv_nsec = 1000000; 783 nanosleep(&sleep_time, NULL); 784 } 785 } 786 787 static TestResult WaitForOneChild(pid_t pid) { 788 int exit_status; 789 pid_t result = TEMP_FAILURE_RETRY(waitpid(pid, &exit_status, 0)); 790 791 TestResult test_result = TEST_SUCCESS; 792 if (result != pid || WEXITSTATUS(exit_status) != 0) { 793 test_result = TEST_FAILED; 794 } 795 return test_result; 796 } 797 798 static void CollectChildTestResult(const ChildProcInfo& child_proc, 799 TestCase& testcase) { 800 int test_id = child_proc.test_id; 801 testcase.SetTestTime(test_id, 802 child_proc.end_time_ns - child_proc.start_time_ns); 803 if (child_proc.timed_out) { 804 // The child process marked as timed_out has not exited, and we should kill 805 // it manually. 806 kill(child_proc.pid, SIGKILL); 807 WaitForOneChild(child_proc.pid); 808 } 809 close(child_proc.child_read_fd); 810 811 if (child_proc.timed_out) { 812 testcase.SetTestResult(test_id, TEST_TIMEOUT); 813 char buf[1024]; 814 snprintf(buf, sizeof(buf), 815 "%s killed because of timeout at %" PRId64 " ms.\n", 816 testcase.GetTestName(test_id).c_str(), 817 testcase.GetTestTime(test_id) / 1000000); 818 testcase.GetTest(test_id).AppendTestOutput(buf); 819 820 } else if (WIFSIGNALED(child_proc.exit_status)) { 821 // Record signal terminated test as failed. 822 testcase.SetTestResult(test_id, TEST_FAILED); 823 char buf[1024]; 824 snprintf(buf, sizeof(buf), "%s terminated by signal: %s.\n", 825 testcase.GetTestName(test_id).c_str(), 826 strsignal(WTERMSIG(child_proc.exit_status))); 827 testcase.GetTest(test_id).AppendTestOutput(buf); 828 829 } else { 830 int exitcode = WEXITSTATUS(child_proc.exit_status); 831 testcase.SetTestResult(test_id, exitcode == 0 ? TEST_SUCCESS : TEST_FAILED); 832 if (exitcode != 0) { 833 char buf[1024]; 834 snprintf(buf, sizeof(buf), "%s exited with exitcode %d.\n", 835 testcase.GetTestName(test_id).c_str(), exitcode); 836 testcase.GetTest(test_id).AppendTestOutput(buf); 837 } 838 } 839 } 840 841 // We choose to use multi-fork and multi-wait here instead of multi-thread, 842 // because it always 843 // makes deadlock to use fork in multi-thread. 844 // Returns true if all tests run successfully, otherwise return false. 845 static bool RunTestInSeparateProc(int argc, char** argv, 846 std::vector<TestCase>& testcase_list, 847 int iteration_count, size_t job_count, 848 const std::string& xml_output_filename) { 849 // Stop default result printer to avoid environment setup/teardown information 850 // for each test. 851 testing::UnitTest::GetInstance()->listeners().Release( 852 testing::UnitTest::GetInstance()->listeners().default_result_printer()); 853 testing::UnitTest::GetInstance()->listeners().Append(new TestResultPrinter); 854 855 if (!RegisterSignalHandler()) { 856 exit(1); 857 } 858 859 bool all_tests_passed = true; 860 861 for (size_t iteration = 1; 862 iteration_count < 0 || iteration <= static_cast<size_t>(iteration_count); 863 ++iteration) { 864 OnTestIterationStartPrint(testcase_list, iteration, iteration_count); 865 int64_t iteration_start_time_ns = NanoTime(); 866 time_t epoch_iteration_start_time = time(NULL); 867 868 // Run up to job_count tests in parallel, each test in a child process. 869 std::vector<ChildProcInfo> child_proc_list; 870 871 // Next test to run is [next_testcase_id:next_test_id]. 872 size_t next_testcase_id = 0; 873 size_t next_test_id = 0; 874 875 // Record how many tests are finished. 876 std::vector<size_t> finished_test_count_list(testcase_list.size(), 0); 877 size_t finished_testcase_count = 0; 878 879 while (finished_testcase_count < testcase_list.size()) { 880 // run up to job_count child processes. 881 while (child_proc_list.size() < job_count && 882 next_testcase_id < testcase_list.size()) { 883 std::string test_name = 884 testcase_list[next_testcase_id].GetTestName(next_test_id); 885 ChildProcInfo child_proc = RunChildProcess(test_name, next_testcase_id, 886 next_test_id, argc, argv); 887 child_proc_list.push_back(child_proc); 888 if (++next_test_id == testcase_list[next_testcase_id].TestCount()) { 889 next_test_id = 0; 890 ++next_testcase_id; 891 } 892 } 893 894 // Wait for any child proc finish or timeout. 895 WaitChildProcs(testcase_list, child_proc_list); 896 897 // Collect result. 898 auto it = child_proc_list.begin(); 899 while (it != child_proc_list.end()) { 900 auto& child_proc = *it; 901 if (child_proc.finished == true) { 902 size_t testcase_id = child_proc.testcase_id; 903 size_t test_id = child_proc.test_id; 904 TestCase& testcase = testcase_list[testcase_id]; 905 906 CollectChildTestResult(child_proc, testcase); 907 OnTestEndPrint(testcase, test_id); 908 909 if (++finished_test_count_list[testcase_id] == testcase.TestCount()) { 910 ++finished_testcase_count; 911 } 912 if (testcase.GetTestResult(test_id) != TEST_SUCCESS) { 913 all_tests_passed = false; 914 } 915 916 it = child_proc_list.erase(it); 917 } else { 918 ++it; 919 } 920 } 921 } 922 923 int64_t elapsed_time_ns = NanoTime() - iteration_start_time_ns; 924 OnTestIterationEndPrint(testcase_list, iteration, elapsed_time_ns); 925 if (!xml_output_filename.empty()) { 926 OnTestIterationEndXmlPrint(xml_output_filename, testcase_list, 927 epoch_iteration_start_time, elapsed_time_ns); 928 } 929 } 930 931 if (!UnregisterSignalHandler()) { 932 exit(1); 933 } 934 935 return all_tests_passed; 936 } 937 938 static size_t GetDefaultJobCount() { 939 return static_cast<size_t>(sysconf(_SC_NPROCESSORS_ONLN)); 940 } 941 942 static void AddPathSeparatorInTestProgramPath(std::vector<char*>& args) { 943 // To run DeathTest in threadsafe mode, gtest requires that the user must 944 // invoke the 945 // test program via a valid path that contains at least one path separator. 946 // The reason is that gtest uses clone() + execve() to run DeathTest in 947 // threadsafe mode, 948 // and execve() doesn't read environment variable PATH, so execve() will not 949 // success 950 // until we specify the absolute path or relative path of the test program 951 // directly. 952 if (strchr(args[0], '/') == NULL) { 953 char path[PATH_MAX]; 954 ssize_t path_len = readlink("/proc/self/exe", path, sizeof(path)); 955 if (path_len <= 0 || path_len >= static_cast<ssize_t>(sizeof(path))) { 956 perror("readlink"); 957 exit(1); 958 } 959 path[path_len] = '\0'; 960 args[0] = strdup(path); 961 } 962 } 963 964 static void AddGtestFilterSynonym(std::vector<char*>& args) { 965 // Support --gtest-filter as a synonym for --gtest_filter. 966 for (size_t i = 1; i < args.size(); ++i) { 967 if (strncmp(args[i], "--gtest-filter", strlen("--gtest-filter")) == 0) { 968 args[i][7] = '_'; 969 } 970 } 971 } 972 973 struct IsolationTestOptions { 974 bool isolate; 975 size_t job_count; 976 int test_deadline_ms; 977 int test_warnline_ms; 978 std::string gtest_color; 979 bool gtest_print_time; 980 int gtest_repeat; 981 std::string gtest_output; 982 }; 983 984 // Pick options not for gtest: There are two parts in args, one part is used in 985 // isolation test mode 986 // as described in PrintHelpInfo(), the other part is handled by 987 // testing::InitGoogleTest() in 988 // gtest. PickOptions() picks the first part into IsolationTestOptions 989 // structure, leaving the second 990 // part in args. 991 // Arguments: 992 // args is used to pass in all command arguments, and pass out only the part 993 // of options for gtest. 994 // options is used to pass out test options in isolation mode. 995 // Return false if there is error in arguments. 996 static bool PickOptions(std::vector<char*>& args, 997 IsolationTestOptions& options) { 998 for (size_t i = 1; i < args.size(); ++i) { 999 if (strcmp(args[i], "--help") == 0 || strcmp(args[i], "-h") == 0) { 1000 PrintHelpInfo(); 1001 options.isolate = false; 1002 return true; 1003 } 1004 } 1005 1006 AddPathSeparatorInTestProgramPath(args); 1007 AddGtestFilterSynonym(args); 1008 1009 // if --vts-selftest argument is used, only enable self tests, otherwise 1010 // remove self tests. 1011 bool enable_selftest = false; 1012 for (size_t i = 1; i < args.size(); ++i) { 1013 if (strcmp(args[i], "--vts-selftest") == 0) { 1014 // This argument is to enable "vts_selftest*" for self test, and is not 1015 // shown in help info. 1016 // Don't remove this option from arguments. 1017 enable_selftest = true; 1018 } 1019 } 1020 std::string gtest_filter_str; 1021 for (size_t i = args.size() - 1; i >= 1; --i) { 1022 if (strncmp(args[i], "--gtest_filter=", strlen("--gtest_filter=")) == 0) { 1023 gtest_filter_str = std::string(args[i]); 1024 args.erase(args.begin() + i); 1025 break; 1026 } 1027 } 1028 if (enable_selftest == true) { 1029 args.push_back(strdup("--gtest_filter=vts_selftest*")); 1030 } else { 1031 if (gtest_filter_str == "") { 1032 gtest_filter_str = "--gtest_filter=-vts_selftest*"; 1033 } else { 1034 // Find if '-' for NEGATIVE_PATTERNS exists. 1035 if (gtest_filter_str.find(":-") != std::string::npos) { 1036 gtest_filter_str += ":vts_selftest*"; 1037 } else { 1038 gtest_filter_str += ":-vts_selftest*"; 1039 } 1040 } 1041 args.push_back(strdup(gtest_filter_str.c_str())); 1042 } 1043 1044 options.isolate = true; 1045 // Parse arguments that make us can't run in isolation mode. 1046 for (size_t i = 1; i < args.size(); ++i) { 1047 if (strcmp(args[i], "--no-isolate") == 0) { 1048 options.isolate = false; 1049 } else if (strcmp(args[i], "--gtest_list_tests") == 0) { 1050 options.isolate = false; 1051 } 1052 } 1053 1054 // Stop parsing if we will not run in isolation mode. 1055 if (options.isolate == false) { 1056 return true; 1057 } 1058 1059 // Init default isolation test options. 1060 options.job_count = GetDefaultJobCount(); 1061 options.test_deadline_ms = DEFAULT_GLOBAL_TEST_RUN_DEADLINE_MS; 1062 options.test_warnline_ms = DEFAULT_GLOBAL_TEST_RUN_WARNLINE_MS; 1063 options.gtest_color = testing::GTEST_FLAG(color); 1064 options.gtest_print_time = testing::GTEST_FLAG(print_time); 1065 options.gtest_repeat = testing::GTEST_FLAG(repeat); 1066 options.gtest_output = testing::GTEST_FLAG(output); 1067 1068 // Parse arguments speficied for isolation mode. 1069 for (size_t i = 1; i < args.size(); ++i) { 1070 if (strncmp(args[i], "-j", strlen("-j")) == 0) { 1071 char* p = args[i] + strlen("-j"); 1072 int count = 0; 1073 if (*p != '\0') { 1074 // Argument like -j5. 1075 count = atoi(p); 1076 } else if (args.size() > i + 1) { 1077 // Arguments like -j 5. 1078 count = atoi(args[i + 1]); 1079 ++i; 1080 } 1081 if (count <= 0) { 1082 fprintf(stderr, "invalid job count: %d\n", count); 1083 return false; 1084 } 1085 options.job_count = static_cast<size_t>(count); 1086 } else if (strncmp(args[i], "--deadline=", strlen("--deadline=")) == 0) { 1087 int time_ms = atoi(args[i] + strlen("--deadline=")); 1088 if (time_ms <= 0) { 1089 fprintf(stderr, "invalid deadline: %d\n", time_ms); 1090 return false; 1091 } 1092 options.test_deadline_ms = time_ms; 1093 } else if (strncmp(args[i], "--warnline=", strlen("--warnline=")) == 0) { 1094 int time_ms = atoi(args[i] + strlen("--warnline=")); 1095 if (time_ms <= 0) { 1096 fprintf(stderr, "invalid warnline: %d\n", time_ms); 1097 return false; 1098 } 1099 options.test_warnline_ms = time_ms; 1100 } else if (strncmp(args[i], "--gtest_color=", strlen("--gtest_color=")) == 1101 0) { 1102 options.gtest_color = args[i] + strlen("--gtest_color="); 1103 } else if (strcmp(args[i], "--gtest_print_time=0") == 0) { 1104 options.gtest_print_time = false; 1105 } else if (strncmp(args[i], "--gtest_repeat=", strlen("--gtest_repeat=")) == 1106 0) { 1107 // If the value of gtest_repeat is < 0, then it indicates the tests 1108 // should be repeated forever. 1109 options.gtest_repeat = atoi(args[i] + strlen("--gtest_repeat=")); 1110 // Remove --gtest_repeat=xx from arguments, so child process only run one 1111 // iteration for a single test. 1112 args.erase(args.begin() + i); 1113 --i; 1114 } else if (strncmp(args[i], "--gtest_output=", strlen("--gtest_output=")) == 1115 0) { 1116 std::string output = args[i] + strlen("--gtest_output="); 1117 // generate output xml file path according to the strategy in gtest. 1118 bool success = true; 1119 if (strncmp(output.c_str(), "xml:", strlen("xml:")) == 0) { 1120 output = output.substr(strlen("xml:")); 1121 if (output.size() == 0) { 1122 success = false; 1123 } 1124 // Make absolute path. 1125 if (success && output[0] != '/') { 1126 char* cwd = getcwd(NULL, 0); 1127 if (cwd != NULL) { 1128 output = std::string(cwd) + "/" + output; 1129 free(cwd); 1130 } else { 1131 success = false; 1132 } 1133 } 1134 // Add file name if output is a directory. 1135 if (success && output.back() == '/') { 1136 output += "test_details.xml"; 1137 } 1138 } 1139 if (success) { 1140 options.gtest_output = output; 1141 } else { 1142 fprintf(stderr, "invalid gtest_output file: %s\n", args[i]); 1143 return false; 1144 } 1145 1146 // Remove --gtest_output=xxx from arguments, so child process will not 1147 // write xml file. 1148 args.erase(args.begin() + i); 1149 --i; 1150 } 1151 } 1152 1153 // Add --no-isolate in args to prevent child process from running in isolation 1154 // mode again. 1155 // As DeathTest will try to call execve(), this argument should always be 1156 // added. 1157 args.insert(args.begin() + 1, strdup("--no-isolate")); 1158 return true; 1159 } 1160 1161 int main(int argc, char** argv) { 1162 std::vector<char*> arg_list; 1163 for (int i = 0; i < argc; ++i) { 1164 arg_list.push_back(argv[i]); 1165 } 1166 1167 IsolationTestOptions options; 1168 if (PickOptions(arg_list, options) == false) { 1169 return 1; 1170 } 1171 1172 if (options.isolate == true) { 1173 // Set global variables. 1174 global_test_run_deadline_ms = options.test_deadline_ms; 1175 global_test_run_warnline_ms = options.test_warnline_ms; 1176 testing::GTEST_FLAG(color) = options.gtest_color.c_str(); 1177 testing::GTEST_FLAG(print_time) = options.gtest_print_time; 1178 std::vector<TestCase> testcase_list; 1179 1180 argc = static_cast<int>(arg_list.size()); 1181 arg_list.push_back(NULL); 1182 if (EnumerateTests(argc, arg_list.data(), testcase_list) == false) { 1183 return 1; 1184 } 1185 bool all_test_passed = RunTestInSeparateProc( 1186 argc, arg_list.data(), testcase_list, options.gtest_repeat, 1187 options.job_count, options.gtest_output); 1188 return all_test_passed ? 0 : 1; 1189 } else { 1190 argc = static_cast<int>(arg_list.size()); 1191 arg_list.push_back(NULL); 1192 testing::InitGoogleTest(&argc, arg_list.data()); 1193 return RUN_ALL_TESTS(); 1194 } 1195 } 1196 1197 //################################################################################ 1198 // VTS Gtest self test, run this by --vts-selftest option. 1199 1200 TEST(vts_selftest, test_success) { ASSERT_EQ(1, 1); } 1201 1202 TEST(vts_selftest, test_fail) { ASSERT_EQ(0, 1); } 1203 1204 TEST(vts_selftest, test_time_warn) { sleep(4); } 1205 1206 TEST(vts_selftest, test_timeout) { 1207 while (1) { 1208 } 1209 } 1210 1211 TEST(vts_selftest, test_signal_SEGV_terminated) { 1212 char* p = reinterpret_cast<char*>(static_cast<intptr_t>(atoi("0"))); 1213 *p = 3; 1214 } 1215 1216 class vts_selftest_DeathTest : public ::testing::Test { 1217 protected: 1218 virtual void SetUp() { 1219 ::testing::FLAGS_gtest_death_test_style = "threadsafe"; 1220 } 1221 }; 1222 1223 static void deathtest_helper_success() { 1224 ASSERT_EQ(1, 1); 1225 exit(0); 1226 } 1227 1228 TEST_F(vts_selftest_DeathTest, success) { 1229 ASSERT_EXIT(deathtest_helper_success(), ::testing::ExitedWithCode(0), ""); 1230 } 1231 1232 static void deathtest_helper_fail() { ASSERT_EQ(1, 0); } 1233 1234 TEST_F(vts_selftest_DeathTest, fail) { 1235 ASSERT_EXIT(deathtest_helper_fail(), ::testing::ExitedWithCode(0), ""); 1236 } 1237