1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/test/launcher/unit_test_launcher.h" 6 7 #include "base/bind.h" 8 #include "base/callback_helpers.h" 9 #include "base/command_line.h" 10 #include "base/compiler_specific.h" 11 #include "base/debug/debugger.h" 12 #include "base/files/file_util.h" 13 #include "base/files/scoped_temp_dir.h" 14 #include "base/format_macros.h" 15 #include "base/message_loop/message_loop.h" 16 #include "base/stl_util.h" 17 #include "base/strings/string_number_conversions.h" 18 #include "base/strings/string_util.h" 19 #include "base/sys_info.h" 20 #include "base/test/gtest_xml_util.h" 21 #include "base/test/launcher/test_launcher.h" 22 #include "base/test/test_switches.h" 23 #include "base/test/test_timeouts.h" 24 #include "base/third_party/dynamic_annotations/dynamic_annotations.h" 25 #include "base/threading/thread_checker.h" 26 #include "testing/gtest/include/gtest/gtest.h" 27 28 namespace base { 29 30 namespace { 31 32 // This constant controls how many tests are run in a single batch by default. 33 const size_t kDefaultTestBatchLimit = 10; 34 35 const char kHelpFlag[] = "help"; 36 37 // Flag to run all tests in a single process. 38 const char kSingleProcessTestsFlag[] = "single-process-tests"; 39 40 void PrintUsage() { 41 fprintf(stdout, 42 "Runs tests using the gtest framework, each batch of tests being\n" 43 "run in their own process. Supported command-line flags:\n" 44 "\n" 45 " Common flags:\n" 46 " --gtest_filter=...\n" 47 " Runs a subset of tests (see --gtest_help for more info).\n" 48 "\n" 49 " --help\n" 50 " Shows this message.\n" 51 "\n" 52 " --gtest_help\n" 53 " Shows the gtest help message.\n" 54 "\n" 55 " --test-launcher-jobs=N\n" 56 " Sets the number of parallel test jobs to N.\n" 57 "\n" 58 " --single-process-tests\n" 59 " Runs the tests and the launcher in the same process. Useful\n" 60 " for debugging a specific test in a debugger.\n" 61 "\n" 62 " Other flags:\n" 63 " --test-launcher-batch-limit=N\n" 64 " Sets the limit of test batch to run in a single process to N.\n" 65 "\n" 66 " --test-launcher-debug-launcher\n" 67 " Disables autodetection of debuggers and similar tools,\n" 68 " making it possible to use them to debug launcher itself.\n" 69 "\n" 70 " --test-launcher-retry-limit=N\n" 71 " Sets the limit of test retries on failures to N.\n" 72 "\n" 73 " --test-launcher-summary-output=PATH\n" 74 " Saves a JSON machine-readable summary of the run.\n" 75 "\n" 76 " --test-launcher-print-test-stdio=auto|always|never\n" 77 " Controls when full test output is printed.\n" 78 " auto means to print it when the test failed.\n" 79 "\n" 80 " --test-launcher-total-shards=N\n" 81 " Sets the total number of shards to N.\n" 82 "\n" 83 " --test-launcher-shard-index=N\n" 84 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"); 85 fflush(stdout); 86 } 87 88 // Returns command line for child GTest process based on the command line 89 // of current process. |test_names| is a vector of test full names 90 // (e.g. "A.B"), |output_file| is path to the GTest XML output file. 91 CommandLine GetCommandLineForChildGTestProcess( 92 const std::vector<std::string>& test_names, 93 const base::FilePath& output_file) { 94 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 95 96 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file); 97 new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":")); 98 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 99 100 return new_cmd_line; 101 } 102 103 class UnitTestLauncherDelegate : public TestLauncherDelegate { 104 public: 105 explicit UnitTestLauncherDelegate(size_t batch_limit, bool use_job_objects) 106 : batch_limit_(batch_limit), 107 use_job_objects_(use_job_objects) { 108 } 109 110 virtual ~UnitTestLauncherDelegate() { 111 DCHECK(thread_checker_.CalledOnValidThread()); 112 } 113 114 private: 115 struct GTestCallbackState { 116 TestLauncher* test_launcher; 117 std::vector<std::string> test_names; 118 FilePath output_file; 119 }; 120 121 virtual bool ShouldRunTest(const testing::TestCase* test_case, 122 const testing::TestInfo* test_info) OVERRIDE { 123 DCHECK(thread_checker_.CalledOnValidThread()); 124 125 // There is no additional logic to disable specific tests. 126 return true; 127 } 128 129 virtual size_t RunTests(TestLauncher* test_launcher, 130 const std::vector<std::string>& test_names) OVERRIDE { 131 DCHECK(thread_checker_.CalledOnValidThread()); 132 133 std::vector<std::string> batch; 134 for (size_t i = 0; i < test_names.size(); i++) { 135 batch.push_back(test_names[i]); 136 137 if (batch.size() >= batch_limit_) { 138 RunBatch(test_launcher, batch); 139 batch.clear(); 140 } 141 } 142 143 RunBatch(test_launcher, batch); 144 145 return test_names.size(); 146 } 147 148 virtual size_t RetryTests( 149 TestLauncher* test_launcher, 150 const std::vector<std::string>& test_names) OVERRIDE { 151 MessageLoop::current()->PostTask( 152 FROM_HERE, 153 Bind(&UnitTestLauncherDelegate::RunSerially, 154 Unretained(this), 155 test_launcher, 156 test_names)); 157 return test_names.size(); 158 } 159 160 void RunSerially(TestLauncher* test_launcher, 161 const std::vector<std::string>& test_names) { 162 if (test_names.empty()) 163 return; 164 165 std::vector<std::string> new_test_names(test_names); 166 std::string test_name(new_test_names.back()); 167 new_test_names.pop_back(); 168 169 // Create a dedicated temporary directory to store the xml result data 170 // per run to ensure clean state and make it possible to launch multiple 171 // processes in parallel. 172 base::FilePath output_file; 173 CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 174 output_file = output_file.AppendASCII("test_results.xml"); 175 176 std::vector<std::string> current_test_names; 177 current_test_names.push_back(test_name); 178 CommandLine cmd_line( 179 GetCommandLineForChildGTestProcess(current_test_names, output_file)); 180 181 GTestCallbackState callback_state; 182 callback_state.test_launcher = test_launcher; 183 callback_state.test_names = current_test_names; 184 callback_state.output_file = output_file; 185 186 test_launcher->LaunchChildGTestProcess( 187 cmd_line, 188 std::string(), 189 TestTimeouts::test_launcher_timeout(), 190 use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0, 191 Bind(&UnitTestLauncherDelegate::SerialGTestCallback, 192 Unretained(this), 193 callback_state, 194 new_test_names)); 195 } 196 197 void RunBatch(TestLauncher* test_launcher, 198 const std::vector<std::string>& test_names) { 199 DCHECK(thread_checker_.CalledOnValidThread()); 200 201 if (test_names.empty()) 202 return; 203 204 // Create a dedicated temporary directory to store the xml result data 205 // per run to ensure clean state and make it possible to launch multiple 206 // processes in parallel. 207 base::FilePath output_file; 208 CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 209 output_file = output_file.AppendASCII("test_results.xml"); 210 211 CommandLine cmd_line( 212 GetCommandLineForChildGTestProcess(test_names, output_file)); 213 214 // Adjust the timeout depending on how many tests we're running 215 // (note that e.g. the last batch of tests will be smaller). 216 // TODO(phajdan.jr): Consider an adaptive timeout, which can change 217 // depending on how many tests ran and how many remain. 218 // Note: do NOT parse child's stdout to do that, it's known to be 219 // unreliable (e.g. buffering issues can mix up the output). 220 base::TimeDelta timeout = 221 test_names.size() * TestTimeouts::test_launcher_timeout(); 222 223 GTestCallbackState callback_state; 224 callback_state.test_launcher = test_launcher; 225 callback_state.test_names = test_names; 226 callback_state.output_file = output_file; 227 228 test_launcher->LaunchChildGTestProcess( 229 cmd_line, 230 std::string(), 231 timeout, 232 use_job_objects_ ? TestLauncher::USE_JOB_OBJECTS : 0, 233 Bind(&UnitTestLauncherDelegate::GTestCallback, 234 Unretained(this), 235 callback_state)); 236 } 237 238 void GTestCallback(const GTestCallbackState& callback_state, 239 int exit_code, 240 const TimeDelta& elapsed_time, 241 bool was_timeout, 242 const std::string& output) { 243 DCHECK(thread_checker_.CalledOnValidThread()); 244 std::vector<std::string> tests_to_relaunch; 245 ProcessTestResults(callback_state.test_launcher, 246 callback_state.test_names, 247 callback_state.output_file, 248 output, 249 exit_code, 250 was_timeout, 251 &tests_to_relaunch); 252 253 // Relaunch requested tests in parallel, but only use single 254 // test per batch for more precise results (crashes, test passes 255 // but non-zero exit codes etc). 256 for (size_t i = 0; i < tests_to_relaunch.size(); i++) { 257 std::vector<std::string> batch; 258 batch.push_back(tests_to_relaunch[i]); 259 RunBatch(callback_state.test_launcher, batch); 260 } 261 262 // The temporary file's directory is also temporary. 263 DeleteFile(callback_state.output_file.DirName(), true); 264 } 265 266 void SerialGTestCallback(const GTestCallbackState& callback_state, 267 const std::vector<std::string>& test_names, 268 int exit_code, 269 const TimeDelta& elapsed_time, 270 bool was_timeout, 271 const std::string& output) { 272 DCHECK(thread_checker_.CalledOnValidThread()); 273 std::vector<std::string> tests_to_relaunch; 274 bool called_any_callbacks = 275 ProcessTestResults(callback_state.test_launcher, 276 callback_state.test_names, 277 callback_state.output_file, 278 output, 279 exit_code, 280 was_timeout, 281 &tests_to_relaunch); 282 283 // There is only one test, there cannot be other tests to relaunch 284 // due to a crash. 285 DCHECK(tests_to_relaunch.empty()); 286 287 // There is only one test, we should have called back with its result. 288 DCHECK(called_any_callbacks); 289 290 // The temporary file's directory is also temporary. 291 DeleteFile(callback_state.output_file.DirName(), true); 292 293 MessageLoop::current()->PostTask( 294 FROM_HERE, 295 Bind(&UnitTestLauncherDelegate::RunSerially, 296 Unretained(this), 297 callback_state.test_launcher, 298 test_names)); 299 } 300 301 static bool ProcessTestResults( 302 TestLauncher* test_launcher, 303 const std::vector<std::string>& test_names, 304 const base::FilePath& output_file, 305 const std::string& output, 306 int exit_code, 307 bool was_timeout, 308 std::vector<std::string>* tests_to_relaunch) { 309 std::vector<TestResult> test_results; 310 bool crashed = false; 311 bool have_test_results = 312 ProcessGTestOutput(output_file, &test_results, &crashed); 313 314 bool called_any_callback = false; 315 316 if (have_test_results) { 317 // TODO(phajdan.jr): Check for duplicates and mismatches between 318 // the results we got from XML file and tests we intended to run. 319 std::map<std::string, TestResult> results_map; 320 for (size_t i = 0; i < test_results.size(); i++) 321 results_map[test_results[i].full_name] = test_results[i]; 322 323 bool had_interrupted_test = false; 324 325 // Results to be reported back to the test launcher. 326 std::vector<TestResult> final_results; 327 328 for (size_t i = 0; i < test_names.size(); i++) { 329 if (ContainsKey(results_map, test_names[i])) { 330 TestResult test_result = results_map[test_names[i]]; 331 if (test_result.status == TestResult::TEST_CRASH) { 332 had_interrupted_test = true; 333 334 if (was_timeout) { 335 // Fix up the test status: we forcibly kill the child process 336 // after the timeout, so from XML results it looks just like 337 // a crash. 338 test_result.status = TestResult::TEST_TIMEOUT; 339 } 340 } else if (test_result.status == TestResult::TEST_SUCCESS || 341 test_result.status == TestResult::TEST_FAILURE) { 342 // We run multiple tests in a batch with a timeout applied 343 // to the entire batch. It is possible that with other tests 344 // running quickly some tests take longer than the per-test timeout. 345 // For consistent handling of tests independent of order and other 346 // factors, mark them as timing out. 347 if (test_result.elapsed_time > 348 TestTimeouts::test_launcher_timeout()) { 349 test_result.status = TestResult::TEST_TIMEOUT; 350 } 351 } 352 test_result.output_snippet = 353 GetTestOutputSnippet(test_result, output); 354 final_results.push_back(test_result); 355 } else if (had_interrupted_test) { 356 tests_to_relaunch->push_back(test_names[i]); 357 } else { 358 // TODO(phajdan.jr): Explicitly pass the info that the test didn't 359 // run for a mysterious reason. 360 LOG(ERROR) << "no test result for " << test_names[i]; 361 TestResult test_result; 362 test_result.full_name = test_names[i]; 363 test_result.status = TestResult::TEST_UNKNOWN; 364 test_result.output_snippet = 365 GetTestOutputSnippet(test_result, output); 366 final_results.push_back(test_result); 367 } 368 } 369 370 // TODO(phajdan.jr): Handle the case where processing XML output 371 // indicates a crash but none of the test results is marked as crashing. 372 373 if (final_results.empty()) 374 return false; 375 376 bool has_non_success_test = false; 377 for (size_t i = 0; i < final_results.size(); i++) { 378 if (final_results[i].status != TestResult::TEST_SUCCESS) { 379 has_non_success_test = true; 380 break; 381 } 382 } 383 384 if (!has_non_success_test && exit_code != 0) { 385 // This is a bit surprising case: all tests are marked as successful, 386 // but the exit code was not zero. This can happen e.g. under memory 387 // tools that report leaks this way. 388 389 if (final_results.size() == 1) { 390 // Easy case. One test only so we know the non-zero exit code 391 // was caused by that one test. 392 final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT; 393 } else { 394 // Harder case. Discard the results and request relaunching all 395 // tests without batching. This will trigger above branch on 396 // relaunch leading to more precise results. 397 LOG(WARNING) << "Not sure which test caused non-zero exit code, " 398 << "relaunching all of them without batching."; 399 400 for (size_t i = 0; i < final_results.size(); i++) 401 tests_to_relaunch->push_back(final_results[i].full_name); 402 403 return false; 404 } 405 } 406 407 for (size_t i = 0; i < final_results.size(); i++) { 408 // Fix the output snippet after possible changes to the test result. 409 final_results[i].output_snippet = 410 GetTestOutputSnippet(final_results[i], output); 411 test_launcher->OnTestFinished(final_results[i]); 412 called_any_callback = true; 413 } 414 } else { 415 fprintf(stdout, 416 "Failed to get out-of-band test success data, " 417 "dumping full stdio below:\n%s\n", 418 output.c_str()); 419 fflush(stdout); 420 421 // We do not have reliable details about test results (parsing test 422 // stdout is known to be unreliable), apply the executable exit code 423 // to all tests. 424 // TODO(phajdan.jr): Be smarter about this, e.g. retry each test 425 // individually. 426 for (size_t i = 0; i < test_names.size(); i++) { 427 TestResult test_result; 428 test_result.full_name = test_names[i]; 429 test_result.status = TestResult::TEST_UNKNOWN; 430 test_launcher->OnTestFinished(test_result); 431 called_any_callback = true; 432 } 433 } 434 435 return called_any_callback; 436 } 437 438 ThreadChecker thread_checker_; 439 440 // Maximum number of tests to run in a single batch. 441 size_t batch_limit_; 442 443 // Determines whether we use job objects on Windows. 444 bool use_job_objects_; 445 }; 446 447 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 448 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 449 return true; 450 451 std::string switch_value = 452 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 453 if (!StringToInt(switch_value, result) || *result < 1) { 454 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 455 return false; 456 } 457 458 return true; 459 } 460 461 int LaunchUnitTestsInternal(const RunTestSuiteCallback& run_test_suite, 462 int default_jobs, 463 bool use_job_objects, 464 const Closure& gtest_init) { 465 #if defined(OS_ANDROID) 466 // We can't easily fork on Android, just run the test suite directly. 467 return run_test_suite.Run(); 468 #else 469 bool force_single_process = false; 470 if (CommandLine::ForCurrentProcess()->HasSwitch( 471 switches::kTestLauncherDebugLauncher)) { 472 fprintf(stdout, "Forcing test launcher debugging mode.\n"); 473 fflush(stdout); 474 } else { 475 if (base::debug::BeingDebugged()) { 476 fprintf(stdout, 477 "Debugger detected, switching to single process mode.\n" 478 "Pass --test-launcher-debug-launcher to debug the launcher " 479 "itself.\n"); 480 fflush(stdout); 481 force_single_process = true; 482 } 483 484 if (RunningOnValgrind()) { 485 fprintf(stdout, 486 "Valgrind detected, switching to single process mode.\n" 487 "Pass --test-launcher-debug-launcher to valgrind the launcher " 488 "itself.\n"); 489 fflush(stdout); 490 force_single_process = true; 491 } 492 } 493 494 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) || 495 CommandLine::ForCurrentProcess()->HasSwitch(kGTestListTestsFlag) || 496 CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) || 497 force_single_process) { 498 return run_test_suite.Run(); 499 } 500 #endif 501 502 if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) { 503 PrintUsage(); 504 return 0; 505 } 506 507 base::TimeTicks start_time(base::TimeTicks::Now()); 508 509 gtest_init.Run(); 510 TestTimeouts::Initialize(); 511 512 int batch_limit = kDefaultTestBatchLimit; 513 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit)) 514 return 1; 515 516 fprintf(stdout, 517 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n" 518 "own process. For debugging a test inside a debugger, use the\n" 519 "--gtest_filter=<your_test_name> flag along with\n" 520 "--single-process-tests.\n"); 521 fflush(stdout); 522 523 MessageLoopForIO message_loop; 524 525 UnitTestLauncherDelegate delegate(batch_limit, use_job_objects); 526 base::TestLauncher launcher(&delegate, default_jobs); 527 bool success = launcher.Run(); 528 529 fprintf(stdout, 530 "Tests took %" PRId64 " seconds.\n", 531 (base::TimeTicks::Now() - start_time).InSeconds()); 532 fflush(stdout); 533 534 return (success ? 0 : 1); 535 } 536 537 void InitGoogleTestChar(int* argc, char** argv) { 538 testing::InitGoogleTest(argc, argv); 539 } 540 541 #if defined(OS_WIN) 542 void InitGoogleTestWChar(int* argc, wchar_t** argv) { 543 testing::InitGoogleTest(argc, argv); 544 } 545 #endif // defined(OS_WIN) 546 547 } // namespace 548 549 int LaunchUnitTests(int argc, 550 char** argv, 551 const RunTestSuiteCallback& run_test_suite) { 552 CommandLine::Init(argc, argv); 553 return LaunchUnitTestsInternal( 554 run_test_suite, 555 SysInfo::NumberOfProcessors(), 556 true, 557 Bind(&InitGoogleTestChar, &argc, argv)); 558 } 559 560 int LaunchUnitTestsSerially(int argc, 561 char** argv, 562 const RunTestSuiteCallback& run_test_suite) { 563 CommandLine::Init(argc, argv); 564 return LaunchUnitTestsInternal( 565 run_test_suite, 566 1, 567 true, 568 Bind(&InitGoogleTestChar, &argc, argv)); 569 } 570 571 #if defined(OS_WIN) 572 int LaunchUnitTests(int argc, 573 wchar_t** argv, 574 bool use_job_objects, 575 const RunTestSuiteCallback& run_test_suite) { 576 // Windows CommandLine::Init ignores argv anyway. 577 CommandLine::Init(argc, NULL); 578 return LaunchUnitTestsInternal( 579 run_test_suite, 580 SysInfo::NumberOfProcessors(), 581 use_job_objects, 582 Bind(&InitGoogleTestWChar, &argc, argv)); 583 } 584 #endif // defined(OS_WIN) 585 586 } // namespace base 587