1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/test/launcher/unit_test_launcher.h" 6 7 #include "base/bind.h" 8 #include "base/callback_helpers.h" 9 #include "base/command_line.h" 10 #include "base/compiler_specific.h" 11 #include "base/debug/debugger.h" 12 #include "base/file_util.h" 13 #include "base/files/scoped_temp_dir.h" 14 #include "base/format_macros.h" 15 #include "base/message_loop/message_loop.h" 16 #include "base/stl_util.h" 17 #include "base/strings/string_number_conversions.h" 18 #include "base/strings/string_util.h" 19 #include "base/sys_info.h" 20 #include "base/test/gtest_xml_util.h" 21 #include "base/test/launcher/test_launcher.h" 22 #include "base/test/test_switches.h" 23 #include "base/test/test_timeouts.h" 24 #include "base/third_party/dynamic_annotations/dynamic_annotations.h" 25 #include "base/threading/thread_checker.h" 26 #include "testing/gtest/include/gtest/gtest.h" 27 28 namespace base { 29 30 namespace { 31 32 // This constant controls how many tests are run in a single batch by default. 33 const size_t kDefaultTestBatchLimit = 10; 34 35 const char kHelpFlag[] = "help"; 36 37 // Flag to run all tests in a single process. 38 const char kSingleProcessTestsFlag[] = "single-process-tests"; 39 40 void PrintUsage() { 41 fprintf(stdout, 42 "Runs tests using the gtest framework, each batch of tests being\n" 43 "run in their own process. Supported command-line flags:\n" 44 "\n" 45 " Common flags:\n" 46 " --gtest_filter=...\n" 47 " Runs a subset of tests (see --gtest_help for more info).\n" 48 "\n" 49 " --help\n" 50 " Shows this message.\n" 51 "\n" 52 " --gtest_help\n" 53 " Shows the gtest help message.\n" 54 "\n" 55 " --test-launcher-jobs=N\n" 56 " Sets the number of parallel test jobs to N.\n" 57 "\n" 58 " --single-process-tests\n" 59 " Runs the tests and the launcher in the same process. Useful\n" 60 " for debugging a specific test in a debugger.\n" 61 "\n" 62 " Other flags:\n" 63 " --test-launcher-batch-limit=N\n" 64 " Sets the limit of test batch to run in a single process to N.\n" 65 "\n" 66 " --test-launcher-debug-launcher\n" 67 " Disables autodetection of debuggers and similar tools,\n" 68 " making it possible to use them to debug launcher itself.\n" 69 "\n" 70 " --test-launcher-retry-limit=N\n" 71 " Sets the limit of test retries on failures to N.\n" 72 "\n" 73 " --test-launcher-summary-output=PATH\n" 74 " Saves a JSON machine-readable summary of the run.\n" 75 "\n" 76 " --test-launcher-print-test-stdio=auto|always|never\n" 77 " Controls when full test output is printed.\n" 78 " auto means to print it when the test failed.\n" 79 "\n" 80 " --test-launcher-total-shards=N\n" 81 " Sets the total number of shards to N.\n" 82 "\n" 83 " --test-launcher-shard-index=N\n" 84 " Sets the shard index to run to N (from 0 to TOTAL - 1).\n"); 85 fflush(stdout); 86 } 87 88 // Returns command line for child GTest process based on the command line 89 // of current process. |test_names| is a vector of test full names 90 // (e.g. "A.B"), |output_file| is path to the GTest XML output file. 91 CommandLine GetCommandLineForChildGTestProcess( 92 const std::vector<std::string>& test_names, 93 const base::FilePath& output_file) { 94 CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 95 96 new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file); 97 new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":")); 98 new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 99 100 return new_cmd_line; 101 } 102 103 class UnitTestLauncherDelegate : public TestLauncherDelegate { 104 public: 105 explicit UnitTestLauncherDelegate(size_t batch_limit) 106 : batch_limit_(batch_limit) { 107 } 108 109 virtual ~UnitTestLauncherDelegate() { 110 DCHECK(thread_checker_.CalledOnValidThread()); 111 } 112 113 private: 114 struct GTestCallbackState { 115 TestLauncher* test_launcher; 116 std::vector<std::string> test_names; 117 FilePath output_file; 118 }; 119 120 virtual void OnTestIterationStarting() OVERRIDE { 121 // Nothing to do. 122 } 123 124 virtual std::string GetTestNameForFiltering( 125 const testing::TestCase* test_case, 126 const testing::TestInfo* test_info) OVERRIDE { 127 DCHECK(thread_checker_.CalledOnValidThread()); 128 129 return std::string(test_case->name()) + "." + test_info->name(); 130 } 131 132 virtual bool ShouldRunTest(const testing::TestCase* test_case, 133 const testing::TestInfo* test_info) OVERRIDE { 134 DCHECK(thread_checker_.CalledOnValidThread()); 135 136 // There is no additional logic to disable specific tests. 137 return true; 138 } 139 140 virtual size_t RunTests(TestLauncher* test_launcher, 141 const std::vector<std::string>& test_names) OVERRIDE { 142 DCHECK(thread_checker_.CalledOnValidThread()); 143 144 std::vector<std::string> batch; 145 for (size_t i = 0; i < test_names.size(); i++) { 146 batch.push_back(test_names[i]); 147 148 if (batch.size() >= batch_limit_) { 149 RunBatch(test_launcher, batch); 150 batch.clear(); 151 } 152 } 153 154 RunBatch(test_launcher, batch); 155 156 return test_names.size(); 157 } 158 159 virtual size_t RetryTests( 160 TestLauncher* test_launcher, 161 const std::vector<std::string>& test_names) OVERRIDE { 162 MessageLoop::current()->PostTask( 163 FROM_HERE, 164 Bind(&UnitTestLauncherDelegate::RunSerially, 165 Unretained(this), 166 test_launcher, 167 test_names)); 168 return test_names.size(); 169 } 170 171 void RunSerially(TestLauncher* test_launcher, 172 const std::vector<std::string>& test_names) { 173 if (test_names.empty()) 174 return; 175 176 std::vector<std::string> new_test_names(test_names); 177 std::string test_name(new_test_names.back()); 178 new_test_names.pop_back(); 179 180 // Create a dedicated temporary directory to store the xml result data 181 // per run to ensure clean state and make it possible to launch multiple 182 // processes in parallel. 183 base::FilePath output_file; 184 CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 185 output_file = output_file.AppendASCII("test_results.xml"); 186 187 std::vector<std::string> current_test_names; 188 current_test_names.push_back(test_name); 189 CommandLine cmd_line( 190 GetCommandLineForChildGTestProcess(current_test_names, output_file)); 191 192 GTestCallbackState callback_state; 193 callback_state.test_launcher = test_launcher; 194 callback_state.test_names = current_test_names; 195 callback_state.output_file = output_file; 196 197 test_launcher->LaunchChildGTestProcess( 198 cmd_line, 199 std::string(), 200 TestTimeouts::test_launcher_timeout(), 201 Bind(&UnitTestLauncherDelegate::SerialGTestCallback, 202 Unretained(this), 203 callback_state, 204 new_test_names)); 205 } 206 207 void RunBatch(TestLauncher* test_launcher, 208 const std::vector<std::string>& test_names) { 209 DCHECK(thread_checker_.CalledOnValidThread()); 210 211 if (test_names.empty()) 212 return; 213 214 // Create a dedicated temporary directory to store the xml result data 215 // per run to ensure clean state and make it possible to launch multiple 216 // processes in parallel. 217 base::FilePath output_file; 218 CHECK(CreateNewTempDirectory(FilePath::StringType(), &output_file)); 219 output_file = output_file.AppendASCII("test_results.xml"); 220 221 CommandLine cmd_line( 222 GetCommandLineForChildGTestProcess(test_names, output_file)); 223 224 // Adjust the timeout depending on how many tests we're running 225 // (note that e.g. the last batch of tests will be smaller). 226 // TODO(phajdan.jr): Consider an adaptive timeout, which can change 227 // depending on how many tests ran and how many remain. 228 // Note: do NOT parse child's stdout to do that, it's known to be 229 // unreliable (e.g. buffering issues can mix up the output). 230 base::TimeDelta timeout = 231 test_names.size() * TestTimeouts::test_launcher_timeout(); 232 233 GTestCallbackState callback_state; 234 callback_state.test_launcher = test_launcher; 235 callback_state.test_names = test_names; 236 callback_state.output_file = output_file; 237 238 test_launcher->LaunchChildGTestProcess( 239 cmd_line, 240 std::string(), 241 timeout, 242 Bind(&UnitTestLauncherDelegate::GTestCallback, 243 Unretained(this), 244 callback_state)); 245 } 246 247 void GTestCallback(const GTestCallbackState& callback_state, 248 int exit_code, 249 const TimeDelta& elapsed_time, 250 bool was_timeout, 251 const std::string& output) { 252 DCHECK(thread_checker_.CalledOnValidThread()); 253 std::vector<std::string> tests_to_relaunch; 254 ProcessTestResults(callback_state.test_launcher, 255 callback_state.test_names, 256 callback_state.output_file, 257 output, 258 exit_code, 259 was_timeout, 260 &tests_to_relaunch); 261 262 // Relaunch requested tests in parallel, but only use single 263 // test per batch for more precise results (crashes, test passes 264 // but non-zero exit codes etc). 265 for (size_t i = 0; i < tests_to_relaunch.size(); i++) { 266 std::vector<std::string> batch; 267 batch.push_back(tests_to_relaunch[i]); 268 RunBatch(callback_state.test_launcher, batch); 269 } 270 271 // The temporary file's directory is also temporary. 272 DeleteFile(callback_state.output_file.DirName(), true); 273 } 274 275 void SerialGTestCallback(const GTestCallbackState& callback_state, 276 const std::vector<std::string>& test_names, 277 int exit_code, 278 const TimeDelta& elapsed_time, 279 bool was_timeout, 280 const std::string& output) { 281 DCHECK(thread_checker_.CalledOnValidThread()); 282 std::vector<std::string> tests_to_relaunch; 283 bool called_any_callbacks = 284 ProcessTestResults(callback_state.test_launcher, 285 callback_state.test_names, 286 callback_state.output_file, 287 output, 288 exit_code, 289 was_timeout, 290 &tests_to_relaunch); 291 292 // There is only one test, there cannot be other tests to relaunch 293 // due to a crash. 294 DCHECK(tests_to_relaunch.empty()); 295 296 // There is only one test, we should have called back with its result. 297 DCHECK(called_any_callbacks); 298 299 // The temporary file's directory is also temporary. 300 DeleteFile(callback_state.output_file.DirName(), true); 301 302 MessageLoop::current()->PostTask( 303 FROM_HERE, 304 Bind(&UnitTestLauncherDelegate::RunSerially, 305 Unretained(this), 306 callback_state.test_launcher, 307 test_names)); 308 } 309 310 static bool ProcessTestResults( 311 TestLauncher* test_launcher, 312 const std::vector<std::string>& test_names, 313 const base::FilePath& output_file, 314 const std::string& output, 315 int exit_code, 316 bool was_timeout, 317 std::vector<std::string>* tests_to_relaunch) { 318 std::vector<TestResult> test_results; 319 bool crashed = false; 320 bool have_test_results = 321 ProcessGTestOutput(output_file, &test_results, &crashed); 322 323 bool called_any_callback = false; 324 325 if (have_test_results) { 326 // TODO(phajdan.jr): Check for duplicates and mismatches between 327 // the results we got from XML file and tests we intended to run. 328 std::map<std::string, TestResult> results_map; 329 for (size_t i = 0; i < test_results.size(); i++) 330 results_map[test_results[i].full_name] = test_results[i]; 331 332 bool had_interrupted_test = false; 333 334 // Results to be reported back to the test launcher. 335 std::vector<TestResult> final_results; 336 337 for (size_t i = 0; i < test_names.size(); i++) { 338 if (ContainsKey(results_map, test_names[i])) { 339 TestResult test_result = results_map[test_names[i]]; 340 if (test_result.status == TestResult::TEST_CRASH) { 341 had_interrupted_test = true; 342 343 if (was_timeout) { 344 // Fix up the test status: we forcibly kill the child process 345 // after the timeout, so from XML results it looks just like 346 // a crash. 347 test_result.status = TestResult::TEST_TIMEOUT; 348 } 349 } else if (test_result.status == TestResult::TEST_SUCCESS || 350 test_result.status == TestResult::TEST_FAILURE) { 351 // We run multiple tests in a batch with a timeout applied 352 // to the entire batch. It is possible that with other tests 353 // running quickly some tests take longer than the per-test timeout. 354 // For consistent handling of tests independent of order and other 355 // factors, mark them as timing out. 356 if (test_result.elapsed_time > 357 TestTimeouts::test_launcher_timeout()) { 358 test_result.status = TestResult::TEST_TIMEOUT; 359 } 360 } 361 test_result.output_snippet = 362 GetTestOutputSnippet(test_result, output); 363 final_results.push_back(test_result); 364 } else if (had_interrupted_test) { 365 tests_to_relaunch->push_back(test_names[i]); 366 } else { 367 // TODO(phajdan.jr): Explicitly pass the info that the test didn't 368 // run for a mysterious reason. 369 LOG(ERROR) << "no test result for " << test_names[i]; 370 TestResult test_result; 371 test_result.full_name = test_names[i]; 372 test_result.status = TestResult::TEST_UNKNOWN; 373 test_result.output_snippet = 374 GetTestOutputSnippet(test_result, output); 375 final_results.push_back(test_result); 376 } 377 } 378 379 // TODO(phajdan.jr): Handle the case where processing XML output 380 // indicates a crash but none of the test results is marked as crashing. 381 382 if (final_results.empty()) 383 return false; 384 385 bool has_non_success_test = false; 386 for (size_t i = 0; i < final_results.size(); i++) { 387 if (final_results[i].status != TestResult::TEST_SUCCESS) { 388 has_non_success_test = true; 389 break; 390 } 391 } 392 393 if (!has_non_success_test && exit_code != 0) { 394 // This is a bit surprising case: all tests are marked as successful, 395 // but the exit code was not zero. This can happen e.g. under memory 396 // tools that report leaks this way. 397 398 if (final_results.size() == 1) { 399 // Easy case. One test only so we know the non-zero exit code 400 // was caused by that one test. 401 final_results[0].status = TestResult::TEST_FAILURE_ON_EXIT; 402 } else { 403 // Harder case. Discard the results and request relaunching all 404 // tests without batching. This will trigger above branch on 405 // relaunch leading to more precise results. 406 LOG(WARNING) << "Not sure which test caused non-zero exit code, " 407 << "relaunching all of them without batching."; 408 409 for (size_t i = 0; i < final_results.size(); i++) 410 tests_to_relaunch->push_back(final_results[i].full_name); 411 412 return false; 413 } 414 } 415 416 for (size_t i = 0; i < final_results.size(); i++) { 417 // Fix the output snippet after possible changes to the test result. 418 final_results[i].output_snippet = 419 GetTestOutputSnippet(final_results[i], output); 420 test_launcher->OnTestFinished(final_results[i]); 421 called_any_callback = true; 422 } 423 } else { 424 fprintf(stdout, 425 "Failed to get out-of-band test success data, " 426 "dumping full stdio below:\n%s\n", 427 output.c_str()); 428 fflush(stdout); 429 430 // We do not have reliable details about test results (parsing test 431 // stdout is known to be unreliable), apply the executable exit code 432 // to all tests. 433 // TODO(phajdan.jr): Be smarter about this, e.g. retry each test 434 // individually. 435 for (size_t i = 0; i < test_names.size(); i++) { 436 TestResult test_result; 437 test_result.full_name = test_names[i]; 438 test_result.status = TestResult::TEST_UNKNOWN; 439 test_launcher->OnTestFinished(test_result); 440 called_any_callback = true; 441 } 442 } 443 444 return called_any_callback; 445 } 446 447 ThreadChecker thread_checker_; 448 449 // Maximum number of tests to run in a single batch. 450 size_t batch_limit_; 451 }; 452 453 bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 454 if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 455 return true; 456 457 std::string switch_value = 458 CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 459 if (!StringToInt(switch_value, result) || *result < 1) { 460 LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 461 return false; 462 } 463 464 return true; 465 } 466 467 int LaunchUnitTestsInternal(int argc, 468 char** argv, 469 const RunTestSuiteCallback& run_test_suite, 470 int default_jobs) { 471 CommandLine::Init(argc, argv); 472 473 #if defined(OS_ANDROID) 474 // We can't easily fork on Android, just run the test suite directly. 475 return run_test_suite.Run(); 476 #else 477 bool force_single_process = false; 478 if (CommandLine::ForCurrentProcess()->HasSwitch( 479 switches::kTestLauncherDebugLauncher)) { 480 fprintf(stdout, "Forcing test launcher debugging mode.\n"); 481 fflush(stdout); 482 } else { 483 if (base::debug::BeingDebugged()) { 484 fprintf(stdout, 485 "Debugger detected, switching to single process mode.\n" 486 "Pass --test-launcher-debug-launcher to debug the launcher " 487 "itself.\n"); 488 fflush(stdout); 489 force_single_process = true; 490 } 491 492 if (RunningOnValgrind()) { 493 fprintf(stdout, 494 "Valgrind detected, switching to single process mode.\n" 495 "Pass --test-launcher-debug-launcher to valgrind the launcher " 496 "itself.\n"); 497 fflush(stdout); 498 force_single_process = true; 499 } 500 } 501 502 if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) || 503 CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) || 504 force_single_process) { 505 return run_test_suite.Run(); 506 } 507 #endif 508 509 if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) { 510 PrintUsage(); 511 return 0; 512 } 513 514 base::TimeTicks start_time(base::TimeTicks::Now()); 515 516 testing::InitGoogleTest(&argc, argv); 517 TestTimeouts::Initialize(); 518 519 int batch_limit = kDefaultTestBatchLimit; 520 if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit)) 521 return 1; 522 523 fprintf(stdout, 524 "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n" 525 "own process. For debugging a test inside a debugger, use the\n" 526 "--gtest_filter=<your_test_name> flag along with\n" 527 "--single-process-tests.\n"); 528 fflush(stdout); 529 530 MessageLoopForIO message_loop; 531 532 UnitTestLauncherDelegate delegate(batch_limit); 533 base::TestLauncher launcher(&delegate, default_jobs); 534 bool success = launcher.Run(argc, argv); 535 536 fprintf(stdout, 537 "Tests took %" PRId64 " seconds.\n", 538 (base::TimeTicks::Now() - start_time).InSeconds()); 539 fflush(stdout); 540 541 return (success ? 0 : 1); 542 } 543 544 } // namespace 545 546 int LaunchUnitTests(int argc, 547 char** argv, 548 const RunTestSuiteCallback& run_test_suite) { 549 return LaunchUnitTestsInternal( 550 argc, argv, run_test_suite, SysInfo::NumberOfProcessors()); 551 } 552 553 int LaunchUnitTestsSerially(int argc, 554 char** argv, 555 const RunTestSuiteCallback& run_test_suite) { 556 return LaunchUnitTestsInternal(argc, argv, run_test_suite, 1); 557 } 558 559 } // namespace base 560
