1 // Copyright 2013 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include "base/test/test_launcher.h" 6 7 #include "base/at_exit.h" 8 #include "base/bind.h" 9 #include "base/command_line.h" 10 #include "base/environment.h" 11 #include "base/file_util.h" 12 #include "base/files/file_path.h" 13 #include "base/format_macros.h" 14 #include "base/logging.h" 15 #include "base/memory/scoped_ptr.h" 16 #include "base/process/kill.h" 17 #include "base/process/launch.h" 18 #include "base/strings/string_number_conversions.h" 19 #include "base/strings/utf_string_conversions.h" 20 #include "base/test/test_timeouts.h" 21 #include "base/time/time.h" 22 #include "testing/gtest/include/gtest/gtest.h" 23 24 #if defined(OS_MACOSX) 25 #include "base/mac/scoped_nsautorelease_pool.h" 26 #endif 27 28 namespace base { 29 30 // See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ . 31 using ::operator<<; 32 33 // The environment variable name for the total number of test shards. 34 const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS"; 35 // The environment variable name for the test shard index. 36 const char kTestShardIndex[] = "GTEST_SHARD_INDEX"; 37 38 // The default output file for XML output. 39 const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL( 40 "test_detail.xml"); 41 42 namespace { 43 44 // Parses the environment variable var as an Int32. If it is unset, returns 45 // default_val. If it is set, unsets it then converts it to Int32 before 46 // returning it. If unsetting or converting to an Int32 fails, print an 47 // error and exit with failure. 48 int32 Int32FromEnvOrDie(const char* const var, int32 default_val) { 49 scoped_ptr<Environment> env(Environment::Create()); 50 std::string str_val; 51 int32 result; 52 if (!env->GetVar(var, &str_val)) 53 return default_val; 54 if (!env->UnSetVar(var)) { 55 LOG(ERROR) << "Invalid environment: we could not unset " << var << ".\n"; 56 exit(EXIT_FAILURE); 57 } 58 if (!StringToInt(str_val, &result)) { 59 LOG(ERROR) << "Invalid environment: " << var << " is not an integer.\n"; 60 exit(EXIT_FAILURE); 61 } 62 return result; 63 } 64 65 // Checks whether sharding is enabled by examining the relevant 66 // environment variable values. If the variables are present, 67 // but inconsistent (i.e., shard_index >= total_shards), prints 68 // an error and exits. 69 void InitSharding(int32* total_shards, int32* shard_index) { 70 *total_shards = Int32FromEnvOrDie(kTestTotalShards, 1); 71 *shard_index = Int32FromEnvOrDie(kTestShardIndex, 0); 72 73 if (*total_shards == -1 && *shard_index != -1) { 74 LOG(ERROR) << "Invalid environment variables: you have " 75 << kTestShardIndex << " = " << *shard_index 76 << ", but have left " << kTestTotalShards << " unset.\n"; 77 exit(EXIT_FAILURE); 78 } else if (*total_shards != -1 && *shard_index == -1) { 79 LOG(ERROR) << "Invalid environment variables: you have " 80 << kTestTotalShards << " = " << *total_shards 81 << ", but have left " << kTestShardIndex << " unset.\n"; 82 exit(EXIT_FAILURE); 83 } else if (*shard_index < 0 || *shard_index >= *total_shards) { 84 LOG(ERROR) << "Invalid environment variables: we require 0 <= " 85 << kTestShardIndex << " < " << kTestTotalShards 86 << ", but you have " << kTestShardIndex << "=" << *shard_index 87 << ", " << kTestTotalShards << "=" << *total_shards << ".\n"; 88 exit(EXIT_FAILURE); 89 } 90 } 91 92 // Given the total number of shards, the shard index, and the test id, returns 93 // true iff the test should be run on this shard. The test id is some arbitrary 94 // but unique non-negative integer assigned by this launcher to each test 95 // method. Assumes that 0 <= shard_index < total_shards, which is first 96 // verified in ShouldShard(). 97 bool ShouldRunTestOnShard(int total_shards, int shard_index, int test_id) { 98 return (test_id % total_shards) == shard_index; 99 } 100 101 // A helper class to output results. 102 // Note: as currently XML is the only supported format by gtest, we don't 103 // check output format (e.g. "xml:" prefix) here and output an XML file 104 // unconditionally. 105 // Note: we don't output per-test-case or total summary info like 106 // total failed_test_count, disabled_test_count, elapsed_time and so on. 107 // Only each test (testcase element in the XML) will have the correct 108 // failed/disabled/elapsed_time information. Each test won't include 109 // detailed failure messages either. 110 class ResultsPrinter { 111 public: 112 explicit ResultsPrinter(const CommandLine& command_line); 113 ~ResultsPrinter(); 114 115 // Adds |result| to the stored test results. 116 void AddTestResult(const TestResult& result); 117 118 // Returns list of full names of failed tests. 119 const std::vector<std::string>& failed_tests() const { return failed_tests_; } 120 121 // Returns total number of tests run. 122 size_t test_run_count() const { return test_run_count_; } 123 124 private: 125 // Test results grouped by test case name. 126 typedef std::map<std::string, std::vector<TestResult> > ResultsMap; 127 ResultsMap results_; 128 129 // List of full names of failed tests. 130 std::vector<std::string> failed_tests_; 131 132 // Total number of tests run. 133 size_t test_run_count_; 134 135 // File handle of output file (can be NULL if no file). 136 FILE* out_; 137 138 DISALLOW_COPY_AND_ASSIGN(ResultsPrinter); 139 }; 140 141 ResultsPrinter::ResultsPrinter(const CommandLine& command_line) 142 : test_run_count_(0), 143 out_(NULL) { 144 if (!command_line.HasSwitch(kGTestOutputFlag)) 145 return; 146 std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag); 147 size_t colon_pos = flag.find(':'); 148 FilePath path; 149 if (colon_pos != std::string::npos) { 150 FilePath flag_path = 151 command_line.GetSwitchValuePath(kGTestOutputFlag); 152 FilePath::StringType path_string = flag_path.value(); 153 path = FilePath(path_string.substr(colon_pos + 1)); 154 // If the given path ends with '/', consider it is a directory. 155 // Note: This does NOT check that a directory (or file) actually exists 156 // (the behavior is same as what gtest does). 157 if (path.EndsWithSeparator()) { 158 FilePath executable = command_line.GetProgram().BaseName(); 159 path = path.Append(executable.ReplaceExtension( 160 FilePath::StringType(FILE_PATH_LITERAL("xml")))); 161 } 162 } 163 if (path.value().empty()) 164 path = FilePath(kDefaultOutputFile); 165 FilePath dir_name = path.DirName(); 166 if (!DirectoryExists(dir_name)) { 167 LOG(WARNING) << "The output directory does not exist. " 168 << "Creating the directory: " << dir_name.value(); 169 // Create the directory if necessary (because the gtest does the same). 170 file_util::CreateDirectory(dir_name); 171 } 172 out_ = file_util::OpenFile(path, "w"); 173 if (!out_) { 174 LOG(ERROR) << "Cannot open output file: " 175 << path.value() << "."; 176 return; 177 } 178 } 179 180 ResultsPrinter::~ResultsPrinter() { 181 if (!out_) 182 return; 183 fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); 184 fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\"" 185 " disabled=\"\" errors=\"\" time=\"\">\n"); 186 for (ResultsMap::iterator i = results_.begin(); i != results_.end(); ++i) { 187 fprintf(out_, " <testsuite name=\"%s\" tests=\"%" PRIuS "\" failures=\"\"" 188 " disabled=\"\" errors=\"\" time=\"\">\n", 189 i->first.c_str(), i->second.size()); 190 for (size_t j = 0; j < i->second.size(); ++j) { 191 const TestResult& result = i->second[j]; 192 fprintf(out_, " <testcase name=\"%s\" status=\"run\" time=\"%.3f\"" 193 " classname=\"%s\">\n", 194 result.test_name.c_str(), 195 result.elapsed_time.InSecondsF(), 196 result.test_case_name.c_str()); 197 if (!result.success) 198 fprintf(out_, " <failure message=\"\" type=\"\"></failure>\n"); 199 fprintf(out_, " </testcase>\n"); 200 } 201 fprintf(out_, " </testsuite>\n"); 202 } 203 fprintf(out_, "</testsuites>\n"); 204 fclose(out_); 205 } 206 207 void ResultsPrinter::AddTestResult(const TestResult& result) { 208 ++test_run_count_; 209 results_[result.test_case_name].push_back(result); 210 211 if (!result.success) { 212 failed_tests_.push_back( 213 std::string(result.test_case_name) + "." + result.test_name); 214 } 215 } 216 217 // For a basic pattern matching for gtest_filter options. (Copied from 218 // gtest.cc, see the comment below and http://crbug.com/44497) 219 bool PatternMatchesString(const char* pattern, const char* str) { 220 switch (*pattern) { 221 case '\0': 222 case ':': // Either ':' or '\0' marks the end of the pattern. 223 return *str == '\0'; 224 case '?': // Matches any single character. 225 return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); 226 case '*': // Matches any string (possibly empty) of characters. 227 return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || 228 PatternMatchesString(pattern + 1, str); 229 default: // Non-special character. Matches itself. 230 return *pattern == *str && 231 PatternMatchesString(pattern + 1, str + 1); 232 } 233 } 234 235 // TODO(phajdan.jr): Avoid duplicating gtest code. (http://crbug.com/44497) 236 // For basic pattern matching for gtest_filter options. (Copied from 237 // gtest.cc) 238 bool MatchesFilter(const std::string& name, const std::string& filter) { 239 const char *cur_pattern = filter.c_str(); 240 for (;;) { 241 if (PatternMatchesString(cur_pattern, name.c_str())) { 242 return true; 243 } 244 245 // Finds the next pattern in the filter. 246 cur_pattern = strchr(cur_pattern, ':'); 247 248 // Returns if no more pattern can be found. 249 if (cur_pattern == NULL) { 250 return false; 251 } 252 253 // Skips the pattern separater (the ':' character). 254 cur_pattern++; 255 } 256 } 257 258 bool RunTests(TestLauncherDelegate* launcher_delegate, 259 int total_shards, 260 int shard_index) { 261 const CommandLine* command_line = CommandLine::ForCurrentProcess(); 262 263 DCHECK(!command_line->HasSwitch(kGTestListTestsFlag)); 264 265 testing::UnitTest* const unit_test = testing::UnitTest::GetInstance(); 266 267 std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag); 268 269 // Split --gtest_filter at '-', if there is one, to separate into 270 // positive filter and negative filter portions. 271 std::string positive_filter = filter; 272 std::string negative_filter; 273 size_t dash_pos = filter.find('-'); 274 if (dash_pos != std::string::npos) { 275 positive_filter = filter.substr(0, dash_pos); // Everything up to the dash. 276 negative_filter = filter.substr(dash_pos + 1); // Everything after the dash. 277 } 278 279 int num_runnable_tests = 0; 280 281 ResultsPrinter printer(*command_line); 282 for (int i = 0; i < unit_test->total_test_case_count(); ++i) { 283 const testing::TestCase* test_case = unit_test->GetTestCase(i); 284 for (int j = 0; j < test_case->total_test_count(); ++j) { 285 const testing::TestInfo* test_info = test_case->GetTestInfo(j); 286 std::string test_name = test_info->test_case_name(); 287 test_name.append("."); 288 test_name.append(test_info->name()); 289 290 // Skip disabled tests. 291 if (test_name.find("DISABLED") != std::string::npos && 292 !command_line->HasSwitch(kGTestRunDisabledTestsFlag)) { 293 continue; 294 } 295 296 // Skip the test that doesn't match the filter string (if given). 297 if ((!positive_filter.empty() && 298 !MatchesFilter(test_name, positive_filter)) || 299 MatchesFilter(test_name, negative_filter)) { 300 continue; 301 } 302 303 if (!launcher_delegate->ShouldRunTest(test_case, test_info)) 304 continue; 305 306 bool should_run = ShouldRunTestOnShard(total_shards, shard_index, 307 num_runnable_tests); 308 num_runnable_tests += 1; 309 if (!should_run) 310 continue; 311 312 launcher_delegate->RunTest(test_case, 313 test_info, 314 base::Bind( 315 &ResultsPrinter::AddTestResult, 316 base::Unretained(&printer))); 317 } 318 } 319 320 launcher_delegate->RunRemainingTests(); 321 322 printf("%" PRIuS " test%s run\n", 323 printer.test_run_count(), 324 printer.test_run_count() > 1 ? "s" : ""); 325 printf("%" PRIuS " test%s failed\n", 326 printer.failed_tests().size(), 327 printer.failed_tests().size() != 1 ? "s" : ""); 328 if (printer.failed_tests().empty()) 329 return true; 330 331 printf("Failing tests:\n"); 332 for (size_t i = 0; i < printer.failed_tests().size(); ++i) 333 printf("%s\n", printer.failed_tests()[i].c_str()); 334 335 return false; 336 } 337 338 } // namespace 339 340 const char kGTestFilterFlag[] = "gtest_filter"; 341 const char kGTestListTestsFlag[] = "gtest_list_tests"; 342 const char kGTestRepeatFlag[] = "gtest_repeat"; 343 const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests"; 344 const char kGTestOutputFlag[] = "gtest_output"; 345 346 const char kHelpFlag[] = "help"; 347 348 TestResult::TestResult() { 349 } 350 351 TestLauncherDelegate::~TestLauncherDelegate() { 352 } 353 354 int LaunchChildGTestProcess(const CommandLine& command_line, 355 const std::string& wrapper, 356 base::TimeDelta timeout, 357 bool* was_timeout) { 358 CommandLine new_command_line(command_line.GetProgram()); 359 CommandLine::SwitchMap switches = command_line.GetSwitches(); 360 361 // Strip out gtest_output flag because otherwise we would overwrite results 362 // of the other tests. 363 switches.erase(kGTestOutputFlag); 364 365 // Strip out gtest_repeat flag - this is handled by the launcher process. 366 switches.erase(kGTestRepeatFlag); 367 368 for (CommandLine::SwitchMap::const_iterator iter = switches.begin(); 369 iter != switches.end(); ++iter) { 370 new_command_line.AppendSwitchNative((*iter).first, (*iter).second); 371 } 372 373 // Prepend wrapper after last CommandLine quasi-copy operation. CommandLine 374 // does not really support removing switches well, and trying to do that 375 // on a CommandLine with a wrapper is known to break. 376 // TODO(phajdan.jr): Give it a try to support CommandLine removing switches. 377 #if defined(OS_WIN) 378 new_command_line.PrependWrapper(ASCIIToWide(wrapper)); 379 #elif defined(OS_POSIX) 380 new_command_line.PrependWrapper(wrapper); 381 #endif 382 383 base::ProcessHandle process_handle; 384 base::LaunchOptions options; 385 386 #if defined(OS_POSIX) 387 // On POSIX, we launch the test in a new process group with pgid equal to 388 // its pid. Any child processes that the test may create will inherit the 389 // same pgid. This way, if the test is abruptly terminated, we can clean up 390 // any orphaned child processes it may have left behind. 391 options.new_process_group = true; 392 #endif 393 394 if (!base::LaunchProcess(new_command_line, options, &process_handle)) 395 return -1; 396 397 int exit_code = 0; 398 if (!base::WaitForExitCodeWithTimeout(process_handle, 399 &exit_code, 400 timeout)) { 401 *was_timeout = true; 402 exit_code = -1; // Set a non-zero exit code to signal a failure. 403 404 // Ensure that the process terminates. 405 base::KillProcess(process_handle, -1, true); 406 } 407 408 #if defined(OS_POSIX) 409 if (exit_code != 0) { 410 // On POSIX, in case the test does not exit cleanly, either due to a crash 411 // or due to it timing out, we need to clean up any child processes that 412 // it might have created. On Windows, child processes are automatically 413 // cleaned up using JobObjects. 414 base::KillProcessGroup(process_handle); 415 } 416 #endif 417 418 base::CloseProcessHandle(process_handle); 419 420 return exit_code; 421 } 422 423 int LaunchTests(TestLauncherDelegate* launcher_delegate, 424 int argc, 425 char** argv) { 426 const CommandLine* command_line = CommandLine::ForCurrentProcess(); 427 428 int32 total_shards; 429 int32 shard_index; 430 InitSharding(&total_shards, &shard_index); 431 432 int cycles = 1; 433 if (command_line->HasSwitch(kGTestRepeatFlag)) 434 StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag), &cycles); 435 436 int exit_code = 0; 437 while (cycles != 0) { 438 if (!RunTests(launcher_delegate, total_shards, shard_index)) { 439 exit_code = 1; 440 break; 441 } 442 443 // Special value "-1" means "repeat indefinitely". 444 if (cycles != -1) 445 cycles--; 446 } 447 448 return exit_code; 449 } 450 451 } // namespace base 452