Home | History | Annotate | Download | only in launcher 
      1 // Copyright 2013 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/test/launcher/test_launcher.h"
      6 
      7 #if defined(OS_POSIX)
      8 #include <fcntl.h>
      9 #endif
     10 
     11 #include "base/at_exit.h"
     12 #include "base/bind.h"
     13 #include "base/command_line.h"
     14 #include "base/environment.h"
     15 #include "base/file_util.h"
     16 #include "base/files/file_path.h"
     17 #include "base/format_macros.h"
     18 #include "base/lazy_instance.h"
     19 #include "base/logging.h"
     20 #include "base/memory/scoped_ptr.h"
     21 #include "base/message_loop/message_loop.h"
     22 #include "base/process/kill.h"
     23 #include "base/process/launch.h"
     24 #include "base/strings/string_number_conversions.h"
     25 #include "base/strings/stringprintf.h"
     26 #include "base/strings/utf_string_conversions.h"
     27 #include "base/test/launcher/test_results_tracker.h"
     28 #include "base/test/sequenced_worker_pool_owner.h"
     29 #include "base/test/test_switches.h"
     30 #include "base/test/test_timeouts.h"
     31 #include "base/threading/thread_checker.h"
     32 #include "base/time/time.h"
     33 #include "testing/gtest/include/gtest/gtest.h"
     34 
     35 #if defined(OS_MACOSX)
     36 #include "base/mac/scoped_nsautorelease_pool.h"
     37 #endif
     38 
     39 namespace base {
     40 
     41 // See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ .
     42 using ::operator<<;
     43 
     44 // The environment variable name for the total number of test shards.
     45 const char kTestTotalShards[] = "GTEST_TOTAL_SHARDS";
     46 // The environment variable name for the test shard index.
     47 const char kTestShardIndex[] = "GTEST_SHARD_INDEX";
     48 
     49 namespace {
     50 
     51 // Maximum time of no output after which we print list of processes still
     52 // running. This deliberately doesn't use TestTimeouts (which is otherwise
     53 // a recommended solution), because they can be increased. This would defeat
     54 // the purpose of this timeout, which is 1) to avoid buildbot "no output for
     55 // X seconds" timeout killing the process 2) help communicate status of
     56 // the test launcher to people looking at the output (no output for a long
     57 // time is mysterious and gives no info about what is happening) 3) help
     58 // debugging in case the process hangs anyway.
     59 const int kOutputTimeoutSeconds = 15;
     60 
     61 // Set of live launch test processes with corresponding lock (it is allowed
     62 // for callers to launch processes on different threads).
     63 LazyInstance<std::map<ProcessHandle, CommandLine> > g_live_processes
     64     = LAZY_INSTANCE_INITIALIZER;
     65 LazyInstance<Lock> g_live_processes_lock = LAZY_INSTANCE_INITIALIZER;
     66 
     67 #if defined(OS_POSIX)
     68 // Self-pipe that makes it possible to do complex shutdown handling
     69 // outside of the signal handler.
     70 int g_shutdown_pipe[2] = { -1, -1 };
     71 
     72 void ShutdownPipeSignalHandler(int signal) {
     73   HANDLE_EINTR(write(g_shutdown_pipe[1], "q", 1));
     74 }
     75 
     76 void KillSpawnedTestProcesses() {
     77   // Keep the lock until exiting the process to prevent further processes
     78   // from being spawned.
     79   AutoLock lock(g_live_processes_lock.Get());
     80 
     81   fprintf(stdout,
     82           "Sending SIGTERM to %" PRIuS " child processes... ",
     83           g_live_processes.Get().size());
     84   fflush(stdout);
     85 
     86   for (std::map<ProcessHandle, CommandLine>::iterator i =
     87            g_live_processes.Get().begin();
     88        i != g_live_processes.Get().end();
     89        ++i) {
     90     // Send the signal to entire process group.
     91     kill((-1) * (i->first), SIGTERM);
     92   }
     93 
     94   fprintf(stdout,
     95           "done.\nGiving processes a chance to terminate cleanly... ");
     96   fflush(stdout);
     97 
     98   PlatformThread::Sleep(TimeDelta::FromMilliseconds(500));
     99 
    100   fprintf(stdout, "done.\n");
    101   fflush(stdout);
    102 
    103   fprintf(stdout,
    104           "Sending SIGKILL to %" PRIuS " child processes... ",
    105           g_live_processes.Get().size());
    106   fflush(stdout);
    107 
    108   for (std::map<ProcessHandle, CommandLine>::iterator i =
    109            g_live_processes.Get().begin();
    110        i != g_live_processes.Get().end();
    111        ++i) {
    112     // Send the signal to entire process group.
    113     kill((-1) * (i->first), SIGKILL);
    114   }
    115 
    116   fprintf(stdout, "done.\n");
    117   fflush(stdout);
    118 }
    119 
    120 // I/O watcher for the reading end of the self-pipe above.
    121 // Terminates any launched child processes and exits the process.
    122 class SignalFDWatcher : public MessageLoopForIO::Watcher {
    123  public:
    124   SignalFDWatcher() {
    125   }
    126 
    127   virtual void OnFileCanReadWithoutBlocking(int fd) OVERRIDE {
    128     fprintf(stdout, "\nCaught signal. Killing spawned test processes...\n");
    129     fflush(stdout);
    130 
    131     KillSpawnedTestProcesses();
    132 
    133     // The signal would normally kill the process, so exit now.
    134     exit(1);
    135   }
    136 
    137   virtual void OnFileCanWriteWithoutBlocking(int fd) OVERRIDE {
    138     NOTREACHED();
    139   }
    140 
    141  private:
    142   DISALLOW_COPY_AND_ASSIGN(SignalFDWatcher);
    143 };
    144 #endif  // defined(OS_POSIX)
    145 
    146 // Parses the environment variable var as an Int32.  If it is unset, returns
    147 // true.  If it is set, unsets it then converts it to Int32 before
    148 // returning it in |result|.  Returns true on success.
    149 bool TakeInt32FromEnvironment(const char* const var, int32* result) {
    150   scoped_ptr<Environment> env(Environment::Create());
    151   std::string str_val;
    152 
    153   if (!env->GetVar(var, &str_val))
    154     return true;
    155 
    156   if (!env->UnSetVar(var)) {
    157     LOG(ERROR) << "Invalid environment: we could not unset " << var << ".\n";
    158     return false;
    159   }
    160 
    161   if (!StringToInt(str_val, result)) {
    162     LOG(ERROR) << "Invalid environment: " << var << " is not an integer.\n";
    163     return false;
    164   }
    165 
    166   return true;
    167 }
    168 
    169 // Unsets the environment variable |name| and returns true on success.
    170 // Also returns true if the variable just doesn't exist.
    171 bool UnsetEnvironmentVariableIfExists(const std::string& name) {
    172   scoped_ptr<Environment> env(Environment::Create());
    173   std::string str_val;
    174 
    175   if (!env->GetVar(name.c_str(), &str_val))
    176     return true;
    177 
    178   return env->UnSetVar(name.c_str());
    179 }
    180 
    181 // Returns true if bot mode has been requested, i.e. defaults optimized
    182 // for continuous integration bots. This way developers don't have to remember
    183 // special command-line flags.
    184 bool BotModeEnabled() {
    185   scoped_ptr<Environment> env(Environment::Create());
    186   return CommandLine::ForCurrentProcess()->HasSwitch(
    187       switches::kTestLauncherBotMode) ||
    188       env->HasVar("CHROMIUM_TEST_LAUNCHER_BOT_MODE");
    189 }
    190 
    191 // For a basic pattern matching for gtest_filter options.  (Copied from
    192 // gtest.cc, see the comment below and http://crbug.com/44497)
    193 bool PatternMatchesString(const char* pattern, const char* str) {
    194   switch (*pattern) {
    195     case '\0':
    196     case ':':  // Either ':' or '\0' marks the end of the pattern.
    197       return *str == '\0';
    198     case '?':  // Matches any single character.
    199       return *str != '\0' && PatternMatchesString(pattern + 1, str + 1);
    200     case '*':  // Matches any string (possibly empty) of characters.
    201       return (*str != '\0' && PatternMatchesString(pattern, str + 1)) ||
    202           PatternMatchesString(pattern + 1, str);
    203     default:  // Non-special character.  Matches itself.
    204       return *pattern == *str &&
    205           PatternMatchesString(pattern + 1, str + 1);
    206   }
    207 }
    208 
    209 // TODO(phajdan.jr): Avoid duplicating gtest code. (http://crbug.com/44497)
    210 // For basic pattern matching for gtest_filter options.  (Copied from
    211 // gtest.cc)
    212 bool MatchesFilter(const std::string& name, const std::string& filter) {
    213   const char *cur_pattern = filter.c_str();
    214   for (;;) {
    215     if (PatternMatchesString(cur_pattern, name.c_str())) {
    216       return true;
    217     }
    218 
    219     // Finds the next pattern in the filter.
    220     cur_pattern = strchr(cur_pattern, ':');
    221 
    222     // Returns if no more pattern can be found.
    223     if (cur_pattern == NULL) {
    224       return false;
    225     }
    226 
    227     // Skips the pattern separater (the ':' character).
    228     cur_pattern++;
    229   }
    230 }
    231 
    232 void RunCallback(
    233     const TestLauncher::LaunchChildGTestProcessCallback& callback,
    234     int exit_code,
    235     const TimeDelta& elapsed_time,
    236     bool was_timeout,
    237     const std::string& output) {
    238   callback.Run(exit_code, elapsed_time, was_timeout, output);
    239 }
    240 
    241 void DoLaunchChildTestProcess(
    242     const CommandLine& command_line,
    243     base::TimeDelta timeout,
    244     bool redirect_stdio,
    245     scoped_refptr<MessageLoopProxy> message_loop_proxy,
    246     const TestLauncher::LaunchChildGTestProcessCallback& callback) {
    247   TimeTicks start_time = TimeTicks::Now();
    248 
    249   // Redirect child process output to a file.
    250   base::FilePath output_file;
    251   CHECK(base::CreateTemporaryFile(&output_file));
    252 
    253   LaunchOptions options;
    254 #if defined(OS_WIN)
    255   win::ScopedHandle handle;
    256 
    257   if (redirect_stdio) {
    258     // Make the file handle inheritable by the child.
    259     SECURITY_ATTRIBUTES sa_attr;
    260     sa_attr.nLength = sizeof(SECURITY_ATTRIBUTES);
    261     sa_attr.lpSecurityDescriptor = NULL;
    262     sa_attr.bInheritHandle = TRUE;
    263 
    264     handle.Set(CreateFile(output_file.value().c_str(),
    265                           GENERIC_WRITE,
    266                           FILE_SHARE_READ | FILE_SHARE_DELETE,
    267                           &sa_attr,
    268                           OPEN_EXISTING,
    269                           FILE_ATTRIBUTE_TEMPORARY,
    270                           NULL));
    271     CHECK(handle.IsValid());
    272     options.inherit_handles = true;
    273     options.stdin_handle = INVALID_HANDLE_VALUE;
    274     options.stdout_handle = handle.Get();
    275     options.stderr_handle = handle.Get();
    276   }
    277 #elif defined(OS_POSIX)
    278   options.new_process_group = true;
    279 
    280   base::FileHandleMappingVector fds_mapping;
    281   file_util::ScopedFD output_file_fd_closer;
    282 
    283   if (redirect_stdio) {
    284     int output_file_fd = open(output_file.value().c_str(), O_RDWR);
    285     CHECK_GE(output_file_fd, 0);
    286 
    287     output_file_fd_closer.reset(&output_file_fd);
    288 
    289     fds_mapping.push_back(std::make_pair(output_file_fd, STDOUT_FILENO));
    290     fds_mapping.push_back(std::make_pair(output_file_fd, STDERR_FILENO));
    291     options.fds_to_remap = &fds_mapping;
    292   }
    293 #endif
    294 
    295   bool was_timeout = false;
    296   int exit_code = LaunchChildTestProcessWithOptions(
    297       command_line, options, timeout, &was_timeout);
    298 
    299   if (redirect_stdio) {
    300 #if defined(OS_WIN)
    301   FlushFileBuffers(handle.Get());
    302   handle.Close();
    303 #elif defined(OS_POSIX)
    304   output_file_fd_closer.reset();
    305 #endif
    306   }
    307 
    308   std::string output_file_contents;
    309   CHECK(base::ReadFileToString(output_file, &output_file_contents));
    310 
    311   if (!base::DeleteFile(output_file, false)) {
    312     // This needs to be non-fatal at least for Windows.
    313     LOG(WARNING) << "Failed to delete " << output_file.AsUTF8Unsafe();
    314   }
    315 
    316   // Run target callback on the thread it was originating from, not on
    317   // a worker pool thread.
    318   message_loop_proxy->PostTask(
    319       FROM_HERE,
    320       Bind(&RunCallback,
    321            callback,
    322            exit_code,
    323            TimeTicks::Now() - start_time,
    324            was_timeout,
    325            output_file_contents));
    326 }
    327 
    328 }  // namespace
    329 
    330 const char kGTestFilterFlag[] = "gtest_filter";
    331 const char kGTestHelpFlag[]   = "gtest_help";
    332 const char kGTestListTestsFlag[] = "gtest_list_tests";
    333 const char kGTestRepeatFlag[] = "gtest_repeat";
    334 const char kGTestRunDisabledTestsFlag[] = "gtest_also_run_disabled_tests";
    335 const char kGTestOutputFlag[] = "gtest_output";
    336 
    337 TestLauncherDelegate::~TestLauncherDelegate() {
    338 }
    339 
    340 TestLauncher::TestLauncher(TestLauncherDelegate* launcher_delegate,
    341                            size_t parallel_jobs)
    342     : launcher_delegate_(launcher_delegate),
    343       total_shards_(1),
    344       shard_index_(0),
    345       cycles_(1),
    346       test_started_count_(0),
    347       test_finished_count_(0),
    348       test_success_count_(0),
    349       test_broken_count_(0),
    350       retry_count_(0),
    351       retry_limit_(0),
    352       run_result_(true),
    353       watchdog_timer_(FROM_HERE,
    354                       TimeDelta::FromSeconds(kOutputTimeoutSeconds),
    355                       this,
    356                       &TestLauncher::OnOutputTimeout),
    357       parallel_jobs_(parallel_jobs) {
    358   if (BotModeEnabled()) {
    359     fprintf(stdout,
    360             "Enabling defaults optimized for continuous integration bots.\n");
    361     fflush(stdout);
    362 
    363     // Enable test retries by default for bots. This can be still overridden
    364     // from command line using --test-launcher-retry-limit flag.
    365     retry_limit_ = 3;
    366   } else {
    367     // Default to serial test execution if not running on a bot. This makes it
    368     // possible to disable stdio redirection and can still be overridden with
    369     // --test-launcher-jobs flag.
    370     parallel_jobs_ = 1;
    371   }
    372 
    373 #if defined(USE_AURA) && defined(OS_LINUX) && !defined(OS_CHROMEOS)
    374   // TODO(erg): Trying to parallelize tests on linux_aura makes things flaky
    375   // because all shards share X11 state. http://crbug.com/326701
    376   parallel_jobs_ = 1;
    377 
    378   fprintf(stdout,
    379           "Disabling parallelization due to the linux_aura flake. "
    380           "http://crbug.com/326701\n");
    381   fflush(stdout);
    382 #endif
    383 }
    384 
    385 TestLauncher::~TestLauncher() {
    386   if (worker_pool_owner_)
    387     worker_pool_owner_->pool()->Shutdown();
    388 }
    389 
    390 bool TestLauncher::Run(int argc, char** argv) {
    391   if (!Init())
    392     return false;
    393 
    394 #if defined(OS_POSIX)
    395   CHECK_EQ(0, pipe(g_shutdown_pipe));
    396 
    397   struct sigaction action;
    398   memset(&action, 0, sizeof(action));
    399   sigemptyset(&action.sa_mask);
    400   action.sa_handler = &ShutdownPipeSignalHandler;
    401 
    402   CHECK_EQ(0, sigaction(SIGINT, &action, NULL));
    403   CHECK_EQ(0, sigaction(SIGQUIT, &action, NULL));
    404   CHECK_EQ(0, sigaction(SIGTERM, &action, NULL));
    405 
    406   MessageLoopForIO::FileDescriptorWatcher controller;
    407   SignalFDWatcher watcher;
    408 
    409   CHECK(MessageLoopForIO::current()->WatchFileDescriptor(
    410             g_shutdown_pipe[0],
    411             true,
    412             MessageLoopForIO::WATCH_READ,
    413             &controller,
    414             &watcher));
    415 #endif  // defined(OS_POSIX)
    416 
    417   // Start the watchdog timer.
    418   watchdog_timer_.Reset();
    419 
    420   MessageLoop::current()->PostTask(
    421       FROM_HERE,
    422       Bind(&TestLauncher::RunTestIteration, Unretained(this)));
    423 
    424   MessageLoop::current()->Run();
    425 
    426   if (cycles_ != 1)
    427     results_tracker_.PrintSummaryOfAllIterations();
    428 
    429   MaybeSaveSummaryAsJSON();
    430 
    431   return run_result_;
    432 }
    433 
    434 void TestLauncher::LaunchChildGTestProcess(
    435     const CommandLine& command_line,
    436     const std::string& wrapper,
    437     base::TimeDelta timeout,
    438     const LaunchChildGTestProcessCallback& callback) {
    439   DCHECK(thread_checker_.CalledOnValidThread());
    440 
    441   // Record the exact command line used to launch the child.
    442   CommandLine new_command_line(
    443       PrepareCommandLineForGTest(command_line, wrapper));
    444 
    445   // When running in parallel mode we need to redirect stdio to avoid mixed-up
    446   // output. We also always redirect on the bots to get the test output into
    447   // JSON summary.
    448   bool redirect_stdio = (parallel_jobs_ > 1) || BotModeEnabled();
    449 
    450   worker_pool_owner_->pool()->PostWorkerTask(
    451       FROM_HERE,
    452       Bind(&DoLaunchChildTestProcess,
    453            new_command_line,
    454            timeout,
    455            redirect_stdio,
    456            MessageLoopProxy::current(),
    457            Bind(&TestLauncher::OnLaunchTestProcessFinished,
    458                 Unretained(this),
    459                 callback)));
    460 }
    461 
    462 void TestLauncher::OnTestFinished(const TestResult& result) {
    463   ++test_finished_count_;
    464 
    465   bool print_snippet = false;
    466   std::string print_test_stdio("auto");
    467   if (CommandLine::ForCurrentProcess()->HasSwitch(
    468           switches::kTestLauncherPrintTestStdio)) {
    469     print_test_stdio = CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
    470         switches::kTestLauncherPrintTestStdio);
    471   }
    472   if (print_test_stdio == "auto") {
    473     print_snippet = (result.status != TestResult::TEST_SUCCESS);
    474   } else if (print_test_stdio == "always") {
    475     print_snippet = true;
    476   } else if (print_test_stdio == "never") {
    477     print_snippet = false;
    478   } else {
    479     LOG(WARNING) << "Invalid value of " << switches::kTestLauncherPrintTestStdio
    480                  << ": " << print_test_stdio;
    481   }
    482   if (print_snippet) {
    483     fprintf(stdout, "%s", result.output_snippet.c_str());
    484     fflush(stdout);
    485   }
    486 
    487   if (result.status == TestResult::TEST_SUCCESS) {
    488     ++test_success_count_;
    489   } else {
    490     tests_to_retry_.insert(result.full_name);
    491   }
    492 
    493   results_tracker_.AddTestResult(result);
    494 
    495   // TODO(phajdan.jr): Align counter (padding).
    496   std::string status_line(
    497       StringPrintf("[%" PRIuS "/%" PRIuS "] %s ",
    498                    test_finished_count_,
    499                    test_started_count_,
    500                    result.full_name.c_str()));
    501   if (result.completed()) {
    502     status_line.append(StringPrintf("(%" PRId64 " ms)",
    503                                     result.elapsed_time.InMilliseconds()));
    504   } else if (result.status == TestResult::TEST_TIMEOUT) {
    505     status_line.append("(TIMED OUT)");
    506   } else if (result.status == TestResult::TEST_CRASH) {
    507     status_line.append("(CRASHED)");
    508   } else if (result.status == TestResult::TEST_SKIPPED) {
    509     status_line.append("(SKIPPED)");
    510   } else if (result.status == TestResult::TEST_UNKNOWN) {
    511     status_line.append("(UNKNOWN)");
    512   } else {
    513     // Fail very loudly so it's not ignored.
    514     CHECK(false) << "Unhandled test result status: " << result.status;
    515   }
    516   fprintf(stdout, "%s\n", status_line.c_str());
    517   fflush(stdout);
    518 
    519   // We just printed a status line, reset the watchdog timer.
    520   watchdog_timer_.Reset();
    521 
    522   if (test_finished_count_ == test_started_count_) {
    523     if (!tests_to_retry_.empty() && retry_count_ < retry_limit_) {
    524       retry_count_++;
    525 
    526       std::vector<std::string> test_names(tests_to_retry_.begin(),
    527                                           tests_to_retry_.end());
    528 
    529       tests_to_retry_.clear();
    530 
    531       size_t retry_started_count =
    532           launcher_delegate_->RetryTests(this, test_names);
    533       if (retry_started_count == 0) {
    534         // Signal failure, but continue to run all requested test iterations.
    535         // With the summary of all iterations at the end this is a good default.
    536         run_result_ = false;
    537 
    538         // The current iteration is done.
    539         fprintf(stdout, "%" PRIuS " test%s run\n",
    540                 test_finished_count_,
    541                 test_finished_count_ > 1 ? "s" : "");
    542         fflush(stdout);
    543 
    544         results_tracker_.PrintSummaryOfCurrentIteration();
    545 
    546         // Kick off the next iteration.
    547         MessageLoop::current()->PostTask(
    548             FROM_HERE,
    549             Bind(&TestLauncher::RunTestIteration, Unretained(this)));
    550       } else {
    551         fprintf(stdout, "Retrying %" PRIuS " test%s (retry #%" PRIuS ")\n",
    552                 retry_started_count,
    553                 retry_started_count > 1 ? "s" : "",
    554                 retry_count_);
    555         fflush(stdout);
    556 
    557         test_started_count_ += retry_started_count;
    558       }
    559     } else {
    560       // The current iteration is done.
    561       fprintf(stdout, "%" PRIuS " test%s run\n",
    562               test_finished_count_,
    563               test_finished_count_ > 1 ? "s" : "");
    564       fflush(stdout);
    565 
    566       results_tracker_.PrintSummaryOfCurrentIteration();
    567 
    568       // When we retry tests, success is determined by having nothing more
    569       // to retry (everything eventually passed), as opposed to having
    570       // no failures at all.
    571       if (!tests_to_retry_.empty()) {
    572         // Signal failure, but continue to run all requested test iterations.
    573         // With the summary of all iterations at the end this is a good default.
    574         run_result_ = false;
    575       }
    576 
    577       // Kick off the next iteration.
    578       MessageLoop::current()->PostTask(
    579           FROM_HERE,
    580           Bind(&TestLauncher::RunTestIteration, Unretained(this)));
    581     }
    582   }
    583 
    584   // Do not waste time on timeouts. We include tests with unknown results here
    585   // because sometimes (e.g. hang in between unit tests) that's how a timeout
    586   // gets reported.
    587   if (result.status == TestResult::TEST_TIMEOUT ||
    588       result.status == TestResult::TEST_UNKNOWN) {
    589     test_broken_count_++;
    590   }
    591   size_t broken_threshold =
    592       std::max(static_cast<size_t>(10), test_started_count_ / 10);
    593   if (test_broken_count_ >= broken_threshold) {
    594     fprintf(stdout, "Too many badly broken tests (%" PRIuS "), exiting now.\n",
    595             test_broken_count_);
    596     fflush(stdout);
    597 
    598 #if defined(OS_POSIX)
    599     KillSpawnedTestProcesses();
    600 #endif  // defined(OS_POSIX)
    601 
    602     results_tracker_.AddGlobalTag("BROKEN_TEST_EARLY_EXIT");
    603     MaybeSaveSummaryAsJSON();
    604 
    605     exit(1);
    606   }
    607 }
    608 
    609 bool TestLauncher::Init() {
    610   const CommandLine* command_line = CommandLine::ForCurrentProcess();
    611 
    612   if (command_line->HasSwitch(kGTestListTestsFlag)) {
    613     // Child gtest processes would list tests instead of running tests.
    614     // TODO(phajdan.jr): Restore support for the flag.
    615     LOG(ERROR) << kGTestListTestsFlag << " is not supported.";
    616     return false;
    617   }
    618 
    619   // Initialize sharding. Command line takes precedence over legacy environment
    620   // variables.
    621   if (command_line->HasSwitch(switches::kTestLauncherTotalShards) &&
    622       command_line->HasSwitch(switches::kTestLauncherShardIndex)) {
    623     if (!StringToInt(
    624             command_line->GetSwitchValueASCII(
    625                 switches::kTestLauncherTotalShards),
    626             &total_shards_)) {
    627       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherTotalShards;
    628       return false;
    629     }
    630     if (!StringToInt(
    631             command_line->GetSwitchValueASCII(
    632                 switches::kTestLauncherShardIndex),
    633             &shard_index_)) {
    634       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherShardIndex;
    635       return false;
    636     }
    637     fprintf(stdout,
    638             "Using sharding settings from command line. This is shard %d/%d\n",
    639             shard_index_, total_shards_);
    640     fflush(stdout);
    641   } else {
    642     if (!TakeInt32FromEnvironment(kTestTotalShards, &total_shards_))
    643       return false;
    644     if (!TakeInt32FromEnvironment(kTestShardIndex, &shard_index_))
    645       return false;
    646     fprintf(stdout,
    647             "Using sharding settings from environment. This is shard %d/%d\n",
    648             shard_index_, total_shards_);
    649     fflush(stdout);
    650   }
    651   if (shard_index_ < 0 ||
    652       total_shards_ < 0 ||
    653       shard_index_ >= total_shards_) {
    654     LOG(ERROR) << "Invalid sharding settings: we require 0 <= "
    655                << kTestShardIndex << " < " << kTestTotalShards
    656                << ", but you have " << kTestShardIndex << "=" << shard_index_
    657                << ", " << kTestTotalShards << "=" << total_shards_ << ".\n";
    658     return false;
    659   }
    660 
    661   // Make sure we don't pass any sharding-related environment to the child
    662   // processes. This test launcher implements the sharding completely.
    663   CHECK(UnsetEnvironmentVariableIfExists("GTEST_TOTAL_SHARDS"));
    664   CHECK(UnsetEnvironmentVariableIfExists("GTEST_SHARD_INDEX"));
    665 
    666   if (command_line->HasSwitch(kGTestRepeatFlag) &&
    667       !StringToInt(command_line->GetSwitchValueASCII(kGTestRepeatFlag),
    668                    &cycles_)) {
    669     LOG(ERROR) << "Invalid value for " << kGTestRepeatFlag;
    670     return false;
    671   }
    672 
    673   if (CommandLine::ForCurrentProcess()->HasSwitch(
    674           switches::kTestLauncherRetryLimit)) {
    675     int retry_limit = -1;
    676     if (!StringToInt(CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
    677                          switches::kTestLauncherRetryLimit), &retry_limit) ||
    678         retry_limit < 0) {
    679       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherRetryLimit;
    680       return false;
    681     }
    682 
    683     retry_limit_ = retry_limit;
    684   }
    685 
    686   if (CommandLine::ForCurrentProcess()->HasSwitch(
    687           switches::kTestLauncherJobs)) {
    688     int jobs = -1;
    689     if (!StringToInt(CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
    690                          switches::kTestLauncherJobs), &jobs) ||
    691         jobs < 0) {
    692       LOG(ERROR) << "Invalid value for " << switches::kTestLauncherJobs;
    693       return false;
    694     }
    695 
    696     parallel_jobs_ = jobs;
    697   }
    698   fprintf(stdout, "Using %" PRIuS " parallel jobs.\n", parallel_jobs_);
    699   fflush(stdout);
    700   worker_pool_owner_.reset(
    701       new SequencedWorkerPoolOwner(parallel_jobs_, "test_launcher"));
    702 
    703   // Split --gtest_filter at '-', if there is one, to separate into
    704   // positive filter and negative filter portions.
    705   std::string filter = command_line->GetSwitchValueASCII(kGTestFilterFlag);
    706   positive_test_filter_ = filter;
    707   size_t dash_pos = filter.find('-');
    708   if (dash_pos != std::string::npos) {
    709     // Everything up to the dash.
    710     positive_test_filter_ = filter.substr(0, dash_pos);
    711 
    712     // Everything after the dash.
    713     negative_test_filter_ = filter.substr(dash_pos + 1);
    714   }
    715 
    716   if (!results_tracker_.Init(*command_line)) {
    717     LOG(ERROR) << "Failed to initialize test results tracker.";
    718     return 1;
    719   }
    720 
    721   return true;
    722 }
    723 
    724 void TestLauncher::RunTests() {
    725   testing::UnitTest* const unit_test = testing::UnitTest::GetInstance();
    726 
    727   int num_runnable_tests = 0;
    728 
    729   std::vector<std::string> test_names;
    730 
    731   for (int i = 0; i < unit_test->total_test_case_count(); ++i) {
    732     const testing::TestCase* test_case = unit_test->GetTestCase(i);
    733     for (int j = 0; j < test_case->total_test_count(); ++j) {
    734       const testing::TestInfo* test_info = test_case->GetTestInfo(j);
    735       std::string test_name = test_info->test_case_name();
    736       test_name.append(".");
    737       test_name.append(test_info->name());
    738 
    739       // Skip disabled tests.
    740       const CommandLine* command_line = CommandLine::ForCurrentProcess();
    741       if (test_name.find("DISABLED") != std::string::npos &&
    742           !command_line->HasSwitch(kGTestRunDisabledTestsFlag)) {
    743         continue;
    744       }
    745 
    746       std::string filtering_test_name =
    747           launcher_delegate_->GetTestNameForFiltering(test_case, test_info);
    748 
    749       // Skip the test that doesn't match the filter string (if given).
    750       if ((!positive_test_filter_.empty() &&
    751            !MatchesFilter(filtering_test_name, positive_test_filter_)) ||
    752           MatchesFilter(filtering_test_name, negative_test_filter_)) {
    753         continue;
    754       }
    755 
    756       if (!launcher_delegate_->ShouldRunTest(test_case, test_info))
    757         continue;
    758 
    759       if (num_runnable_tests++ % total_shards_ != shard_index_)
    760         continue;
    761 
    762       test_names.push_back(test_name);
    763     }
    764   }
    765 
    766   test_started_count_ = launcher_delegate_->RunTests(this, test_names);
    767 
    768   if (test_started_count_ == 0) {
    769     fprintf(stdout, "0 tests run\n");
    770     fflush(stdout);
    771 
    772     // No tests have actually been started, so kick off the next iteration.
    773     MessageLoop::current()->PostTask(
    774         FROM_HERE,
    775         Bind(&TestLauncher::RunTestIteration, Unretained(this)));
    776   }
    777 }
    778 
    779 void TestLauncher::RunTestIteration() {
    780   if (cycles_ == 0) {
    781     MessageLoop::current()->Quit();
    782     return;
    783   }
    784 
    785   // Special value "-1" means "repeat indefinitely".
    786   cycles_ = (cycles_ == -1) ? cycles_ : cycles_ - 1;
    787 
    788   test_started_count_ = 0;
    789   test_finished_count_ = 0;
    790   test_success_count_ = 0;
    791   test_broken_count_ = 0;
    792   retry_count_ = 0;
    793   tests_to_retry_.clear();
    794   results_tracker_.OnTestIterationStarting();
    795   launcher_delegate_->OnTestIterationStarting();
    796 
    797   MessageLoop::current()->PostTask(
    798       FROM_HERE, Bind(&TestLauncher::RunTests, Unretained(this)));
    799 }
    800 
    801 void TestLauncher::MaybeSaveSummaryAsJSON() {
    802   const CommandLine* command_line = CommandLine::ForCurrentProcess();
    803   if (command_line->HasSwitch(switches::kTestLauncherSummaryOutput)) {
    804     FilePath summary_path(command_line->GetSwitchValuePath(
    805                               switches::kTestLauncherSummaryOutput));
    806     if (!results_tracker_.SaveSummaryAsJSON(summary_path)) {
    807       LOG(ERROR) << "Failed to save test launcher output summary.";
    808     }
    809   }
    810 }
    811 
    812 void TestLauncher::OnLaunchTestProcessFinished(
    813     const LaunchChildGTestProcessCallback& callback,
    814     int exit_code,
    815     const TimeDelta& elapsed_time,
    816     bool was_timeout,
    817     const std::string& output) {
    818   DCHECK(thread_checker_.CalledOnValidThread());
    819 
    820   callback.Run(exit_code, elapsed_time, was_timeout, output);
    821 }
    822 
    823 void TestLauncher::OnOutputTimeout() {
    824   DCHECK(thread_checker_.CalledOnValidThread());
    825 
    826   AutoLock lock(g_live_processes_lock.Get());
    827 
    828   fprintf(stdout, "Still waiting for the following processes to finish:\n");
    829 
    830   for (std::map<ProcessHandle, CommandLine>::iterator i =
    831            g_live_processes.Get().begin();
    832        i != g_live_processes.Get().end();
    833        ++i) {
    834 #if defined(OS_WIN)
    835     fwprintf(stdout, L"\t%s\n", i->second.GetCommandLineString().c_str());
    836 #else
    837     fprintf(stdout, "\t%s\n", i->second.GetCommandLineString().c_str());
    838 #endif
    839   }
    840 
    841   fflush(stdout);
    842 
    843   // Arm the timer again - otherwise it would fire only once.
    844   watchdog_timer_.Reset();
    845 }
    846 
    847 std::string GetTestOutputSnippet(const TestResult& result,
    848                                  const std::string& full_output) {
    849   size_t run_pos = full_output.find(std::string("[ RUN      ] ") +
    850                                     result.full_name);
    851   if (run_pos == std::string::npos)
    852     return std::string();
    853 
    854   size_t end_pos = full_output.find(std::string("[  FAILED  ] ") +
    855                                     result.full_name,
    856                                     run_pos);
    857   // Only clip the snippet to the "OK" message if the test really
    858   // succeeded. It still might have e.g. crashed after printing it.
    859   if (end_pos == std::string::npos &&
    860       result.status == TestResult::TEST_SUCCESS) {
    861     end_pos = full_output.find(std::string("[       OK ] ") +
    862                                result.full_name,
    863                                run_pos);
    864   }
    865   if (end_pos != std::string::npos) {
    866     size_t newline_pos = full_output.find("\n", end_pos);
    867     if (newline_pos != std::string::npos)
    868       end_pos = newline_pos + 1;
    869   }
    870 
    871   std::string snippet(full_output.substr(run_pos));
    872   if (end_pos != std::string::npos)
    873     snippet = full_output.substr(run_pos, end_pos - run_pos);
    874 
    875   return snippet;
    876 }
    877 
    878 int LaunchChildGTestProcess(const CommandLine& command_line,
    879                             const std::string& wrapper,
    880                             base::TimeDelta timeout,
    881                             bool* was_timeout) {
    882   LaunchOptions options;
    883 
    884 #if defined(OS_POSIX)
    885   // On POSIX, we launch the test in a new process group with pgid equal to
    886   // its pid. Any child processes that the test may create will inherit the
    887   // same pgid. This way, if the test is abruptly terminated, we can clean up
    888   // any orphaned child processes it may have left behind.
    889   options.new_process_group = true;
    890 #endif
    891 
    892   return LaunchChildTestProcessWithOptions(
    893       PrepareCommandLineForGTest(command_line, wrapper),
    894       options,
    895       timeout,
    896       was_timeout);
    897 }
    898 
    899 CommandLine PrepareCommandLineForGTest(const CommandLine& command_line,
    900                                        const std::string& wrapper) {
    901   CommandLine new_command_line(command_line.GetProgram());
    902   CommandLine::SwitchMap switches = command_line.GetSwitches();
    903 
    904   // Strip out gtest_repeat flag - this is handled by the launcher process.
    905   switches.erase(kGTestRepeatFlag);
    906 
    907   for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
    908        iter != switches.end(); ++iter) {
    909     new_command_line.AppendSwitchNative((*iter).first, (*iter).second);
    910   }
    911 
    912   // Prepend wrapper after last CommandLine quasi-copy operation. CommandLine
    913   // does not really support removing switches well, and trying to do that
    914   // on a CommandLine with a wrapper is known to break.
    915   // TODO(phajdan.jr): Give it a try to support CommandLine removing switches.
    916 #if defined(OS_WIN)
    917   new_command_line.PrependWrapper(ASCIIToWide(wrapper));
    918 #elif defined(OS_POSIX)
    919   new_command_line.PrependWrapper(wrapper);
    920 #endif
    921 
    922   return new_command_line;
    923 }
    924 
    925 int LaunchChildTestProcessWithOptions(const CommandLine& command_line,
    926                                       const LaunchOptions& options,
    927                                       base::TimeDelta timeout,
    928                                       bool* was_timeout) {
    929 #if defined(OS_POSIX)
    930   // Make sure an option we rely on is present - see LaunchChildGTestProcess.
    931   DCHECK(options.new_process_group);
    932 #endif
    933 
    934   LaunchOptions new_options(options);
    935 
    936 #if defined(OS_WIN)
    937   DCHECK(!new_options.job_handle);
    938 
    939   win::ScopedHandle job_handle(CreateJobObject(NULL, NULL));
    940   if (!job_handle.IsValid()) {
    941     LOG(ERROR) << "Could not create JobObject.";
    942     return -1;
    943   }
    944 
    945   // Allow break-away from job since sandbox and few other places rely on it
    946   // on Windows versions prior to Windows 8 (which supports nested jobs).
    947   // TODO(phajdan.jr): Do not allow break-away on Windows 8.
    948   if (!SetJobObjectLimitFlags(job_handle.Get(),
    949                               JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE |
    950                               JOB_OBJECT_LIMIT_BREAKAWAY_OK)) {
    951     LOG(ERROR) << "Could not SetJobObjectLimitFlags.";
    952     return -1;
    953   }
    954 
    955   new_options.job_handle = job_handle.Get();
    956 #endif  // defined(OS_WIN)
    957 
    958   base::ProcessHandle process_handle;
    959 
    960   {
    961     // Note how we grab the lock before the process possibly gets created.
    962     // This ensures that when the lock is held, ALL the processes are registered
    963     // in the set.
    964     AutoLock lock(g_live_processes_lock.Get());
    965 
    966     if (!base::LaunchProcess(command_line, new_options, &process_handle))
    967       return -1;
    968 
    969     g_live_processes.Get().insert(std::make_pair(process_handle, command_line));
    970   }
    971 
    972   int exit_code = 0;
    973   if (!base::WaitForExitCodeWithTimeout(process_handle,
    974                                         &exit_code,
    975                                         timeout)) {
    976     *was_timeout = true;
    977     exit_code = -1;  // Set a non-zero exit code to signal a failure.
    978 
    979     // Ensure that the process terminates.
    980     base::KillProcess(process_handle, -1, true);
    981   }
    982 
    983   {
    984     // Note how we grab the log before issuing a possibly broad process kill.
    985     // Other code parts that grab the log kill processes, so avoid trying
    986     // to do that twice and trigger all kinds of log messages.
    987     AutoLock lock(g_live_processes_lock.Get());
    988 
    989 #if defined(OS_POSIX)
    990     if (exit_code != 0) {
    991       // On POSIX, in case the test does not exit cleanly, either due to a crash
    992       // or due to it timing out, we need to clean up any child processes that
    993       // it might have created. On Windows, child processes are automatically
    994       // cleaned up using JobObjects.
    995       base::KillProcessGroup(process_handle);
    996     }
    997 #endif
    998 
    999     g_live_processes.Get().erase(process_handle);
   1000   }
   1001 
   1002   base::CloseProcessHandle(process_handle);
   1003 
   1004   return exit_code;
   1005 }
   1006 
   1007 }  // namespace base
   1008