Home | History | Annotate | Download | only in test 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/public/test/test_launcher.h"
      6 
      7 #include <map>
      8 #include <string>
      9 #include <vector>
     10 
     11 #include "base/command_line.h"
     12 #include "base/containers/hash_tables.h"
     13 #include "base/environment.h"
     14 #include "base/file_util.h"
     15 #include "base/files/scoped_temp_dir.h"
     16 #include "base/logging.h"
     17 #include "base/memory/linked_ptr.h"
     18 #include "base/memory/scoped_ptr.h"
     19 #include "base/message_loop/message_loop.h"
     20 #include "base/stl_util.h"
     21 #include "base/strings/string_number_conversions.h"
     22 #include "base/strings/string_util.h"
     23 #include "base/strings/utf_string_conversions.h"
     24 #include "base/test/launcher/test_launcher.h"
     25 #include "base/test/test_suite.h"
     26 #include "base/test/test_switches.h"
     27 #include "base/test/test_timeouts.h"
     28 #include "base/time/time.h"
     29 #include "content/public/app/content_main.h"
     30 #include "content/public/app/content_main_delegate.h"
     31 #include "content/public/app/startup_helper_win.h"
     32 #include "content/public/common/content_switches.h"
     33 #include "content/public/common/sandbox_init.h"
     34 #include "content/public/test/browser_test.h"
     35 #include "net/base/escape.h"
     36 #include "testing/gtest/include/gtest/gtest.h"
     37 
     38 #if defined(OS_WIN)
     39 #include "base/base_switches.h"
     40 #include "content/common/sandbox_win.h"
     41 #include "sandbox/win/src/sandbox_factory.h"
     42 #include "sandbox/win/src/sandbox_types.h"
     43 #elif defined(OS_MACOSX)
     44 #include "base/mac/scoped_nsautorelease_pool.h"
     45 #endif
     46 
     47 namespace content {
     48 
     49 namespace {
     50 
     51 // Tests with this prefix run before the same test without it, and use the same
     52 // profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
     53 // that span browser restarts.
     54 const char kPreTestPrefix[] = "PRE_";
     55 
     56 // Manual tests only run when --run-manual is specified. This allows writing
     57 // tests that don't run automatically but are still in the same test binary.
     58 // This is useful so that a team that wants to run a few tests doesn't have to
     59 // add a new binary that must be compiled on all builds.
     60 const char kManualTestPrefix[] = "MANUAL_";
     61 
     62 TestLauncherDelegate* g_launcher_delegate;
     63 ContentMainParams* g_params;
     64 
     65 std::string RemoveAnyPrePrefixes(const std::string& test_name) {
     66   std::string result(test_name);
     67   ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string());
     68   return result;
     69 }
     70 
     71 void PrintUsage() {
     72   fprintf(stdout,
     73           "Runs tests using the gtest framework, each batch of tests being\n"
     74           "run in their own process. Supported command-line flags:\n"
     75           "\n"
     76           " Common flags:\n"
     77           "  --gtest_filter=...\n"
     78           "    Runs a subset of tests (see --gtest_help for more info).\n"
     79           "\n"
     80           "  --help\n"
     81           "    Shows this message.\n"
     82           "\n"
     83           "  --gtest_help\n"
     84           "    Shows the gtest help message.\n"
     85           "\n"
     86           "  --test-launcher-jobs=N\n"
     87           "    Sets the number of parallel test jobs to N.\n"
     88           "\n"
     89           "  --single_process\n"
     90           "    Runs the tests and the launcher in the same process. Useful\n"
     91           "    for debugging a specific test in a debugger.\n"
     92           "\n"
     93           " Other flags:\n"
     94           "  --test-launcher-retry-limit=N\n"
     95           "    Sets the limit of test retries on failures to N.\n"
     96           "\n"
     97           "  --test-launcher-summary-output=PATH\n"
     98           "    Saves a JSON machine-readable summary of the run.\n"
     99           "\n"
    100           "  --test-launcher-print-test-stdio=auto|always|never\n"
    101           "    Controls when full test output is printed.\n"
    102           "    auto means to print it when the test failed.\n"
    103           "\n"
    104           "  --test-launcher-total-shards=N\n"
    105           "    Sets the total number of shards to N.\n"
    106           "\n"
    107           "  --test-launcher-shard-index=N\n"
    108           "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
    109 }
    110 
    111 // Implementation of base::TestLauncherDelegate. This is also a test launcher,
    112 // wrapping a lower-level test launcher with content-specific code.
    113 class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
    114  public:
    115   explicit WrapperTestLauncherDelegate(
    116       content::TestLauncherDelegate* launcher_delegate)
    117       : launcher_delegate_(launcher_delegate) {
    118     CHECK(temp_dir_.CreateUniqueTempDir());
    119   }
    120 
    121   // base::TestLauncherDelegate:
    122   virtual bool ShouldRunTest(const testing::TestCase* test_case,
    123                              const testing::TestInfo* test_info) OVERRIDE;
    124   virtual size_t RunTests(base::TestLauncher* test_launcher,
    125                           const std::vector<std::string>& test_names) OVERRIDE;
    126   virtual size_t RetryTests(
    127       base::TestLauncher* test_launcher,
    128       const std::vector<std::string>& test_names) OVERRIDE;
    129 
    130  private:
    131   void DoRunTest(base::TestLauncher* test_launcher,
    132                  const std::string& test_name);
    133 
    134   // Launches test named |test_name| using parallel launcher,
    135   // given result of PRE_ test |pre_test_result|.
    136   void RunDependentTest(base::TestLauncher* test_launcher,
    137                         const std::string test_name,
    138                         const base::TestResult& pre_test_result);
    139 
    140   // Callback to receive result of a test.
    141   void GTestCallback(
    142       base::TestLauncher* test_launcher,
    143       const std::string& test_name,
    144       int exit_code,
    145       const base::TimeDelta& elapsed_time,
    146       bool was_timeout,
    147       const std::string& output);
    148 
    149   content::TestLauncherDelegate* launcher_delegate_;
    150 
    151   // Store dependent test name (map is indexed by full test name).
    152   typedef std::map<std::string, std::string> DependentTestMap;
    153   DependentTestMap dependent_test_map_;
    154   DependentTestMap reverse_dependent_test_map_;
    155 
    156   // Store unique data directory prefix for test names (without PRE_ prefixes).
    157   // PRE_ tests and tests that depend on them must share the same
    158   // data directory. Using test name as directory name leads to too long
    159   // names (exceeding UNIX_PATH_MAX, which creates a problem with
    160   // process_singleton_linux). Create a randomly-named temporary directory
    161   // and keep track of the names so that PRE_ tests can still re-use them.
    162   typedef std::map<std::string, base::FilePath> UserDataDirMap;
    163   UserDataDirMap user_data_dir_map_;
    164 
    165   // Store names of all seen tests to properly handle PRE_ tests.
    166   std::set<std::string> all_test_names_;
    167 
    168   // Temporary directory for user data directories.
    169   base::ScopedTempDir temp_dir_;
    170 
    171   DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
    172 };
    173 
    174 bool WrapperTestLauncherDelegate::ShouldRunTest(
    175     const testing::TestCase* test_case,
    176     const testing::TestInfo* test_info) {
    177   all_test_names_.insert(
    178       std::string(test_case->name()) + "." + test_info->name());
    179 
    180   if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
    181       !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
    182     return false;
    183   }
    184 
    185   if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) {
    186     // We will actually run PRE_ tests, but to ensure they run on the same shard
    187     // as dependent tests, handle all these details internally.
    188     return false;
    189   }
    190 
    191   return true;
    192 }
    193 
    194 std::string GetPreTestName(const std::string& full_name) {
    195   size_t dot_pos = full_name.find('.');
    196   CHECK_NE(dot_pos, std::string::npos);
    197   std::string test_case_name = full_name.substr(0, dot_pos);
    198   std::string test_name = full_name.substr(dot_pos + 1);
    199   return test_case_name + "." + kPreTestPrefix + test_name;
    200 }
    201 
    202 size_t WrapperTestLauncherDelegate::RunTests(
    203     base::TestLauncher* test_launcher,
    204     const std::vector<std::string>& test_names) {
    205   dependent_test_map_.clear();
    206   reverse_dependent_test_map_.clear();
    207   user_data_dir_map_.clear();
    208 
    209   // Number of additional tests to run because of dependencies.
    210   size_t additional_tests_to_run_count = 0;
    211 
    212   // Compute dependencies of tests to be run.
    213   for (size_t i = 0; i < test_names.size(); i++) {
    214     std::string full_name(test_names[i]);
    215     std::string pre_test_name(GetPreTestName(full_name));
    216 
    217     while (ContainsKey(all_test_names_, pre_test_name)) {
    218       additional_tests_to_run_count++;
    219 
    220       DCHECK(!ContainsKey(dependent_test_map_, pre_test_name));
    221       dependent_test_map_[pre_test_name] = full_name;
    222 
    223       DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name));
    224       reverse_dependent_test_map_[full_name] = pre_test_name;
    225 
    226       full_name = pre_test_name;
    227       pre_test_name = GetPreTestName(pre_test_name);
    228     }
    229   }
    230 
    231   for (size_t i = 0; i < test_names.size(); i++) {
    232     std::string full_name(test_names[i]);
    233 
    234     // Make sure no PRE_ tests were requested explicitly.
    235     DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name));
    236 
    237     if (!ContainsKey(user_data_dir_map_, full_name)) {
    238       base::FilePath temp_dir;
    239       CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
    240                                           FILE_PATH_LITERAL("d"), &temp_dir));
    241       user_data_dir_map_[full_name] = temp_dir;
    242     }
    243 
    244     // If the test has any dependencies, get to the root and start with that.
    245     while (ContainsKey(reverse_dependent_test_map_, full_name))
    246       full_name = GetPreTestName(full_name);
    247 
    248     DoRunTest(test_launcher, full_name);
    249   }
    250 
    251   return test_names.size() + additional_tests_to_run_count;
    252 }
    253 
    254 size_t WrapperTestLauncherDelegate::RetryTests(
    255     base::TestLauncher* test_launcher,
    256     const std::vector<std::string>& test_names) {
    257   // List of tests we can kick off right now, depending on no other tests.
    258   std::vector<std::string> tests_to_run_now;
    259 
    260   // We retry at least the tests requested to retry.
    261   std::set<std::string> test_names_set(test_names.begin(), test_names.end());
    262 
    263   // In the face of PRE_ tests, we need to retry the entire chain of tests,
    264   // from the very first one.
    265   for (size_t i = 0; i < test_names.size(); i++) {
    266     std::string test_name(test_names[i]);
    267     while (ContainsKey(reverse_dependent_test_map_, test_name)) {
    268       test_name = reverse_dependent_test_map_[test_name];
    269       test_names_set.insert(test_name);
    270     }
    271   }
    272 
    273   // Discard user data directories from any previous runs. Start with
    274   // fresh state.
    275   for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin();
    276        i != user_data_dir_map_.end();
    277        ++i) {
    278     // Delete temporary directories now to avoid using too much space in /tmp.
    279     if (!base::DeleteFile(i->second, true)) {
    280       LOG(WARNING) << "Failed to delete " << i->second.value();
    281     }
    282   }
    283   user_data_dir_map_.clear();
    284 
    285   for (std::set<std::string>::const_iterator i = test_names_set.begin();
    286        i != test_names_set.end();
    287        ++i) {
    288     std::string full_name(*i);
    289 
    290     // Make sure PRE_ tests and tests that depend on them share the same
    291     // data directory - based it on the test name without prefixes.
    292     std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name));
    293     if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) {
    294       base::FilePath temp_dir;
    295       CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
    296                                           FILE_PATH_LITERAL("d"), &temp_dir));
    297       user_data_dir_map_[test_name_no_pre] = temp_dir;
    298     }
    299 
    300     size_t dot_pos = full_name.find('.');
    301     CHECK_NE(dot_pos, std::string::npos);
    302     std::string test_case_name = full_name.substr(0, dot_pos);
    303     std::string test_name = full_name.substr(dot_pos + 1);
    304     std::string pre_test_name(
    305         test_case_name + "." + kPreTestPrefix + test_name);
    306     if (!ContainsKey(test_names_set, pre_test_name))
    307       tests_to_run_now.push_back(full_name);
    308   }
    309 
    310   for (size_t i = 0; i < tests_to_run_now.size(); i++)
    311     DoRunTest(test_launcher, tests_to_run_now[i]);
    312 
    313   return test_names_set.size();
    314 }
    315 
    316 void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher,
    317                                             const std::string& test_name) {
    318   std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
    319 
    320   CommandLine cmd_line(*CommandLine::ForCurrentProcess());
    321   CHECK(launcher_delegate_->AdjustChildProcessCommandLine(
    322             &cmd_line, user_data_dir_map_[test_name_no_pre]));
    323 
    324   CommandLine new_cmd_line(cmd_line.GetProgram());
    325   CommandLine::SwitchMap switches = cmd_line.GetSwitches();
    326 
    327   // Strip out gtest_output flag because otherwise we would overwrite results
    328   // of the other tests.
    329   switches.erase(base::kGTestOutputFlag);
    330 
    331   for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
    332        iter != switches.end(); ++iter) {
    333     new_cmd_line.AppendSwitchNative(iter->first, iter->second);
    334   }
    335 
    336   // Always enable disabled tests.  This method is not called with disabled
    337   // tests unless this flag was specified to the browser test executable.
    338   new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
    339   new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
    340   new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
    341 
    342   char* browser_wrapper = getenv("BROWSER_WRAPPER");
    343 
    344   test_launcher->LaunchChildGTestProcess(
    345       new_cmd_line,
    346       browser_wrapper ? browser_wrapper : std::string(),
    347       TestTimeouts::action_max_timeout(),
    348       true,
    349       base::Bind(&WrapperTestLauncherDelegate::GTestCallback,
    350                  base::Unretained(this),
    351                  test_launcher,
    352                  test_name));
    353 }
    354 
    355 void WrapperTestLauncherDelegate::RunDependentTest(
    356     base::TestLauncher* test_launcher,
    357     const std::string test_name,
    358     const base::TestResult& pre_test_result) {
    359   if (pre_test_result.status == base::TestResult::TEST_SUCCESS) {
    360     // Only run the dependent test if PRE_ test succeeded.
    361     DoRunTest(test_launcher, test_name);
    362   } else {
    363     // Otherwise skip the test.
    364     base::TestResult test_result;
    365     test_result.full_name = test_name;
    366     test_result.status = base::TestResult::TEST_SKIPPED;
    367     test_launcher->OnTestFinished(test_result);
    368 
    369     if (ContainsKey(dependent_test_map_, test_name)) {
    370       RunDependentTest(test_launcher,
    371                        dependent_test_map_[test_name],
    372                        test_result);
    373     }
    374   }
    375 }
    376 
    377 void WrapperTestLauncherDelegate::GTestCallback(
    378     base::TestLauncher* test_launcher,
    379     const std::string& test_name,
    380     int exit_code,
    381     const base::TimeDelta& elapsed_time,
    382     bool was_timeout,
    383     const std::string& output) {
    384   base::TestResult result;
    385   result.full_name = test_name;
    386 
    387   // TODO(phajdan.jr): Recognize crashes.
    388   if (exit_code == 0)
    389     result.status = base::TestResult::TEST_SUCCESS;
    390   else if (was_timeout)
    391     result.status = base::TestResult::TEST_TIMEOUT;
    392   else
    393     result.status = base::TestResult::TEST_FAILURE;
    394 
    395   result.elapsed_time = elapsed_time;
    396 
    397   result.output_snippet = GetTestOutputSnippet(result, output);
    398 
    399   if (ContainsKey(dependent_test_map_, test_name)) {
    400     RunDependentTest(test_launcher, dependent_test_map_[test_name], result);
    401   } else {
    402     // No other tests depend on this, we can delete the temporary directory now.
    403     // Do so to avoid too many temporary files using lots of disk space.
    404     std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
    405     if (ContainsKey(user_data_dir_map_, test_name_no_pre)) {
    406       if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) {
    407         LOG(WARNING) << "Failed to delete "
    408                      << user_data_dir_map_[test_name_no_pre].value();
    409       }
    410       user_data_dir_map_.erase(test_name_no_pre);
    411     }
    412   }
    413 
    414   test_launcher->OnTestFinished(result);
    415 }
    416 
    417 }  // namespace
    418 
    419 const char kHelpFlag[]   = "help";
    420 
    421 const char kLaunchAsBrowser[] = "as-browser";
    422 
    423 // See kManualTestPrefix above.
    424 const char kRunManualTestsFlag[] = "run-manual";
    425 
    426 const char kSingleProcessTestsFlag[]   = "single_process";
    427 
    428 
    429 TestLauncherDelegate::~TestLauncherDelegate() {
    430 }
    431 
    432 int LaunchTests(TestLauncherDelegate* launcher_delegate,
    433                 int default_jobs,
    434                 int argc,
    435                 char** argv) {
    436   DCHECK(!g_launcher_delegate);
    437   g_launcher_delegate = launcher_delegate;
    438 
    439   CommandLine::Init(argc, argv);
    440   const CommandLine* command_line = CommandLine::ForCurrentProcess();
    441 
    442   if (command_line->HasSwitch(kHelpFlag)) {
    443     PrintUsage();
    444     return 0;
    445   }
    446 
    447   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
    448       launcher_delegate->CreateContentMainDelegate());
    449   ContentMainParams params(chrome_main_delegate.get());
    450 
    451 #if defined(OS_WIN)
    452   sandbox::SandboxInterfaceInfo sandbox_info = {0};
    453   InitializeSandboxInfo(&sandbox_info);
    454 
    455   params.instance = GetModuleHandle(NULL);
    456   params.sandbox_info = &sandbox_info;
    457 #elif !defined(OS_ANDROID)
    458   params.argc = argc;
    459   params.argv = const_cast<const char**>(argv);
    460 #endif  // defined(OS_WIN)
    461 
    462   if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
    463       (command_line->HasSwitch(switches::kSingleProcess) &&
    464        command_line->HasSwitch(base::kGTestFilterFlag)) ||
    465       command_line->HasSwitch(base::kGTestListTestsFlag) ||
    466       command_line->HasSwitch(base::kGTestHelpFlag)) {
    467     g_params = &params;
    468     return launcher_delegate->RunTestSuite(argc, argv);
    469   }
    470 
    471 #if !defined(OS_ANDROID)
    472   if (command_line->HasSwitch(switches::kProcessType) ||
    473       command_line->HasSwitch(kLaunchAsBrowser)) {
    474     return ContentMain(params);
    475   }
    476 #endif
    477 
    478   base::AtExitManager at_exit;
    479   testing::InitGoogleTest(&argc, argv);
    480   TestTimeouts::Initialize();
    481 
    482   fprintf(stdout,
    483       "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
    484       "For debugging a test inside a debugger, use the\n"
    485       "--gtest_filter=<your_test_name> flag along with either\n"
    486       "--single_process (to run the test in one launcher/browser process) or\n"
    487       "--single-process (to do the above, and also run Chrome in single-"
    488           "process mode).\n");
    489 
    490   base::MessageLoopForIO message_loop;
    491 
    492   // Allow the |launcher_delegate| to modify |default_jobs|.
    493   launcher_delegate->AdjustDefaultParallelJobs(&default_jobs);
    494 
    495   WrapperTestLauncherDelegate delegate(launcher_delegate);
    496   base::TestLauncher launcher(&delegate, default_jobs);
    497   return (launcher.Run() ? 0 : 1);
    498 }
    499 
    500 TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
    501   return g_launcher_delegate;
    502 }
    503 
    504 ContentMainParams* GetContentMainParams() {
    505   return g_params;
    506 }
    507 
    508 }  // namespace content
    509