Home | History | Annotate | Download | only in test 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/public/test/test_launcher.h"
      6 
      7 #include <map>
      8 #include <string>
      9 #include <vector>
     10 
     11 #include "base/command_line.h"
     12 #include "base/containers/hash_tables.h"
     13 #include "base/environment.h"
     14 #include "base/file_util.h"
     15 #include "base/files/scoped_temp_dir.h"
     16 #include "base/logging.h"
     17 #include "base/memory/linked_ptr.h"
     18 #include "base/memory/scoped_ptr.h"
     19 #include "base/message_loop/message_loop.h"
     20 #include "base/stl_util.h"
     21 #include "base/strings/string_number_conversions.h"
     22 #include "base/strings/string_util.h"
     23 #include "base/strings/utf_string_conversions.h"
     24 #include "base/test/launcher/test_launcher.h"
     25 #include "base/test/test_suite.h"
     26 #include "base/test/test_switches.h"
     27 #include "base/test/test_timeouts.h"
     28 #include "base/time/time.h"
     29 #include "content/public/app/content_main.h"
     30 #include "content/public/app/content_main_delegate.h"
     31 #include "content/public/app/startup_helper_win.h"
     32 #include "content/public/common/content_switches.h"
     33 #include "content/public/common/sandbox_init.h"
     34 #include "content/public/test/browser_test.h"
     35 #include "net/base/escape.h"
     36 #include "testing/gtest/include/gtest/gtest.h"
     37 
     38 #if defined(OS_WIN)
     39 #include "base/base_switches.h"
     40 #include "content/common/sandbox_win.h"
     41 #include "sandbox/win/src/sandbox_factory.h"
     42 #include "sandbox/win/src/sandbox_types.h"
     43 #elif defined(OS_MACOSX)
     44 #include "base/mac/scoped_nsautorelease_pool.h"
     45 #endif
     46 
     47 namespace content {
     48 
     49 namespace {
     50 
     51 // Tests with this prefix run before the same test without it, and use the same
     52 // profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
     53 // that span browser restarts.
     54 const char kPreTestPrefix[] = "PRE_";
     55 
     56 // Manual tests only run when --run-manual is specified. This allows writing
     57 // tests that don't run automatically but are still in the same test binary.
     58 // This is useful so that a team that wants to run a few tests doesn't have to
     59 // add a new binary that must be compiled on all builds.
     60 const char kManualTestPrefix[] = "MANUAL_";
     61 
     62 TestLauncherDelegate* g_launcher_delegate;
     63 
     64 std::string RemoveAnyPrePrefixes(const std::string& test_name) {
     65   std::string result(test_name);
     66   ReplaceSubstringsAfterOffset(&result, 0, kPreTestPrefix, std::string());
     67   return result;
     68 }
     69 
     70 void PrintUsage() {
     71   fprintf(stdout,
     72           "Runs tests using the gtest framework, each batch of tests being\n"
     73           "run in their own process. Supported command-line flags:\n"
     74           "\n"
     75           " Common flags:\n"
     76           "  --gtest_filter=...\n"
     77           "    Runs a subset of tests (see --gtest_help for more info).\n"
     78           "\n"
     79           "  --help\n"
     80           "    Shows this message.\n"
     81           "\n"
     82           "  --gtest_help\n"
     83           "    Shows the gtest help message.\n"
     84           "\n"
     85           "  --test-launcher-jobs=N\n"
     86           "    Sets the number of parallel test jobs to N.\n"
     87           "\n"
     88           "  --single_process\n"
     89           "    Runs the tests and the launcher in the same process. Useful\n"
     90           "    for debugging a specific test in a debugger.\n"
     91           "\n"
     92           " Other flags:\n"
     93           "  --test-launcher-retry-limit=N\n"
     94           "    Sets the limit of test retries on failures to N.\n"
     95           "\n"
     96           "  --test-launcher-summary-output=PATH\n"
     97           "    Saves a JSON machine-readable summary of the run.\n"
     98           "\n"
     99           "  --test-launcher-print-test-stdio=auto|always|never\n"
    100           "    Controls when full test output is printed.\n"
    101           "    auto means to print it when the test failed.\n"
    102           "\n"
    103           "  --test-launcher-total-shards=N\n"
    104           "    Sets the total number of shards to N.\n"
    105           "\n"
    106           "  --test-launcher-shard-index=N\n"
    107           "    Sets the shard index to run to N (from 0 to TOTAL - 1).\n");
    108 }
    109 
    110 // Implementation of base::TestLauncherDelegate. This is also a test launcher,
    111 // wrapping a lower-level test launcher with content-specific code.
    112 class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
    113  public:
    114   explicit WrapperTestLauncherDelegate(
    115       content::TestLauncherDelegate* launcher_delegate)
    116       : launcher_delegate_(launcher_delegate) {
    117     CHECK(temp_dir_.CreateUniqueTempDir());
    118   }
    119 
    120   // base::TestLauncherDelegate:
    121   virtual void OnTestIterationStarting() OVERRIDE;
    122   virtual std::string GetTestNameForFiltering(
    123       const testing::TestCase* test_case,
    124       const testing::TestInfo* test_info) OVERRIDE;
    125   virtual bool ShouldRunTest(const testing::TestCase* test_case,
    126                              const testing::TestInfo* test_info) OVERRIDE;
    127   virtual size_t RunTests(base::TestLauncher* test_launcher,
    128                           const std::vector<std::string>& test_names) OVERRIDE;
    129   virtual size_t RetryTests(
    130       base::TestLauncher* test_launcher,
    131       const std::vector<std::string>& test_names) OVERRIDE;
    132 
    133  private:
    134   void DoRunTest(base::TestLauncher* test_launcher,
    135                  const std::string& test_name);
    136 
    137   // Launches test named |test_name| using parallel launcher,
    138   // given result of PRE_ test |pre_test_result|.
    139   void RunDependentTest(base::TestLauncher* test_launcher,
    140                         const std::string test_name,
    141                         const base::TestResult& pre_test_result);
    142 
    143   // Callback to receive result of a test.
    144   void GTestCallback(
    145       base::TestLauncher* test_launcher,
    146       const std::string& test_name,
    147       int exit_code,
    148       const base::TimeDelta& elapsed_time,
    149       bool was_timeout,
    150       const std::string& output);
    151 
    152   content::TestLauncherDelegate* launcher_delegate_;
    153 
    154   // Store dependent test name (map is indexed by full test name).
    155   typedef std::map<std::string, std::string> DependentTestMap;
    156   DependentTestMap dependent_test_map_;
    157   DependentTestMap reverse_dependent_test_map_;
    158 
    159   // Store unique data directory prefix for test names (without PRE_ prefixes).
    160   // PRE_ tests and tests that depend on them must share the same
    161   // data directory. Using test name as directory name leads to too long
    162   // names (exceeding UNIX_PATH_MAX, which creates a problem with
    163   // process_singleton_linux). Create a randomly-named temporary directory
    164   // and keep track of the names so that PRE_ tests can still re-use them.
    165   typedef std::map<std::string, base::FilePath> UserDataDirMap;
    166   UserDataDirMap user_data_dir_map_;
    167 
    168   // Store names of all seen tests to properly handle PRE_ tests.
    169   std::set<std::string> all_test_names_;
    170 
    171   // Temporary directory for user data directories.
    172   base::ScopedTempDir temp_dir_;
    173 
    174   DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
    175 };
    176 
    177 void WrapperTestLauncherDelegate::OnTestIterationStarting() {
    178   dependent_test_map_.clear();
    179   user_data_dir_map_.clear();
    180 }
    181 
    182 std::string WrapperTestLauncherDelegate::GetTestNameForFiltering(
    183     const testing::TestCase* test_case,
    184     const testing::TestInfo* test_info) {
    185   return RemoveAnyPrePrefixes(
    186       std::string(test_case->name()) + "." + test_info->name());
    187 }
    188 
    189 bool WrapperTestLauncherDelegate::ShouldRunTest(
    190     const testing::TestCase* test_case,
    191     const testing::TestInfo* test_info) {
    192   all_test_names_.insert(
    193       std::string(test_case->name()) + "." + test_info->name());
    194 
    195   if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
    196       !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
    197     return false;
    198   }
    199 
    200   if (StartsWithASCII(test_info->name(), kPreTestPrefix, true)) {
    201     // We will actually run PRE_ tests, but to ensure they run on the same shard
    202     // as dependent tests, handle all these details internally.
    203     return false;
    204   }
    205 
    206   return true;
    207 }
    208 
    209 std::string GetPreTestName(const std::string& full_name) {
    210   size_t dot_pos = full_name.find('.');
    211   CHECK_NE(dot_pos, std::string::npos);
    212   std::string test_case_name = full_name.substr(0, dot_pos);
    213   std::string test_name = full_name.substr(dot_pos + 1);
    214   return test_case_name + "." + kPreTestPrefix + test_name;
    215 }
    216 
    217 size_t WrapperTestLauncherDelegate::RunTests(
    218     base::TestLauncher* test_launcher,
    219     const std::vector<std::string>& test_names) {
    220   // Number of additional tests to run because of dependencies.
    221   size_t additional_tests_to_run_count = 0;
    222 
    223   // Compute dependencies of tests to be run.
    224   for (size_t i = 0; i < test_names.size(); i++) {
    225     std::string full_name(test_names[i]);
    226     std::string pre_test_name(GetPreTestName(full_name));
    227 
    228     while (ContainsKey(all_test_names_, pre_test_name)) {
    229       additional_tests_to_run_count++;
    230 
    231       DCHECK(!ContainsKey(dependent_test_map_, pre_test_name));
    232       dependent_test_map_[pre_test_name] = full_name;
    233 
    234       DCHECK(!ContainsKey(reverse_dependent_test_map_, full_name));
    235       reverse_dependent_test_map_[full_name] = pre_test_name;
    236 
    237       full_name = pre_test_name;
    238       pre_test_name = GetPreTestName(pre_test_name);
    239     }
    240   }
    241 
    242   for (size_t i = 0; i < test_names.size(); i++) {
    243     std::string full_name(test_names[i]);
    244 
    245     // Make sure no PRE_ tests were requested explicitly.
    246     DCHECK_EQ(full_name, RemoveAnyPrePrefixes(full_name));
    247 
    248     if (!ContainsKey(user_data_dir_map_, full_name)) {
    249       base::FilePath temp_dir;
    250       CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
    251                                           FILE_PATH_LITERAL("d"), &temp_dir));
    252       user_data_dir_map_[full_name] = temp_dir;
    253     }
    254 
    255     // If the test has any dependencies, get to the root and start with that.
    256     while (ContainsKey(reverse_dependent_test_map_, full_name))
    257       full_name = GetPreTestName(full_name);
    258 
    259     DoRunTest(test_launcher, full_name);
    260   }
    261 
    262   return test_names.size() + additional_tests_to_run_count;
    263 }
    264 
    265 size_t WrapperTestLauncherDelegate::RetryTests(
    266     base::TestLauncher* test_launcher,
    267     const std::vector<std::string>& test_names) {
    268   // List of tests we can kick off right now, depending on no other tests.
    269   std::vector<std::string> tests_to_run_now;
    270 
    271   // We retry at least the tests requested to retry.
    272   std::set<std::string> test_names_set(test_names.begin(), test_names.end());
    273 
    274   // In the face of PRE_ tests, we need to retry the entire chain of tests,
    275   // from the very first one.
    276   for (size_t i = 0; i < test_names.size(); i++) {
    277     std::string test_name(test_names[i]);
    278     while (ContainsKey(reverse_dependent_test_map_, test_name)) {
    279       test_name = reverse_dependent_test_map_[test_name];
    280       test_names_set.insert(test_name);
    281     }
    282   }
    283 
    284   // Discard user data directories from any previous runs. Start with
    285   // fresh state.
    286   for (UserDataDirMap::const_iterator i = user_data_dir_map_.begin();
    287        i != user_data_dir_map_.end();
    288        ++i) {
    289     // Delete temporary directories now to avoid using too much space in /tmp.
    290     if (!base::DeleteFile(i->second, true)) {
    291       LOG(WARNING) << "Failed to delete " << i->second.value();
    292     }
    293   }
    294   user_data_dir_map_.clear();
    295 
    296   for (std::set<std::string>::const_iterator i = test_names_set.begin();
    297        i != test_names_set.end();
    298        ++i) {
    299     std::string full_name(*i);
    300 
    301     // Make sure PRE_ tests and tests that depend on them share the same
    302     // data directory - based it on the test name without prefixes.
    303     std::string test_name_no_pre(RemoveAnyPrePrefixes(full_name));
    304     if (!ContainsKey(user_data_dir_map_, test_name_no_pre)) {
    305       base::FilePath temp_dir;
    306       CHECK(base::CreateTemporaryDirInDir(temp_dir_.path(),
    307                                           FILE_PATH_LITERAL("d"), &temp_dir));
    308       user_data_dir_map_[test_name_no_pre] = temp_dir;
    309     }
    310 
    311     size_t dot_pos = full_name.find('.');
    312     CHECK_NE(dot_pos, std::string::npos);
    313     std::string test_case_name = full_name.substr(0, dot_pos);
    314     std::string test_name = full_name.substr(dot_pos + 1);
    315     std::string pre_test_name(
    316         test_case_name + "." + kPreTestPrefix + test_name);
    317     if (!ContainsKey(test_names_set, pre_test_name))
    318       tests_to_run_now.push_back(full_name);
    319   }
    320 
    321   for (size_t i = 0; i < tests_to_run_now.size(); i++)
    322     DoRunTest(test_launcher, tests_to_run_now[i]);
    323 
    324   return test_names_set.size();
    325 }
    326 
    327 void WrapperTestLauncherDelegate::DoRunTest(base::TestLauncher* test_launcher,
    328                                             const std::string& test_name) {
    329   std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
    330 
    331   CommandLine cmd_line(*CommandLine::ForCurrentProcess());
    332   CHECK(launcher_delegate_->AdjustChildProcessCommandLine(
    333             &cmd_line, user_data_dir_map_[test_name_no_pre]));
    334 
    335   CommandLine new_cmd_line(cmd_line.GetProgram());
    336   CommandLine::SwitchMap switches = cmd_line.GetSwitches();
    337 
    338   // Strip out gtest_output flag because otherwise we would overwrite results
    339   // of the other tests.
    340   switches.erase(base::kGTestOutputFlag);
    341 
    342   for (CommandLine::SwitchMap::const_iterator iter = switches.begin();
    343        iter != switches.end(); ++iter) {
    344     new_cmd_line.AppendSwitchNative(iter->first, iter->second);
    345   }
    346 
    347   // Always enable disabled tests.  This method is not called with disabled
    348   // tests unless this flag was specified to the browser test executable.
    349   new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
    350   new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
    351   new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
    352 
    353   char* browser_wrapper = getenv("BROWSER_WRAPPER");
    354 
    355   test_launcher->LaunchChildGTestProcess(
    356       new_cmd_line,
    357       browser_wrapper ? browser_wrapper : std::string(),
    358       TestTimeouts::action_max_timeout(),
    359       base::Bind(&WrapperTestLauncherDelegate::GTestCallback,
    360                  base::Unretained(this),
    361                  test_launcher,
    362                  test_name));
    363 }
    364 
    365 void WrapperTestLauncherDelegate::RunDependentTest(
    366     base::TestLauncher* test_launcher,
    367     const std::string test_name,
    368     const base::TestResult& pre_test_result) {
    369   if (pre_test_result.status == base::TestResult::TEST_SUCCESS) {
    370     // Only run the dependent test if PRE_ test succeeded.
    371     DoRunTest(test_launcher, test_name);
    372   } else {
    373     // Otherwise skip the test.
    374     base::TestResult test_result;
    375     test_result.full_name = test_name;
    376     test_result.status = base::TestResult::TEST_SKIPPED;
    377     test_launcher->OnTestFinished(test_result);
    378 
    379     if (ContainsKey(dependent_test_map_, test_name)) {
    380       RunDependentTest(test_launcher,
    381                        dependent_test_map_[test_name],
    382                        test_result);
    383     }
    384   }
    385 }
    386 
    387 void WrapperTestLauncherDelegate::GTestCallback(
    388     base::TestLauncher* test_launcher,
    389     const std::string& test_name,
    390     int exit_code,
    391     const base::TimeDelta& elapsed_time,
    392     bool was_timeout,
    393     const std::string& output) {
    394   base::TestResult result;
    395   result.full_name = test_name;
    396 
    397   // TODO(phajdan.jr): Recognize crashes.
    398   if (exit_code == 0)
    399     result.status = base::TestResult::TEST_SUCCESS;
    400   else if (was_timeout)
    401     result.status = base::TestResult::TEST_TIMEOUT;
    402   else
    403     result.status = base::TestResult::TEST_FAILURE;
    404 
    405   result.elapsed_time = elapsed_time;
    406 
    407   result.output_snippet = GetTestOutputSnippet(result, output);
    408 
    409   if (ContainsKey(dependent_test_map_, test_name)) {
    410     RunDependentTest(test_launcher, dependent_test_map_[test_name], result);
    411   } else {
    412     // No other tests depend on this, we can delete the temporary directory now.
    413     // Do so to avoid too many temporary files using lots of disk space.
    414     std::string test_name_no_pre(RemoveAnyPrePrefixes(test_name));
    415     if (ContainsKey(user_data_dir_map_, test_name_no_pre)) {
    416       if (!base::DeleteFile(user_data_dir_map_[test_name_no_pre], true)) {
    417         LOG(WARNING) << "Failed to delete "
    418                      << user_data_dir_map_[test_name_no_pre].value();
    419       }
    420       user_data_dir_map_.erase(test_name_no_pre);
    421     }
    422   }
    423 
    424   test_launcher->OnTestFinished(result);
    425 }
    426 
    427 }  // namespace
    428 
    429 const char kHelpFlag[]   = "help";
    430 
    431 const char kLaunchAsBrowser[] = "as-browser";
    432 
    433 // See kManualTestPrefix above.
    434 const char kRunManualTestsFlag[] = "run-manual";
    435 
    436 const char kSingleProcessTestsFlag[]   = "single_process";
    437 
    438 
    439 TestLauncherDelegate::~TestLauncherDelegate() {
    440 }
    441 
    442 bool ShouldRunContentMain() {
    443 #if defined(OS_WIN) || defined(OS_LINUX)
    444   CommandLine* command_line = CommandLine::ForCurrentProcess();
    445   return command_line->HasSwitch(switches::kProcessType) ||
    446          command_line->HasSwitch(kLaunchAsBrowser);
    447 #else
    448   return false;
    449 #endif  // defined(OS_WIN) || defined(OS_LINUX)
    450 }
    451 
    452 int RunContentMain(int argc, char** argv,
    453                    TestLauncherDelegate* launcher_delegate) {
    454 #if defined(OS_WIN)
    455   sandbox::SandboxInterfaceInfo sandbox_info = {0};
    456   InitializeSandboxInfo(&sandbox_info);
    457   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
    458       launcher_delegate->CreateContentMainDelegate());
    459   return ContentMain(GetModuleHandle(NULL),
    460                      &sandbox_info,
    461                      chrome_main_delegate.get());
    462 #elif defined(OS_LINUX)
    463   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
    464       launcher_delegate->CreateContentMainDelegate());
    465   return ContentMain(argc, const_cast<const char**>(argv),
    466                      chrome_main_delegate.get());
    467 #endif  // defined(OS_WIN)
    468   NOTREACHED();
    469   return 0;
    470 }
    471 
    472 int LaunchTests(TestLauncherDelegate* launcher_delegate,
    473                 int default_jobs,
    474                 int argc,
    475                 char** argv) {
    476   DCHECK(!g_launcher_delegate);
    477   g_launcher_delegate = launcher_delegate;
    478 
    479   CommandLine::Init(argc, argv);
    480   const CommandLine* command_line = CommandLine::ForCurrentProcess();
    481 
    482   if (command_line->HasSwitch(kHelpFlag)) {
    483     PrintUsage();
    484     return 0;
    485   }
    486 
    487   if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
    488       (command_line->HasSwitch(switches::kSingleProcess) &&
    489        command_line->HasSwitch(base::kGTestFilterFlag)) ||
    490       command_line->HasSwitch(base::kGTestListTestsFlag) ||
    491       command_line->HasSwitch(base::kGTestHelpFlag)) {
    492 #if defined(OS_WIN)
    493     if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
    494       sandbox::SandboxInterfaceInfo sandbox_info;
    495       InitializeSandboxInfo(&sandbox_info);
    496       InitializeSandbox(&sandbox_info);
    497     }
    498 #endif
    499     return launcher_delegate->RunTestSuite(argc, argv);
    500   }
    501 
    502   if (ShouldRunContentMain())
    503     return RunContentMain(argc, argv, launcher_delegate);
    504 
    505   base::AtExitManager at_exit;
    506   testing::InitGoogleTest(&argc, argv);
    507   TestTimeouts::Initialize();
    508 
    509   fprintf(stdout,
    510       "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
    511       "For debugging a test inside a debugger, use the\n"
    512       "--gtest_filter=<your_test_name> flag along with either\n"
    513       "--single_process (to run the test in one launcher/browser process) or\n"
    514       "--single-process (to do the above, and also run Chrome in single-"
    515           "process mode).\n");
    516 
    517   base::MessageLoopForIO message_loop;
    518 
    519   WrapperTestLauncherDelegate delegate(launcher_delegate);
    520   base::TestLauncher launcher(&delegate, default_jobs);
    521   bool success = launcher.Run(argc, argv);
    522   return (success ? 0 : 1);
    523 }
    524 
    525 TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
    526   return g_launcher_delegate;
    527 }
    528 
    529 }  // namespace content
    530