Home | History | Annotate | Download | only in test 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "content/public/test/test_launcher.h"
      6 
      7 #include <string>
      8 #include <vector>
      9 
     10 #include "base/command_line.h"
     11 #include "base/containers/hash_tables.h"
     12 #include "base/environment.h"
     13 #include "base/file_util.h"
     14 #include "base/files/scoped_temp_dir.h"
     15 #include "base/logging.h"
     16 #include "base/memory/linked_ptr.h"
     17 #include "base/memory/scoped_ptr.h"
     18 #include "base/strings/string_number_conversions.h"
     19 #include "base/strings/string_util.h"
     20 #include "base/strings/utf_string_conversions.h"
     21 #include "base/test/test_launcher.h"
     22 #include "base/test/test_suite.h"
     23 #include "base/test/test_timeouts.h"
     24 #include "base/time/time.h"
     25 #include "content/public/app/content_main.h"
     26 #include "content/public/app/content_main_delegate.h"
     27 #include "content/public/app/startup_helper_win.h"
     28 #include "content/public/common/content_switches.h"
     29 #include "content/public/common/sandbox_init.h"
     30 #include "content/public/test/browser_test.h"
     31 #include "net/base/escape.h"
     32 #include "testing/gtest/include/gtest/gtest.h"
     33 
     34 #if defined(OS_WIN)
     35 #include "base/base_switches.h"
     36 #include "content/common/sandbox_win.h"
     37 #include "sandbox/win/src/sandbox_factory.h"
     38 #include "sandbox/win/src/sandbox_types.h"
     39 #elif defined(OS_MACOSX)
     40 #include "base/mac/scoped_nsautorelease_pool.h"
     41 #endif
     42 
     43 namespace content {
     44 
     45 namespace {
     46 
     47 // Tests with this prefix run before the same test without it, and use the same
     48 // profile. i.e. Foo.PRE_Test runs and then Foo.Test. This allows writing tests
     49 // that span browser restarts.
     50 const char kPreTestPrefix[] = "PRE_";
     51 
     52 // Manual tests only run when --run-manual is specified. This allows writing
     53 // tests that don't run automatically but are still in the same test binary.
     54 // This is useful so that a team that wants to run a few tests doesn't have to
     55 // add a new binary that must be compiled on all builds.
     56 const char kManualTestPrefix[] = "MANUAL_";
     57 
     58 TestLauncherDelegate* g_launcher_delegate;
     59 }
     60 
     61 namespace {
     62 
     63 int DoRunTestInternal(const testing::TestCase* test_case,
     64                       const std::string& test_name,
     65                       const CommandLine& command_line,
     66                       base::TimeDelta default_timeout,
     67                       bool* was_timeout) {
     68   if (test_case) {
     69     std::string pre_test_name = test_name;
     70     std::string replace_string = std::string(".") + kPreTestPrefix;
     71     ReplaceFirstSubstringAfterOffset(&pre_test_name, 0, ".", replace_string);
     72     for (int i = 0; i < test_case->total_test_count(); ++i) {
     73       const testing::TestInfo* test_info = test_case->GetTestInfo(i);
     74       std::string cur_test_name = test_info->test_case_name();
     75       cur_test_name.append(".");
     76       cur_test_name.append(test_info->name());
     77       if (cur_test_name == pre_test_name) {
     78         int exit_code = DoRunTestInternal(test_case,
     79                                           pre_test_name,
     80                                           command_line,
     81                                           default_timeout,
     82                                           was_timeout);
     83         if (exit_code != 0)
     84           return exit_code;
     85       }
     86     }
     87   }
     88 
     89   CommandLine new_cmd_line(command_line);
     90 
     91   // Always enable disabled tests.  This method is not called with disabled
     92   // tests unless this flag was specified to the browser test executable.
     93   new_cmd_line.AppendSwitch("gtest_also_run_disabled_tests");
     94   new_cmd_line.AppendSwitchASCII("gtest_filter", test_name);
     95   new_cmd_line.AppendSwitch(kSingleProcessTestsFlag);
     96 
     97   char* browser_wrapper = getenv("BROWSER_WRAPPER");
     98   int exit_code = base::LaunchChildGTestProcess(
     99       new_cmd_line,
    100       browser_wrapper ? browser_wrapper : std::string(),
    101       default_timeout,
    102       was_timeout);
    103   if (*was_timeout) {
    104     LOG(ERROR) << "Test timeout (" << default_timeout.InMilliseconds()
    105                << " ms) exceeded for " << test_name;
    106   }
    107 
    108   return exit_code;
    109 }
    110 
    111 // Runs test specified by |test_name| in a child process,
    112 // and returns the exit code.
    113 int DoRunTest(TestLauncherDelegate* launcher_delegate,
    114               const testing::TestCase* test_case,
    115               const std::string& test_name,
    116               base::TimeDelta default_timeout,
    117               bool* was_timeout) {
    118   if (was_timeout)
    119     *was_timeout = false;
    120 
    121 #if defined(OS_MACOSX)
    122   // Some of the below method calls will leak objects if there is no
    123   // autorelease pool in place.
    124   base::mac::ScopedNSAutoreleasePool pool;
    125 #endif
    126 
    127   base::ScopedTempDir temp_dir;
    128   // Create a new data dir and pass it to the child.
    129   if (!temp_dir.CreateUniqueTempDir() || !temp_dir.IsValid()) {
    130     LOG(ERROR) << "Error creating temp data directory";
    131     return -1;
    132   }
    133 
    134   CommandLine new_cmd_line(*CommandLine::ForCurrentProcess());
    135   if (!launcher_delegate->AdjustChildProcessCommandLine(&new_cmd_line,
    136                                                         temp_dir.path())) {
    137     return -1;
    138   }
    139 
    140   return DoRunTestInternal(
    141       test_case, test_name, new_cmd_line, default_timeout, was_timeout);
    142 }
    143 
    144 void PrintUsage() {
    145   fprintf(stdout,
    146       "Runs tests using the gtest framework, each test being run in its own\n"
    147       "process.  Any gtest flags can be specified.\n"
    148       "  --single_process\n"
    149       "    Runs the tests and the launcher in the same process. Useful for \n"
    150       "    debugging a specific test in a debugger.\n"
    151       "  --single-process\n"
    152       "    Same as above, and also runs Chrome in single-process mode.\n"
    153       "  --help\n"
    154       "    Shows this message.\n"
    155       "  --gtest_help\n"
    156       "    Shows the gtest help message.\n");
    157 }
    158 
    159 // Implementation of base::TestLauncherDelegate. This is also a test launcher,
    160 // wrapping a lower-level test launcher with content-specific code.
    161 class WrapperTestLauncherDelegate : public base::TestLauncherDelegate {
    162  public:
    163   explicit WrapperTestLauncherDelegate(
    164       content::TestLauncherDelegate* launcher_delegate)
    165       : launcher_delegate_(launcher_delegate),
    166         timeout_count_(0),
    167         printed_timeout_message_(false) {
    168   }
    169 
    170   // base::TestLauncherDelegate:
    171   virtual bool ShouldRunTest(const testing::TestCase* test_case,
    172                              const testing::TestInfo* test_info) OVERRIDE;
    173   virtual void RunTest(
    174       const testing::TestCase* test_case,
    175       const testing::TestInfo* test_info,
    176       const base::TestLauncherDelegate::TestResultCallback& callback) OVERRIDE;
    177   virtual void RunRemainingTests() OVERRIDE;
    178 
    179  private:
    180   content::TestLauncherDelegate* launcher_delegate_;
    181 
    182   // Number of times a test timeout occurred.
    183   size_t timeout_count_;
    184 
    185   // True after a message about too many timeouts has been printed,
    186   // to avoid doing it more than once.
    187   bool printed_timeout_message_;
    188 
    189   DISALLOW_COPY_AND_ASSIGN(WrapperTestLauncherDelegate);
    190 };
    191 
    192 bool WrapperTestLauncherDelegate::ShouldRunTest(
    193     const testing::TestCase* test_case,
    194     const testing::TestInfo* test_info) {
    195   std::string test_name =
    196       std::string(test_case->name()) + "." + test_info->name();
    197 
    198   if (StartsWithASCII(test_info->name(), kPreTestPrefix, true))
    199     return false;
    200 
    201   if (StartsWithASCII(test_info->name(), kManualTestPrefix, true) &&
    202       !CommandLine::ForCurrentProcess()->HasSwitch(kRunManualTestsFlag)) {
    203     return false;
    204   }
    205 
    206   // Stop test execution after too many timeouts.
    207   if (timeout_count_ > 5) {
    208     if (!printed_timeout_message_) {
    209       printed_timeout_message_ = true;
    210       printf("Too many timeouts, aborting test\n");
    211     }
    212     return false;
    213   }
    214 
    215   return true;
    216 }
    217 
    218 void WrapperTestLauncherDelegate::RunTest(
    219     const testing::TestCase* test_case,
    220     const testing::TestInfo* test_info,
    221     const base::TestLauncherDelegate::TestResultCallback& callback) {
    222   base::TimeTicks start_time = base::TimeTicks::Now();
    223   bool was_timeout = false;
    224   std::string test_name =
    225       std::string(test_case->name()) + "." + test_info->name();
    226   int exit_code = DoRunTest(launcher_delegate_,
    227                             test_case,
    228                             test_name,
    229                             TestTimeouts::action_max_timeout(),
    230                             &was_timeout);
    231   if (was_timeout)
    232     timeout_count_++;
    233 
    234   base::TestResult result;
    235   result.test_case_name = test_case->name();
    236   result.test_name = test_info->name();
    237   result.success = (exit_code == 0);
    238   result.elapsed_time = (base::TimeTicks::Now() - start_time);
    239 
    240   callback.Run(result);
    241 }
    242 
    243 void WrapperTestLauncherDelegate::RunRemainingTests() {
    244   // No need to do anything here, we launch tests synchronously.
    245 }
    246 
    247 }  // namespace
    248 
    249 // The following is kept for historical reasons (so people that are used to
    250 // using it don't get surprised).
    251 const char kChildProcessFlag[]   = "child";
    252 
    253 const char kGTestHelpFlag[]   = "gtest_help";
    254 
    255 const char kHelpFlag[]   = "help";
    256 
    257 const char kLaunchAsBrowser[] = "as-browser";
    258 
    259 // See kManualTestPrefix above.
    260 const char kRunManualTestsFlag[] = "run-manual";
    261 
    262 const char kSingleProcessTestsFlag[]   = "single_process";
    263 
    264 
    265 TestLauncherDelegate::~TestLauncherDelegate() {
    266 }
    267 
    268 bool ShouldRunContentMain() {
    269 #if defined(OS_WIN) || defined(OS_LINUX)
    270   CommandLine* command_line = CommandLine::ForCurrentProcess();
    271   return command_line->HasSwitch(switches::kProcessType) ||
    272          command_line->HasSwitch(kLaunchAsBrowser);
    273 #else
    274   return false;
    275 #endif  // defined(OS_WIN) || defined(OS_LINUX)
    276 }
    277 
    278 int RunContentMain(int argc, char** argv,
    279                    TestLauncherDelegate* launcher_delegate) {
    280 #if defined(OS_WIN)
    281   sandbox::SandboxInterfaceInfo sandbox_info = {0};
    282   InitializeSandboxInfo(&sandbox_info);
    283   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
    284       launcher_delegate->CreateContentMainDelegate());
    285   return ContentMain(GetModuleHandle(NULL),
    286                      &sandbox_info,
    287                      chrome_main_delegate.get());
    288 #elif defined(OS_LINUX)
    289   scoped_ptr<ContentMainDelegate> chrome_main_delegate(
    290       launcher_delegate->CreateContentMainDelegate());
    291   return ContentMain(argc, const_cast<const char**>(argv),
    292                      chrome_main_delegate.get());
    293 #endif  // defined(OS_WIN)
    294   NOTREACHED();
    295   return 0;
    296 }
    297 
    298 int LaunchTests(TestLauncherDelegate* launcher_delegate,
    299                 int argc,
    300                 char** argv) {
    301   DCHECK(!g_launcher_delegate);
    302   g_launcher_delegate = launcher_delegate;
    303 
    304   CommandLine::Init(argc, argv);
    305   const CommandLine* command_line = CommandLine::ForCurrentProcess();
    306 
    307   if (command_line->HasSwitch(kHelpFlag)) {
    308     PrintUsage();
    309     return 0;
    310   }
    311 
    312   if (command_line->HasSwitch(kSingleProcessTestsFlag) ||
    313       (command_line->HasSwitch(switches::kSingleProcess) &&
    314        command_line->HasSwitch(base::kGTestFilterFlag)) ||
    315       command_line->HasSwitch(base::kGTestListTestsFlag) ||
    316       command_line->HasSwitch(kGTestHelpFlag)) {
    317 #if defined(OS_WIN)
    318     if (command_line->HasSwitch(kSingleProcessTestsFlag)) {
    319       sandbox::SandboxInterfaceInfo sandbox_info;
    320       InitializeSandboxInfo(&sandbox_info);
    321       InitializeSandbox(&sandbox_info);
    322     }
    323 #endif
    324     return launcher_delegate->RunTestSuite(argc, argv);
    325   }
    326 
    327   if (ShouldRunContentMain())
    328     return RunContentMain(argc, argv, launcher_delegate);
    329 
    330   fprintf(stdout,
    331       "Starting tests...\n"
    332       "IMPORTANT DEBUGGING NOTE: each test is run inside its own process.\n"
    333       "For debugging a test inside a debugger, use the\n"
    334       "--gtest_filter=<your_test_name> flag along with either\n"
    335       "--single_process (to run the test in one launcher/browser process) or\n"
    336       "--single-process (to do the above, and also run Chrome in single-"
    337       "process mode).\n");
    338 
    339   base::AtExitManager at_exit;
    340   testing::InitGoogleTest(&argc, argv);
    341   TestTimeouts::Initialize();
    342 
    343   WrapperTestLauncherDelegate delegate(launcher_delegate);
    344   return base::LaunchTests(&delegate, argc, argv);
    345 }
    346 
    347 TestLauncherDelegate* GetCurrentTestLauncherDelegate() {
    348   return g_launcher_delegate;
    349 }
    350 
    351 }  // namespace content
    352