Home | History | Annotate | Download | only in test 
      1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "base/test/test_suite.h"
      6 
      7 #include "base/at_exit.h"
      8 #include "base/base_paths.h"
      9 #include "base/base_switches.h"
     10 #include "base/command_line.h"
     11 #include "base/debug/debug_on_start_win.h"
     12 #include "base/debug/debugger.h"
     13 #include "base/file_path.h"
     14 #include "base/i18n/icu_util.h"
     15 #include "base/logging.h"
     16 #include "base/mac/scoped_nsautorelease_pool.h"
     17 #include "base/memory/scoped_ptr.h"
     18 #include "base/path_service.h"
     19 #include "base/process_util.h"
     20 #include "base/test/multiprocess_test.h"
     21 #include "base/test/test_timeouts.h"
     22 #include "base/time.h"
     23 #include "testing/gtest/include/gtest/gtest.h"
     24 #include "testing/multiprocess_func_list.h"
     25 
     26 #if defined(OS_MACOSX)
     27 #include "base/test/mock_chrome_application_mac.h"
     28 #endif
     29 
     30 #if defined(TOOLKIT_USES_GTK)
     31 #include <gtk/gtk.h>
     32 #endif
     33 
     34 namespace {
     35 
     36 class MaybeTestDisabler : public testing::EmptyTestEventListener {
     37  public:
     38   virtual void OnTestStart(const testing::TestInfo& test_info) {
     39     ASSERT_FALSE(TestSuite::IsMarkedMaybe(test_info))
     40         << "Probably the OS #ifdefs don't include all of the necessary "
     41            "platforms.\nPlease ensure that no tests have the MAYBE_ prefix "
     42            "after the code is preprocessed.";
     43   }
     44 };
     45 
     46 }  // namespace
     47 
     48 const char TestSuite::kStrictFailureHandling[] = "strict_failure_handling";
     49 
     50 TestSuite::TestSuite(int argc, char** argv) {
     51 #if defined(OS_WIN)
     52   testing::GTEST_FLAG(catch_exceptions) = false;
     53 #endif
     54   base::EnableTerminationOnHeapCorruption();
     55   CommandLine::Init(argc, argv);
     56   testing::InitGoogleTest(&argc, argv);
     57 #if defined(TOOLKIT_USES_GTK)
     58   g_thread_init(NULL);
     59   gtk_init_check(&argc, &argv);
     60 #endif  // defined(TOOLKIT_USES_GTK)
     61   // Don't add additional code to this constructor.  Instead add it to
     62   // Initialize().  See bug 6436.
     63 }
     64 
     65 TestSuite::~TestSuite() {
     66   CommandLine::Reset();
     67 }
     68 
     69 // static
     70 bool TestSuite::IsMarkedFlaky(const testing::TestInfo& test) {
     71   return strncmp(test.name(), "FLAKY_", 6) == 0;
     72 }
     73 
     74 // static
     75 bool TestSuite::IsMarkedFailing(const testing::TestInfo& test) {
     76   return strncmp(test.name(), "FAILS_", 6) == 0;
     77 }
     78 
     79 // static
     80 bool TestSuite::IsMarkedMaybe(const testing::TestInfo& test) {
     81   return strncmp(test.name(), "MAYBE_", 6) == 0;
     82 }
     83 
     84 // static
     85 bool TestSuite::ShouldIgnoreFailure(const testing::TestInfo& test) {
     86   if (CommandLine::ForCurrentProcess()->HasSwitch(kStrictFailureHandling))
     87     return false;
     88   return IsMarkedFlaky(test) || IsMarkedFailing(test);
     89 }
     90 
     91 // static
     92 bool TestSuite::NonIgnoredFailures(const testing::TestInfo& test) {
     93   return test.should_run() && test.result()->Failed() &&
     94       !ShouldIgnoreFailure(test);
     95 }
     96 
     97 int TestSuite::GetTestCount(TestMatch test_match) {
     98   testing::UnitTest* instance = testing::UnitTest::GetInstance();
     99   int count = 0;
    100 
    101   for (int i = 0; i < instance->total_test_case_count(); ++i) {
    102     const testing::TestCase& test_case = *instance->GetTestCase(i);
    103     for (int j = 0; j < test_case.total_test_count(); ++j) {
    104       if (test_match(*test_case.GetTestInfo(j))) {
    105         count++;
    106       }
    107     }
    108   }
    109 
    110   return count;
    111 }
    112 
    113 void TestSuite::CatchMaybeTests() {
    114   testing::TestEventListeners& listeners =
    115       testing::UnitTest::GetInstance()->listeners();
    116   listeners.Append(new MaybeTestDisabler);
    117 }
    118 
    119 // Don't add additional code to this method.  Instead add it to
    120 // Initialize().  See bug 6436.
    121 int TestSuite::Run() {
    122   base::mac::ScopedNSAutoreleasePool scoped_pool;
    123 
    124   Initialize();
    125   std::string client_func =
    126       CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
    127           switches::kTestChildProcess);
    128   // Check to see if we are being run as a client process.
    129   if (!client_func.empty())
    130     return multi_process_function_list::InvokeChildProcessTest(client_func);
    131   int result = RUN_ALL_TESTS();
    132 
    133   // If there are failed tests, see if we should ignore the failures.
    134   if (result != 0 && GetTestCount(&TestSuite::NonIgnoredFailures) == 0)
    135     result = 0;
    136 
    137   // Display the number of flaky tests.
    138   int flaky_count = GetTestCount(&TestSuite::IsMarkedFlaky);
    139   if (flaky_count) {
    140     printf("  YOU HAVE %d FLAKY %s\n\n", flaky_count,
    141            flaky_count == 1 ? "TEST" : "TESTS");
    142   }
    143 
    144   // Display the number of tests with ignored failures (FAILS).
    145   int failing_count = GetTestCount(&TestSuite::IsMarkedFailing);
    146   if (failing_count) {
    147     printf("  YOU HAVE %d %s with ignored failures (FAILS prefix)\n\n",
    148            failing_count, failing_count == 1 ? "test" : "tests");
    149   }
    150 
    151   // This MUST happen before Shutdown() since Shutdown() tears down
    152   // objects (such as NotificationService::current()) that Cocoa
    153   // objects use to remove themselves as observers.
    154   scoped_pool.Recycle();
    155 
    156   Shutdown();
    157 
    158   return result;
    159 }
    160 
    161 // static
    162 void TestSuite::UnitTestAssertHandler(const std::string& str) {
    163   RAW_LOG(FATAL, str.c_str());
    164 }
    165 
    166 void TestSuite::SuppressErrorDialogs() {
    167 #if defined(OS_WIN)
    168   UINT new_flags = SEM_FAILCRITICALERRORS |
    169                    SEM_NOGPFAULTERRORBOX |
    170                    SEM_NOOPENFILEERRORBOX;
    171 
    172   // Preserve existing error mode, as discussed at
    173   // http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx
    174   UINT existing_flags = SetErrorMode(new_flags);
    175   SetErrorMode(existing_flags | new_flags);
    176 #endif  // defined(OS_WIN)
    177 }
    178 
    179 void TestSuite::Initialize() {
    180 #if defined(OS_MACOSX)
    181   // Some of the app unit tests spin runloops.
    182   mock_cr_app::RegisterMockCrApp();
    183 #endif
    184 
    185   // Initialize logging.
    186   FilePath exe;
    187   PathService::Get(base::FILE_EXE, &exe);
    188   FilePath log_filename = exe.ReplaceExtension(FILE_PATH_LITERAL("log"));
    189   logging::InitLogging(
    190       log_filename.value().c_str(),
    191       logging::LOG_TO_BOTH_FILE_AND_SYSTEM_DEBUG_LOG,
    192       logging::LOCK_LOG_FILE,
    193       logging::DELETE_OLD_LOG_FILE,
    194       logging::DISABLE_DCHECK_FOR_NON_OFFICIAL_RELEASE_BUILDS);
    195   // We want process and thread IDs because we may have multiple processes.
    196   // Note: temporarily enabled timestamps in an effort to catch bug 6361.
    197   logging::SetLogItems(true, true, true, true);
    198 
    199   CHECK(base::EnableInProcessStackDumping());
    200 #if defined(OS_WIN)
    201   // Make sure we run with high resolution timer to minimize differences
    202   // between production code and test code.
    203   base::Time::EnableHighResolutionTimer(true);
    204 #endif  // defined(OS_WIN)
    205 
    206   // In some cases, we do not want to see standard error dialogs.
    207   if (!base::debug::BeingDebugged() &&
    208       !CommandLine::ForCurrentProcess()->HasSwitch("show-error-dialogs")) {
    209     SuppressErrorDialogs();
    210     base::debug::SetSuppressDebugUI(true);
    211     logging::SetLogAssertHandler(UnitTestAssertHandler);
    212   }
    213 
    214   icu_util::Initialize();
    215 
    216   CatchMaybeTests();
    217 
    218   TestTimeouts::Initialize();
    219 }
    220 
    221 void TestSuite::Shutdown() {
    222 }
    223