Home | History | Annotate | Download | only in metrics 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include <math.h>
      6 
      7 #include "base/basictypes.h"
      8 #include "base/bind.h"
      9 #include "base/logging.h"
     10 #include "base/memory/scoped_ptr.h"
     11 #include "base/message_loop/message_loop.h"
     12 #include "base/message_loop/message_loop_proxy.h"
     13 #include "base/strings/string_number_conversions.h"
     14 #include "base/strings/string_split.h"
     15 #include "base/strings/string_tokenizer.h"
     16 #include "base/synchronization/condition_variable.h"
     17 #include "base/synchronization/lock.h"
     18 #include "base/threading/platform_thread.h"
     19 #include "base/time/time.h"
     20 #include "build/build_config.h"
     21 #include "chrome/browser/metrics/thread_watcher.h"
     22 #include "chrome/common/chrome_switches.h"
     23 #include "content/public/test/test_browser_thread.h"
     24 #include "testing/gtest/include/gtest/gtest.h"
     25 #include "testing/platform_test.h"
     26 
     27 using base::TimeDelta;
     28 using base::TimeTicks;
     29 using content::BrowserThread;
     30 
     31 enum State {
     32   INITIALIZED,        // Created ThreadWatch object.
     33   ACTIVATED,          // Thread watching activated.
     34   SENT_PING,          // Sent ping message to watched thread.
     35   RECEIVED_PONG,      // Received Pong message.
     36   DEACTIVATED,        // Thread watching de-activated.
     37 };
     38 
     39 enum WaitState {
     40   UNINITIALIZED,
     41   STARTED_WAITING,    // Start waiting for state_ to change to expected_state.
     42   STOPPED_WAITING,    // Done with the waiting.
     43   ALL_DONE,           // Done with waiting for STOPPED_WAITING.
     44 };
     45 
     46 enum CheckResponseState {
     47   UNKNOWN,
     48   SUCCESSFUL,         // CheckResponse was successful.
     49   FAILED,             // CheckResponse has failed.
     50 };
     51 
     52 // This class helps to track and manipulate thread state during tests. This
     53 // class also has utility method to simulate hanging of watched thread by making
     54 // the watched thread wait for a very long time by posting a task on watched
     55 // thread that keeps it busy. It also has an utility method to block running of
     56 // tests until ThreadWatcher object's post-condition state changes to an
     57 // expected state.
     58 class CustomThreadWatcher : public ThreadWatcher {
     59  public:
     60   base::Lock custom_lock_;
     61   base::ConditionVariable state_changed_;
     62   State thread_watcher_state_;
     63   WaitState wait_state_;
     64   CheckResponseState check_response_state_;
     65   uint64 ping_sent_;
     66   uint64 pong_received_;
     67   base::subtle::Atomic32 success_response_;
     68   base::subtle::Atomic32 failed_response_;
     69   base::TimeTicks saved_ping_time_;
     70   uint64 saved_ping_sequence_number_;
     71 
     72   CustomThreadWatcher(const BrowserThread::ID thread_id,
     73                       const std::string thread_name,
     74                       const TimeDelta& sleep_time,
     75                       const TimeDelta& unresponsive_time)
     76       : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time,
     77                       unresponsive_time, ThreadWatcherList::kUnresponsiveCount,
     78                       true, ThreadWatcherList::kLiveThreadsThreshold)),
     79         state_changed_(&custom_lock_),
     80         thread_watcher_state_(INITIALIZED),
     81         wait_state_(UNINITIALIZED),
     82         check_response_state_(UNKNOWN),
     83         ping_sent_(0),
     84         pong_received_(0),
     85         success_response_(0),
     86         failed_response_(0),
     87         saved_ping_time_(base::TimeTicks::Now()),
     88         saved_ping_sequence_number_(0) {
     89   }
     90 
     91   State UpdateState(State new_state) {
     92     State old_state;
     93     {
     94       base::AutoLock auto_lock(custom_lock_);
     95       old_state = thread_watcher_state_;
     96       if (old_state != DEACTIVATED)
     97         thread_watcher_state_ = new_state;
     98       if (new_state == SENT_PING)
     99         ++ping_sent_;
    100       if (new_state == RECEIVED_PONG)
    101         ++pong_received_;
    102       saved_ping_time_ = ping_time();
    103       saved_ping_sequence_number_ = ping_sequence_number();
    104     }
    105     state_changed_.Broadcast();
    106     return old_state;
    107   }
    108 
    109   WaitState UpdateWaitState(WaitState new_state) {
    110     WaitState old_state;
    111     {
    112       base::AutoLock auto_lock(custom_lock_);
    113       old_state = wait_state_;
    114       wait_state_ = new_state;
    115     }
    116     state_changed_.Broadcast();
    117     return old_state;
    118   }
    119 
    120   virtual void ActivateThreadWatching() OVERRIDE {
    121     State old_state = UpdateState(ACTIVATED);
    122     EXPECT_EQ(old_state, INITIALIZED);
    123     ThreadWatcher::ActivateThreadWatching();
    124   }
    125 
    126   virtual void DeActivateThreadWatching() OVERRIDE {
    127     State old_state = UpdateState(DEACTIVATED);
    128     EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING ||
    129                 old_state == RECEIVED_PONG);
    130     ThreadWatcher::DeActivateThreadWatching();
    131   }
    132 
    133   virtual void PostPingMessage() OVERRIDE {
    134     State old_state = UpdateState(SENT_PING);
    135     EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG);
    136     ThreadWatcher::PostPingMessage();
    137   }
    138 
    139   virtual void OnPongMessage(uint64 ping_sequence_number) OVERRIDE {
    140     State old_state = UpdateState(RECEIVED_PONG);
    141     EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED);
    142     ThreadWatcher::OnPongMessage(ping_sequence_number);
    143   }
    144 
    145   virtual void OnCheckResponsiveness(uint64 ping_sequence_number) OVERRIDE {
    146     ThreadWatcher::OnCheckResponsiveness(ping_sequence_number);
    147     {
    148       base::AutoLock auto_lock(custom_lock_);
    149       if (responsive_) {
    150         base::subtle::Release_Store(&success_response_,
    151             base::subtle::Acquire_Load(&success_response_) + 1);
    152         check_response_state_ = SUCCESSFUL;
    153       } else {
    154         base::subtle::Release_Store(&failed_response_,
    155             base::subtle::Acquire_Load(&failed_response_) + 1);
    156         check_response_state_ = FAILED;
    157       }
    158     }
    159     // Broadcast to indicate we have checked responsiveness of the thread that
    160     // is watched.
    161     state_changed_.Broadcast();
    162   }
    163 
    164   void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) {
    165     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
    166     TimeTicks end_time = TimeTicks::Now() + wait_time;
    167     {
    168       base::AutoLock auto_lock(custom_lock_);
    169       while (wait_state_ != expected_state && TimeTicks::Now() < end_time)
    170         state_changed_.TimedWait(end_time - TimeTicks::Now());
    171     }
    172   }
    173 
    174   void VeryLongMethod(TimeDelta wait_time) {
    175     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
    176     WaitForWaitStateChange(wait_time, STOPPED_WAITING);
    177     UpdateWaitState(ALL_DONE);
    178   }
    179 
    180   State WaitForStateChange(const TimeDelta& wait_time, State expected_state) {
    181     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
    182     UpdateWaitState(STARTED_WAITING);
    183 
    184     State exit_state = INITIALIZED;
    185     // Keep the thread that is running the tests waiting until ThreadWatcher
    186     // object's state changes to the expected_state or until wait_time elapses.
    187     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
    188         TimeTicks end_time = TimeTicks::Now() + wait_time;
    189         {
    190           base::AutoLock auto_lock(custom_lock_);
    191           while (thread_watcher_state_ != expected_state &&
    192                  TimeTicks::Now() < end_time) {
    193             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
    194             state_changed_.TimedWait(state_change_wait_time);
    195           }
    196           // Capture the thread_watcher_state_ before it changes and return it
    197           // to the caller.
    198           exit_state = thread_watcher_state_;
    199           if (exit_state == expected_state)
    200             break;
    201         }
    202     }
    203     UpdateWaitState(STOPPED_WAITING);
    204     return exit_state;
    205   }
    206 
    207   CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time,
    208                                           CheckResponseState expected_state) {
    209     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
    210     UpdateWaitState(STARTED_WAITING);
    211 
    212     CheckResponseState exit_state = UNKNOWN;
    213     // Keep the thread that is running the tests waiting until ThreadWatcher
    214     // object's check_response_state_ changes to the expected_state or until
    215     // wait_time elapses.
    216     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
    217         TimeTicks end_time = TimeTicks::Now() + wait_time;
    218         {
    219           base::AutoLock auto_lock(custom_lock_);
    220           while (check_response_state_ != expected_state &&
    221                  TimeTicks::Now() < end_time) {
    222             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
    223             state_changed_.TimedWait(state_change_wait_time);
    224           }
    225           // Capture the check_response_state_ before it changes and return it
    226           // to the caller.
    227           exit_state = check_response_state_;
    228           if (exit_state == expected_state)
    229             break;
    230         }
    231     }
    232     UpdateWaitState(STOPPED_WAITING);
    233     return exit_state;
    234   }
    235 };
    236 
    237 class ThreadWatcherTest : public ::testing::Test {
    238  public:
    239   static const TimeDelta kSleepTime;
    240   static const TimeDelta kUnresponsiveTime;
    241   static const BrowserThread::ID io_thread_id;
    242   static const std::string io_thread_name;
    243   static const BrowserThread::ID db_thread_id;
    244   static const std::string db_thread_name;
    245   static const std::string crash_on_hang_seconds;
    246   static const std::string crash_on_hang_thread_names;
    247   static const std::string thread_names_and_live_threshold;
    248   static const std::string crash_on_hang_thread_data;
    249   CustomThreadWatcher* io_watcher_;
    250   CustomThreadWatcher* db_watcher_;
    251   ThreadWatcherList* thread_watcher_list_;
    252 
    253   ThreadWatcherTest()
    254       : setup_complete_(&lock_),
    255         initialized_(false) {
    256     db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB));
    257     io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO));
    258     watchdog_thread_.reset(new WatchDogThread());
    259     db_thread_->Start();
    260     io_thread_->Start();
    261     watchdog_thread_->Start();
    262 
    263     WatchDogThread::PostTask(
    264         FROM_HERE,
    265         base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this)));
    266 
    267     WaitForSetUp(TimeDelta::FromMinutes(1));
    268   }
    269 
    270   void SetUpObjects() {
    271     DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    272 
    273     // Setup the registry for thread watchers.
    274     thread_watcher_list_ = new ThreadWatcherList();
    275 
    276     // Create thread watcher object for the IO thread.
    277     io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name,
    278                                           kSleepTime, kUnresponsiveTime);
    279     EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id));
    280 
    281     // Create thread watcher object for the DB thread.
    282     db_watcher_ = new CustomThreadWatcher(
    283         db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime);
    284     EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id));
    285 
    286     {
    287       base::AutoLock lock(lock_);
    288       initialized_ = true;
    289     }
    290     setup_complete_.Signal();
    291   }
    292 
    293   void WaitForSetUp(TimeDelta wait_time) {
    294     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
    295     TimeTicks end_time = TimeTicks::Now() + wait_time;
    296     {
    297       base::AutoLock auto_lock(lock_);
    298       while (!initialized_ && TimeTicks::Now() < end_time)
    299         setup_complete_.TimedWait(end_time - TimeTicks::Now());
    300     }
    301   }
    302 
    303   virtual ~ThreadWatcherTest() {
    304     ThreadWatcherList::DeleteAll();
    305     io_watcher_ = NULL;
    306     db_watcher_ = NULL;
    307     io_thread_.reset();
    308     db_thread_.reset();
    309     watchdog_thread_.reset();
    310     thread_watcher_list_ = NULL;
    311   }
    312 
    313  private:
    314   base::Lock lock_;
    315   base::ConditionVariable setup_complete_;
    316   bool initialized_;
    317   scoped_ptr<content::TestBrowserThread> db_thread_;
    318   scoped_ptr<content::TestBrowserThread> io_thread_;
    319   scoped_ptr<WatchDogThread> watchdog_thread_;
    320 };
    321 
    322 // Define static constants.
    323 const TimeDelta ThreadWatcherTest::kSleepTime =
    324     TimeDelta::FromMilliseconds(50);
    325 const TimeDelta ThreadWatcherTest::kUnresponsiveTime =
    326     TimeDelta::FromMilliseconds(500);
    327 const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO;
    328 const std::string ThreadWatcherTest::io_thread_name = "IO";
    329 const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB;
    330 const std::string ThreadWatcherTest::db_thread_name = "DB";
    331 const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO";
    332 const std::string ThreadWatcherTest::thread_names_and_live_threshold =
    333     "UI:4,IO:4";
    334 const std::string ThreadWatcherTest::crash_on_hang_thread_data =
    335     "UI:5:12,IO:5:12,FILE:5:12";
    336 
    337 TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) {
    338   // Setup command_line arguments.
    339   CommandLine command_line(CommandLine::NO_PROGRAM);
    340   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
    341                                  crash_on_hang_thread_names);
    342 
    343   // Parse command_line arguments.
    344   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
    345   uint32 unresponsive_threshold;
    346   ThreadWatcherList::ParseCommandLine(command_line,
    347                                       &unresponsive_threshold,
    348                                       &crash_on_hang_threads);
    349 
    350   // Verify the data.
    351   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
    352   std::vector<std::string> values;
    353   while (tokens.GetNext()) {
    354     const std::string& token = tokens.token();
    355     base::SplitString(token, ':', &values);
    356     std::string thread_name = values[0];
    357 
    358     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
    359         crash_on_hang_threads.find(thread_name);
    360     bool crash_on_hang = (it != crash_on_hang_threads.end());
    361     EXPECT_TRUE(crash_on_hang);
    362     EXPECT_LT(0u, it->second.live_threads_threshold);
    363     EXPECT_LT(0u, it->second.unresponsive_threshold);
    364   }
    365 }
    366 
    367 TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) {
    368   // Setup command_line arguments.
    369   CommandLine command_line(CommandLine::NO_PROGRAM);
    370   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
    371                                  thread_names_and_live_threshold);
    372 
    373   // Parse command_line arguments.
    374   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
    375   uint32 unresponsive_threshold;
    376   ThreadWatcherList::ParseCommandLine(command_line,
    377                                       &unresponsive_threshold,
    378                                       &crash_on_hang_threads);
    379 
    380   // Verify the data.
    381   base::StringTokenizer tokens(thread_names_and_live_threshold, ",");
    382   std::vector<std::string> values;
    383   while (tokens.GetNext()) {
    384     const std::string& token = tokens.token();
    385     base::SplitString(token, ':', &values);
    386     std::string thread_name = values[0];
    387 
    388     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
    389         crash_on_hang_threads.find(thread_name);
    390     bool crash_on_hang = (it != crash_on_hang_threads.end());
    391     EXPECT_TRUE(crash_on_hang);
    392     EXPECT_EQ(4u, it->second.live_threads_threshold);
    393     EXPECT_LT(0u, it->second.unresponsive_threshold);
    394   }
    395 }
    396 
    397 TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) {
    398   // Setup command_line arguments.
    399   CommandLine command_line(CommandLine::NO_PROGRAM);
    400   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
    401                                  crash_on_hang_thread_data);
    402 
    403   // Parse command_line arguments.
    404   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
    405   uint32 unresponsive_threshold;
    406   ThreadWatcherList::ParseCommandLine(command_line,
    407                                       &unresponsive_threshold,
    408                                       &crash_on_hang_threads);
    409 
    410   // Verify the data.
    411   base::StringTokenizer tokens(crash_on_hang_thread_data, ",");
    412   std::vector<std::string> values;
    413   while (tokens.GetNext()) {
    414     const std::string& token = tokens.token();
    415     base::SplitString(token, ':', &values);
    416     std::string thread_name = values[0];
    417 
    418     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
    419         crash_on_hang_threads.find(thread_name);
    420 
    421     bool crash_on_hang = (it != crash_on_hang_threads.end());
    422     EXPECT_TRUE(crash_on_hang);
    423 
    424     uint32 crash_live_threads_threshold = it->second.live_threads_threshold;
    425     EXPECT_EQ(5u, crash_live_threads_threshold);
    426 
    427     uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold;
    428     uint32 crash_on_unresponsive_seconds =
    429         ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold;
    430     EXPECT_EQ(12u, crash_on_unresponsive_seconds);
    431   }
    432 }
    433 
    434 // Test registration. When thread_watcher_list_ goes out of scope after
    435 // TearDown, all thread watcher objects will be deleted.
    436 TEST_F(ThreadWatcherTest, Registration) {
    437   // Check ThreadWatcher object has all correct parameters.
    438   EXPECT_EQ(io_thread_id, io_watcher_->thread_id());
    439   EXPECT_EQ(io_thread_name, io_watcher_->thread_name());
    440   EXPECT_EQ(kSleepTime, io_watcher_->sleep_time());
    441   EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time());
    442   EXPECT_FALSE(io_watcher_->active());
    443 
    444   // Check ThreadWatcher object of watched DB thread has correct data.
    445   EXPECT_EQ(db_thread_id, db_watcher_->thread_id());
    446   EXPECT_EQ(db_thread_name, db_watcher_->thread_name());
    447   EXPECT_EQ(kSleepTime, db_watcher_->sleep_time());
    448   EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time());
    449   EXPECT_FALSE(db_watcher_->active());
    450 }
    451 
    452 // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This
    453 // method also checks that pong message was sent by the watched thread and pong
    454 // message was received by the WatchDogThread. It also checks that
    455 // OnCheckResponsiveness has verified the ping-pong mechanism and the watched
    456 // thread is not hung.
    457 TEST_F(ThreadWatcherTest, ThreadResponding) {
    458   TimeTicks time_before_ping = TimeTicks::Now();
    459   // Activate watching IO thread.
    460   WatchDogThread::PostTask(
    461       FROM_HERE,
    462       base::Bind(&ThreadWatcher::ActivateThreadWatching,
    463                  base::Unretained(io_watcher_)));
    464 
    465   // Activate would have started ping/pong messaging. Expect atleast one
    466   // ping/pong messaging sequence to happen.
    467   io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1),
    468                                   RECEIVED_PONG);
    469   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
    470   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
    471   EXPECT_TRUE(io_watcher_->active());
    472   EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping);
    473   EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0));
    474 
    475   // Verify watched thread is responding with ping/pong messaging.
    476   io_watcher_->WaitForCheckResponse(
    477       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
    478   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
    479       static_cast<base::subtle::Atomic32>(0));
    480   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
    481       static_cast<base::subtle::Atomic32>(0));
    482 
    483   // DeActivate thread watching for shutdown.
    484   WatchDogThread::PostTask(
    485       FROM_HERE,
    486       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
    487       base::Unretained(io_watcher_)));
    488 }
    489 
    490 // This test posts a task on watched thread that takes very long time (this is
    491 // to simulate hanging of watched thread). It then checks for
    492 // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false
    493 // if the watched thread is not responding).
    494 TEST_F(ThreadWatcherTest, ThreadNotResponding) {
    495   // Simulate hanging of watched thread by making the watched thread wait for a
    496   // very long time by posting a task on watched thread that keeps it busy.
    497   // It is safe to use base::Unretained because test is waiting for the method
    498   // to finish.
    499   BrowserThread::PostTask(
    500       io_thread_id,
    501       FROM_HERE,
    502       base::Bind(&CustomThreadWatcher::VeryLongMethod,
    503                  base::Unretained(io_watcher_),
    504                  kUnresponsiveTime * 10));
    505 
    506   // Activate thread watching.
    507   WatchDogThread::PostTask(
    508       FROM_HERE,
    509       base::Bind(&ThreadWatcher::ActivateThreadWatching,
    510                  base::Unretained(io_watcher_)));
    511 
    512   // Verify watched thread is not responding for ping messages.
    513   io_watcher_->WaitForCheckResponse(
    514       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
    515   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
    516       static_cast<base::subtle::Atomic32>(0));
    517   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
    518       static_cast<base::subtle::Atomic32>(0));
    519 
    520   // DeActivate thread watching for shutdown.
    521   WatchDogThread::PostTask(
    522       FROM_HERE,
    523       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
    524                  base::Unretained(io_watcher_)));
    525 
    526   // Wait for the io_watcher_'s VeryLongMethod to finish.
    527   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
    528 }
    529 
    530 // Test watching of multiple threads with all threads not responding.
    531 TEST_F(ThreadWatcherTest, MultipleThreadsResponding) {
    532   // Check for DB thread to perform ping/pong messaging.
    533   WatchDogThread::PostTask(
    534       FROM_HERE,
    535       base::Bind(&ThreadWatcher::ActivateThreadWatching,
    536                  base::Unretained(db_watcher_)));
    537 
    538   // Check for IO thread to perform ping/pong messaging.
    539   WatchDogThread::PostTask(
    540       FROM_HERE,
    541       base::Bind(&ThreadWatcher::ActivateThreadWatching,
    542                  base::Unretained(io_watcher_)));
    543 
    544   // Verify DB thread is responding with ping/pong messaging.
    545   db_watcher_->WaitForCheckResponse(
    546       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
    547   EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0));
    548   EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0));
    549   EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0));
    550   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
    551       static_cast<base::subtle::Atomic32>(0));
    552   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
    553       static_cast<base::subtle::Atomic32>(0));
    554 
    555   // Verify IO thread is responding with ping/pong messaging.
    556   io_watcher_->WaitForCheckResponse(
    557       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
    558   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
    559   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
    560   EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0));
    561   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
    562       static_cast<base::subtle::Atomic32>(0));
    563   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
    564       static_cast<base::subtle::Atomic32>(0));
    565 
    566   // DeActivate thread watching for shutdown.
    567   WatchDogThread::PostTask(
    568       FROM_HERE,
    569       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
    570                  base::Unretained(io_watcher_)));
    571 
    572   WatchDogThread::PostTask(
    573       FROM_HERE,
    574       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
    575                  base::Unretained(db_watcher_)));
    576 }
    577 
    578 // Test watching of multiple threads with one of the threads not responding.
    579 TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) {
    580   // Simulate hanging of watched thread by making the watched thread wait for a
    581   // very long time by posting a task on watched thread that keeps it busy.
    582   // It is safe ot use base::Unretained because test is waiting for the method
    583   // to finish.
    584   BrowserThread::PostTask(
    585       io_thread_id,
    586       FROM_HERE,
    587       base::Bind(&CustomThreadWatcher::VeryLongMethod,
    588                  base::Unretained(io_watcher_),
    589                  kUnresponsiveTime * 10));
    590 
    591   // Activate watching of DB thread.
    592   WatchDogThread::PostTask(
    593       FROM_HERE,
    594       base::Bind(&ThreadWatcher::ActivateThreadWatching,
    595                  base::Unretained(db_watcher_)));
    596 
    597   // Activate watching of IO thread.
    598   WatchDogThread::PostTask(
    599       FROM_HERE,
    600       base::Bind(&ThreadWatcher::ActivateThreadWatching,
    601                  base::Unretained(io_watcher_)));
    602 
    603   // Verify DB thread is responding with ping/pong messaging.
    604   db_watcher_->WaitForCheckResponse(
    605       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
    606   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
    607       static_cast<base::subtle::Atomic32>(0));
    608   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
    609       static_cast<base::subtle::Atomic32>(0));
    610 
    611   // Verify IO thread is not responding for ping messages.
    612   io_watcher_->WaitForCheckResponse(
    613       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
    614   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
    615       static_cast<base::subtle::Atomic32>(0));
    616   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
    617       static_cast<base::subtle::Atomic32>(0));
    618 
    619   // DeActivate thread watching for shutdown.
    620   WatchDogThread::PostTask(
    621       FROM_HERE,
    622       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
    623                  base::Unretained(io_watcher_)));
    624   WatchDogThread::PostTask(
    625       FROM_HERE,
    626       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
    627                  base::Unretained(db_watcher_)));
    628 
    629   // Wait for the io_watcher_'s VeryLongMethod to finish.
    630   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
    631 }
    632