1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <math.h> 6 7 #include "base/basictypes.h" 8 #include "base/bind.h" 9 #include "base/logging.h" 10 #include "base/memory/scoped_ptr.h" 11 #include "base/message_loop/message_loop.h" 12 #include "base/message_loop/message_loop_proxy.h" 13 #include "base/strings/string_number_conversions.h" 14 #include "base/strings/string_split.h" 15 #include "base/strings/string_tokenizer.h" 16 #include "base/synchronization/condition_variable.h" 17 #include "base/synchronization/lock.h" 18 #include "base/threading/platform_thread.h" 19 #include "base/time/time.h" 20 #include "build/build_config.h" 21 #include "chrome/browser/metrics/thread_watcher.h" 22 #include "chrome/common/chrome_switches.h" 23 #include "content/public/test/test_browser_thread.h" 24 #include "testing/gtest/include/gtest/gtest.h" 25 #include "testing/platform_test.h" 26 27 using base::TimeDelta; 28 using base::TimeTicks; 29 using content::BrowserThread; 30 31 enum State { 32 INITIALIZED, // Created ThreadWatch object. 33 ACTIVATED, // Thread watching activated. 34 SENT_PING, // Sent ping message to watched thread. 35 RECEIVED_PONG, // Received Pong message. 36 DEACTIVATED, // Thread watching de-activated. 37 }; 38 39 enum WaitState { 40 UNINITIALIZED, 41 STARTED_WAITING, // Start waiting for state_ to change to expected_state. 42 STOPPED_WAITING, // Done with the waiting. 43 ALL_DONE, // Done with waiting for STOPPED_WAITING. 44 }; 45 46 enum CheckResponseState { 47 UNKNOWN, 48 SUCCESSFUL, // CheckResponse was successful. 49 FAILED, // CheckResponse has failed. 50 }; 51 52 // This class helps to track and manipulate thread state during tests. This 53 // class also has utility method to simulate hanging of watched thread by making 54 // the watched thread wait for a very long time by posting a task on watched 55 // thread that keeps it busy. It also has an utility method to block running of 56 // tests until ThreadWatcher object's post-condition state changes to an 57 // expected state. 58 class CustomThreadWatcher : public ThreadWatcher { 59 public: 60 base::Lock custom_lock_; 61 base::ConditionVariable state_changed_; 62 State thread_watcher_state_; 63 WaitState wait_state_; 64 CheckResponseState check_response_state_; 65 uint64 ping_sent_; 66 uint64 pong_received_; 67 uint64 success_response_; 68 uint64 failed_response_; 69 base::TimeTicks saved_ping_time_; 70 uint64 saved_ping_sequence_number_; 71 72 CustomThreadWatcher(const BrowserThread::ID thread_id, 73 const std::string thread_name, 74 const TimeDelta& sleep_time, 75 const TimeDelta& unresponsive_time) 76 : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time, 77 unresponsive_time, ThreadWatcherList::kUnresponsiveCount, 78 true, ThreadWatcherList::kLiveThreadsThreshold)), 79 state_changed_(&custom_lock_), 80 thread_watcher_state_(INITIALIZED), 81 wait_state_(UNINITIALIZED), 82 check_response_state_(UNKNOWN), 83 ping_sent_(0), 84 pong_received_(0), 85 success_response_(0), 86 failed_response_(0), 87 saved_ping_time_(base::TimeTicks::Now()), 88 saved_ping_sequence_number_(0) { 89 } 90 91 State UpdateState(State new_state) { 92 State old_state; 93 { 94 base::AutoLock auto_lock(custom_lock_); 95 old_state = thread_watcher_state_; 96 if (old_state != DEACTIVATED) 97 thread_watcher_state_ = new_state; 98 if (new_state == SENT_PING) 99 ++ping_sent_; 100 if (new_state == RECEIVED_PONG) 101 ++pong_received_; 102 saved_ping_time_ = ping_time(); 103 saved_ping_sequence_number_ = ping_sequence_number(); 104 } 105 state_changed_.Broadcast(); 106 return old_state; 107 } 108 109 WaitState UpdateWaitState(WaitState new_state) { 110 WaitState old_state; 111 { 112 base::AutoLock auto_lock(custom_lock_); 113 old_state = wait_state_; 114 wait_state_ = new_state; 115 } 116 state_changed_.Broadcast(); 117 return old_state; 118 } 119 120 virtual void ActivateThreadWatching() OVERRIDE { 121 State old_state = UpdateState(ACTIVATED); 122 EXPECT_EQ(old_state, INITIALIZED); 123 ThreadWatcher::ActivateThreadWatching(); 124 } 125 126 virtual void DeActivateThreadWatching() OVERRIDE { 127 State old_state = UpdateState(DEACTIVATED); 128 EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING || 129 old_state == RECEIVED_PONG); 130 ThreadWatcher::DeActivateThreadWatching(); 131 } 132 133 virtual void PostPingMessage() OVERRIDE { 134 State old_state = UpdateState(SENT_PING); 135 EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG); 136 ThreadWatcher::PostPingMessage(); 137 } 138 139 virtual void OnPongMessage(uint64 ping_sequence_number) OVERRIDE { 140 State old_state = UpdateState(RECEIVED_PONG); 141 EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED); 142 ThreadWatcher::OnPongMessage(ping_sequence_number); 143 } 144 145 virtual void OnCheckResponsiveness(uint64 ping_sequence_number) OVERRIDE { 146 ThreadWatcher::OnCheckResponsiveness(ping_sequence_number); 147 { 148 base::AutoLock auto_lock(custom_lock_); 149 if (responsive_) { 150 ++success_response_; 151 check_response_state_ = SUCCESSFUL; 152 } else { 153 ++failed_response_; 154 check_response_state_ = FAILED; 155 } 156 } 157 // Broadcast to indicate we have checked responsiveness of the thread that 158 // is watched. 159 state_changed_.Broadcast(); 160 } 161 162 void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) { 163 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 164 TimeTicks end_time = TimeTicks::Now() + wait_time; 165 { 166 base::AutoLock auto_lock(custom_lock_); 167 while (wait_state_ != expected_state && TimeTicks::Now() < end_time) 168 state_changed_.TimedWait(end_time - TimeTicks::Now()); 169 } 170 } 171 172 void VeryLongMethod(TimeDelta wait_time) { 173 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 174 WaitForWaitStateChange(wait_time, STOPPED_WAITING); 175 UpdateWaitState(ALL_DONE); 176 } 177 178 State WaitForStateChange(const TimeDelta& wait_time, State expected_state) { 179 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 180 UpdateWaitState(STARTED_WAITING); 181 182 State exit_state = INITIALIZED; 183 // Keep the thread that is running the tests waiting until ThreadWatcher 184 // object's state changes to the expected_state or until wait_time elapses. 185 for (uint32 i = 0; i < unresponsive_threshold_; ++i) { 186 TimeTicks end_time = TimeTicks::Now() + wait_time; 187 { 188 base::AutoLock auto_lock(custom_lock_); 189 while (thread_watcher_state_ != expected_state && 190 TimeTicks::Now() < end_time) { 191 TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); 192 state_changed_.TimedWait(state_change_wait_time); 193 } 194 // Capture the thread_watcher_state_ before it changes and return it 195 // to the caller. 196 exit_state = thread_watcher_state_; 197 if (exit_state == expected_state) 198 break; 199 } 200 } 201 UpdateWaitState(STOPPED_WAITING); 202 return exit_state; 203 } 204 205 CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time, 206 CheckResponseState expected_state) { 207 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 208 UpdateWaitState(STARTED_WAITING); 209 210 CheckResponseState exit_state = UNKNOWN; 211 // Keep the thread that is running the tests waiting until ThreadWatcher 212 // object's check_response_state_ changes to the expected_state or until 213 // wait_time elapses. 214 for (uint32 i = 0; i < unresponsive_threshold_; ++i) { 215 TimeTicks end_time = TimeTicks::Now() + wait_time; 216 { 217 base::AutoLock auto_lock(custom_lock_); 218 while (check_response_state_ != expected_state && 219 TimeTicks::Now() < end_time) { 220 TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); 221 state_changed_.TimedWait(state_change_wait_time); 222 } 223 // Capture the check_response_state_ before it changes and return it 224 // to the caller. 225 exit_state = check_response_state_; 226 if (exit_state == expected_state) 227 break; 228 } 229 } 230 UpdateWaitState(STOPPED_WAITING); 231 return exit_state; 232 } 233 }; 234 235 class ThreadWatcherTest : public ::testing::Test { 236 public: 237 static const TimeDelta kSleepTime; 238 static const TimeDelta kUnresponsiveTime; 239 static const BrowserThread::ID io_thread_id; 240 static const std::string io_thread_name; 241 static const BrowserThread::ID db_thread_id; 242 static const std::string db_thread_name; 243 static const std::string crash_on_hang_seconds; 244 static const std::string crash_on_hang_thread_names; 245 static const std::string thread_names_and_live_threshold; 246 static const std::string crash_on_hang_thread_data; 247 CustomThreadWatcher* io_watcher_; 248 CustomThreadWatcher* db_watcher_; 249 ThreadWatcherList* thread_watcher_list_; 250 251 ThreadWatcherTest() 252 : setup_complete_(&lock_), 253 initialized_(false) { 254 db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB)); 255 io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO)); 256 watchdog_thread_.reset(new WatchDogThread()); 257 db_thread_->Start(); 258 io_thread_->Start(); 259 watchdog_thread_->Start(); 260 261 WatchDogThread::PostTask( 262 FROM_HERE, 263 base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this))); 264 265 WaitForSetUp(TimeDelta::FromMinutes(1)); 266 } 267 268 void SetUpObjects() { 269 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread()); 270 271 // Setup the registry for thread watchers. 272 thread_watcher_list_ = new ThreadWatcherList(); 273 274 // Create thread watcher object for the IO thread. 275 io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name, 276 kSleepTime, kUnresponsiveTime); 277 EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id)); 278 279 // Create thread watcher object for the DB thread. 280 db_watcher_ = new CustomThreadWatcher( 281 db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime); 282 EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id)); 283 284 { 285 base::AutoLock lock(lock_); 286 initialized_ = true; 287 } 288 setup_complete_.Signal(); 289 } 290 291 void WaitForSetUp(TimeDelta wait_time) { 292 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 293 TimeTicks end_time = TimeTicks::Now() + wait_time; 294 { 295 base::AutoLock auto_lock(lock_); 296 while (!initialized_ && TimeTicks::Now() < end_time) 297 setup_complete_.TimedWait(end_time - TimeTicks::Now()); 298 } 299 } 300 301 virtual ~ThreadWatcherTest() { 302 ThreadWatcherList::DeleteAll(); 303 io_watcher_ = NULL; 304 db_watcher_ = NULL; 305 io_thread_.reset(); 306 db_thread_.reset(); 307 watchdog_thread_.reset(); 308 thread_watcher_list_ = NULL; 309 } 310 311 private: 312 base::Lock lock_; 313 base::ConditionVariable setup_complete_; 314 bool initialized_; 315 scoped_ptr<content::TestBrowserThread> db_thread_; 316 scoped_ptr<content::TestBrowserThread> io_thread_; 317 scoped_ptr<WatchDogThread> watchdog_thread_; 318 }; 319 320 // Define static constants. 321 const TimeDelta ThreadWatcherTest::kSleepTime = 322 TimeDelta::FromMilliseconds(50); 323 const TimeDelta ThreadWatcherTest::kUnresponsiveTime = 324 TimeDelta::FromMilliseconds(500); 325 const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO; 326 const std::string ThreadWatcherTest::io_thread_name = "IO"; 327 const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB; 328 const std::string ThreadWatcherTest::db_thread_name = "DB"; 329 const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO"; 330 const std::string ThreadWatcherTest::thread_names_and_live_threshold = 331 "UI:4,IO:4"; 332 const std::string ThreadWatcherTest::crash_on_hang_thread_data = 333 "UI:5:12,IO:5:12,FILE:5:12"; 334 335 TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) { 336 // Setup command_line arguments. 337 CommandLine command_line(CommandLine::NO_PROGRAM); 338 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 339 crash_on_hang_thread_names); 340 341 // Parse command_line arguments. 342 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 343 uint32 unresponsive_threshold; 344 ThreadWatcherList::ParseCommandLine(command_line, 345 &unresponsive_threshold, 346 &crash_on_hang_threads); 347 348 // Verify the data. 349 base::StringTokenizer tokens(crash_on_hang_thread_names, ","); 350 std::vector<std::string> values; 351 while (tokens.GetNext()) { 352 const std::string& token = tokens.token(); 353 base::SplitString(token, ':', &values); 354 std::string thread_name = values[0]; 355 356 ThreadWatcherList::CrashOnHangThreadMap::iterator it = 357 crash_on_hang_threads.find(thread_name); 358 bool crash_on_hang = (it != crash_on_hang_threads.end()); 359 EXPECT_TRUE(crash_on_hang); 360 EXPECT_LT(0u, it->second.live_threads_threshold); 361 EXPECT_LT(0u, it->second.unresponsive_threshold); 362 } 363 } 364 365 TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) { 366 // Setup command_line arguments. 367 CommandLine command_line(CommandLine::NO_PROGRAM); 368 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 369 thread_names_and_live_threshold); 370 371 // Parse command_line arguments. 372 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 373 uint32 unresponsive_threshold; 374 ThreadWatcherList::ParseCommandLine(command_line, 375 &unresponsive_threshold, 376 &crash_on_hang_threads); 377 378 // Verify the data. 379 base::StringTokenizer tokens(thread_names_and_live_threshold, ","); 380 std::vector<std::string> values; 381 while (tokens.GetNext()) { 382 const std::string& token = tokens.token(); 383 base::SplitString(token, ':', &values); 384 std::string thread_name = values[0]; 385 386 ThreadWatcherList::CrashOnHangThreadMap::iterator it = 387 crash_on_hang_threads.find(thread_name); 388 bool crash_on_hang = (it != crash_on_hang_threads.end()); 389 EXPECT_TRUE(crash_on_hang); 390 EXPECT_EQ(4u, it->second.live_threads_threshold); 391 EXPECT_LT(0u, it->second.unresponsive_threshold); 392 } 393 } 394 395 TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) { 396 // Setup command_line arguments. 397 CommandLine command_line(CommandLine::NO_PROGRAM); 398 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 399 crash_on_hang_thread_data); 400 401 // Parse command_line arguments. 402 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 403 uint32 unresponsive_threshold; 404 ThreadWatcherList::ParseCommandLine(command_line, 405 &unresponsive_threshold, 406 &crash_on_hang_threads); 407 408 // Verify the data. 409 base::StringTokenizer tokens(crash_on_hang_thread_data, ","); 410 std::vector<std::string> values; 411 while (tokens.GetNext()) { 412 const std::string& token = tokens.token(); 413 base::SplitString(token, ':', &values); 414 std::string thread_name = values[0]; 415 416 ThreadWatcherList::CrashOnHangThreadMap::iterator it = 417 crash_on_hang_threads.find(thread_name); 418 419 bool crash_on_hang = (it != crash_on_hang_threads.end()); 420 EXPECT_TRUE(crash_on_hang); 421 422 uint32 crash_live_threads_threshold = it->second.live_threads_threshold; 423 EXPECT_EQ(5u, crash_live_threads_threshold); 424 425 uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold; 426 uint32 crash_on_unresponsive_seconds = 427 ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold; 428 EXPECT_EQ(12u, crash_on_unresponsive_seconds); 429 } 430 } 431 432 // Test registration. When thread_watcher_list_ goes out of scope after 433 // TearDown, all thread watcher objects will be deleted. 434 TEST_F(ThreadWatcherTest, Registration) { 435 // Check ThreadWatcher object has all correct parameters. 436 EXPECT_EQ(io_thread_id, io_watcher_->thread_id()); 437 EXPECT_EQ(io_thread_name, io_watcher_->thread_name()); 438 EXPECT_EQ(kSleepTime, io_watcher_->sleep_time()); 439 EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time()); 440 EXPECT_FALSE(io_watcher_->active()); 441 442 // Check ThreadWatcher object of watched DB thread has correct data. 443 EXPECT_EQ(db_thread_id, db_watcher_->thread_id()); 444 EXPECT_EQ(db_thread_name, db_watcher_->thread_name()); 445 EXPECT_EQ(kSleepTime, db_watcher_->sleep_time()); 446 EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time()); 447 EXPECT_FALSE(db_watcher_->active()); 448 } 449 450 // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This 451 // method also checks that pong message was sent by the watched thread and pong 452 // message was received by the WatchDogThread. It also checks that 453 // OnCheckResponsiveness has verified the ping-pong mechanism and the watched 454 // thread is not hung. 455 TEST_F(ThreadWatcherTest, ThreadResponding) { 456 TimeTicks time_before_ping = TimeTicks::Now(); 457 // Activate watching IO thread. 458 WatchDogThread::PostTask( 459 FROM_HERE, 460 base::Bind(&ThreadWatcher::ActivateThreadWatching, 461 base::Unretained(io_watcher_))); 462 463 // Activate would have started ping/pong messaging. Expect atleast one 464 // ping/pong messaging sequence to happen. 465 io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1), 466 RECEIVED_PONG); 467 EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0)); 468 EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0)); 469 EXPECT_TRUE(io_watcher_->active()); 470 EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping); 471 EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0)); 472 473 // Verify watched thread is responding with ping/pong messaging. 474 io_watcher_->WaitForCheckResponse( 475 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 476 EXPECT_GT(io_watcher_->success_response_, static_cast<uint64>(0)); 477 EXPECT_EQ(io_watcher_->failed_response_, static_cast<uint64>(0)); 478 479 // DeActivate thread watching for shutdown. 480 WatchDogThread::PostTask( 481 FROM_HERE, 482 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 483 base::Unretained(io_watcher_))); 484 } 485 486 // This test posts a task on watched thread that takes very long time (this is 487 // to simulate hanging of watched thread). It then checks for 488 // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false 489 // if the watched thread is not responding). 490 TEST_F(ThreadWatcherTest, ThreadNotResponding) { 491 // Simulate hanging of watched thread by making the watched thread wait for a 492 // very long time by posting a task on watched thread that keeps it busy. 493 // It is safe to use base::Unretained because test is waiting for the method 494 // to finish. 495 BrowserThread::PostTask( 496 io_thread_id, 497 FROM_HERE, 498 base::Bind(&CustomThreadWatcher::VeryLongMethod, 499 base::Unretained(io_watcher_), 500 kUnresponsiveTime * 10)); 501 502 // Activate thread watching. 503 WatchDogThread::PostTask( 504 FROM_HERE, 505 base::Bind(&ThreadWatcher::ActivateThreadWatching, 506 base::Unretained(io_watcher_))); 507 508 // Verify watched thread is not responding for ping messages. 509 io_watcher_->WaitForCheckResponse( 510 kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); 511 EXPECT_EQ(io_watcher_->success_response_, static_cast<uint64>(0)); 512 EXPECT_GT(io_watcher_->failed_response_, static_cast<uint64>(0)); 513 514 // DeActivate thread watching for shutdown. 515 WatchDogThread::PostTask( 516 FROM_HERE, 517 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 518 base::Unretained(io_watcher_))); 519 520 // Wait for the io_watcher_'s VeryLongMethod to finish. 521 io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); 522 } 523 524 // Test watching of multiple threads with all threads not responding. 525 TEST_F(ThreadWatcherTest, MultipleThreadsResponding) { 526 // Check for DB thread to perform ping/pong messaging. 527 WatchDogThread::PostTask( 528 FROM_HERE, 529 base::Bind(&ThreadWatcher::ActivateThreadWatching, 530 base::Unretained(db_watcher_))); 531 532 // Check for IO thread to perform ping/pong messaging. 533 WatchDogThread::PostTask( 534 FROM_HERE, 535 base::Bind(&ThreadWatcher::ActivateThreadWatching, 536 base::Unretained(io_watcher_))); 537 538 // Verify DB thread is responding with ping/pong messaging. 539 db_watcher_->WaitForCheckResponse( 540 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 541 EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0)); 542 EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0)); 543 EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0)); 544 EXPECT_GT(db_watcher_->success_response_, static_cast<uint64>(0)); 545 EXPECT_EQ(db_watcher_->failed_response_, static_cast<uint64>(0)); 546 547 // Verify IO thread is responding with ping/pong messaging. 548 io_watcher_->WaitForCheckResponse( 549 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 550 EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0)); 551 EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0)); 552 EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0)); 553 EXPECT_GT(io_watcher_->success_response_, static_cast<uint64>(0)); 554 EXPECT_EQ(io_watcher_->failed_response_, static_cast<uint64>(0)); 555 556 // DeActivate thread watching for shutdown. 557 WatchDogThread::PostTask( 558 FROM_HERE, 559 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 560 base::Unretained(io_watcher_))); 561 562 WatchDogThread::PostTask( 563 FROM_HERE, 564 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 565 base::Unretained(db_watcher_))); 566 } 567 568 // Test watching of multiple threads with one of the threads not responding. 569 TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) { 570 // Simulate hanging of watched thread by making the watched thread wait for a 571 // very long time by posting a task on watched thread that keeps it busy. 572 // It is safe ot use base::Unretained because test is waiting for the method 573 // to finish. 574 BrowserThread::PostTask( 575 io_thread_id, 576 FROM_HERE, 577 base::Bind(&CustomThreadWatcher::VeryLongMethod, 578 base::Unretained(io_watcher_), 579 kUnresponsiveTime * 10)); 580 581 // Activate watching of DB thread. 582 WatchDogThread::PostTask( 583 FROM_HERE, 584 base::Bind(&ThreadWatcher::ActivateThreadWatching, 585 base::Unretained(db_watcher_))); 586 587 // Activate watching of IO thread. 588 WatchDogThread::PostTask( 589 FROM_HERE, 590 base::Bind(&ThreadWatcher::ActivateThreadWatching, 591 base::Unretained(io_watcher_))); 592 593 // Verify DB thread is responding with ping/pong messaging. 594 db_watcher_->WaitForCheckResponse( 595 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 596 EXPECT_GT(db_watcher_->success_response_, static_cast<uint64>(0)); 597 EXPECT_EQ(db_watcher_->failed_response_, static_cast<uint64>(0)); 598 599 // Verify IO thread is not responding for ping messages. 600 io_watcher_->WaitForCheckResponse( 601 kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); 602 EXPECT_EQ(io_watcher_->success_response_, static_cast<uint64>(0)); 603 EXPECT_GT(io_watcher_->failed_response_, static_cast<uint64>(0)); 604 605 // DeActivate thread watching for shutdown. 606 WatchDogThread::PostTask( 607 FROM_HERE, 608 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 609 base::Unretained(io_watcher_))); 610 WatchDogThread::PostTask( 611 FROM_HERE, 612 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 613 base::Unretained(db_watcher_))); 614 615 // Wait for the io_watcher_'s VeryLongMethod to finish. 616 io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); 617 } 618