1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 #include <math.h> 6 7 #include "base/basictypes.h" 8 #include "base/bind.h" 9 #include "base/logging.h" 10 #include "base/memory/scoped_ptr.h" 11 #include "base/message_loop/message_loop.h" 12 #include "base/message_loop/message_loop_proxy.h" 13 #include "base/run_loop.h" 14 #include "base/strings/string_number_conversions.h" 15 #include "base/strings/string_split.h" 16 #include "base/strings/string_tokenizer.h" 17 #include "base/synchronization/condition_variable.h" 18 #include "base/synchronization/lock.h" 19 #include "base/threading/platform_thread.h" 20 #include "base/time/time.h" 21 #include "build/build_config.h" 22 #include "chrome/browser/metrics/thread_watcher.h" 23 #include "chrome/common/chrome_switches.h" 24 #include "content/public/test/test_browser_thread.h" 25 #include "testing/gtest/include/gtest/gtest.h" 26 #include "testing/platform_test.h" 27 28 using base::TimeDelta; 29 using base::TimeTicks; 30 using content::BrowserThread; 31 32 enum State { 33 INITIALIZED, // Created ThreadWatch object. 34 ACTIVATED, // Thread watching activated. 35 SENT_PING, // Sent ping message to watched thread. 36 RECEIVED_PONG, // Received Pong message. 37 DEACTIVATED, // Thread watching de-activated. 38 }; 39 40 enum WaitState { 41 UNINITIALIZED, 42 STARTED_WAITING, // Start waiting for state_ to change to expected_state. 43 STOPPED_WAITING, // Done with the waiting. 44 ALL_DONE, // Done with waiting for STOPPED_WAITING. 45 }; 46 47 enum CheckResponseState { 48 UNKNOWN, 49 SUCCESSFUL, // CheckResponse was successful. 50 FAILED, // CheckResponse has failed. 51 }; 52 53 // This class helps to track and manipulate thread state during tests. This 54 // class also has utility method to simulate hanging of watched thread by making 55 // the watched thread wait for a very long time by posting a task on watched 56 // thread that keeps it busy. It also has an utility method to block running of 57 // tests until ThreadWatcher object's post-condition state changes to an 58 // expected state. 59 class CustomThreadWatcher : public ThreadWatcher { 60 public: 61 base::Lock custom_lock_; 62 base::ConditionVariable state_changed_; 63 State thread_watcher_state_; 64 WaitState wait_state_; 65 CheckResponseState check_response_state_; 66 uint64 ping_sent_; 67 uint64 pong_received_; 68 base::subtle::Atomic32 success_response_; 69 base::subtle::Atomic32 failed_response_; 70 base::TimeTicks saved_ping_time_; 71 uint64 saved_ping_sequence_number_; 72 73 CustomThreadWatcher(const BrowserThread::ID thread_id, 74 const std::string thread_name, 75 const TimeDelta& sleep_time, 76 const TimeDelta& unresponsive_time) 77 : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time, 78 unresponsive_time, ThreadWatcherList::kUnresponsiveCount, 79 true, ThreadWatcherList::kLiveThreadsThreshold)), 80 state_changed_(&custom_lock_), 81 thread_watcher_state_(INITIALIZED), 82 wait_state_(UNINITIALIZED), 83 check_response_state_(UNKNOWN), 84 ping_sent_(0), 85 pong_received_(0), 86 success_response_(0), 87 failed_response_(0), 88 saved_ping_time_(base::TimeTicks::Now()), 89 saved_ping_sequence_number_(0) { 90 } 91 92 State UpdateState(State new_state) { 93 State old_state; 94 { 95 base::AutoLock auto_lock(custom_lock_); 96 old_state = thread_watcher_state_; 97 if (old_state != DEACTIVATED) 98 thread_watcher_state_ = new_state; 99 if (new_state == SENT_PING) 100 ++ping_sent_; 101 if (new_state == RECEIVED_PONG) 102 ++pong_received_; 103 saved_ping_time_ = ping_time(); 104 saved_ping_sequence_number_ = ping_sequence_number(); 105 } 106 state_changed_.Broadcast(); 107 return old_state; 108 } 109 110 WaitState UpdateWaitState(WaitState new_state) { 111 WaitState old_state; 112 { 113 base::AutoLock auto_lock(custom_lock_); 114 old_state = wait_state_; 115 wait_state_ = new_state; 116 } 117 state_changed_.Broadcast(); 118 return old_state; 119 } 120 121 virtual void ActivateThreadWatching() OVERRIDE { 122 State old_state = UpdateState(ACTIVATED); 123 EXPECT_EQ(old_state, INITIALIZED); 124 ThreadWatcher::ActivateThreadWatching(); 125 } 126 127 virtual void DeActivateThreadWatching() OVERRIDE { 128 State old_state = UpdateState(DEACTIVATED); 129 EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING || 130 old_state == RECEIVED_PONG); 131 ThreadWatcher::DeActivateThreadWatching(); 132 } 133 134 virtual void PostPingMessage() OVERRIDE { 135 State old_state = UpdateState(SENT_PING); 136 EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG); 137 ThreadWatcher::PostPingMessage(); 138 } 139 140 virtual void OnPongMessage(uint64 ping_sequence_number) OVERRIDE { 141 State old_state = UpdateState(RECEIVED_PONG); 142 EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED); 143 ThreadWatcher::OnPongMessage(ping_sequence_number); 144 } 145 146 virtual void OnCheckResponsiveness(uint64 ping_sequence_number) OVERRIDE { 147 ThreadWatcher::OnCheckResponsiveness(ping_sequence_number); 148 { 149 base::AutoLock auto_lock(custom_lock_); 150 if (responsive_) { 151 base::subtle::Release_Store(&success_response_, 152 base::subtle::Acquire_Load(&success_response_) + 1); 153 check_response_state_ = SUCCESSFUL; 154 } else { 155 base::subtle::Release_Store(&failed_response_, 156 base::subtle::Acquire_Load(&failed_response_) + 1); 157 check_response_state_ = FAILED; 158 } 159 } 160 // Broadcast to indicate we have checked responsiveness of the thread that 161 // is watched. 162 state_changed_.Broadcast(); 163 } 164 165 void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) { 166 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 167 TimeTicks end_time = TimeTicks::Now() + wait_time; 168 { 169 base::AutoLock auto_lock(custom_lock_); 170 while (wait_state_ != expected_state && TimeTicks::Now() < end_time) 171 state_changed_.TimedWait(end_time - TimeTicks::Now()); 172 } 173 } 174 175 void VeryLongMethod(TimeDelta wait_time) { 176 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 177 WaitForWaitStateChange(wait_time, STOPPED_WAITING); 178 UpdateWaitState(ALL_DONE); 179 } 180 181 State WaitForStateChange(const TimeDelta& wait_time, State expected_state) { 182 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 183 UpdateWaitState(STARTED_WAITING); 184 185 State exit_state = INITIALIZED; 186 // Keep the thread that is running the tests waiting until ThreadWatcher 187 // object's state changes to the expected_state or until wait_time elapses. 188 for (uint32 i = 0; i < unresponsive_threshold_; ++i) { 189 TimeTicks end_time = TimeTicks::Now() + wait_time; 190 { 191 base::AutoLock auto_lock(custom_lock_); 192 while (thread_watcher_state_ != expected_state && 193 TimeTicks::Now() < end_time) { 194 TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); 195 state_changed_.TimedWait(state_change_wait_time); 196 } 197 // Capture the thread_watcher_state_ before it changes and return it 198 // to the caller. 199 exit_state = thread_watcher_state_; 200 if (exit_state == expected_state) 201 break; 202 } 203 } 204 UpdateWaitState(STOPPED_WAITING); 205 return exit_state; 206 } 207 208 CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time, 209 CheckResponseState expected_state) { 210 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 211 UpdateWaitState(STARTED_WAITING); 212 213 CheckResponseState exit_state = UNKNOWN; 214 // Keep the thread that is running the tests waiting until ThreadWatcher 215 // object's check_response_state_ changes to the expected_state or until 216 // wait_time elapses. 217 for (uint32 i = 0; i < unresponsive_threshold_; ++i) { 218 TimeTicks end_time = TimeTicks::Now() + wait_time; 219 { 220 base::AutoLock auto_lock(custom_lock_); 221 while (check_response_state_ != expected_state && 222 TimeTicks::Now() < end_time) { 223 TimeDelta state_change_wait_time = end_time - TimeTicks::Now(); 224 state_changed_.TimedWait(state_change_wait_time); 225 } 226 // Capture the check_response_state_ before it changes and return it 227 // to the caller. 228 exit_state = check_response_state_; 229 if (exit_state == expected_state) 230 break; 231 } 232 } 233 UpdateWaitState(STOPPED_WAITING); 234 return exit_state; 235 } 236 }; 237 238 class ThreadWatcherTest : public ::testing::Test { 239 public: 240 static const TimeDelta kSleepTime; 241 static const TimeDelta kUnresponsiveTime; 242 static const BrowserThread::ID io_thread_id; 243 static const std::string io_thread_name; 244 static const BrowserThread::ID db_thread_id; 245 static const std::string db_thread_name; 246 static const std::string crash_on_hang_seconds; 247 static const std::string crash_on_hang_thread_names; 248 static const std::string thread_names_and_live_threshold; 249 static const std::string crash_on_hang_thread_data; 250 CustomThreadWatcher* io_watcher_; 251 CustomThreadWatcher* db_watcher_; 252 ThreadWatcherList* thread_watcher_list_; 253 254 ThreadWatcherTest() 255 : setup_complete_(&lock_), 256 initialized_(false) { 257 db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB)); 258 io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO)); 259 watchdog_thread_.reset(new WatchDogThread()); 260 db_thread_->Start(); 261 io_thread_->Start(); 262 watchdog_thread_->Start(); 263 264 WatchDogThread::PostTask( 265 FROM_HERE, 266 base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this))); 267 268 WaitForSetUp(TimeDelta::FromMinutes(1)); 269 } 270 271 void SetUpObjects() { 272 DCHECK(WatchDogThread::CurrentlyOnWatchDogThread()); 273 274 // Setup the registry for thread watchers. 275 thread_watcher_list_ = new ThreadWatcherList(); 276 277 // Create thread watcher object for the IO thread. 278 io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name, 279 kSleepTime, kUnresponsiveTime); 280 EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id)); 281 282 // Create thread watcher object for the DB thread. 283 db_watcher_ = new CustomThreadWatcher( 284 db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime); 285 EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id)); 286 287 { 288 base::AutoLock lock(lock_); 289 initialized_ = true; 290 } 291 setup_complete_.Signal(); 292 } 293 294 void WaitForSetUp(TimeDelta wait_time) { 295 DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 296 TimeTicks end_time = TimeTicks::Now() + wait_time; 297 { 298 base::AutoLock auto_lock(lock_); 299 while (!initialized_ && TimeTicks::Now() < end_time) 300 setup_complete_.TimedWait(end_time - TimeTicks::Now()); 301 } 302 } 303 304 virtual ~ThreadWatcherTest() { 305 ThreadWatcherList::DeleteAll(); 306 io_watcher_ = NULL; 307 db_watcher_ = NULL; 308 io_thread_.reset(); 309 db_thread_.reset(); 310 watchdog_thread_.reset(); 311 thread_watcher_list_ = NULL; 312 } 313 314 private: 315 base::Lock lock_; 316 base::ConditionVariable setup_complete_; 317 bool initialized_; 318 scoped_ptr<content::TestBrowserThread> db_thread_; 319 scoped_ptr<content::TestBrowserThread> io_thread_; 320 scoped_ptr<WatchDogThread> watchdog_thread_; 321 }; 322 323 // Define static constants. 324 const TimeDelta ThreadWatcherTest::kSleepTime = 325 TimeDelta::FromMilliseconds(50); 326 const TimeDelta ThreadWatcherTest::kUnresponsiveTime = 327 TimeDelta::FromMilliseconds(500); 328 const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO; 329 const std::string ThreadWatcherTest::io_thread_name = "IO"; 330 const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB; 331 const std::string ThreadWatcherTest::db_thread_name = "DB"; 332 const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO"; 333 const std::string ThreadWatcherTest::thread_names_and_live_threshold = 334 "UI:4,IO:4"; 335 const std::string ThreadWatcherTest::crash_on_hang_thread_data = 336 "UI:5:12,IO:5:12,FILE:5:12"; 337 338 TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) { 339 // Setup command_line arguments. 340 CommandLine command_line(CommandLine::NO_PROGRAM); 341 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 342 crash_on_hang_thread_names); 343 344 // Parse command_line arguments. 345 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 346 uint32 unresponsive_threshold; 347 ThreadWatcherList::ParseCommandLine(command_line, 348 &unresponsive_threshold, 349 &crash_on_hang_threads); 350 351 // Verify the data. 352 base::StringTokenizer tokens(crash_on_hang_thread_names, ","); 353 std::vector<std::string> values; 354 while (tokens.GetNext()) { 355 const std::string& token = tokens.token(); 356 base::SplitString(token, ':', &values); 357 std::string thread_name = values[0]; 358 359 ThreadWatcherList::CrashOnHangThreadMap::iterator it = 360 crash_on_hang_threads.find(thread_name); 361 bool crash_on_hang = (it != crash_on_hang_threads.end()); 362 EXPECT_TRUE(crash_on_hang); 363 EXPECT_LT(0u, it->second.live_threads_threshold); 364 EXPECT_LT(0u, it->second.unresponsive_threshold); 365 } 366 } 367 368 TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) { 369 // Setup command_line arguments. 370 CommandLine command_line(CommandLine::NO_PROGRAM); 371 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 372 thread_names_and_live_threshold); 373 374 // Parse command_line arguments. 375 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 376 uint32 unresponsive_threshold; 377 ThreadWatcherList::ParseCommandLine(command_line, 378 &unresponsive_threshold, 379 &crash_on_hang_threads); 380 381 // Verify the data. 382 base::StringTokenizer tokens(thread_names_and_live_threshold, ","); 383 std::vector<std::string> values; 384 while (tokens.GetNext()) { 385 const std::string& token = tokens.token(); 386 base::SplitString(token, ':', &values); 387 std::string thread_name = values[0]; 388 389 ThreadWatcherList::CrashOnHangThreadMap::iterator it = 390 crash_on_hang_threads.find(thread_name); 391 bool crash_on_hang = (it != crash_on_hang_threads.end()); 392 EXPECT_TRUE(crash_on_hang); 393 EXPECT_EQ(4u, it->second.live_threads_threshold); 394 EXPECT_LT(0u, it->second.unresponsive_threshold); 395 } 396 } 397 398 TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) { 399 // Setup command_line arguments. 400 CommandLine command_line(CommandLine::NO_PROGRAM); 401 command_line.AppendSwitchASCII(switches::kCrashOnHangThreads, 402 crash_on_hang_thread_data); 403 404 // Parse command_line arguments. 405 ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads; 406 uint32 unresponsive_threshold; 407 ThreadWatcherList::ParseCommandLine(command_line, 408 &unresponsive_threshold, 409 &crash_on_hang_threads); 410 411 // Verify the data. 412 base::StringTokenizer tokens(crash_on_hang_thread_data, ","); 413 std::vector<std::string> values; 414 while (tokens.GetNext()) { 415 const std::string& token = tokens.token(); 416 base::SplitString(token, ':', &values); 417 std::string thread_name = values[0]; 418 419 ThreadWatcherList::CrashOnHangThreadMap::iterator it = 420 crash_on_hang_threads.find(thread_name); 421 422 bool crash_on_hang = (it != crash_on_hang_threads.end()); 423 EXPECT_TRUE(crash_on_hang); 424 425 uint32 crash_live_threads_threshold = it->second.live_threads_threshold; 426 EXPECT_EQ(5u, crash_live_threads_threshold); 427 428 uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold; 429 uint32 crash_on_unresponsive_seconds = 430 ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold; 431 EXPECT_EQ(12u, crash_on_unresponsive_seconds); 432 } 433 } 434 435 // Test registration. When thread_watcher_list_ goes out of scope after 436 // TearDown, all thread watcher objects will be deleted. 437 TEST_F(ThreadWatcherTest, Registration) { 438 // Check ThreadWatcher object has all correct parameters. 439 EXPECT_EQ(io_thread_id, io_watcher_->thread_id()); 440 EXPECT_EQ(io_thread_name, io_watcher_->thread_name()); 441 EXPECT_EQ(kSleepTime, io_watcher_->sleep_time()); 442 EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time()); 443 EXPECT_FALSE(io_watcher_->active()); 444 445 // Check ThreadWatcher object of watched DB thread has correct data. 446 EXPECT_EQ(db_thread_id, db_watcher_->thread_id()); 447 EXPECT_EQ(db_thread_name, db_watcher_->thread_name()); 448 EXPECT_EQ(kSleepTime, db_watcher_->sleep_time()); 449 EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time()); 450 EXPECT_FALSE(db_watcher_->active()); 451 } 452 453 // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This 454 // method also checks that pong message was sent by the watched thread and pong 455 // message was received by the WatchDogThread. It also checks that 456 // OnCheckResponsiveness has verified the ping-pong mechanism and the watched 457 // thread is not hung. 458 TEST_F(ThreadWatcherTest, ThreadResponding) { 459 TimeTicks time_before_ping = TimeTicks::Now(); 460 // Activate watching IO thread. 461 WatchDogThread::PostTask( 462 FROM_HERE, 463 base::Bind(&ThreadWatcher::ActivateThreadWatching, 464 base::Unretained(io_watcher_))); 465 466 // Activate would have started ping/pong messaging. Expect atleast one 467 // ping/pong messaging sequence to happen. 468 io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1), 469 RECEIVED_PONG); 470 EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0)); 471 EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0)); 472 EXPECT_TRUE(io_watcher_->active()); 473 EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping); 474 EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0)); 475 476 // Verify watched thread is responding with ping/pong messaging. 477 io_watcher_->WaitForCheckResponse( 478 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 479 EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 480 static_cast<base::subtle::Atomic32>(0)); 481 EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 482 static_cast<base::subtle::Atomic32>(0)); 483 484 // DeActivate thread watching for shutdown. 485 WatchDogThread::PostTask( 486 FROM_HERE, 487 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 488 base::Unretained(io_watcher_))); 489 } 490 491 // This test posts a task on watched thread that takes very long time (this is 492 // to simulate hanging of watched thread). It then checks for 493 // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false 494 // if the watched thread is not responding). 495 TEST_F(ThreadWatcherTest, ThreadNotResponding) { 496 // Simulate hanging of watched thread by making the watched thread wait for a 497 // very long time by posting a task on watched thread that keeps it busy. 498 // It is safe to use base::Unretained because test is waiting for the method 499 // to finish. 500 BrowserThread::PostTask( 501 io_thread_id, 502 FROM_HERE, 503 base::Bind(&CustomThreadWatcher::VeryLongMethod, 504 base::Unretained(io_watcher_), 505 kUnresponsiveTime * 10)); 506 507 // Activate thread watching. 508 WatchDogThread::PostTask( 509 FROM_HERE, 510 base::Bind(&ThreadWatcher::ActivateThreadWatching, 511 base::Unretained(io_watcher_))); 512 513 // Verify watched thread is not responding for ping messages. 514 io_watcher_->WaitForCheckResponse( 515 kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); 516 EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 517 static_cast<base::subtle::Atomic32>(0)); 518 EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 519 static_cast<base::subtle::Atomic32>(0)); 520 521 // DeActivate thread watching for shutdown. 522 WatchDogThread::PostTask( 523 FROM_HERE, 524 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 525 base::Unretained(io_watcher_))); 526 527 // Wait for the io_watcher_'s VeryLongMethod to finish. 528 io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); 529 } 530 531 // Test watching of multiple threads with all threads not responding. 532 TEST_F(ThreadWatcherTest, MultipleThreadsResponding) { 533 // Check for DB thread to perform ping/pong messaging. 534 WatchDogThread::PostTask( 535 FROM_HERE, 536 base::Bind(&ThreadWatcher::ActivateThreadWatching, 537 base::Unretained(db_watcher_))); 538 539 // Check for IO thread to perform ping/pong messaging. 540 WatchDogThread::PostTask( 541 FROM_HERE, 542 base::Bind(&ThreadWatcher::ActivateThreadWatching, 543 base::Unretained(io_watcher_))); 544 545 // Verify DB thread is responding with ping/pong messaging. 546 db_watcher_->WaitForCheckResponse( 547 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 548 EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0)); 549 EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0)); 550 EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0)); 551 EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)), 552 static_cast<base::subtle::Atomic32>(0)); 553 EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)), 554 static_cast<base::subtle::Atomic32>(0)); 555 556 // Verify IO thread is responding with ping/pong messaging. 557 io_watcher_->WaitForCheckResponse( 558 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 559 EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0)); 560 EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0)); 561 EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0)); 562 EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 563 static_cast<base::subtle::Atomic32>(0)); 564 EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 565 static_cast<base::subtle::Atomic32>(0)); 566 567 // DeActivate thread watching for shutdown. 568 WatchDogThread::PostTask( 569 FROM_HERE, 570 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 571 base::Unretained(io_watcher_))); 572 573 WatchDogThread::PostTask( 574 FROM_HERE, 575 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 576 base::Unretained(db_watcher_))); 577 } 578 579 // Test watching of multiple threads with one of the threads not responding. 580 TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) { 581 // Simulate hanging of watched thread by making the watched thread wait for a 582 // very long time by posting a task on watched thread that keeps it busy. 583 // It is safe ot use base::Unretained because test is waiting for the method 584 // to finish. 585 BrowserThread::PostTask( 586 io_thread_id, 587 FROM_HERE, 588 base::Bind(&CustomThreadWatcher::VeryLongMethod, 589 base::Unretained(io_watcher_), 590 kUnresponsiveTime * 10)); 591 592 // Activate watching of DB thread. 593 WatchDogThread::PostTask( 594 FROM_HERE, 595 base::Bind(&ThreadWatcher::ActivateThreadWatching, 596 base::Unretained(db_watcher_))); 597 598 // Activate watching of IO thread. 599 WatchDogThread::PostTask( 600 FROM_HERE, 601 base::Bind(&ThreadWatcher::ActivateThreadWatching, 602 base::Unretained(io_watcher_))); 603 604 // Verify DB thread is responding with ping/pong messaging. 605 db_watcher_->WaitForCheckResponse( 606 kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL); 607 EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)), 608 static_cast<base::subtle::Atomic32>(0)); 609 EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)), 610 static_cast<base::subtle::Atomic32>(0)); 611 612 // Verify IO thread is not responding for ping messages. 613 io_watcher_->WaitForCheckResponse( 614 kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED); 615 EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)), 616 static_cast<base::subtle::Atomic32>(0)); 617 EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)), 618 static_cast<base::subtle::Atomic32>(0)); 619 620 // DeActivate thread watching for shutdown. 621 WatchDogThread::PostTask( 622 FROM_HERE, 623 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 624 base::Unretained(io_watcher_))); 625 WatchDogThread::PostTask( 626 FROM_HERE, 627 base::Bind(&ThreadWatcher::DeActivateThreadWatching, 628 base::Unretained(db_watcher_))); 629 630 // Wait for the io_watcher_'s VeryLongMethod to finish. 631 io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE); 632 } 633 634 class ThreadWatcherListTest : public ::testing::Test { 635 protected: 636 ThreadWatcherListTest() 637 : done_(&lock_), 638 state_available_(false), 639 has_thread_watcher_list_(false), 640 stopped_(false) { 641 } 642 643 void ReadStateOnWatchDogThread() { 644 CHECK(WatchDogThread::CurrentlyOnWatchDogThread()); 645 { 646 base::AutoLock auto_lock(lock_); 647 has_thread_watcher_list_ = 648 ThreadWatcherList::g_thread_watcher_list_ != NULL; 649 stopped_ = ThreadWatcherList::g_stopped_; 650 state_available_ = true; 651 } 652 done_.Signal(); 653 } 654 655 void CheckState(bool has_thread_watcher_list, 656 bool stopped, 657 const char* const msg) { 658 CHECK(!WatchDogThread::CurrentlyOnWatchDogThread()); 659 { 660 base::AutoLock auto_lock(lock_); 661 state_available_ = false; 662 } 663 664 WatchDogThread::PostTask( 665 FROM_HERE, 666 base::Bind(&ThreadWatcherListTest::ReadStateOnWatchDogThread, 667 base::Unretained(this))); 668 { 669 base::AutoLock auto_lock(lock_); 670 while (!state_available_) 671 done_.Wait(); 672 673 EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg; 674 EXPECT_EQ(stopped, stopped_) << msg; 675 } 676 } 677 678 base::Lock lock_; 679 base::ConditionVariable done_; 680 681 bool state_available_; 682 bool has_thread_watcher_list_; 683 bool stopped_; 684 }; 685 686 TEST_F(ThreadWatcherListTest, Restart) { 687 ThreadWatcherList::g_initialize_delay_seconds = 1; 688 689 base::MessageLoopForUI message_loop_for_ui; 690 content::TestBrowserThread ui_thread(BrowserThread::UI, &message_loop_for_ui); 691 692 scoped_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread()); 693 watchdog_thread_->Start(); 694 695 // See http://crbug.com/347887. 696 // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_, 697 // whilst StopWatchingAll() will just PostTask to destroy it. 698 // Ensure that when Stop is called, Start will NOT create 699 // g_thread_watcher_list_ later on. 700 ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess()); 701 ThreadWatcherList::StopWatchingAll(); 702 message_loop_for_ui.PostDelayedTask( 703 FROM_HERE, 704 message_loop_for_ui.QuitClosure(), 705 base::TimeDelta::FromSeconds( 706 ThreadWatcherList::g_initialize_delay_seconds)); 707 message_loop_for_ui.Run(); 708 709 CheckState(false /* has_thread_watcher_list */, 710 true /* stopped */, 711 "Start / Stopped"); 712 713 // Proceed with just |StartWatchingAll| and ensure it'll be started. 714 ThreadWatcherList::StartWatchingAll(*CommandLine::ForCurrentProcess()); 715 message_loop_for_ui.PostDelayedTask( 716 FROM_HERE, 717 message_loop_for_ui.QuitClosure(), 718 base::TimeDelta::FromSeconds( 719 ThreadWatcherList::g_initialize_delay_seconds + 1)); 720 message_loop_for_ui.Run(); 721 722 CheckState(true /* has_thread_watcher_list */, 723 false /* stopped */, 724 "Started"); 725 726 // Finally, StopWatchingAll() must stop. 727 ThreadWatcherList::StopWatchingAll(); 728 message_loop_for_ui.PostDelayedTask( 729 FROM_HERE, 730 message_loop_for_ui.QuitClosure(), 731 base::TimeDelta::FromSeconds( 732 ThreadWatcherList::g_initialize_delay_seconds)); 733 message_loop_for_ui.Run(); 734 735 CheckState(false /* has_thread_watcher_list */, 736 true /* stopped */, 737 "Stopped"); 738 } 739
