Home | History | Annotate | Download | only in metrics 
      1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
      2 // Use of this source code is governed by a BSD-style license that can be
      3 // found in the LICENSE file.
      4 
      5 #include "chrome/browser/metrics/thread_watcher.h"
      6 
      7 #include <math.h>  // ceil
      8 
      9 #include "base/bind.h"
     10 #include "base/compiler_specific.h"
     11 #include "base/debug/alias.h"
     12 #include "base/lazy_instance.h"
     13 #include "base/strings/string_number_conversions.h"
     14 #include "base/strings/string_split.h"
     15 #include "base/strings/string_tokenizer.h"
     16 #include "base/strings/stringprintf.h"
     17 #include "base/threading/thread_restrictions.h"
     18 #include "build/build_config.h"
     19 #include "chrome/browser/metrics/metrics_service.h"
     20 #include "chrome/common/chrome_switches.h"
     21 #include "chrome/common/chrome_version_info.h"
     22 #include "chrome/common/dump_without_crashing.h"
     23 #include "chrome/common/logging_chrome.h"
     24 
     25 #if defined(OS_WIN)
     26 #include "base/win/windows_version.h"
     27 #endif
     28 
     29 using content::BrowserThread;
     30 
     31 namespace {
     32 
     33 // The following are unique function names for forcing the crash when a thread
     34 // is unresponsive. This makes it possible to tell from the callstack alone what
     35 // thread was unresponsive.
     36 //
     37 // We disable optimizations for this block of functions so the compiler doesn't
     38 // merge them all together.
     39 MSVC_DISABLE_OPTIMIZE()
     40 MSVC_PUSH_DISABLE_WARNING(4748)
     41 
     42 #ifndef NDEBUG
     43 int* NullPointer() {
     44   return reinterpret_cast<int*>(NULL);
     45 }
     46 #endif
     47 
     48 void NullPointerCrash(int line_number) {
     49 #ifndef NDEBUG
     50   *NullPointer() = line_number;  // Crash.
     51 #else
     52   logging::DumpWithoutCrashing();
     53 #endif
     54 }
     55 
     56 NOINLINE void ShutdownCrash() {
     57   NullPointerCrash(__LINE__);
     58 }
     59 
     60 NOINLINE void ThreadUnresponsive_UI() {
     61   NullPointerCrash(__LINE__);
     62 }
     63 
     64 NOINLINE void ThreadUnresponsive_DB() {
     65   NullPointerCrash(__LINE__);
     66 }
     67 
     68 NOINLINE void ThreadUnresponsive_FILE() {
     69   NullPointerCrash(__LINE__);
     70 }
     71 
     72 NOINLINE void ThreadUnresponsive_FILE_USER_BLOCKING() {
     73   NullPointerCrash(__LINE__);
     74 }
     75 
     76 NOINLINE void ThreadUnresponsive_PROCESS_LAUNCHER() {
     77   NullPointerCrash(__LINE__);
     78 }
     79 
     80 NOINLINE void ThreadUnresponsive_CACHE() {
     81   NullPointerCrash(__LINE__);
     82 }
     83 
     84 NOINLINE void ThreadUnresponsive_IO() {
     85   NullPointerCrash(__LINE__);
     86 }
     87 
     88 MSVC_POP_WARNING()
     89 MSVC_ENABLE_OPTIMIZE();
     90 
     91 void CrashBecauseThreadWasUnresponsive(BrowserThread::ID thread_id) {
     92   base::debug::Alias(&thread_id);
     93 
     94   switch (thread_id) {
     95     case BrowserThread::UI:
     96       return ThreadUnresponsive_UI();
     97     case BrowserThread::DB:
     98       return ThreadUnresponsive_DB();
     99     case BrowserThread::FILE:
    100       return ThreadUnresponsive_FILE();
    101     case BrowserThread::FILE_USER_BLOCKING:
    102       return ThreadUnresponsive_FILE_USER_BLOCKING();
    103     case BrowserThread::PROCESS_LAUNCHER:
    104       return ThreadUnresponsive_PROCESS_LAUNCHER();
    105     case BrowserThread::CACHE:
    106       return ThreadUnresponsive_CACHE();
    107     case BrowserThread::IO:
    108       return ThreadUnresponsive_IO();
    109     case BrowserThread::ID_COUNT:
    110       CHECK(false);  // This shouldn't actually be reached!
    111       break;
    112 
    113     // Omission of the default hander is intentional -- that way the compiler
    114     // should warn if our switch becomes outdated.
    115   }
    116 
    117   CHECK(false) << "Unknown thread was unresponsive.";  // Shouldn't be reached.
    118 }
    119 
    120 }  // namespace
    121 
    122 // ThreadWatcher methods and members.
    123 ThreadWatcher::ThreadWatcher(const WatchingParams& params)
    124     : thread_id_(params.thread_id),
    125       thread_name_(params.thread_name),
    126       watched_loop_(
    127           BrowserThread::GetMessageLoopProxyForThread(params.thread_id)),
    128       sleep_time_(params.sleep_time),
    129       unresponsive_time_(params.unresponsive_time),
    130       ping_time_(base::TimeTicks::Now()),
    131       pong_time_(ping_time_),
    132       ping_sequence_number_(0),
    133       active_(false),
    134       ping_count_(params.unresponsive_threshold),
    135       response_time_histogram_(NULL),
    136       unresponsive_time_histogram_(NULL),
    137       unresponsive_count_(0),
    138       hung_processing_complete_(false),
    139       unresponsive_threshold_(params.unresponsive_threshold),
    140       crash_on_hang_(params.crash_on_hang),
    141       live_threads_threshold_(params.live_threads_threshold),
    142       weak_ptr_factory_(this) {
    143   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    144   Initialize();
    145 }
    146 
    147 ThreadWatcher::~ThreadWatcher() {}
    148 
    149 // static
    150 void ThreadWatcher::StartWatching(const WatchingParams& params) {
    151   DCHECK_GE(params.sleep_time.InMilliseconds(), 0);
    152   DCHECK_GE(params.unresponsive_time.InMilliseconds(),
    153             params.sleep_time.InMilliseconds());
    154 
    155   // If we are not on WatchDogThread, then post a task to call StartWatching on
    156   // WatchDogThread.
    157   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
    158     WatchDogThread::PostTask(
    159         FROM_HERE,
    160         base::Bind(&ThreadWatcher::StartWatching, params));
    161     return;
    162   }
    163 
    164   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    165 
    166   // Create a new thread watcher object for the given thread and activate it.
    167   ThreadWatcher* watcher = new ThreadWatcher(params);
    168 
    169   DCHECK(watcher);
    170   // If we couldn't register the thread watcher object, we are shutting down,
    171   // then don't activate thread watching.
    172   if (!ThreadWatcherList::IsRegistered(params.thread_id))
    173     return;
    174   watcher->ActivateThreadWatching();
    175 }
    176 
    177 void ThreadWatcher::ActivateThreadWatching() {
    178   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    179   if (active_) return;
    180   active_ = true;
    181   ping_count_ = unresponsive_threshold_;
    182   ResetHangCounters();
    183   base::MessageLoop::current()->PostTask(
    184       FROM_HERE,
    185       base::Bind(&ThreadWatcher::PostPingMessage,
    186                  weak_ptr_factory_.GetWeakPtr()));
    187 }
    188 
    189 void ThreadWatcher::DeActivateThreadWatching() {
    190   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    191   active_ = false;
    192   ping_count_ = 0;
    193   weak_ptr_factory_.InvalidateWeakPtrs();
    194 }
    195 
    196 void ThreadWatcher::WakeUp() {
    197   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    198   // There is some user activity, PostPingMessage task of thread watcher if
    199   // needed.
    200   if (!active_) return;
    201 
    202   // Throw away the previous |unresponsive_count_| and start over again. Just
    203   // before going to sleep, |unresponsive_count_| could be very close to
    204   // |unresponsive_threshold_| and when user becomes active,
    205   // |unresponsive_count_| can go over |unresponsive_threshold_| if there was no
    206   // response for ping messages. Reset |unresponsive_count_| to start measuring
    207   // the unresponsiveness of the threads when system becomes active.
    208   unresponsive_count_ = 0;
    209 
    210   if (ping_count_ <= 0) {
    211     ping_count_ = unresponsive_threshold_;
    212     ResetHangCounters();
    213     PostPingMessage();
    214   } else {
    215     ping_count_ = unresponsive_threshold_;
    216   }
    217 }
    218 
    219 void ThreadWatcher::PostPingMessage() {
    220   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    221   // If we have stopped watching or if the user is idle, then stop sending
    222   // ping messages.
    223   if (!active_ || ping_count_ <= 0)
    224     return;
    225 
    226   // Save the current time when we have sent ping message.
    227   ping_time_ = base::TimeTicks::Now();
    228 
    229   // Send a ping message to the watched thread. Callback will be called on
    230   // the WatchDogThread.
    231   base::Closure callback(
    232       base::Bind(&ThreadWatcher::OnPongMessage, weak_ptr_factory_.GetWeakPtr(),
    233                  ping_sequence_number_));
    234   if (watched_loop_->PostTask(
    235           FROM_HERE,
    236           base::Bind(&ThreadWatcher::OnPingMessage, thread_id_,
    237                      callback))) {
    238       // Post a task to check the responsiveness of watched thread.
    239       base::MessageLoop::current()->PostDelayedTask(
    240           FROM_HERE,
    241           base::Bind(&ThreadWatcher::OnCheckResponsiveness,
    242                      weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
    243           unresponsive_time_);
    244   } else {
    245     // Watched thread might have gone away, stop watching it.
    246     DeActivateThreadWatching();
    247   }
    248 }
    249 
    250 void ThreadWatcher::OnPongMessage(uint64 ping_sequence_number) {
    251   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    252 
    253   // Record watched thread's response time.
    254   base::TimeTicks now = base::TimeTicks::Now();
    255   base::TimeDelta response_time = now - ping_time_;
    256   response_time_histogram_->AddTime(response_time);
    257 
    258   // Save the current time when we have got pong message.
    259   pong_time_ = now;
    260 
    261   // Check if there are any extra pings in flight.
    262   DCHECK_EQ(ping_sequence_number_, ping_sequence_number);
    263   if (ping_sequence_number_ != ping_sequence_number)
    264     return;
    265 
    266   // Increment sequence number for the next ping message to indicate watched
    267   // thread is responsive.
    268   ++ping_sequence_number_;
    269 
    270   // If we have stopped watching or if the user is idle, then stop sending
    271   // ping messages.
    272   if (!active_ || --ping_count_ <= 0)
    273     return;
    274 
    275   base::MessageLoop::current()->PostDelayedTask(
    276       FROM_HERE,
    277       base::Bind(&ThreadWatcher::PostPingMessage,
    278                  weak_ptr_factory_.GetWeakPtr()),
    279       sleep_time_);
    280 }
    281 
    282 void ThreadWatcher::OnCheckResponsiveness(uint64 ping_sequence_number) {
    283   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    284   // If we have stopped watching then consider thread as responding.
    285   if (!active_) {
    286     responsive_ = true;
    287     return;
    288   }
    289   // If the latest ping_sequence_number_ is not same as the ping_sequence_number
    290   // that is passed in, then we can assume OnPongMessage was called.
    291   // OnPongMessage increments ping_sequence_number_.
    292   if (ping_sequence_number_ != ping_sequence_number) {
    293     // Reset unresponsive_count_ to zero because we got a response from the
    294     // watched thread.
    295     ResetHangCounters();
    296 
    297     responsive_ = true;
    298     return;
    299   }
    300   // Record that we got no response from watched thread.
    301   GotNoResponse();
    302 
    303   // Post a task to check the responsiveness of watched thread.
    304   base::MessageLoop::current()->PostDelayedTask(
    305       FROM_HERE,
    306       base::Bind(&ThreadWatcher::OnCheckResponsiveness,
    307                  weak_ptr_factory_.GetWeakPtr(), ping_sequence_number_),
    308       unresponsive_time_);
    309   responsive_ = false;
    310 }
    311 
    312 void ThreadWatcher::Initialize() {
    313   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    314   ThreadWatcherList::Register(this);
    315 
    316   const std::string response_time_histogram_name =
    317       "ThreadWatcher.ResponseTime." + thread_name_;
    318   response_time_histogram_ = base::Histogram::FactoryTimeGet(
    319       response_time_histogram_name,
    320       base::TimeDelta::FromMilliseconds(1),
    321       base::TimeDelta::FromSeconds(100), 50,
    322       base::Histogram::kUmaTargetedHistogramFlag);
    323 
    324   const std::string unresponsive_time_histogram_name =
    325       "ThreadWatcher.Unresponsive." + thread_name_;
    326   unresponsive_time_histogram_ = base::Histogram::FactoryTimeGet(
    327       unresponsive_time_histogram_name,
    328       base::TimeDelta::FromMilliseconds(1),
    329       base::TimeDelta::FromSeconds(100), 50,
    330       base::Histogram::kUmaTargetedHistogramFlag);
    331 
    332   const std::string responsive_count_histogram_name =
    333       "ThreadWatcher.ResponsiveThreads." + thread_name_;
    334   responsive_count_histogram_ = base::LinearHistogram::FactoryGet(
    335       responsive_count_histogram_name, 1, 10, 11,
    336       base::Histogram::kUmaTargetedHistogramFlag);
    337 
    338   const std::string unresponsive_count_histogram_name =
    339       "ThreadWatcher.UnresponsiveThreads." + thread_name_;
    340   unresponsive_count_histogram_ = base::LinearHistogram::FactoryGet(
    341       unresponsive_count_histogram_name, 1, 10, 11,
    342       base::Histogram::kUmaTargetedHistogramFlag);
    343 }
    344 
    345 // static
    346 void ThreadWatcher::OnPingMessage(const BrowserThread::ID& thread_id,
    347                                   const base::Closure& callback_task) {
    348   // This method is called on watched thread.
    349   DCHECK(BrowserThread::CurrentlyOn(thread_id));
    350   WatchDogThread::PostTask(FROM_HERE, callback_task);
    351 }
    352 
    353 void ThreadWatcher::ResetHangCounters() {
    354   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    355   unresponsive_count_ = 0;
    356   hung_processing_complete_ = false;
    357 }
    358 
    359 void ThreadWatcher::GotNoResponse() {
    360   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    361 
    362   ++unresponsive_count_;
    363   if (!IsVeryUnresponsive())
    364     return;
    365 
    366   // Record total unresponsive_time since last pong message.
    367   base::TimeDelta unresponse_time = base::TimeTicks::Now() - pong_time_;
    368   unresponsive_time_histogram_->AddTime(unresponse_time);
    369 
    370   // We have already collected stats for the non-responding watched thread.
    371   if (hung_processing_complete_)
    372     return;
    373 
    374   // Record how other threads are responding.
    375   uint32 responding_thread_count = 0;
    376   uint32 unresponding_thread_count = 0;
    377   ThreadWatcherList::GetStatusOfThreads(&responding_thread_count,
    378                                         &unresponding_thread_count);
    379 
    380   // Record how many watched threads are responding.
    381   responsive_count_histogram_->Add(responding_thread_count);
    382 
    383   // Record how many watched threads are not responding.
    384   unresponsive_count_histogram_->Add(unresponding_thread_count);
    385 
    386   // Crash the browser if the watched thread is to be crashed on hang and if the
    387   // number of other threads responding is less than or equal to
    388   // live_threads_threshold_ and at least one other thread is responding.
    389   if (crash_on_hang_ &&
    390       responding_thread_count > 0 &&
    391       responding_thread_count <= live_threads_threshold_) {
    392     static bool crashed_once = false;
    393     if (!crashed_once) {
    394       crashed_once = true;
    395       CrashBecauseThreadWasUnresponsive(thread_id_);
    396     }
    397   }
    398 
    399   hung_processing_complete_ = true;
    400 }
    401 
    402 bool ThreadWatcher::IsVeryUnresponsive() {
    403   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    404   return unresponsive_count_ >= unresponsive_threshold_;
    405 }
    406 
    407 // ThreadWatcherList methods and members.
    408 //
    409 // static
    410 ThreadWatcherList* ThreadWatcherList::g_thread_watcher_list_ = NULL;
    411 // static
    412 const int ThreadWatcherList::kSleepSeconds = 1;
    413 // static
    414 const int ThreadWatcherList::kUnresponsiveSeconds = 2;
    415 // static
    416 const int ThreadWatcherList::kUnresponsiveCount = 9;
    417 // static
    418 const int ThreadWatcherList::kLiveThreadsThreshold = 2;
    419 
    420 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds(
    421     uint32 live_threads_threshold,
    422     uint32 unresponsive_threshold)
    423     : live_threads_threshold(live_threads_threshold),
    424       unresponsive_threshold(unresponsive_threshold) {
    425 }
    426 
    427 ThreadWatcherList::CrashDataThresholds::CrashDataThresholds()
    428     : live_threads_threshold(kLiveThreadsThreshold),
    429       unresponsive_threshold(kUnresponsiveCount) {
    430 }
    431 
    432 // static
    433 void ThreadWatcherList::StartWatchingAll(const CommandLine& command_line) {
    434   // TODO(rtenneti): Enable ThreadWatcher.
    435   uint32 unresponsive_threshold;
    436   CrashOnHangThreadMap crash_on_hang_threads;
    437   ParseCommandLine(command_line,
    438                    &unresponsive_threshold,
    439                    &crash_on_hang_threads);
    440 
    441   ThreadWatcherObserver::SetupNotifications(
    442       base::TimeDelta::FromSeconds(kSleepSeconds * unresponsive_threshold));
    443 
    444   WatchDogThread::PostDelayedTask(
    445       FROM_HERE,
    446       base::Bind(&ThreadWatcherList::InitializeAndStartWatching,
    447                  unresponsive_threshold,
    448                  crash_on_hang_threads),
    449       base::TimeDelta::FromSeconds(120));
    450 }
    451 
    452 // static
    453 void ThreadWatcherList::StopWatchingAll() {
    454   // TODO(rtenneti): Enable ThreadWatcher.
    455   ThreadWatcherObserver::RemoveNotifications();
    456   DeleteAll();
    457 }
    458 
    459 // static
    460 void ThreadWatcherList::Register(ThreadWatcher* watcher) {
    461   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    462   if (!g_thread_watcher_list_)
    463     return;
    464   DCHECK(!g_thread_watcher_list_->Find(watcher->thread_id()));
    465   g_thread_watcher_list_->registered_[watcher->thread_id()] = watcher;
    466 }
    467 
    468 // static
    469 bool ThreadWatcherList::IsRegistered(const BrowserThread::ID thread_id) {
    470   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    471   return NULL != ThreadWatcherList::Find(thread_id);
    472 }
    473 
    474 // static
    475 void ThreadWatcherList::GetStatusOfThreads(uint32* responding_thread_count,
    476                                            uint32* unresponding_thread_count) {
    477   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    478   *responding_thread_count = 0;
    479   *unresponding_thread_count = 0;
    480   if (!g_thread_watcher_list_)
    481     return;
    482 
    483   for (RegistrationList::iterator it =
    484            g_thread_watcher_list_->registered_.begin();
    485        g_thread_watcher_list_->registered_.end() != it;
    486        ++it) {
    487     if (it->second->IsVeryUnresponsive())
    488       ++(*unresponding_thread_count);
    489     else
    490       ++(*responding_thread_count);
    491   }
    492 }
    493 
    494 // static
    495 void ThreadWatcherList::WakeUpAll() {
    496   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    497   if (!g_thread_watcher_list_)
    498     return;
    499 
    500   for (RegistrationList::iterator it =
    501            g_thread_watcher_list_->registered_.begin();
    502        g_thread_watcher_list_->registered_.end() != it;
    503        ++it)
    504     it->second->WakeUp();
    505 }
    506 
    507 ThreadWatcherList::ThreadWatcherList() {
    508   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    509   CHECK(!g_thread_watcher_list_);
    510   g_thread_watcher_list_ = this;
    511 }
    512 
    513 ThreadWatcherList::~ThreadWatcherList() {
    514   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    515   DCHECK(this == g_thread_watcher_list_);
    516   g_thread_watcher_list_ = NULL;
    517 }
    518 
    519 // static
    520 void ThreadWatcherList::ParseCommandLine(
    521     const CommandLine& command_line,
    522     uint32* unresponsive_threshold,
    523     CrashOnHangThreadMap* crash_on_hang_threads) {
    524   // Initialize |unresponsive_threshold| to a default value.
    525   *unresponsive_threshold = kUnresponsiveCount;
    526 
    527   // Increase the unresponsive_threshold on the Stable and Beta channels to
    528   // reduce the number of crashes due to ThreadWatcher.
    529   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
    530   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
    531     *unresponsive_threshold *= 4;
    532   } else if (channel == chrome::VersionInfo::CHANNEL_BETA) {
    533     *unresponsive_threshold *= 2;
    534   }
    535 
    536 #if defined(OS_WIN)
    537   // For Windows XP (old systems), double the unresponsive_threshold to give
    538   // the OS a chance to schedule UI/IO threads a time slice to respond with a
    539   // pong message (to get around limitations with the OS).
    540   if (base::win::GetVersion() <= base::win::VERSION_XP)
    541     *unresponsive_threshold *= 2;
    542 #endif
    543 
    544   uint32 crash_seconds = *unresponsive_threshold * kUnresponsiveSeconds;
    545   std::string crash_on_hang_thread_names;
    546   bool has_command_line_overwrite = false;
    547   if (command_line.HasSwitch(switches::kCrashOnHangThreads)) {
    548     crash_on_hang_thread_names =
    549         command_line.GetSwitchValueASCII(switches::kCrashOnHangThreads);
    550     has_command_line_overwrite = true;
    551   } else if (channel != chrome::VersionInfo::CHANNEL_STABLE) {
    552     // Default to crashing the browser if UI or IO or FILE threads are not
    553     // responsive except in stable channel.
    554     crash_on_hang_thread_names = base::StringPrintf(
    555         "UI:%d:%d,IO:%d:%d,FILE:%d:%d",
    556         kLiveThreadsThreshold, crash_seconds,
    557         kLiveThreadsThreshold, crash_seconds,
    558         kLiveThreadsThreshold, crash_seconds * 5);
    559   }
    560 
    561   ParseCommandLineCrashOnHangThreads(crash_on_hang_thread_names,
    562                                      kLiveThreadsThreshold,
    563                                      crash_seconds,
    564                                      crash_on_hang_threads);
    565 
    566   if (channel != chrome::VersionInfo::CHANNEL_CANARY ||
    567       has_command_line_overwrite) {
    568     return;
    569   }
    570 
    571   // Set up a field trial for 100% of the users to crash if either UI or IO
    572   // thread is not responsive for 30 seconds (or 15 pings).
    573   scoped_refptr<base::FieldTrial> field_trial(
    574       base::FieldTrialList::FactoryGetFieldTrial(
    575           "ThreadWatcher", 100, "default_hung_threads",
    576           2014, 10, 30, base::FieldTrial::SESSION_RANDOMIZED, NULL));
    577   int hung_thread_group = field_trial->AppendGroup("hung_thread", 100);
    578   if (field_trial->group() == hung_thread_group) {
    579     for (CrashOnHangThreadMap::iterator it = crash_on_hang_threads->begin();
    580          crash_on_hang_threads->end() != it;
    581          ++it) {
    582       if (it->first == "FILE")
    583         continue;
    584       it->second.live_threads_threshold = INT_MAX;
    585       if (it->first == "UI") {
    586         // TODO(rtenneti): set unresponsive threshold to 120 seconds to catch
    587         // the worst UI hangs and for fewer crashes due to ThreadWatcher. Reduce
    588         // it to a more reasonable time ala IO thread.
    589         it->second.unresponsive_threshold = 60;
    590       } else {
    591         it->second.unresponsive_threshold = 15;
    592       }
    593     }
    594   }
    595 }
    596 
    597 // static
    598 void ThreadWatcherList::ParseCommandLineCrashOnHangThreads(
    599     const std::string& crash_on_hang_thread_names,
    600     uint32 default_live_threads_threshold,
    601     uint32 default_crash_seconds,
    602     CrashOnHangThreadMap* crash_on_hang_threads) {
    603   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
    604   std::vector<std::string> values;
    605   while (tokens.GetNext()) {
    606     const std::string& token = tokens.token();
    607     base::SplitString(token, ':', &values);
    608     std::string thread_name = values[0];
    609 
    610     uint32 live_threads_threshold = default_live_threads_threshold;
    611     uint32 crash_seconds = default_crash_seconds;
    612     if (values.size() >= 2 &&
    613         (!base::StringToUint(values[1], &live_threads_threshold))) {
    614       continue;
    615     }
    616     if (values.size() >= 3 &&
    617         (!base::StringToUint(values[2], &crash_seconds))) {
    618       continue;
    619     }
    620     uint32 unresponsive_threshold = static_cast<uint32>(
    621         ceil(static_cast<float>(crash_seconds) / kUnresponsiveSeconds));
    622 
    623     CrashDataThresholds crash_data(live_threads_threshold,
    624                                    unresponsive_threshold);
    625     // Use the last specifier.
    626     (*crash_on_hang_threads)[thread_name] = crash_data;
    627   }
    628 }
    629 
    630 // static
    631 void ThreadWatcherList::InitializeAndStartWatching(
    632     uint32 unresponsive_threshold,
    633     const CrashOnHangThreadMap& crash_on_hang_threads) {
    634   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    635 
    636   ThreadWatcherList* thread_watcher_list = new ThreadWatcherList();
    637   CHECK(thread_watcher_list);
    638 
    639   BrowserThread::PostTask(
    640       BrowserThread::UI,
    641       FROM_HERE,
    642       base::Bind(&StartupTimeBomb::DisarmStartupTimeBomb));
    643 
    644   const base::TimeDelta kSleepTime =
    645       base::TimeDelta::FromSeconds(kSleepSeconds);
    646   const base::TimeDelta kUnresponsiveTime =
    647       base::TimeDelta::FromSeconds(kUnresponsiveSeconds);
    648 
    649   StartWatching(BrowserThread::UI, "UI", kSleepTime, kUnresponsiveTime,
    650                 unresponsive_threshold, crash_on_hang_threads);
    651   StartWatching(BrowserThread::IO, "IO", kSleepTime, kUnresponsiveTime,
    652                 unresponsive_threshold, crash_on_hang_threads);
    653   StartWatching(BrowserThread::DB, "DB", kSleepTime, kUnresponsiveTime,
    654                 unresponsive_threshold, crash_on_hang_threads);
    655   StartWatching(BrowserThread::FILE, "FILE", kSleepTime, kUnresponsiveTime,
    656                 unresponsive_threshold, crash_on_hang_threads);
    657   StartWatching(BrowserThread::CACHE, "CACHE", kSleepTime, kUnresponsiveTime,
    658                 unresponsive_threshold, crash_on_hang_threads);
    659 }
    660 
    661 // static
    662 void ThreadWatcherList::StartWatching(
    663     const BrowserThread::ID& thread_id,
    664     const std::string& thread_name,
    665     const base::TimeDelta& sleep_time,
    666     const base::TimeDelta& unresponsive_time,
    667     uint32 unresponsive_threshold,
    668     const CrashOnHangThreadMap& crash_on_hang_threads) {
    669   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    670 
    671   CrashOnHangThreadMap::const_iterator it =
    672       crash_on_hang_threads.find(thread_name);
    673   bool crash_on_hang = false;
    674   uint32 live_threads_threshold = 0;
    675   if (it != crash_on_hang_threads.end()) {
    676     crash_on_hang = true;
    677     live_threads_threshold = it->second.live_threads_threshold;
    678     unresponsive_threshold = it->second.unresponsive_threshold;
    679   }
    680 
    681   ThreadWatcher::StartWatching(
    682       ThreadWatcher::WatchingParams(thread_id,
    683                                     thread_name,
    684                                     sleep_time,
    685                                     unresponsive_time,
    686                                     unresponsive_threshold,
    687                                     crash_on_hang,
    688                                     live_threads_threshold));
    689 }
    690 
    691 // static
    692 void ThreadWatcherList::DeleteAll() {
    693   if (!WatchDogThread::CurrentlyOnWatchDogThread()) {
    694     WatchDogThread::PostTask(
    695         FROM_HERE,
    696         base::Bind(&ThreadWatcherList::DeleteAll));
    697     return;
    698   }
    699 
    700   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    701   if (!g_thread_watcher_list_)
    702     return;
    703 
    704   // Delete all thread watcher objects.
    705   while (!g_thread_watcher_list_->registered_.empty()) {
    706     RegistrationList::iterator it = g_thread_watcher_list_->registered_.begin();
    707     delete it->second;
    708     g_thread_watcher_list_->registered_.erase(it);
    709   }
    710 
    711   delete g_thread_watcher_list_;
    712 }
    713 
    714 // static
    715 ThreadWatcher* ThreadWatcherList::Find(const BrowserThread::ID& thread_id) {
    716   DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
    717   if (!g_thread_watcher_list_)
    718     return NULL;
    719   RegistrationList::iterator it =
    720       g_thread_watcher_list_->registered_.find(thread_id);
    721   if (g_thread_watcher_list_->registered_.end() == it)
    722     return NULL;
    723   return it->second;
    724 }
    725 
    726 // ThreadWatcherObserver methods and members.
    727 //
    728 // static
    729 ThreadWatcherObserver* ThreadWatcherObserver::g_thread_watcher_observer_ = NULL;
    730 
    731 ThreadWatcherObserver::ThreadWatcherObserver(
    732     const base::TimeDelta& wakeup_interval)
    733     : last_wakeup_time_(base::TimeTicks::Now()),
    734       wakeup_interval_(wakeup_interval) {
    735   CHECK(!g_thread_watcher_observer_);
    736   g_thread_watcher_observer_ = this;
    737 }
    738 
    739 ThreadWatcherObserver::~ThreadWatcherObserver() {
    740   DCHECK(this == g_thread_watcher_observer_);
    741   g_thread_watcher_observer_ = NULL;
    742 }
    743 
    744 // static
    745 void ThreadWatcherObserver::SetupNotifications(
    746     const base::TimeDelta& wakeup_interval) {
    747   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    748   ThreadWatcherObserver* observer = new ThreadWatcherObserver(wakeup_interval);
    749   MetricsService::SetUpNotifications(&observer->registrar_, observer);
    750 }
    751 
    752 // static
    753 void ThreadWatcherObserver::RemoveNotifications() {
    754   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    755   if (!g_thread_watcher_observer_)
    756     return;
    757   g_thread_watcher_observer_->registrar_.RemoveAll();
    758   delete g_thread_watcher_observer_;
    759 }
    760 
    761 void ThreadWatcherObserver::Observe(
    762     int type,
    763     const content::NotificationSource& source,
    764     const content::NotificationDetails& details) {
    765   // There is some user activity, see if thread watchers are to be awakened.
    766   base::TimeTicks now = base::TimeTicks::Now();
    767   if ((now - last_wakeup_time_) < wakeup_interval_)
    768     return;
    769   last_wakeup_time_ = now;
    770   WatchDogThread::PostTask(
    771       FROM_HERE,
    772       base::Bind(&ThreadWatcherList::WakeUpAll));
    773 }
    774 
    775 // WatchDogThread methods and members.
    776 
    777 // This lock protects g_watchdog_thread.
    778 static base::LazyInstance<base::Lock>::Leaky
    779     g_watchdog_lock = LAZY_INSTANCE_INITIALIZER;
    780 
    781 // The singleton of this class.
    782 static WatchDogThread* g_watchdog_thread = NULL;
    783 
    784 WatchDogThread::WatchDogThread() : Thread("BrowserWatchdog") {
    785 }
    786 
    787 WatchDogThread::~WatchDogThread() {
    788   Stop();
    789 }
    790 
    791 // static
    792 bool WatchDogThread::CurrentlyOnWatchDogThread() {
    793   base::AutoLock lock(g_watchdog_lock.Get());
    794   return g_watchdog_thread &&
    795       g_watchdog_thread->message_loop() == base::MessageLoop::current();
    796 }
    797 
    798 // static
    799 bool WatchDogThread::PostTask(const tracked_objects::Location& from_here,
    800                               const base::Closure& task) {
    801   return PostTaskHelper(from_here, task, base::TimeDelta());
    802 }
    803 
    804 // static
    805 bool WatchDogThread::PostDelayedTask(const tracked_objects::Location& from_here,
    806                                      const base::Closure& task,
    807                                      base::TimeDelta delay) {
    808   return PostTaskHelper(from_here, task, delay);
    809 }
    810 
    811 // static
    812 bool WatchDogThread::PostTaskHelper(
    813     const tracked_objects::Location& from_here,
    814     const base::Closure& task,
    815     base::TimeDelta delay) {
    816   {
    817     base::AutoLock lock(g_watchdog_lock.Get());
    818 
    819     base::MessageLoop* message_loop = g_watchdog_thread ?
    820         g_watchdog_thread->message_loop() : NULL;
    821     if (message_loop) {
    822       message_loop->PostDelayedTask(from_here, task, delay);
    823       return true;
    824     }
    825   }
    826 
    827   return false;
    828 }
    829 
    830 void WatchDogThread::Init() {
    831   // This thread shouldn't be allowed to perform any blocking disk I/O.
    832   base::ThreadRestrictions::SetIOAllowed(false);
    833 
    834   base::AutoLock lock(g_watchdog_lock.Get());
    835   CHECK(!g_watchdog_thread);
    836   g_watchdog_thread = this;
    837 }
    838 
    839 void WatchDogThread::CleanUp() {
    840   base::AutoLock lock(g_watchdog_lock.Get());
    841   g_watchdog_thread = NULL;
    842 }
    843 
    844 namespace {
    845 
    846 // StartupWatchDogThread methods and members.
    847 //
    848 // Class for detecting hangs during startup.
    849 class StartupWatchDogThread : public base::Watchdog {
    850  public:
    851   // Constructor specifies how long the StartupWatchDogThread will wait before
    852   // alarming.
    853   explicit StartupWatchDogThread(const base::TimeDelta& duration)
    854       : base::Watchdog(duration, "Startup watchdog thread", true) {
    855   }
    856 
    857   // Alarm is called if the time expires after an Arm() without someone calling
    858   // Disarm(). When Alarm goes off, in release mode we get the crash dump
    859   // without crashing and in debug mode we break into the debugger.
    860   virtual void Alarm() OVERRIDE {
    861 #ifndef NDEBUG
    862     DCHECK(false);
    863 #else
    864     logging::DumpWithoutCrashing();
    865 #endif
    866   }
    867 
    868   DISALLOW_COPY_AND_ASSIGN(StartupWatchDogThread);
    869 };
    870 
    871 // ShutdownWatchDogThread methods and members.
    872 //
    873 // Class for detecting hangs during shutdown.
    874 class ShutdownWatchDogThread : public base::Watchdog {
    875  public:
    876   // Constructor specifies how long the ShutdownWatchDogThread will wait before
    877   // alarming.
    878   explicit ShutdownWatchDogThread(const base::TimeDelta& duration)
    879       : base::Watchdog(duration, "Shutdown watchdog thread", true) {
    880   }
    881 
    882   // Alarm is called if the time expires after an Arm() without someone calling
    883   // Disarm(). We crash the browser if this method is called.
    884   virtual void Alarm() OVERRIDE {
    885     ShutdownCrash();
    886   }
    887 
    888   DISALLOW_COPY_AND_ASSIGN(ShutdownWatchDogThread);
    889 };
    890 }  // namespace
    891 
    892 // StartupTimeBomb methods and members.
    893 //
    894 // static
    895 StartupTimeBomb* StartupTimeBomb::g_startup_timebomb_ = NULL;
    896 
    897 StartupTimeBomb::StartupTimeBomb()
    898     : startup_watchdog_(NULL),
    899       thread_id_(base::PlatformThread::CurrentId()) {
    900   CHECK(!g_startup_timebomb_);
    901   g_startup_timebomb_ = this;
    902 }
    903 
    904 StartupTimeBomb::~StartupTimeBomb() {
    905   DCHECK(this == g_startup_timebomb_);
    906   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
    907   if (startup_watchdog_)
    908     Disarm();
    909   g_startup_timebomb_ = NULL;
    910 }
    911 
    912 void StartupTimeBomb::Arm(const base::TimeDelta& duration) {
    913   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
    914   DCHECK(!startup_watchdog_);
    915   startup_watchdog_ = new StartupWatchDogThread(duration);
    916   startup_watchdog_->Arm();
    917   return;
    918 }
    919 
    920 void StartupTimeBomb::Disarm() {
    921   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
    922   if (startup_watchdog_) {
    923     startup_watchdog_->Disarm();
    924     startup_watchdog_->Cleanup();
    925     DeleteStartupWatchdog();
    926   }
    927 }
    928 
    929 void StartupTimeBomb::DeleteStartupWatchdog() {
    930   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
    931   if (startup_watchdog_->IsJoinable()) {
    932     // Allow the watchdog thread to shutdown on UI. Watchdog thread shutdowns
    933     // very fast.
    934     base::ThreadRestrictions::SetIOAllowed(true);
    935     delete startup_watchdog_;
    936     startup_watchdog_ = NULL;
    937     return;
    938   }
    939   base::MessageLoop::current()->PostDelayedTask(
    940       FROM_HERE,
    941       base::Bind(&StartupTimeBomb::DeleteStartupWatchdog,
    942                  base::Unretained(this)),
    943       base::TimeDelta::FromSeconds(10));
    944 }
    945 
    946 // static
    947 void StartupTimeBomb::DisarmStartupTimeBomb() {
    948   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
    949   if (g_startup_timebomb_)
    950     g_startup_timebomb_->Disarm();
    951 }
    952 
    953 // ShutdownWatcherHelper methods and members.
    954 //
    955 // ShutdownWatcherHelper is a wrapper class for detecting hangs during
    956 // shutdown.
    957 ShutdownWatcherHelper::ShutdownWatcherHelper()
    958     : shutdown_watchdog_(NULL),
    959       thread_id_(base::PlatformThread::CurrentId()) {
    960 }
    961 
    962 ShutdownWatcherHelper::~ShutdownWatcherHelper() {
    963   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
    964   if (shutdown_watchdog_) {
    965     shutdown_watchdog_->Disarm();
    966     delete shutdown_watchdog_;
    967     shutdown_watchdog_ = NULL;
    968   }
    969 }
    970 
    971 void ShutdownWatcherHelper::Arm(const base::TimeDelta& duration) {
    972   DCHECK_EQ(thread_id_, base::PlatformThread::CurrentId());
    973   DCHECK(!shutdown_watchdog_);
    974   base::TimeDelta actual_duration = duration;
    975 
    976   chrome::VersionInfo::Channel channel = chrome::VersionInfo::GetChannel();
    977   if (channel == chrome::VersionInfo::CHANNEL_STABLE) {
    978     actual_duration *= 20;
    979   } else if (channel == chrome::VersionInfo::CHANNEL_BETA ||
    980              channel == chrome::VersionInfo::CHANNEL_DEV) {
    981     actual_duration *= 10;
    982   }
    983 
    984 #if defined(OS_WIN)
    985   // On Windows XP, give twice the time for shutdown.
    986   if (base::win::GetVersion() <= base::win::VERSION_XP)
    987     actual_duration *= 2;
    988 #endif
    989 
    990   shutdown_watchdog_ = new ShutdownWatchDogThread(actual_duration);
    991   shutdown_watchdog_->Arm();
    992 }
    993