Home | History | Annotate | Download | only in concurrent_events
      1 """
      2 A stress-test of sorts for LLDB's handling of threads in the inferior.
      3 
      4 This test sets a breakpoint in the main thread where test parameters (numbers of
      5 threads) can be adjusted, runs the inferior to that point, and modifies the
      6 locals that control the event thread counts. This test also sets a breakpoint in
      7 breakpoint_func (the function executed by each 'breakpoint' thread) and a
      8 watchpoint on a global modified in watchpoint_func. The inferior is continued
      9 until exit or a crash takes place, and the number of events seen by LLDB is
     10 verified to match the expected number of events.
     11 """
     12 
     13 import os, signal, time
     14 import unittest2
     15 import lldb
     16 from lldbtest import *
     17 import lldbutil
     18 
     19 # ==================================================
     20 # Dictionary of signal names
     21 # ==================================================
     22 signal_names = dict((getattr(signal, n), n) \
     23         for n in dir(signal) if n.startswith('SIG') and '_' not in n )
     24 
     25 
     26 class ConcurrentEventsTestCase(TestBase):
     27 
     28     mydir = os.path.join("functionalities", "thread", "concurrent_events")
     29 
     30     #
     31     ## Tests for multiple threads that generate a single event.
     32     #
     33     @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
     34     @dwarf_test
     35     def test_many_breakpoints_dwarf(self):
     36         """Test 100 breakpoints from 100 threads."""
     37         self.buildDwarf(dictionary=self.getBuildFlags())
     38         self.do_thread_actions(num_breakpoint_threads=100)
     39 
     40     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
     41     @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
     42     @dwarf_test
     43     def test_many_watchpoints_dwarf(self):
     44         """Test 100 watchpoints from 100 threads."""
     45         self.buildDwarf(dictionary=self.getBuildFlags())
     46         self.do_thread_actions(num_watchpoint_threads=100)
     47 
     48     @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
     49     @dwarf_test
     50     def test_many_signals_dwarf(self):
     51         """Test 100 signals from 100 threads."""
     52         self.buildDwarf(dictionary=self.getBuildFlags())
     53         self.do_thread_actions(num_signal_threads=100)
     54 
     55     @unittest2.skipIf(TestBase.skipLongRunningTest(), "Skip this long running test")
     56     @dwarf_test
     57     def test_many_crash_dwarf(self):
     58         """Test 100 threads that cause a segfault."""
     59         self.buildDwarf(dictionary=self.getBuildFlags())
     60         self.do_thread_actions(num_crash_threads=100)
     61 
     62 
     63     #
     64     ## Tests for concurrent signal and breakpoint
     65     #
     66     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
     67     @dwarf_test
     68     def test_signal_break_dwarf(self):
     69         """Test signal and a breakpoint in multiple threads."""
     70         self.buildDwarf(dictionary=self.getBuildFlags())
     71         self.do_thread_actions(num_breakpoint_threads=1, num_signal_threads=1)
     72 
     73     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
     74     @dwarf_test
     75     def test_delay_signal_break_dwarf(self):
     76         """Test (1-second delay) signal and a breakpoint in multiple threads."""
     77         self.buildDwarf(dictionary=self.getBuildFlags())
     78         self.do_thread_actions(num_breakpoint_threads=1, num_delay_signal_threads=1)
     79 
     80     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
     81     @dwarf_test
     82     def test_signal_delay_break_dwarf(self):
     83         """Test signal and a (1 second delay) breakpoint in multiple threads."""
     84         self.buildDwarf(dictionary=self.getBuildFlags())
     85         self.do_thread_actions(num_delay_breakpoint_threads=1, num_signal_threads=1)
     86 
     87 
     88     #
     89     ## Tests for concurrent watchpoint and breakpoint
     90     #
     91     @dwarf_test
     92     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
     93     def test_watch_break_dwarf(self):
     94         """Test watchpoint and a breakpoint in multiple threads."""
     95         self.buildDwarf(dictionary=self.getBuildFlags())
     96         self.do_thread_actions(num_breakpoint_threads=1, num_watchpoint_threads=1)
     97 
     98     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
     99     @dwarf_test
    100     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    101     def test_delay_watch_break_dwarf(self):
    102         """Test (1-second delay) watchpoint and a breakpoint in multiple threads."""
    103         self.buildDwarf(dictionary=self.getBuildFlags())
    104         self.do_thread_actions(num_breakpoint_threads=1, num_delay_watchpoint_threads=1)
    105 
    106     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    107     @dwarf_test
    108     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    109     def test_watch_break_dwarf(self):
    110         """Test watchpoint and a (1 second delay) breakpoint in multiple threads."""
    111         self.buildDwarf(dictionary=self.getBuildFlags())
    112         self.do_thread_actions(num_delay_breakpoint_threads=1, num_watchpoint_threads=1)
    113 
    114     #
    115     ## Tests for concurrent signal and watchpoint
    116     #
    117     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    118     @dwarf_test
    119     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    120     def test_signal_watch_dwarf(self):
    121         """Test a watchpoint and a signal in multiple threads."""
    122         self.buildDwarf(dictionary=self.getBuildFlags())
    123         self.do_thread_actions(num_signal_threads=1, num_watchpoint_threads=1)
    124 
    125     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    126     @dwarf_test
    127     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    128     def test_delay_signal_watch_dwarf(self):
    129         """Test a watchpoint and a (1 second delay) signal in multiple threads."""
    130         self.buildDwarf(dictionary=self.getBuildFlags())
    131         self.do_thread_actions(num_delay_signal_threads=1, num_watchpoint_threads=1)
    132 
    133     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    134     @dwarf_test
    135     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    136     def test_signal_delay_watch_dwarf(self):
    137         """Test a (1 second delay) watchpoint and a signal in multiple threads."""
    138         self.buildDwarf(dictionary=self.getBuildFlags())
    139         self.do_thread_actions(num_signal_threads=1, num_delay_watchpoint_threads=1)
    140 
    141 
    142     #
    143     ## Tests for multiple breakpoint threads
    144     #
    145     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    146     @dwarf_test
    147     def test_two_breakpoint_threads_dwarf(self):
    148         """Test two threads that trigger a breakpoint. """
    149         self.buildDwarf(dictionary=self.getBuildFlags())
    150         self.do_thread_actions(num_breakpoint_threads=2)
    151 
    152     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    153     @dwarf_test
    154     def test_breakpoint_one_delay_breakpoint_threads_dwarf(self):
    155         """Test threads that trigger a breakpoint where one thread has a 1 second delay. """
    156         self.buildDwarf(dictionary=self.getBuildFlags())
    157         self.do_thread_actions(num_breakpoint_threads=1,
    158                                num_delay_breakpoint_threads=1)
    159 
    160     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    161     @dwarf_test
    162     def test_two_breakpoints_one_signal_dwarf(self):
    163         """Test two threads that trigger a breakpoint and one signal thread. """
    164         self.buildDwarf(dictionary=self.getBuildFlags())
    165         self.do_thread_actions(num_breakpoint_threads=2, num_signal_threads=1)
    166 
    167     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    168     @dwarf_test
    169     def test_breakpoint_delay_breakpoint_one_signal_dwarf(self):
    170         """Test two threads that trigger a breakpoint (one with a 1 second delay) and one signal thread. """
    171         self.buildDwarf(dictionary=self.getBuildFlags())
    172         self.do_thread_actions(num_breakpoint_threads=1,
    173                                num_delay_breakpoint_threads=1,
    174                                num_signal_threads=1)
    175 
    176     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    177     @dwarf_test
    178     def test_two_breakpoints_one_delay_signal_dwarf(self):
    179         """Test two threads that trigger a breakpoint and one (1 second delay) signal thread. """
    180         self.buildDwarf(dictionary=self.getBuildFlags())
    181         self.do_thread_actions(num_breakpoint_threads=2, num_delay_signal_threads=1)
    182 
    183     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    184     @dwarf_test
    185     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    186     def test_two_breakpoints_one_watchpoint_dwarf(self):
    187         """Test two threads that trigger a breakpoint and one watchpoint thread. """
    188         self.buildDwarf(dictionary=self.getBuildFlags())
    189         self.do_thread_actions(num_breakpoint_threads=2, num_watchpoint_threads=1)
    190 
    191     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    192     @dwarf_test
    193     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    194     def test_breakpoints_delayed_breakpoint_one_watchpoint_dwarf(self):
    195         """Test a breakpoint, a delayed breakpoint, and one watchpoint thread. """
    196         self.buildDwarf(dictionary=self.getBuildFlags())
    197         self.do_thread_actions(num_breakpoint_threads=1,
    198                                num_delay_breakpoint_threads=1,
    199                                num_watchpoint_threads=1)
    200 
    201     #
    202     ## Tests for multiple watchpoint threads
    203     #
    204     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    205     @dwarf_test
    206     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    207     def test_two_watchpoint_threads_dwarf(self):
    208         """Test two threads that trigger a watchpoint. """
    209         self.buildDwarf(dictionary=self.getBuildFlags())
    210         self.do_thread_actions(num_watchpoint_threads=2)
    211 
    212     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    213     @dwarf_test
    214     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    215     def test_watchpoint_with_delay_waychpoint_threads_dwarf(self):
    216         """Test two threads that trigger a watchpoint where one thread has a 1 second delay. """
    217         self.buildDwarf(dictionary=self.getBuildFlags())
    218         self.do_thread_actions(num_watchpoint_threads=1,
    219                                num_delay_watchpoint_threads=1)
    220 
    221     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    222     @dwarf_test
    223     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    224     def test_two_watchpoints_one_breakpoint_dwarf(self):
    225         """Test two threads that trigger a watchpoint and one breakpoint thread. """
    226         self.buildDwarf(dictionary=self.getBuildFlags())
    227         self.do_thread_actions(num_watchpoint_threads=2, num_breakpoint_threads=1)
    228 
    229     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    230     @dwarf_test
    231     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    232     def test_two_watchpoints_one_delay_breakpoint_dwarf(self):
    233         """Test two threads that trigger a watchpoint and one (1 second delay) breakpoint thread. """
    234         self.buildDwarf(dictionary=self.getBuildFlags())
    235         self.do_thread_actions(num_watchpoint_threads=2, num_delay_breakpoint_threads=1)
    236 
    237     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    238     @dwarf_test
    239     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    240     def test_watchpoint_delay_watchpoint_one_breakpoint_dwarf(self):
    241         """Test two threads that trigger a watchpoint (one with a 1 second delay) and one breakpoint thread. """
    242         self.buildDwarf(dictionary=self.getBuildFlags())
    243         self.do_thread_actions(num_watchpoint_threads=1,
    244                                num_delay_watchpoint_threads=1,
    245                                num_breakpoint_threads=1)
    246 
    247     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    248     @dwarf_test
    249     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    250     def test_two_watchpoints_one_signal_dwarf(self):
    251         """Test two threads that trigger a watchpoint and one signal thread. """
    252         self.buildDwarf(dictionary=self.getBuildFlags())
    253         self.do_thread_actions(num_watchpoint_threads=2, num_signal_threads=1)
    254 
    255     #
    256     ## Test for watchpoint, signal and breakpoint happening concurrently
    257     #
    258     @dwarf_test
    259     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    260     def test_signal_watch_break_dwarf(self):
    261         """Test a signal/watchpoint/breakpoint in multiple threads."""
    262         self.buildDwarf(dictionary=self.getBuildFlags())
    263         self.do_thread_actions(num_signal_threads=1,
    264                                num_watchpoint_threads=1,
    265                                num_breakpoint_threads=1)
    266 
    267     @dwarf_test
    268     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    269     def test_signal_watch_break_dwarf(self):
    270         """Test one signal thread with 5 watchpoint and breakpoint threads."""
    271         self.buildDwarf(dictionary=self.getBuildFlags())
    272         self.do_thread_actions(num_signal_threads=1,
    273                                num_watchpoint_threads=5,
    274                                num_breakpoint_threads=5)
    275 
    276     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    277     @dwarf_test
    278     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    279     def test_signal_watch_break_dwarf(self):
    280         """Test with 5 watchpoint and breakpoint threads."""
    281         self.buildDwarf(dictionary=self.getBuildFlags())
    282         self.do_thread_actions(num_watchpoint_threads=5,
    283                                num_breakpoint_threads=5)
    284 
    285 
    286     #
    287     ## Test for crashing threads happening concurrently with other events
    288     #
    289     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    290     @dwarf_test
    291     def test_crash_with_break_dwarf(self):
    292         """ Test a thread that crashes while another thread hits a breakpoint."""
    293         self.buildDwarf(dictionary=self.getBuildFlags())
    294         self.do_thread_actions(num_crash_threads=1, num_breakpoint_threads=1)
    295 
    296     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    297     @dwarf_test
    298     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    299     def test_crash_with_watchpoint_dwarf(self):
    300         """ Test a thread that crashes while another thread hits a watchpoint."""
    301         self.buildDwarf(dictionary=self.getBuildFlags())
    302         self.do_thread_actions(num_crash_threads=1, num_watchpoint_threads=1)
    303 
    304     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    305     @dwarf_test
    306     def test_crash_with_signal_dwarf(self):
    307         """ Test a thread that crashes while another thread generates a signal."""
    308         self.buildDwarf(dictionary=self.getBuildFlags())
    309         self.do_thread_actions(num_crash_threads=1, num_signal_threads=1)
    310 
    311     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    312     @dwarf_test
    313     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    314     def test_crash_with_watchpoint_breakpoint_signal_dwarf(self):
    315         """ Test a thread that crashes while other threads generate a signal and hit a watchpoint and breakpoint. """
    316         self.buildDwarf(dictionary=self.getBuildFlags())
    317         self.do_thread_actions(num_crash_threads=1,
    318                                num_breakpoint_threads=1,
    319                                num_signal_threads=1,
    320                                num_watchpoint_threads=1)
    321 
    322     @expectedFailureFreeBSD('llvm.org/pr16706') # Watchpoints fail on FreeBSD
    323     @dwarf_test
    324     @skipIfLinux # llvm.org/pr16714 - LLDB sometimes crashes when setting watchpoints in multithreaded programs
    325     def test_delayed_crash_with_breakpoint_watchpoint_dwarf(self):
    326         """ Test a thread with a delayed crash while other threads hit a watchpoint and a breakpoint. """
    327         self.buildDwarf(dictionary=self.getBuildFlags())
    328         self.do_thread_actions(num_delay_crash_threads=1,
    329                                num_breakpoint_threads=1,
    330                                num_watchpoint_threads=1)
    331 
    332     @expectedFailureFreeBSD("llvm.org/pr16696") # threaded inferior not yet implemented on FreeBSD
    333     @dwarf_test
    334     def test_delayed_crash_with_breakpoint_signal_dwarf(self):
    335         """ Test a thread with a delayed crash while other threads generate a signal and hit a breakpoint. """
    336         self.buildDwarf(dictionary=self.getBuildFlags())
    337         self.do_thread_actions(num_delay_crash_threads=1,
    338                                num_breakpoint_threads=1,
    339                                num_signal_threads=1)
    340 
    341     def setUp(self):
    342         # Call super's setUp().
    343         TestBase.setUp(self)
    344         # Find the line number for our breakpoint.
    345         self.filename = 'main.cpp'
    346         self.thread_breakpoint_line = line_number(self.filename, '// Set breakpoint here')
    347         self.setup_breakpoint_line = line_number(self.filename, '// Break here and adjust num')
    348         self.finish_breakpoint_line = line_number(self.filename, '// Break here and verify one thread is active')
    349 
    350     def describe_threads(self):
    351         ret = []
    352         for x in self.inferior_process:
    353             id = x.GetIndexID()
    354             reason = x.GetStopReason()
    355             status = "stopped" if x.IsStopped() else "running"
    356             reason_str = lldbutil.stop_reason_to_str(reason)
    357             if reason == lldb.eStopReasonBreakpoint:
    358                 bpid = x.GetStopReasonDataAtIndex(0)
    359                 bp = self.inferior_target.FindBreakpointByID(bpid)
    360                 reason_str = "%s hit %d times" % (lldbutil.get_description(bp), bp.GetHitCount())
    361             elif reason == lldb.eStopReasonWatchpoint:
    362                 watchid = x.GetStopReasonDataAtIndex(0)
    363                 watch = self.inferior_target.FindWatchpointByID(watchid)
    364                 reason_str = "%s hit %d times" % (lldbutil.get_description(watch), watch.GetHitCount())
    365             elif reason == lldb.eStopReasonSignal:
    366                 reason_str = "signal %s" % (signal_names[x.GetStopReasonDataAtIndex(0)])
    367 
    368             location = "\t".join([lldbutil.get_description(x.GetFrameAtIndex(i)) for i in range(x.GetNumFrames())])
    369             ret.append("thread %d %s due to %s at\n\t%s" % (id, status, reason_str, location))
    370         return ret
    371 
    372     def add_breakpoint(self, line, descriptions):
    373         """ Adds a breakpoint at self.filename:line and appends its description to descriptions, and
    374             returns the LLDB SBBreakpoint object.
    375         """
    376 
    377         bpno = lldbutil.run_break_set_by_file_and_line(self, self.filename, line, num_expected_locations=-1)
    378         bp = self.inferior_target.FindBreakpointByID(bpno)
    379         descriptions.append(": file = 'main.cpp', line = %d" % self.finish_breakpoint_line)
    380         return bp
    381 
    382     def inferior_done(self):
    383         """ Returns true if the inferior is done executing all the event threads (and is stopped at self.finish_breakpoint, 
    384             or has terminated execution.
    385         """
    386         return self.finish_breakpoint.GetHitCount() > 0 or \
    387                 self.crash_count > 0 or \
    388                 self.inferior_process.GetState == lldb.eStateExited
    389 
    390     def do_thread_actions(self,
    391                           num_breakpoint_threads = 0,
    392                           num_signal_threads = 0,
    393                           num_watchpoint_threads = 0,
    394                           num_crash_threads = 0,
    395                           num_delay_breakpoint_threads = 0,
    396                           num_delay_signal_threads = 0,
    397                           num_delay_watchpoint_threads = 0,
    398                           num_delay_crash_threads = 0):
    399         """ Sets a breakpoint in the main thread where test parameters (numbers of threads) can be adjusted, runs the inferior
    400             to that point, and modifies the locals that control the event thread counts. Also sets a breakpoint in
    401             breakpoint_func (the function executed by each 'breakpoint' thread) and a watchpoint on a global modified in
    402             watchpoint_func. The inferior is continued until exit or a crash takes place, and the number of events seen by LLDB
    403             is verified to match the expected number of events.
    404         """
    405         exe = os.path.join(os.getcwd(), "a.out")
    406         self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
    407 
    408         # Get the target
    409         self.inferior_target = self.dbg.GetSelectedTarget()
    410 
    411         expected_bps = []
    412 
    413         # Initialize all the breakpoints (main thread/aux thread)
    414         self.setup_breakpoint = self.add_breakpoint(self.setup_breakpoint_line, expected_bps)
    415         self.finish_breakpoint = self.add_breakpoint(self.finish_breakpoint_line, expected_bps)
    416 
    417         # Set the thread breakpoint
    418         if num_breakpoint_threads + num_delay_breakpoint_threads > 0:
    419             self.thread_breakpoint = self.add_breakpoint(self.thread_breakpoint_line, expected_bps)
    420 
    421         # Verify breakpoints
    422         self.expect("breakpoint list -f", "Breakpoint locations shown correctly", substrs = expected_bps)
    423 
    424         # Run the program.
    425         self.runCmd("run", RUN_SUCCEEDED)
    426 
    427         # Check we are at line self.setup_breakpoint
    428         self.expect("thread backtrace", STOPPED_DUE_TO_BREAKPOINT,
    429             substrs = ["stop reason = breakpoint 1."])
    430 
    431         # Initialize the (single) watchpoint on the global variable (g_watchme)
    432         if num_watchpoint_threads + num_delay_watchpoint_threads > 0:
    433             self.runCmd("watchpoint set variable g_watchme")
    434             for w in self.inferior_target.watchpoint_iter():
    435                 self.thread_watchpoint = w
    436                 self.assertTrue("g_watchme" in str(self.thread_watchpoint), "Watchpoint location not shown correctly")
    437 
    438         # Get the process
    439         self.inferior_process = self.inferior_target.GetProcess()
    440 
    441         # We should be stopped at the setup site where we can set the number of
    442         # threads doing each action (break/crash/signal/watch)
    443         self.assertEqual(self.inferior_process.GetNumThreads(), 1, 'Expected to stop before any additional threads are spawned.')
    444 
    445         self.runCmd("expr num_breakpoint_threads=%d" % num_breakpoint_threads)
    446         self.runCmd("expr num_crash_threads=%d" % num_crash_threads)
    447         self.runCmd("expr num_signal_threads=%d" % num_signal_threads)
    448         self.runCmd("expr num_watchpoint_threads=%d" % num_watchpoint_threads)
    449 
    450         self.runCmd("expr num_delay_breakpoint_threads=%d" % num_delay_breakpoint_threads)
    451         self.runCmd("expr num_delay_crash_threads=%d" % num_delay_crash_threads)
    452         self.runCmd("expr num_delay_signal_threads=%d" % num_delay_signal_threads)
    453         self.runCmd("expr num_delay_watchpoint_threads=%d" % num_delay_watchpoint_threads)
    454 
    455         # Continue the inferior so threads are spawned
    456         self.runCmd("continue")
    457 
    458         # Make sure we see all the threads. The inferior program's threads all synchronize with a pseudo-barrier; that is,
    459         # the inferior program ensures all threads are started and running before any thread triggers its 'event'.
    460         num_threads = self.inferior_process.GetNumThreads()
    461         expected_num_threads = num_breakpoint_threads + num_delay_breakpoint_threads \
    462                              + num_signal_threads + num_delay_signal_threads \
    463                              + num_watchpoint_threads + num_delay_watchpoint_threads \
    464                              + num_crash_threads + num_delay_crash_threads + 1
    465         self.assertEqual(num_threads, expected_num_threads,
    466             'Expected to see %d threads, but seeing %d. Details:\n%s' % (expected_num_threads,
    467                                                                          num_threads,
    468                                                                          "\n\t".join(self.describe_threads())))
    469 
    470         self.signal_count = len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonSignal))
    471         self.crash_count = len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonException))
    472 
    473         # Run to completion (or crash)
    474         while not self.inferior_done(): 
    475             if self.TraceOn():
    476                 self.runCmd("thread backtrace all")
    477             self.runCmd("continue")
    478             self.signal_count += len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonSignal))
    479             self.crash_count += len(lldbutil.get_stopped_threads(self.inferior_process, lldb.eStopReasonException))
    480 
    481         if num_crash_threads > 0 or num_delay_crash_threads > 0:
    482             # Expecting a crash
    483             self.assertTrue(self.crash_count > 0,
    484                 "Expecting at least one thread to crash. Details: %s" % "\t\n".join(self.describe_threads()))
    485 
    486             # Ensure the zombie process is reaped
    487             self.runCmd("process kill")
    488 
    489         elif num_crash_threads == 0 and num_delay_crash_threads == 0:
    490             # There should be a single active thread (the main one) which hit the breakpoint after joining
    491             self.assertEqual(1, self.finish_breakpoint.GetHitCount(), "Expected main thread (finish) breakpoint to be hit once")
    492 
    493             num_threads = self.inferior_process.GetNumThreads()
    494             self.assertEqual(1, num_threads, "Expecting 1 thread but seeing %d. Details:%s" % (num_threads,
    495                                                                                              "\n\t".join(self.describe_threads())))
    496             self.runCmd("continue")
    497 
    498             # The inferior process should have exited without crashing
    499             self.assertEqual(0, self.crash_count, "Unexpected thread(s) in crashed state")
    500             self.assertTrue(self.inferior_process.GetState() == lldb.eStateExited, PROCESS_EXITED)
    501 
    502             # Verify the number of actions took place matches expected numbers
    503             expected_breakpoint_threads = num_delay_breakpoint_threads + num_breakpoint_threads
    504             breakpoint_hit_count = self.thread_breakpoint.GetHitCount() if expected_breakpoint_threads > 0 else 0
    505             self.assertEqual(expected_breakpoint_threads, breakpoint_hit_count,
    506                 "Expected %d breakpoint hits, but got %d" % (expected_breakpoint_threads, breakpoint_hit_count))
    507 
    508             expected_signal_threads = num_delay_signal_threads + num_signal_threads
    509             self.assertEqual(expected_signal_threads, self.signal_count,
    510                 "Expected %d stops due to signal delivery, but got %d" % (expected_signal_threads, self.signal_count))
    511 
    512             expected_watchpoint_threads = num_delay_watchpoint_threads + num_watchpoint_threads
    513             watchpoint_hit_count = self.thread_watchpoint.GetHitCount() if expected_watchpoint_threads > 0 else 0
    514             self.assertEqual(expected_watchpoint_threads, watchpoint_hit_count,
    515                 "Expected %d watchpoint hits, got %d" % (expected_watchpoint_threads, watchpoint_hit_count))
    516 
    517 
    518 if __name__ == '__main__':
    519     import atexit
    520     lldb.SBDebugger.Initialize()
    521     atexit.register(lambda: lldb.SBDebugger.Terminate())
    522     unittest2.main()
    523