1 """ 2 Watches the key ``paste.httpserver.thread_pool`` to see how many 3 threads there are and report on any wedged threads. 4 """ 5 import sys 6 import cgi 7 import time 8 import traceback 9 from cStringIO import StringIO 10 from thread import get_ident 11 from paste import httpexceptions 12 from paste.request import construct_url, parse_formvars 13 from paste.util.template import HTMLTemplate, bunch 14 15 page_template = HTMLTemplate(''' 16 <html> 17 <head> 18 <style type="text/css"> 19 body { 20 font-family: sans-serif; 21 } 22 table.environ tr td { 23 border-bottom: #bbb 1px solid; 24 } 25 table.environ tr td.bottom { 26 border-bottom: none; 27 } 28 table.thread { 29 border: 1px solid #000; 30 margin-bottom: 1em; 31 } 32 table.thread tr td { 33 border-bottom: #999 1px solid; 34 padding-right: 1em; 35 } 36 table.thread tr td.bottom { 37 border-bottom: none; 38 } 39 table.thread tr.this_thread td { 40 background-color: #006; 41 color: #fff; 42 } 43 a.button { 44 background-color: #ddd; 45 border: #aaa outset 2px; 46 text-decoration: none; 47 margin-top: 10px; 48 font-size: 80%; 49 color: #000; 50 } 51 a.button:hover { 52 background-color: #eee; 53 border: #bbb outset 2px; 54 } 55 a.button:active { 56 border: #bbb inset 2px; 57 } 58 </style> 59 <title>{{title}}</title> 60 </head> 61 <body> 62 <h1>{{title}}</h1> 63 {{if kill_thread_id}} 64 <div style="background-color: #060; color: #fff; 65 border: 2px solid #000;"> 66 Thread {{kill_thread_id}} killed 67 </div> 68 {{endif}} 69 <div>Pool size: {{nworkers}} 70 {{if actual_workers > nworkers}} 71 + {{actual_workers-nworkers}} extra 72 {{endif}} 73 ({{nworkers_used}} used including current request)<br> 74 idle: {{len(track_threads["idle"])}}, 75 busy: {{len(track_threads["busy"])}}, 76 hung: {{len(track_threads["hung"])}}, 77 dying: {{len(track_threads["dying"])}}, 78 zombie: {{len(track_threads["zombie"])}}</div> 79 80 {{for thread in threads}} 81 82 <table class="thread"> 83 <tr {{if thread.thread_id == this_thread_id}}class="this_thread"{{endif}}> 84 <td> 85 <b>Thread</b> 86 {{if thread.thread_id == this_thread_id}} 87 (<i>this</i> request) 88 {{endif}}</td> 89 <td> 90 <b>{{thread.thread_id}} 91 {{if allow_kill}} 92 <form action="{{script_name}}/kill" method="POST" 93 style="display: inline"> 94 <input type="hidden" name="thread_id" value="{{thread.thread_id}}"> 95 <input type="submit" value="kill"> 96 </form> 97 {{endif}} 98 </b> 99 </td> 100 </tr> 101 <tr> 102 <td>Time processing request</td> 103 <td>{{thread.time_html|html}}</td> 104 </tr> 105 <tr> 106 <td>URI</td> 107 <td>{{if thread.uri == 'unknown'}} 108 unknown 109 {{else}}<a href="{{thread.uri}}">{{thread.uri_short}}</a> 110 {{endif}} 111 </td> 112 <tr> 113 <td colspan="2" class="bottom"> 114 <a href="#" class="button" style="width: 9em; display: block" 115 onclick=" 116 var el = document.getElementById('environ-{{thread.thread_id}}'); 117 if (el.style.display) { 118 el.style.display = ''; 119 this.innerHTML = \'▾ Hide environ\'; 120 } else { 121 el.style.display = 'none'; 122 this.innerHTML = \'▸ Show environ\'; 123 } 124 return false 125 ">▸ Show environ</a> 126 127 <div id="environ-{{thread.thread_id}}" style="display: none"> 128 {{if thread.environ:}} 129 <table class="environ"> 130 {{for loop, item in looper(sorted(thread.environ.items()))}} 131 {{py:key, value=item}} 132 <tr> 133 <td {{if loop.last}}class="bottom"{{endif}}>{{key}}</td> 134 <td {{if loop.last}}class="bottom"{{endif}}>{{value}}</td> 135 </tr> 136 {{endfor}} 137 </table> 138 {{else}} 139 Thread is in process of starting 140 {{endif}} 141 </div> 142 143 {{if thread.traceback}} 144 <a href="#" class="button" style="width: 9em; display: block" 145 onclick=" 146 var el = document.getElementById('traceback-{{thread.thread_id}}'); 147 if (el.style.display) { 148 el.style.display = ''; 149 this.innerHTML = \'▾ Hide traceback\'; 150 } else { 151 el.style.display = 'none'; 152 this.innerHTML = \'▸ Show traceback\'; 153 } 154 return false 155 ">▸ Show traceback</a> 156 157 <div id="traceback-{{thread.thread_id}}" style="display: none"> 158 <pre class="traceback">{{thread.traceback}}</pre> 159 </div> 160 {{endif}} 161 162 </td> 163 </tr> 164 </table> 165 166 {{endfor}} 167 168 </body> 169 </html> 170 ''', name='watchthreads.page_template') 171 172 class WatchThreads(object): 173 174 """ 175 Application that watches the threads in ``paste.httpserver``, 176 showing the length each thread has been working on a request. 177 178 If allow_kill is true, then you can kill errant threads through 179 this application. 180 181 This application can expose private information (specifically in 182 the environment, like cookies), so it should be protected. 183 """ 184 185 def __init__(self, allow_kill=False): 186 self.allow_kill = allow_kill 187 188 def __call__(self, environ, start_response): 189 if 'paste.httpserver.thread_pool' not in environ: 190 start_response('403 Forbidden', [('Content-type', 'text/plain')]) 191 return ['You must use the threaded Paste HTTP server to use this application'] 192 if environ.get('PATH_INFO') == '/kill': 193 return self.kill(environ, start_response) 194 else: 195 return self.show(environ, start_response) 196 197 def show(self, environ, start_response): 198 start_response('200 OK', [('Content-type', 'text/html')]) 199 form = parse_formvars(environ) 200 if form.get('kill'): 201 kill_thread_id = form['kill'] 202 else: 203 kill_thread_id = None 204 thread_pool = environ['paste.httpserver.thread_pool'] 205 nworkers = thread_pool.nworkers 206 now = time.time() 207 208 209 workers = thread_pool.worker_tracker.items() 210 workers.sort(key=lambda v: v[1][0]) 211 threads = [] 212 for thread_id, (time_started, worker_environ) in workers: 213 thread = bunch() 214 threads.append(thread) 215 if worker_environ: 216 thread.uri = construct_url(worker_environ) 217 else: 218 thread.uri = 'unknown' 219 thread.thread_id = thread_id 220 thread.time_html = format_time(now-time_started) 221 thread.uri_short = shorten(thread.uri) 222 thread.environ = worker_environ 223 thread.traceback = traceback_thread(thread_id) 224 225 page = page_template.substitute( 226 title="Thread Pool Worker Tracker", 227 nworkers=nworkers, 228 actual_workers=len(thread_pool.workers), 229 nworkers_used=len(workers), 230 script_name=environ['SCRIPT_NAME'], 231 kill_thread_id=kill_thread_id, 232 allow_kill=self.allow_kill, 233 threads=threads, 234 this_thread_id=get_ident(), 235 track_threads=thread_pool.track_threads()) 236 237 return [page] 238 239 def kill(self, environ, start_response): 240 if not self.allow_kill: 241 exc = httpexceptions.HTTPForbidden( 242 'Killing threads has not been enabled. Shame on you ' 243 'for trying!') 244 return exc(environ, start_response) 245 vars = parse_formvars(environ) 246 thread_id = int(vars['thread_id']) 247 thread_pool = environ['paste.httpserver.thread_pool'] 248 if thread_id not in thread_pool.worker_tracker: 249 exc = httpexceptions.PreconditionFailed( 250 'You tried to kill thread %s, but it is not working on ' 251 'any requests' % thread_id) 252 return exc(environ, start_response) 253 thread_pool.kill_worker(thread_id) 254 script_name = environ['SCRIPT_NAME'] or '/' 255 exc = httpexceptions.HTTPFound( 256 headers=[('Location', script_name+'?kill=%s' % thread_id)]) 257 return exc(environ, start_response) 258 259 def traceback_thread(thread_id): 260 """ 261 Returns a plain-text traceback of the given thread, or None if it 262 can't get a traceback. 263 """ 264 if not hasattr(sys, '_current_frames'): 265 # Only 2.5 has support for this, with this special function 266 return None 267 frames = sys._current_frames() 268 if not thread_id in frames: 269 return None 270 frame = frames[thread_id] 271 out = StringIO() 272 traceback.print_stack(frame, file=out) 273 return out.getvalue() 274 275 hide_keys = ['paste.httpserver.thread_pool'] 276 277 def format_environ(environ): 278 if environ is None: 279 return environ_template.substitute( 280 key='---', 281 value='No environment registered for this thread yet') 282 environ_rows = [] 283 for key, value in sorted(environ.items()): 284 if key in hide_keys: 285 continue 286 try: 287 if key.upper() != key: 288 value = repr(value) 289 environ_rows.append( 290 environ_template.substitute( 291 key=cgi.escape(str(key)), 292 value=cgi.escape(str(value)))) 293 except Exception as e: 294 environ_rows.append( 295 environ_template.substitute( 296 key=cgi.escape(str(key)), 297 value='Error in <code>repr()</code>: %s' % e)) 298 return ''.join(environ_rows) 299 300 def format_time(time_length): 301 if time_length >= 60*60: 302 # More than an hour 303 time_string = '%i:%02i:%02i' % (int(time_length/60/60), 304 int(time_length/60) % 60, 305 time_length % 60) 306 elif time_length >= 120: 307 time_string = '%i:%02i' % (int(time_length/60), 308 time_length % 60) 309 elif time_length > 60: 310 time_string = '%i sec' % time_length 311 elif time_length > 1: 312 time_string = '%0.1f sec' % time_length 313 else: 314 time_string = '%0.2f sec' % time_length 315 if time_length < 5: 316 return time_string 317 elif time_length < 120: 318 return '<span style="color: #900">%s</span>' % time_string 319 else: 320 return '<span style="background-color: #600; color: #fff">%s</span>' % time_string 321 322 def shorten(s): 323 if len(s) > 60: 324 return s[:40]+'...'+s[-10:] 325 else: 326 return s 327 328 def make_watch_threads(global_conf, allow_kill=False): 329 from paste.deploy.converters import asbool 330 return WatchThreads(allow_kill=asbool(allow_kill)) 331 make_watch_threads.__doc__ = WatchThreads.__doc__ 332 333 def make_bad_app(global_conf, pause=0): 334 pause = int(pause) 335 def bad_app(environ, start_response): 336 import thread 337 if pause: 338 time.sleep(pause) 339 else: 340 count = 0 341 while 1: 342 print("I'm alive %s (%s)" % (count, thread.get_ident())) 343 time.sleep(10) 344 count += 1 345 start_response('200 OK', [('content-type', 'text/plain')]) 346 return ['OK, paused %s seconds' % pause] 347 return bad_app 348
