1 """Class for printing reports on profiled python code.""" 2 3 # Class for printing reports on profiled python code. rev 1.0 4/1/94 4 # 5 # Based on prior profile module by Sjoerd Mullender... 6 # which was hacked somewhat by: Guido van Rossum 7 # 8 # see profile.py for more info. 9 10 # Copyright 1994, by InfoSeek Corporation, all rights reserved. 11 # Written by James Roskind 12 # 13 # Permission to use, copy, modify, and distribute this Python software 14 # and its associated documentation for any purpose (subject to the 15 # restriction in the following sentence) without fee is hereby granted, 16 # provided that the above copyright notice appears in all copies, and 17 # that both that copyright notice and this permission notice appear in 18 # supporting documentation, and that the name of InfoSeek not be used in 19 # advertising or publicity pertaining to distribution of the software 20 # without specific, written prior permission. This permission is 21 # explicitly restricted to the copying and modification of the software 22 # to remain in Python, compiled Python, or other languages (such as C) 23 # wherein the modified or derived code is exclusively imported into a 24 # Python module. 25 # 26 # INFOSEEK CORPORATION DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS 27 # SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND 28 # FITNESS. IN NO EVENT SHALL INFOSEEK CORPORATION BE LIABLE FOR ANY 29 # SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER 30 # RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF 31 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN 32 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 33 34 35 import sys 36 import os 37 import time 38 import marshal 39 import re 40 from functools import cmp_to_key 41 42 __all__ = ["Stats"] 43 44 class Stats: 45 """This class is used for creating reports from data generated by the 46 Profile class. It is a "friend" of that class, and imports data either 47 by direct access to members of Profile class, or by reading in a dictionary 48 that was emitted (via marshal) from the Profile class. 49 50 The big change from the previous Profiler (in terms of raw functionality) 51 is that an "add()" method has been provided to combine Stats from 52 several distinct profile runs. Both the constructor and the add() 53 method now take arbitrarily many file names as arguments. 54 55 All the print methods now take an argument that indicates how many lines 56 to print. If the arg is a floating point number between 0 and 1.0, then 57 it is taken as a decimal percentage of the available lines to be printed 58 (e.g., .1 means print 10% of all available lines). If it is an integer, 59 it is taken to mean the number of lines of data that you wish to have 60 printed. 61 62 The sort_stats() method now processes some additional options (i.e., in 63 addition to the old -1, 0, 1, or 2). It takes an arbitrary number of 64 quoted strings to select the sort order. For example sort_stats('time', 65 'name') sorts on the major key of 'internal function time', and on the 66 minor key of 'the name of the function'. Look at the two tables in 67 sort_stats() and get_sort_arg_defs(self) for more examples. 68 69 All methods return self, so you can string together commands like: 70 Stats('foo', 'goo').strip_dirs().sort_stats('calls').\ 71 print_stats(5).print_callers(5) 72 """ 73 74 def __init__(self, *args, **kwds): 75 # I can't figure out how to explictly specify a stream keyword arg 76 # with *args: 77 # def __init__(self, *args, stream=sys.stdout): ... 78 # so I use **kwds and sqauwk if something unexpected is passed in. 79 self.stream = sys.stdout 80 if "stream" in kwds: 81 self.stream = kwds["stream"] 82 del kwds["stream"] 83 if kwds: 84 keys = kwds.keys() 85 keys.sort() 86 extras = ", ".join(["%s=%s" % (k, kwds[k]) for k in keys]) 87 raise ValueError, "unrecognized keyword args: %s" % extras 88 if not len(args): 89 arg = None 90 else: 91 arg = args[0] 92 args = args[1:] 93 self.init(arg) 94 self.add(*args) 95 96 def init(self, arg): 97 self.all_callees = None # calc only if needed 98 self.files = [] 99 self.fcn_list = None 100 self.total_tt = 0 101 self.total_calls = 0 102 self.prim_calls = 0 103 self.max_name_len = 0 104 self.top_level = {} 105 self.stats = {} 106 self.sort_arg_dict = {} 107 self.load_stats(arg) 108 trouble = 1 109 try: 110 self.get_top_level_stats() 111 trouble = 0 112 finally: 113 if trouble: 114 print >> self.stream, "Invalid timing data", 115 if self.files: print >> self.stream, self.files[-1], 116 print >> self.stream 117 118 def load_stats(self, arg): 119 if not arg: self.stats = {} 120 elif isinstance(arg, basestring): 121 f = open(arg, 'rb') 122 self.stats = marshal.load(f) 123 f.close() 124 try: 125 file_stats = os.stat(arg) 126 arg = time.ctime(file_stats.st_mtime) + " " + arg 127 except: # in case this is not unix 128 pass 129 self.files = [ arg ] 130 elif hasattr(arg, 'create_stats'): 131 arg.create_stats() 132 self.stats = arg.stats 133 arg.stats = {} 134 if not self.stats: 135 raise TypeError, "Cannot create or construct a %r object from '%r''" % ( 136 self.__class__, arg) 137 return 138 139 def get_top_level_stats(self): 140 for func, (cc, nc, tt, ct, callers) in self.stats.items(): 141 self.total_calls += nc 142 self.prim_calls += cc 143 self.total_tt += tt 144 if ("jprofile", 0, "profiler") in callers: 145 self.top_level[func] = None 146 if len(func_std_string(func)) > self.max_name_len: 147 self.max_name_len = len(func_std_string(func)) 148 149 def add(self, *arg_list): 150 if not arg_list: return self 151 if len(arg_list) > 1: self.add(*arg_list[1:]) 152 other = arg_list[0] 153 if type(self) != type(other) or self.__class__ != other.__class__: 154 other = Stats(other) 155 self.files += other.files 156 self.total_calls += other.total_calls 157 self.prim_calls += other.prim_calls 158 self.total_tt += other.total_tt 159 for func in other.top_level: 160 self.top_level[func] = None 161 162 if self.max_name_len < other.max_name_len: 163 self.max_name_len = other.max_name_len 164 165 self.fcn_list = None 166 167 for func, stat in other.stats.iteritems(): 168 if func in self.stats: 169 old_func_stat = self.stats[func] 170 else: 171 old_func_stat = (0, 0, 0, 0, {},) 172 self.stats[func] = add_func_stats(old_func_stat, stat) 173 return self 174 175 def dump_stats(self, filename): 176 """Write the profile data to a file we know how to load back.""" 177 f = file(filename, 'wb') 178 try: 179 marshal.dump(self.stats, f) 180 finally: 181 f.close() 182 183 # list the tuple indices and directions for sorting, 184 # along with some printable description 185 sort_arg_dict_default = { 186 "calls" : (((1,-1), ), "call count"), 187 "cumulative": (((3,-1), ), "cumulative time"), 188 "file" : (((4, 1), ), "file name"), 189 "line" : (((5, 1), ), "line number"), 190 "module" : (((4, 1), ), "file name"), 191 "name" : (((6, 1), ), "function name"), 192 "nfl" : (((6, 1),(4, 1),(5, 1),), "name/file/line"), 193 "pcalls" : (((0,-1), ), "call count"), 194 "stdname" : (((7, 1), ), "standard name"), 195 "time" : (((2,-1), ), "internal time"), 196 } 197 198 def get_sort_arg_defs(self): 199 """Expand all abbreviations that are unique.""" 200 if not self.sort_arg_dict: 201 self.sort_arg_dict = dict = {} 202 bad_list = {} 203 for word, tup in self.sort_arg_dict_default.iteritems(): 204 fragment = word 205 while fragment: 206 if not fragment: 207 break 208 if fragment in dict: 209 bad_list[fragment] = 0 210 break 211 dict[fragment] = tup 212 fragment = fragment[:-1] 213 for word in bad_list: 214 del dict[word] 215 return self.sort_arg_dict 216 217 def sort_stats(self, *field): 218 if not field: 219 self.fcn_list = 0 220 return self 221 if len(field) == 1 and isinstance(field[0], (int, long)): 222 # Be compatible with old profiler 223 field = [ {-1: "stdname", 224 0: "calls", 225 1: "time", 226 2: "cumulative"}[field[0]] ] 227 228 sort_arg_defs = self.get_sort_arg_defs() 229 sort_tuple = () 230 self.sort_type = "" 231 connector = "" 232 for word in field: 233 sort_tuple = sort_tuple + sort_arg_defs[word][0] 234 self.sort_type += connector + sort_arg_defs[word][1] 235 connector = ", " 236 237 stats_list = [] 238 for func, (cc, nc, tt, ct, callers) in self.stats.iteritems(): 239 stats_list.append((cc, nc, tt, ct) + func + 240 (func_std_string(func), func)) 241 242 stats_list.sort(key=cmp_to_key(TupleComp(sort_tuple).compare)) 243 244 self.fcn_list = fcn_list = [] 245 for tuple in stats_list: 246 fcn_list.append(tuple[-1]) 247 return self 248 249 def reverse_order(self): 250 if self.fcn_list: 251 self.fcn_list.reverse() 252 return self 253 254 def strip_dirs(self): 255 oldstats = self.stats 256 self.stats = newstats = {} 257 max_name_len = 0 258 for func, (cc, nc, tt, ct, callers) in oldstats.iteritems(): 259 newfunc = func_strip_path(func) 260 if len(func_std_string(newfunc)) > max_name_len: 261 max_name_len = len(func_std_string(newfunc)) 262 newcallers = {} 263 for func2, caller in callers.iteritems(): 264 newcallers[func_strip_path(func2)] = caller 265 266 if newfunc in newstats: 267 newstats[newfunc] = add_func_stats( 268 newstats[newfunc], 269 (cc, nc, tt, ct, newcallers)) 270 else: 271 newstats[newfunc] = (cc, nc, tt, ct, newcallers) 272 old_top = self.top_level 273 self.top_level = new_top = {} 274 for func in old_top: 275 new_top[func_strip_path(func)] = None 276 277 self.max_name_len = max_name_len 278 279 self.fcn_list = None 280 self.all_callees = None 281 return self 282 283 def calc_callees(self): 284 if self.all_callees: return 285 self.all_callees = all_callees = {} 286 for func, (cc, nc, tt, ct, callers) in self.stats.iteritems(): 287 if not func in all_callees: 288 all_callees[func] = {} 289 for func2, caller in callers.iteritems(): 290 if not func2 in all_callees: 291 all_callees[func2] = {} 292 all_callees[func2][func] = caller 293 return 294 295 #****************************************************************** 296 # The following functions support actual printing of reports 297 #****************************************************************** 298 299 # Optional "amount" is either a line count, or a percentage of lines. 300 301 def eval_print_amount(self, sel, list, msg): 302 new_list = list 303 if isinstance(sel, basestring): 304 try: 305 rex = re.compile(sel) 306 except re.error: 307 msg += " <Invalid regular expression %r>\n" % sel 308 return new_list, msg 309 new_list = [] 310 for func in list: 311 if rex.search(func_std_string(func)): 312 new_list.append(func) 313 else: 314 count = len(list) 315 if isinstance(sel, float) and 0.0 <= sel < 1.0: 316 count = int(count * sel + .5) 317 new_list = list[:count] 318 elif isinstance(sel, (int, long)) and 0 <= sel < count: 319 count = sel 320 new_list = list[:count] 321 if len(list) != len(new_list): 322 msg += " List reduced from %r to %r due to restriction <%r>\n" % ( 323 len(list), len(new_list), sel) 324 325 return new_list, msg 326 327 def get_print_list(self, sel_list): 328 width = self.max_name_len 329 if self.fcn_list: 330 stat_list = self.fcn_list[:] 331 msg = " Ordered by: " + self.sort_type + '\n' 332 else: 333 stat_list = self.stats.keys() 334 msg = " Random listing order was used\n" 335 336 for selection in sel_list: 337 stat_list, msg = self.eval_print_amount(selection, stat_list, msg) 338 339 count = len(stat_list) 340 341 if not stat_list: 342 return 0, stat_list 343 print >> self.stream, msg 344 if count < len(self.stats): 345 width = 0 346 for func in stat_list: 347 if len(func_std_string(func)) > width: 348 width = len(func_std_string(func)) 349 return width+2, stat_list 350 351 def print_stats(self, *amount): 352 for filename in self.files: 353 print >> self.stream, filename 354 if self.files: print >> self.stream 355 indent = ' ' * 8 356 for func in self.top_level: 357 print >> self.stream, indent, func_get_function_name(func) 358 359 print >> self.stream, indent, self.total_calls, "function calls", 360 if self.total_calls != self.prim_calls: 361 print >> self.stream, "(%d primitive calls)" % self.prim_calls, 362 print >> self.stream, "in %.3f seconds" % self.total_tt 363 print >> self.stream 364 width, list = self.get_print_list(amount) 365 if list: 366 self.print_title() 367 for func in list: 368 self.print_line(func) 369 print >> self.stream 370 print >> self.stream 371 return self 372 373 def print_callees(self, *amount): 374 width, list = self.get_print_list(amount) 375 if list: 376 self.calc_callees() 377 378 self.print_call_heading(width, "called...") 379 for func in list: 380 if func in self.all_callees: 381 self.print_call_line(width, func, self.all_callees[func]) 382 else: 383 self.print_call_line(width, func, {}) 384 print >> self.stream 385 print >> self.stream 386 return self 387 388 def print_callers(self, *amount): 389 width, list = self.get_print_list(amount) 390 if list: 391 self.print_call_heading(width, "was called by...") 392 for func in list: 393 cc, nc, tt, ct, callers = self.stats[func] 394 self.print_call_line(width, func, callers, "<-") 395 print >> self.stream 396 print >> self.stream 397 return self 398 399 def print_call_heading(self, name_size, column_title): 400 print >> self.stream, "Function ".ljust(name_size) + column_title 401 # print sub-header only if we have new-style callers 402 subheader = False 403 for cc, nc, tt, ct, callers in self.stats.itervalues(): 404 if callers: 405 value = callers.itervalues().next() 406 subheader = isinstance(value, tuple) 407 break 408 if subheader: 409 print >> self.stream, " "*name_size + " ncalls tottime cumtime" 410 411 def print_call_line(self, name_size, source, call_dict, arrow="->"): 412 print >> self.stream, func_std_string(source).ljust(name_size) + arrow, 413 if not call_dict: 414 print >> self.stream 415 return 416 clist = call_dict.keys() 417 clist.sort() 418 indent = "" 419 for func in clist: 420 name = func_std_string(func) 421 value = call_dict[func] 422 if isinstance(value, tuple): 423 nc, cc, tt, ct = value 424 if nc != cc: 425 substats = '%d/%d' % (nc, cc) 426 else: 427 substats = '%d' % (nc,) 428 substats = '%s %s %s %s' % (substats.rjust(7+2*len(indent)), 429 f8(tt), f8(ct), name) 430 left_width = name_size + 1 431 else: 432 substats = '%s(%r) %s' % (name, value, f8(self.stats[func][3])) 433 left_width = name_size + 3 434 print >> self.stream, indent*left_width + substats 435 indent = " " 436 437 def print_title(self): 438 print >> self.stream, ' ncalls tottime percall cumtime percall', 439 print >> self.stream, 'filename:lineno(function)' 440 441 def print_line(self, func): # hack : should print percentages 442 cc, nc, tt, ct, callers = self.stats[func] 443 c = str(nc) 444 if nc != cc: 445 c = c + '/' + str(cc) 446 print >> self.stream, c.rjust(9), 447 print >> self.stream, f8(tt), 448 if nc == 0: 449 print >> self.stream, ' '*8, 450 else: 451 print >> self.stream, f8(float(tt)/nc), 452 print >> self.stream, f8(ct), 453 if cc == 0: 454 print >> self.stream, ' '*8, 455 else: 456 print >> self.stream, f8(float(ct)/cc), 457 print >> self.stream, func_std_string(func) 458 459 class TupleComp: 460 """This class provides a generic function for comparing any two tuples. 461 Each instance records a list of tuple-indices (from most significant 462 to least significant), and sort direction (ascending or decending) for 463 each tuple-index. The compare functions can then be used as the function 464 argument to the system sort() function when a list of tuples need to be 465 sorted in the instances order.""" 466 467 def __init__(self, comp_select_list): 468 self.comp_select_list = comp_select_list 469 470 def compare (self, left, right): 471 for index, direction in self.comp_select_list: 472 l = left[index] 473 r = right[index] 474 if l < r: 475 return -direction 476 if l > r: 477 return direction 478 return 0 479 480 #************************************************************************** 481 # func_name is a triple (file:string, line:int, name:string) 482 483 def func_strip_path(func_name): 484 filename, line, name = func_name 485 return os.path.basename(filename), line, name 486 487 def func_get_function_name(func): 488 return func[2] 489 490 def func_std_string(func_name): # match what old profile produced 491 if func_name[:2] == ('~', 0): 492 # special case for built-in functions 493 name = func_name[2] 494 if name.startswith('<') and name.endswith('>'): 495 return '{%s}' % name[1:-1] 496 else: 497 return name 498 else: 499 return "%s:%d(%s)" % func_name 500 501 #************************************************************************** 502 # The following functions combine statists for pairs functions. 503 # The bulk of the processing involves correctly handling "call" lists, 504 # such as callers and callees. 505 #************************************************************************** 506 507 def add_func_stats(target, source): 508 """Add together all the stats for two profile entries.""" 509 cc, nc, tt, ct, callers = source 510 t_cc, t_nc, t_tt, t_ct, t_callers = target 511 return (cc+t_cc, nc+t_nc, tt+t_tt, ct+t_ct, 512 add_callers(t_callers, callers)) 513 514 def add_callers(target, source): 515 """Combine two caller lists in a single list.""" 516 new_callers = {} 517 for func, caller in target.iteritems(): 518 new_callers[func] = caller 519 for func, caller in source.iteritems(): 520 if func in new_callers: 521 if isinstance(caller, tuple): 522 # format used by cProfile 523 new_callers[func] = tuple([i[0] + i[1] for i in 524 zip(caller, new_callers[func])]) 525 else: 526 # format used by profile 527 new_callers[func] += caller 528 else: 529 new_callers[func] = caller 530 return new_callers 531 532 def count_calls(callers): 533 """Sum the caller statistics to get total number of calls received.""" 534 nc = 0 535 for calls in callers.itervalues(): 536 nc += calls 537 return nc 538 539 #************************************************************************** 540 # The following functions support printing of reports 541 #************************************************************************** 542 543 def f8(x): 544 return "%8.3f" % x 545 546 #************************************************************************** 547 # Statistics browser added by ESR, April 2001 548 #************************************************************************** 549 550 if __name__ == '__main__': 551 import cmd 552 try: 553 import readline 554 except ImportError: 555 pass 556 557 class ProfileBrowser(cmd.Cmd): 558 def __init__(self, profile=None): 559 cmd.Cmd.__init__(self) 560 self.prompt = "% " 561 self.stats = None 562 self.stream = sys.stdout 563 if profile is not None: 564 self.do_read(profile) 565 566 def generic(self, fn, line): 567 args = line.split() 568 processed = [] 569 for term in args: 570 try: 571 processed.append(int(term)) 572 continue 573 except ValueError: 574 pass 575 try: 576 frac = float(term) 577 if frac > 1 or frac < 0: 578 print >> self.stream, "Fraction argument must be in [0, 1]" 579 continue 580 processed.append(frac) 581 continue 582 except ValueError: 583 pass 584 processed.append(term) 585 if self.stats: 586 getattr(self.stats, fn)(*processed) 587 else: 588 print >> self.stream, "No statistics object is loaded." 589 return 0 590 def generic_help(self): 591 print >> self.stream, "Arguments may be:" 592 print >> self.stream, "* An integer maximum number of entries to print." 593 print >> self.stream, "* A decimal fractional number between 0 and 1, controlling" 594 print >> self.stream, " what fraction of selected entries to print." 595 print >> self.stream, "* A regular expression; only entries with function names" 596 print >> self.stream, " that match it are printed." 597 598 def do_add(self, line): 599 if self.stats: 600 self.stats.add(line) 601 else: 602 print >> self.stream, "No statistics object is loaded." 603 return 0 604 def help_add(self): 605 print >> self.stream, "Add profile info from given file to current statistics object." 606 607 def do_callees(self, line): 608 return self.generic('print_callees', line) 609 def help_callees(self): 610 print >> self.stream, "Print callees statistics from the current stat object." 611 self.generic_help() 612 613 def do_callers(self, line): 614 return self.generic('print_callers', line) 615 def help_callers(self): 616 print >> self.stream, "Print callers statistics from the current stat object." 617 self.generic_help() 618 619 def do_EOF(self, line): 620 print >> self.stream, "" 621 return 1 622 def help_EOF(self): 623 print >> self.stream, "Leave the profile brower." 624 625 def do_quit(self, line): 626 return 1 627 def help_quit(self): 628 print >> self.stream, "Leave the profile brower." 629 630 def do_read(self, line): 631 if line: 632 try: 633 self.stats = Stats(line) 634 except IOError, args: 635 print >> self.stream, args[1] 636 return 637 except Exception as err: 638 print >> self.stream, err.__class__.__name__ + ':', err 639 return 640 self.prompt = line + "% " 641 elif len(self.prompt) > 2: 642 line = self.prompt[:-2] 643 self.do_read(line) 644 else: 645 print >> self.stream, "No statistics object is current -- cannot reload." 646 return 0 647 def help_read(self): 648 print >> self.stream, "Read in profile data from a specified file." 649 print >> self.stream, "Without argument, reload the current file." 650 651 def do_reverse(self, line): 652 if self.stats: 653 self.stats.reverse_order() 654 else: 655 print >> self.stream, "No statistics object is loaded." 656 return 0 657 def help_reverse(self): 658 print >> self.stream, "Reverse the sort order of the profiling report." 659 660 def do_sort(self, line): 661 if not self.stats: 662 print >> self.stream, "No statistics object is loaded." 663 return 664 abbrevs = self.stats.get_sort_arg_defs() 665 if line and all((x in abbrevs) for x in line.split()): 666 self.stats.sort_stats(*line.split()) 667 else: 668 print >> self.stream, "Valid sort keys (unique prefixes are accepted):" 669 for (key, value) in Stats.sort_arg_dict_default.iteritems(): 670 print >> self.stream, "%s -- %s" % (key, value[1]) 671 return 0 672 def help_sort(self): 673 print >> self.stream, "Sort profile data according to specified keys." 674 print >> self.stream, "(Typing `sort' without arguments lists valid keys.)" 675 def complete_sort(self, text, *args): 676 return [a for a in Stats.sort_arg_dict_default if a.startswith(text)] 677 678 def do_stats(self, line): 679 return self.generic('print_stats', line) 680 def help_stats(self): 681 print >> self.stream, "Print statistics from the current stat object." 682 self.generic_help() 683 684 def do_strip(self, line): 685 if self.stats: 686 self.stats.strip_dirs() 687 else: 688 print >> self.stream, "No statistics object is loaded." 689 def help_strip(self): 690 print >> self.stream, "Strip leading path information from filenames in the report." 691 692 def help_help(self): 693 print >> self.stream, "Show help for a given command." 694 695 def postcmd(self, stop, line): 696 if stop: 697 return stop 698 return None 699 700 import sys 701 if len(sys.argv) > 1: 702 initprofile = sys.argv[1] 703 else: 704 initprofile = None 705 try: 706 browser = ProfileBrowser(initprofile) 707 print >> browser.stream, "Welcome to the profile statistics browser." 708 browser.cmdloop() 709 print >> browser.stream, "Goodbye." 710 except KeyboardInterrupt: 711 pass 712 713 # That's all, folks. 714
