1 """Parse a Python module and describe its classes and methods. 2 3 Parse enough of a Python file to recognize imports and class and 4 method definitions, and to find out the superclasses of a class. 5 6 The interface consists of a single function: 7 readmodule_ex(module [, path]) 8 where module is the name of a Python module, and path is an optional 9 list of directories where the module is to be searched. If present, 10 path is prepended to the system search path sys.path. The return 11 value is a dictionary. The keys of the dictionary are the names of 12 the classes defined in the module (including classes that are defined 13 via the from XXX import YYY construct). The values are class 14 instances of the class Class defined here. One special key/value pair 15 is present for packages: the key '__path__' has a list as its value 16 which contains the package search path. 17 18 A class is described by the class Class in this module. Instances 19 of this class have the following instance variables: 20 module -- the module name 21 name -- the name of the class 22 super -- a list of super classes (Class instances) 23 methods -- a dictionary of methods 24 file -- the file in which the class was defined 25 lineno -- the line in the file on which the class statement occurred 26 The dictionary of methods uses the method names as keys and the line 27 numbers on which the method was defined as values. 28 If the name of a super class is not recognized, the corresponding 29 entry in the list of super classes is not a class instance but a 30 string giving the name of the super class. Since import statements 31 are recognized and imported modules are scanned as well, this 32 shouldn't happen often. 33 34 A function is described by the class Function in this module. 35 Instances of this class have the following instance variables: 36 module -- the module name 37 name -- the name of the class 38 file -- the file in which the class was defined 39 lineno -- the line in the file on which the class statement occurred 40 """ 41 42 import sys 43 import imp 44 import tokenize 45 from token import NAME, DEDENT, OP 46 from operator import itemgetter 47 48 __all__ = ["readmodule", "readmodule_ex", "Class", "Function"] 49 50 _modules = {} # cache of modules we've seen 51 52 # each Python class is represented by an instance of this class 53 class Class: 54 '''Class to represent a Python class.''' 55 def __init__(self, module, name, super, file, lineno): 56 self.module = module 57 self.name = name 58 if super is None: 59 super = [] 60 self.super = super 61 self.methods = {} 62 self.file = file 63 self.lineno = lineno 64 65 def _addmethod(self, name, lineno): 66 self.methods[name] = lineno 67 68 class Function: 69 '''Class to represent a top-level Python function''' 70 def __init__(self, module, name, file, lineno): 71 self.module = module 72 self.name = name 73 self.file = file 74 self.lineno = lineno 75 76 def readmodule(module, path=None): 77 '''Backwards compatible interface. 78 79 Call readmodule_ex() and then only keep Class objects from the 80 resulting dictionary.''' 81 82 res = {} 83 for key, value in _readmodule(module, path or []).items(): 84 if isinstance(value, Class): 85 res[key] = value 86 return res 87 88 def readmodule_ex(module, path=None): 89 '''Read a module file and return a dictionary of classes. 90 91 Search for MODULE in PATH and sys.path, read and parse the 92 module and return a dictionary with one entry for each class 93 found in the module. 94 ''' 95 return _readmodule(module, path or []) 96 97 def _readmodule(module, path, inpackage=None): 98 '''Do the hard work for readmodule[_ex]. 99 100 If INPACKAGE is given, it must be the dotted name of the package in 101 which we are searching for a submodule, and then PATH must be the 102 package search path; otherwise, we are searching for a top-level 103 module, and PATH is combined with sys.path. 104 ''' 105 # Compute the full module name (prepending inpackage if set) 106 if inpackage is not None: 107 fullmodule = "%s.%s" % (inpackage, module) 108 else: 109 fullmodule = module 110 111 # Check in the cache 112 if fullmodule in _modules: 113 return _modules[fullmodule] 114 115 # Initialize the dict for this module's contents 116 dict = {} 117 118 # Check if it is a built-in module; we don't do much for these 119 if module in sys.builtin_module_names and inpackage is None: 120 _modules[module] = dict 121 return dict 122 123 # Check for a dotted module name 124 i = module.rfind('.') 125 if i >= 0: 126 package = module[:i] 127 submodule = module[i+1:] 128 parent = _readmodule(package, path, inpackage) 129 if inpackage is not None: 130 package = "%s.%s" % (inpackage, package) 131 return _readmodule(submodule, parent['__path__'], package) 132 133 # Search the path for the module 134 f = None 135 if inpackage is not None: 136 f, fname, (_s, _m, ty) = imp.find_module(module, path) 137 else: 138 f, fname, (_s, _m, ty) = imp.find_module(module, path + sys.path) 139 if ty == imp.PKG_DIRECTORY: 140 dict['__path__'] = [fname] 141 path = [fname] + path 142 f, fname, (_s, _m, ty) = imp.find_module('__init__', [fname]) 143 _modules[fullmodule] = dict 144 if ty != imp.PY_SOURCE: 145 # not Python source, can't do anything with this module 146 f.close() 147 return dict 148 149 stack = [] # stack of (class, indent) pairs 150 151 g = tokenize.generate_tokens(f.readline) 152 try: 153 for tokentype, token, start, _end, _line in g: 154 if tokentype == DEDENT: 155 lineno, thisindent = start 156 # close nested classes and defs 157 while stack and stack[-1][1] >= thisindent: 158 del stack[-1] 159 elif token == 'def': 160 lineno, thisindent = start 161 # close previous nested classes and defs 162 while stack and stack[-1][1] >= thisindent: 163 del stack[-1] 164 tokentype, meth_name, start = g.next()[0:3] 165 if tokentype != NAME: 166 continue # Syntax error 167 if stack: 168 cur_class = stack[-1][0] 169 if isinstance(cur_class, Class): 170 # it's a method 171 cur_class._addmethod(meth_name, lineno) 172 # else it's a nested def 173 else: 174 # it's a function 175 dict[meth_name] = Function(fullmodule, meth_name, 176 fname, lineno) 177 stack.append((None, thisindent)) # Marker for nested fns 178 elif token == 'class': 179 lineno, thisindent = start 180 # close previous nested classes and defs 181 while stack and stack[-1][1] >= thisindent: 182 del stack[-1] 183 tokentype, class_name, start = g.next()[0:3] 184 if tokentype != NAME: 185 continue # Syntax error 186 # parse what follows the class name 187 tokentype, token, start = g.next()[0:3] 188 inherit = None 189 if token == '(': 190 names = [] # List of superclasses 191 # there's a list of superclasses 192 level = 1 193 super = [] # Tokens making up current superclass 194 while True: 195 tokentype, token, start = g.next()[0:3] 196 if token in (')', ',') and level == 1: 197 n = "".join(super) 198 if n in dict: 199 # we know this super class 200 n = dict[n] 201 else: 202 c = n.split('.') 203 if len(c) > 1: 204 # super class is of the form 205 # module.class: look in module for 206 # class 207 m = c[-2] 208 c = c[-1] 209 if m in _modules: 210 d = _modules[m] 211 if c in d: 212 n = d[c] 213 names.append(n) 214 super = [] 215 if token == '(': 216 level += 1 217 elif token == ')': 218 level -= 1 219 if level == 0: 220 break 221 elif token == ',' and level == 1: 222 pass 223 # only use NAME and OP (== dot) tokens for type name 224 elif tokentype in (NAME, OP) and level == 1: 225 super.append(token) 226 # expressions in the base list are not supported 227 inherit = names 228 cur_class = Class(fullmodule, class_name, inherit, 229 fname, lineno) 230 if not stack: 231 dict[class_name] = cur_class 232 stack.append((cur_class, thisindent)) 233 elif token == 'import' and start[1] == 0: 234 modules = _getnamelist(g) 235 for mod, _mod2 in modules: 236 try: 237 # Recursively read the imported module 238 if inpackage is None: 239 _readmodule(mod, path) 240 else: 241 try: 242 _readmodule(mod, path, inpackage) 243 except ImportError: 244 _readmodule(mod, []) 245 except: 246 # If we can't find or parse the imported module, 247 # too bad -- don't die here. 248 pass 249 elif token == 'from' and start[1] == 0: 250 mod, token = _getname(g) 251 if not mod or token != "import": 252 continue 253 names = _getnamelist(g) 254 try: 255 # Recursively read the imported module 256 d = _readmodule(mod, path, inpackage) 257 except: 258 # If we can't find or parse the imported module, 259 # too bad -- don't die here. 260 continue 261 # add any classes that were defined in the imported module 262 # to our name space if they were mentioned in the list 263 for n, n2 in names: 264 if n in d: 265 dict[n2 or n] = d[n] 266 elif n == '*': 267 # don't add names that start with _ 268 for n in d: 269 if n[0] != '_': 270 dict[n] = d[n] 271 except StopIteration: 272 pass 273 274 f.close() 275 return dict 276 277 def _getnamelist(g): 278 # Helper to get a comma-separated list of dotted names plus 'as' 279 # clauses. Return a list of pairs (name, name2) where name2 is 280 # the 'as' name, or None if there is no 'as' clause. 281 names = [] 282 while True: 283 name, token = _getname(g) 284 if not name: 285 break 286 if token == 'as': 287 name2, token = _getname(g) 288 else: 289 name2 = None 290 names.append((name, name2)) 291 while token != "," and "\n" not in token: 292 token = g.next()[1] 293 if token != ",": 294 break 295 return names 296 297 def _getname(g): 298 # Helper to get a dotted name, return a pair (name, token) where 299 # name is the dotted name, or None if there was no dotted name, 300 # and token is the next input token. 301 parts = [] 302 tokentype, token = g.next()[0:2] 303 if tokentype != NAME and token != '*': 304 return (None, token) 305 parts.append(token) 306 while True: 307 tokentype, token = g.next()[0:2] 308 if token != '.': 309 break 310 tokentype, token = g.next()[0:2] 311 if tokentype != NAME: 312 break 313 parts.append(token) 314 return (".".join(parts), token) 315 316 def _main(): 317 # Main program for testing. 318 import os 319 mod = sys.argv[1] 320 if os.path.exists(mod): 321 path = [os.path.dirname(mod)] 322 mod = os.path.basename(mod) 323 if mod.lower().endswith(".py"): 324 mod = mod[:-3] 325 else: 326 path = [] 327 dict = readmodule_ex(mod, path) 328 objs = dict.values() 329 objs.sort(lambda a, b: cmp(getattr(a, 'lineno', 0), 330 getattr(b, 'lineno', 0))) 331 for obj in objs: 332 if isinstance(obj, Class): 333 print "class", obj.name, obj.super, obj.lineno 334 methods = sorted(obj.methods.iteritems(), key=itemgetter(1)) 335 for name, lineno in methods: 336 if name != "__path__": 337 print " def", name, lineno 338 elif isinstance(obj, Function): 339 print "def", obj.name, obj.lineno 340 341 if __name__ == "__main__": 342 _main() 343
