1 """Find modules used by a script, using introspection.""" 2 # This module should be kept compatible with Python 2.2, see PEP 291. 3 4 from __future__ import generators 5 import dis 6 import imp 7 import marshal 8 import os 9 import sys 10 import types 11 import struct 12 13 if hasattr(sys.__stdout__, "newlines"): 14 READ_MODE = "U" # universal line endings 15 else: 16 # remain compatible with Python < 2.3 17 READ_MODE = "r" 18 19 LOAD_CONST = chr(dis.opname.index('LOAD_CONST')) 20 IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME')) 21 STORE_NAME = chr(dis.opname.index('STORE_NAME')) 22 STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL')) 23 STORE_OPS = [STORE_NAME, STORE_GLOBAL] 24 HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT) 25 26 # Modulefinder does a good job at simulating Python's, but it can not 27 # handle __path__ modifications packages make at runtime. Therefore there 28 # is a mechanism whereby you can register extra paths in this map for a 29 # package, and it will be honored. 30 31 # Note this is a mapping is lists of paths. 32 packagePathMap = {} 33 34 # A Public interface 35 def AddPackagePath(packagename, path): 36 paths = packagePathMap.get(packagename, []) 37 paths.append(path) 38 packagePathMap[packagename] = paths 39 40 replacePackageMap = {} 41 42 # This ReplacePackage mechanism allows modulefinder to work around the 43 # way the _xmlplus package injects itself under the name "xml" into 44 # sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml") 45 # before running ModuleFinder. 46 47 def ReplacePackage(oldname, newname): 48 replacePackageMap[oldname] = newname 49 50 51 class Module: 52 53 def __init__(self, name, file=None, path=None): 54 self.__name__ = name 55 self.__file__ = file 56 self.__path__ = path 57 self.__code__ = None 58 # The set of global names that are assigned to in the module. 59 # This includes those names imported through starimports of 60 # Python modules. 61 self.globalnames = {} 62 # The set of starimports this module did that could not be 63 # resolved, ie. a starimport from a non-Python module. 64 self.starimports = {} 65 66 def __repr__(self): 67 s = "Module(%r" % (self.__name__,) 68 if self.__file__ is not None: 69 s = s + ", %r" % (self.__file__,) 70 if self.__path__ is not None: 71 s = s + ", %r" % (self.__path__,) 72 s = s + ")" 73 return s 74 75 class ModuleFinder: 76 77 def __init__(self, path=None, debug=0, excludes=[], replace_paths=[]): 78 if path is None: 79 path = sys.path 80 self.path = path 81 self.modules = {} 82 self.badmodules = {} 83 self.debug = debug 84 self.indent = 0 85 self.excludes = excludes 86 self.replace_paths = replace_paths 87 self.processed_paths = [] # Used in debugging only 88 89 def msg(self, level, str, *args): 90 if level <= self.debug: 91 for i in range(self.indent): 92 print " ", 93 print str, 94 for arg in args: 95 print repr(arg), 96 print 97 98 def msgin(self, *args): 99 level = args[0] 100 if level <= self.debug: 101 self.indent = self.indent + 1 102 self.msg(*args) 103 104 def msgout(self, *args): 105 level = args[0] 106 if level <= self.debug: 107 self.indent = self.indent - 1 108 self.msg(*args) 109 110 def run_script(self, pathname): 111 self.msg(2, "run_script", pathname) 112 with open(pathname, READ_MODE) as fp: 113 stuff = ("", "r", imp.PY_SOURCE) 114 self.load_module('__main__', fp, pathname, stuff) 115 116 def load_file(self, pathname): 117 dir, name = os.path.split(pathname) 118 name, ext = os.path.splitext(name) 119 with open(pathname, READ_MODE) as fp: 120 stuff = (ext, "r", imp.PY_SOURCE) 121 self.load_module(name, fp, pathname, stuff) 122 123 def import_hook(self, name, caller=None, fromlist=None, level=-1): 124 self.msg(3, "import_hook", name, caller, fromlist, level) 125 parent = self.determine_parent(caller, level=level) 126 q, tail = self.find_head_package(parent, name) 127 m = self.load_tail(q, tail) 128 if not fromlist: 129 return q 130 if m.__path__: 131 self.ensure_fromlist(m, fromlist) 132 return None 133 134 def determine_parent(self, caller, level=-1): 135 self.msgin(4, "determine_parent", caller, level) 136 if not caller or level == 0: 137 self.msgout(4, "determine_parent -> None") 138 return None 139 pname = caller.__name__ 140 if level >= 1: # relative import 141 if caller.__path__: 142 level -= 1 143 if level == 0: 144 parent = self.modules[pname] 145 assert parent is caller 146 self.msgout(4, "determine_parent ->", parent) 147 return parent 148 if pname.count(".") < level: 149 raise ImportError, "relative importpath too deep" 150 pname = ".".join(pname.split(".")[:-level]) 151 parent = self.modules[pname] 152 self.msgout(4, "determine_parent ->", parent) 153 return parent 154 if caller.__path__: 155 parent = self.modules[pname] 156 assert caller is parent 157 self.msgout(4, "determine_parent ->", parent) 158 return parent 159 if '.' in pname: 160 i = pname.rfind('.') 161 pname = pname[:i] 162 parent = self.modules[pname] 163 assert parent.__name__ == pname 164 self.msgout(4, "determine_parent ->", parent) 165 return parent 166 self.msgout(4, "determine_parent -> None") 167 return None 168 169 def find_head_package(self, parent, name): 170 self.msgin(4, "find_head_package", parent, name) 171 if '.' in name: 172 i = name.find('.') 173 head = name[:i] 174 tail = name[i+1:] 175 else: 176 head = name 177 tail = "" 178 if parent: 179 qname = "%s.%s" % (parent.__name__, head) 180 else: 181 qname = head 182 q = self.import_module(head, qname, parent) 183 if q: 184 self.msgout(4, "find_head_package ->", (q, tail)) 185 return q, tail 186 if parent: 187 qname = head 188 parent = None 189 q = self.import_module(head, qname, parent) 190 if q: 191 self.msgout(4, "find_head_package ->", (q, tail)) 192 return q, tail 193 self.msgout(4, "raise ImportError: No module named", qname) 194 raise ImportError, "No module named " + qname 195 196 def load_tail(self, q, tail): 197 self.msgin(4, "load_tail", q, tail) 198 m = q 199 while tail: 200 i = tail.find('.') 201 if i < 0: i = len(tail) 202 head, tail = tail[:i], tail[i+1:] 203 mname = "%s.%s" % (m.__name__, head) 204 m = self.import_module(head, mname, m) 205 if not m: 206 self.msgout(4, "raise ImportError: No module named", mname) 207 raise ImportError, "No module named " + mname 208 self.msgout(4, "load_tail ->", m) 209 return m 210 211 def ensure_fromlist(self, m, fromlist, recursive=0): 212 self.msg(4, "ensure_fromlist", m, fromlist, recursive) 213 for sub in fromlist: 214 if sub == "*": 215 if not recursive: 216 all = self.find_all_submodules(m) 217 if all: 218 self.ensure_fromlist(m, all, 1) 219 elif not hasattr(m, sub): 220 subname = "%s.%s" % (m.__name__, sub) 221 submod = self.import_module(sub, subname, m) 222 if not submod: 223 raise ImportError, "No module named " + subname 224 225 def find_all_submodules(self, m): 226 if not m.__path__: 227 return 228 modules = {} 229 # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. 230 # But we must also collect Python extension modules - although 231 # we cannot separate normal dlls from Python extensions. 232 suffixes = [] 233 for triple in imp.get_suffixes(): 234 suffixes.append(triple[0]) 235 for dir in m.__path__: 236 try: 237 names = os.listdir(dir) 238 except os.error: 239 self.msg(2, "can't list directory", dir) 240 continue 241 for name in names: 242 mod = None 243 for suff in suffixes: 244 n = len(suff) 245 if name[-n:] == suff: 246 mod = name[:-n] 247 break 248 if mod and mod != "__init__": 249 modules[mod] = mod 250 return modules.keys() 251 252 def import_module(self, partname, fqname, parent): 253 self.msgin(3, "import_module", partname, fqname, parent) 254 try: 255 m = self.modules[fqname] 256 except KeyError: 257 pass 258 else: 259 self.msgout(3, "import_module ->", m) 260 return m 261 if fqname in self.badmodules: 262 self.msgout(3, "import_module -> None") 263 return None 264 if parent and parent.__path__ is None: 265 self.msgout(3, "import_module -> None") 266 return None 267 try: 268 fp, pathname, stuff = self.find_module(partname, 269 parent and parent.__path__, parent) 270 except ImportError: 271 self.msgout(3, "import_module ->", None) 272 return None 273 try: 274 m = self.load_module(fqname, fp, pathname, stuff) 275 finally: 276 if fp: fp.close() 277 if parent: 278 setattr(parent, partname, m) 279 self.msgout(3, "import_module ->", m) 280 return m 281 282 def load_module(self, fqname, fp, pathname, file_info): 283 suffix, mode, type = file_info 284 self.msgin(2, "load_module", fqname, fp and "fp", pathname) 285 if type == imp.PKG_DIRECTORY: 286 m = self.load_package(fqname, pathname) 287 self.msgout(2, "load_module ->", m) 288 return m 289 if type == imp.PY_SOURCE: 290 co = compile(fp.read()+'\n', pathname, 'exec') 291 elif type == imp.PY_COMPILED: 292 if fp.read(4) != imp.get_magic(): 293 self.msgout(2, "raise ImportError: Bad magic number", pathname) 294 raise ImportError, "Bad magic number in %s" % pathname 295 fp.read(4) 296 co = marshal.load(fp) 297 else: 298 co = None 299 m = self.add_module(fqname) 300 m.__file__ = pathname 301 if co: 302 if self.replace_paths: 303 co = self.replace_paths_in_code(co) 304 m.__code__ = co 305 self.scan_code(co, m) 306 self.msgout(2, "load_module ->", m) 307 return m 308 309 def _add_badmodule(self, name, caller): 310 if name not in self.badmodules: 311 self.badmodules[name] = {} 312 if caller: 313 self.badmodules[name][caller.__name__] = 1 314 else: 315 self.badmodules[name]["-"] = 1 316 317 def _safe_import_hook(self, name, caller, fromlist, level=-1): 318 # wrapper for self.import_hook() that won't raise ImportError 319 if name in self.badmodules: 320 self._add_badmodule(name, caller) 321 return 322 try: 323 self.import_hook(name, caller, level=level) 324 except ImportError, msg: 325 self.msg(2, "ImportError:", str(msg)) 326 self._add_badmodule(name, caller) 327 else: 328 if fromlist: 329 for sub in fromlist: 330 if sub in self.badmodules: 331 self._add_badmodule(sub, caller) 332 continue 333 try: 334 self.import_hook(name, caller, [sub], level=level) 335 except ImportError, msg: 336 self.msg(2, "ImportError:", str(msg)) 337 fullname = name + "." + sub 338 self._add_badmodule(fullname, caller) 339 340 def scan_opcodes(self, co, 341 unpack = struct.unpack): 342 # Scan the code, and yield 'interesting' opcode combinations 343 # Version for Python 2.4 and older 344 code = co.co_code 345 names = co.co_names 346 consts = co.co_consts 347 while code: 348 c = code[0] 349 if c in STORE_OPS: 350 oparg, = unpack('<H', code[1:3]) 351 yield "store", (names[oparg],) 352 code = code[3:] 353 continue 354 if c == LOAD_CONST and code[3] == IMPORT_NAME: 355 oparg_1, oparg_2 = unpack('<xHxH', code[:6]) 356 yield "import", (consts[oparg_1], names[oparg_2]) 357 code = code[6:] 358 continue 359 if c >= HAVE_ARGUMENT: 360 code = code[3:] 361 else: 362 code = code[1:] 363 364 def scan_opcodes_25(self, co, 365 unpack = struct.unpack): 366 # Scan the code, and yield 'interesting' opcode combinations 367 # Python 2.5 version (has absolute and relative imports) 368 code = co.co_code 369 names = co.co_names 370 consts = co.co_consts 371 LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME 372 while code: 373 c = code[0] 374 if c in STORE_OPS: 375 oparg, = unpack('<H', code[1:3]) 376 yield "store", (names[oparg],) 377 code = code[3:] 378 continue 379 if code[:9:3] == LOAD_LOAD_AND_IMPORT: 380 oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9]) 381 level = consts[oparg_1] 382 if level == -1: # normal import 383 yield "import", (consts[oparg_2], names[oparg_3]) 384 elif level == 0: # absolute import 385 yield "absolute_import", (consts[oparg_2], names[oparg_3]) 386 else: # relative import 387 yield "relative_import", (level, consts[oparg_2], names[oparg_3]) 388 code = code[9:] 389 continue 390 if c >= HAVE_ARGUMENT: 391 code = code[3:] 392 else: 393 code = code[1:] 394 395 def scan_code(self, co, m): 396 code = co.co_code 397 if sys.version_info >= (2, 5): 398 scanner = self.scan_opcodes_25 399 else: 400 scanner = self.scan_opcodes 401 for what, args in scanner(co): 402 if what == "store": 403 name, = args 404 m.globalnames[name] = 1 405 elif what in ("import", "absolute_import"): 406 fromlist, name = args 407 have_star = 0 408 if fromlist is not None: 409 if "*" in fromlist: 410 have_star = 1 411 fromlist = [f for f in fromlist if f != "*"] 412 if what == "absolute_import": level = 0 413 else: level = -1 414 self._safe_import_hook(name, m, fromlist, level=level) 415 if have_star: 416 # We've encountered an "import *". If it is a Python module, 417 # the code has already been parsed and we can suck out the 418 # global names. 419 mm = None 420 if m.__path__: 421 # At this point we don't know whether 'name' is a 422 # submodule of 'm' or a global module. Let's just try 423 # the full name first. 424 mm = self.modules.get(m.__name__ + "." + name) 425 if mm is None: 426 mm = self.modules.get(name) 427 if mm is not None: 428 m.globalnames.update(mm.globalnames) 429 m.starimports.update(mm.starimports) 430 if mm.__code__ is None: 431 m.starimports[name] = 1 432 else: 433 m.starimports[name] = 1 434 elif what == "relative_import": 435 level, fromlist, name = args 436 if name: 437 self._safe_import_hook(name, m, fromlist, level=level) 438 else: 439 parent = self.determine_parent(m, level=level) 440 self._safe_import_hook(parent.__name__, None, fromlist, level=0) 441 else: 442 # We don't expect anything else from the generator. 443 raise RuntimeError(what) 444 445 for c in co.co_consts: 446 if isinstance(c, type(co)): 447 self.scan_code(c, m) 448 449 def load_package(self, fqname, pathname): 450 self.msgin(2, "load_package", fqname, pathname) 451 newname = replacePackageMap.get(fqname) 452 if newname: 453 fqname = newname 454 m = self.add_module(fqname) 455 m.__file__ = pathname 456 m.__path__ = [pathname] 457 458 # As per comment at top of file, simulate runtime __path__ additions. 459 m.__path__ = m.__path__ + packagePathMap.get(fqname, []) 460 461 fp, buf, stuff = self.find_module("__init__", m.__path__) 462 self.load_module(fqname, fp, buf, stuff) 463 self.msgout(2, "load_package ->", m) 464 if fp: 465 fp.close() 466 return m 467 468 def add_module(self, fqname): 469 if fqname in self.modules: 470 return self.modules[fqname] 471 self.modules[fqname] = m = Module(fqname) 472 return m 473 474 def find_module(self, name, path, parent=None): 475 if parent is not None: 476 # assert path is not None 477 fullname = parent.__name__+'.'+name 478 else: 479 fullname = name 480 if fullname in self.excludes: 481 self.msgout(3, "find_module -> Excluded", fullname) 482 raise ImportError, name 483 484 if path is None: 485 if name in sys.builtin_module_names: 486 return (None, None, ("", "", imp.C_BUILTIN)) 487 488 path = self.path 489 return imp.find_module(name, path) 490 491 def report(self): 492 """Print a report to stdout, listing the found modules with their 493 paths, as well as modules that are missing, or seem to be missing. 494 """ 495 print 496 print " %-25s %s" % ("Name", "File") 497 print " %-25s %s" % ("----", "----") 498 # Print modules found 499 keys = self.modules.keys() 500 keys.sort() 501 for key in keys: 502 m = self.modules[key] 503 if m.__path__: 504 print "P", 505 else: 506 print "m", 507 print "%-25s" % key, m.__file__ or "" 508 509 # Print missing modules 510 missing, maybe = self.any_missing_maybe() 511 if missing: 512 print 513 print "Missing modules:" 514 for name in missing: 515 mods = self.badmodules[name].keys() 516 mods.sort() 517 print "?", name, "imported from", ', '.join(mods) 518 # Print modules that may be missing, but then again, maybe not... 519 if maybe: 520 print 521 print "Submodules that appear to be missing, but could also be", 522 print "global names in the parent package:" 523 for name in maybe: 524 mods = self.badmodules[name].keys() 525 mods.sort() 526 print "?", name, "imported from", ', '.join(mods) 527 528 def any_missing(self): 529 """Return a list of modules that appear to be missing. Use 530 any_missing_maybe() if you want to know which modules are 531 certain to be missing, and which *may* be missing. 532 """ 533 missing, maybe = self.any_missing_maybe() 534 return missing + maybe 535 536 def any_missing_maybe(self): 537 """Return two lists, one with modules that are certainly missing 538 and one with modules that *may* be missing. The latter names could 539 either be submodules *or* just global names in the package. 540 541 The reason it can't always be determined is that it's impossible to 542 tell which names are imported when "from module import *" is done 543 with an extension module, short of actually importing it. 544 """ 545 missing = [] 546 maybe = [] 547 for name in self.badmodules: 548 if name in self.excludes: 549 continue 550 i = name.rfind(".") 551 if i < 0: 552 missing.append(name) 553 continue 554 subname = name[i+1:] 555 pkgname = name[:i] 556 pkg = self.modules.get(pkgname) 557 if pkg is not None: 558 if pkgname in self.badmodules[name]: 559 # The package tried to import this module itself and 560 # failed. It's definitely missing. 561 missing.append(name) 562 elif subname in pkg.globalnames: 563 # It's a global in the package: definitely not missing. 564 pass 565 elif pkg.starimports: 566 # It could be missing, but the package did an "import *" 567 # from a non-Python module, so we simply can't be sure. 568 maybe.append(name) 569 else: 570 # It's not a global in the package, the package didn't 571 # do funny star imports, it's very likely to be missing. 572 # The symbol could be inserted into the package from the 573 # outside, but since that's not good style we simply list 574 # it missing. 575 missing.append(name) 576 else: 577 missing.append(name) 578 missing.sort() 579 maybe.sort() 580 return missing, maybe 581 582 def replace_paths_in_code(self, co): 583 new_filename = original_filename = os.path.normpath(co.co_filename) 584 for f, r in self.replace_paths: 585 if original_filename.startswith(f): 586 new_filename = r + original_filename[len(f):] 587 break 588 589 if self.debug and original_filename not in self.processed_paths: 590 if new_filename != original_filename: 591 self.msgout(2, "co_filename %r changed to %r" \ 592 % (original_filename,new_filename,)) 593 else: 594 self.msgout(2, "co_filename %r remains unchanged" \ 595 % (original_filename,)) 596 self.processed_paths.append(original_filename) 597 598 consts = list(co.co_consts) 599 for i in range(len(consts)): 600 if isinstance(consts[i], type(co)): 601 consts[i] = self.replace_paths_in_code(consts[i]) 602 603 return types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, 604 co.co_flags, co.co_code, tuple(consts), co.co_names, 605 co.co_varnames, new_filename, co.co_name, 606 co.co_firstlineno, co.co_lnotab, 607 co.co_freevars, co.co_cellvars) 608 609 610 def test(): 611 # Parse command line 612 import getopt 613 try: 614 opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") 615 except getopt.error, msg: 616 print msg 617 return 618 619 # Process options 620 debug = 1 621 domods = 0 622 addpath = [] 623 exclude = [] 624 for o, a in opts: 625 if o == '-d': 626 debug = debug + 1 627 if o == '-m': 628 domods = 1 629 if o == '-p': 630 addpath = addpath + a.split(os.pathsep) 631 if o == '-q': 632 debug = 0 633 if o == '-x': 634 exclude.append(a) 635 636 # Provide default arguments 637 if not args: 638 script = "hello.py" 639 else: 640 script = args[0] 641 642 # Set the path based on sys.path and the script directory 643 path = sys.path[:] 644 path[0] = os.path.dirname(script) 645 path = addpath + path 646 if debug > 1: 647 print "path:" 648 for item in path: 649 print " ", repr(item) 650 651 # Create the module finder and turn its crank 652 mf = ModuleFinder(path, debug, exclude) 653 for arg in args[1:]: 654 if arg == '-m': 655 domods = 1 656 continue 657 if domods: 658 if arg[-2:] == '.*': 659 mf.import_hook(arg[:-2], None, ["*"]) 660 else: 661 mf.import_hook(arg) 662 else: 663 mf.load_file(arg) 664 mf.run_script(script) 665 mf.report() 666 return mf # for -i debugging 667 668 669 if __name__ == '__main__': 670 try: 671 mf = test() 672 except KeyboardInterrupt: 673 print "\n[interrupt]" 674
