1 """ 2 Find modules used by a script, using bytecode analysis. 3 4 Based on the stdlib modulefinder by Thomas Heller and Just van Rossum, 5 but uses a graph data structure and 2.3 features 6 7 XXX: Verify all calls to import_hook (and variants) to ensure that 8 imports are done in the right way. 9 """ 10 from __future__ import absolute_import, print_function 11 12 import pkg_resources 13 14 import dis 15 import imp 16 import marshal 17 import os 18 import sys 19 import struct 20 import zipimport 21 import re 22 from collections import deque, namedtuple 23 import ast 24 25 from altgraph.ObjectGraph import ObjectGraph 26 from altgraph import GraphError 27 28 from itertools import count 29 30 from modulegraph import util 31 from modulegraph import zipio 32 33 if sys.version_info[0] == 2: 34 from StringIO import StringIO as BytesIO 35 from StringIO import StringIO 36 from urllib import pathname2url 37 def _Bchr(value): 38 return chr(value) 39 40 else: 41 from urllib.request import pathname2url 42 from io import BytesIO, StringIO 43 44 def _Bchr(value): 45 return value 46 47 48 # File open mode for reading (univeral newlines) 49 if sys.version_info[0] == 2: 50 _READ_MODE = "rU" 51 else: 52 _READ_MODE = "r" 53 54 55 56 57 # Modulegraph does a good job at simulating Python's, but it can not 58 # handle packagepath modifications packages make at runtime. Therefore there 59 # is a mechanism whereby you can register extra paths in this map for a 60 # package, and it will be honored. 61 # 62 # Note this is a mapping is lists of paths. 63 _packagePathMap = {} 64 65 # Prefix used in magic .pth files used by setuptools to create namespace 66 # packages without an __init__.py file. 67 # 68 # The value is a list of such prefixes as the prefix varies with versions of 69 # setuptools. 70 _SETUPTOOLS_NAMESPACEPKG_PTHs=( 71 "import sys,types,os; p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('", 72 "import sys,new,os; p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('", 73 "import sys, types, os;p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('", 74 ) 75 76 77 def _namespace_package_path(fqname, pathnames, path=None): 78 """ 79 Return the __path__ for the python package in *fqname*. 80 81 This function uses setuptools metadata to extract information 82 about namespace packages from installed eggs. 83 """ 84 working_set = pkg_resources.WorkingSet(path) 85 86 path = list(pathnames) 87 88 for dist in working_set: 89 if dist.has_metadata('namespace_packages.txt'): 90 namespaces = dist.get_metadata( 91 'namespace_packages.txt').splitlines() 92 if fqname in namespaces: 93 nspath = os.path.join(dist.location, *fqname.split('.')) 94 if nspath not in path: 95 path.append(nspath) 96 97 return path 98 99 _strs = re.compile(r'''^\s*["']([A-Za-z0-9_]+)["'],?\s*''') # "<- emacs happy 100 101 def _eval_str_tuple(value): 102 """ 103 Input is the repr of a tuple of strings, output 104 is that tuple. 105 106 This only works with a tuple where the members are 107 python identifiers. 108 """ 109 if not (value.startswith('(') and value.endswith(')')): 110 raise ValueError(value) 111 112 orig_value = value 113 value = value[1:-1] 114 115 result = [] 116 while value: 117 m = _strs.match(value) 118 if m is None: 119 raise ValueError(orig_value) 120 121 result.append(m.group(1)) 122 value = value[len(m.group(0)):] 123 124 return tuple(result) 125 126 def _path_from_importerror(exc, default): 127 # This is a hack, but sadly enough the necessary information 128 # isn't available otherwise. 129 m = re.match('^No module named (\S+)$', str(exc)) 130 if m is not None: 131 return m.group(1) 132 133 return default 134 135 def os_listdir(path): 136 """ 137 Deprecated name 138 """ 139 warnings.warn("Use zipio.listdir instead of os_listdir", 140 DeprecationWarning) 141 return zipio.listdir(path) 142 143 144 def _code_to_file(co): 145 """ Convert code object to a .pyc pseudo-file """ 146 return BytesIO( 147 imp.get_magic() + b'\0\0\0\0' + marshal.dumps(co)) 148 149 150 def find_module(name, path=None): 151 """ 152 A version of imp.find_module that works with zipped packages. 153 """ 154 if path is None: 155 path = sys.path 156 157 # Support for the PEP302 importer for normal imports: 158 # - Python 2.5 has pkgutil.ImpImporter 159 # - In setuptools 0.7 and later there's _pkgutil.ImpImporter 160 # - In earlier setuptools versions you pkg_resources.ImpWrapper 161 # 162 # XXX: This is a bit of a hack, should check if we can just rely on 163 # PEP302's get_code() method with all recent versions of pkgutil and/or 164 # setuptools (setuptools 0.6.latest, setuptools trunk and python2.[45]) 165 # 166 # For python 3.3 this code should be replaced by code using importlib, 167 # for python 3.2 and 2.7 this should be cleaned up a lot. 168 try: 169 from pkgutil import ImpImporter 170 except ImportError: 171 try: 172 from _pkgutil import ImpImporter 173 except ImportError: 174 ImpImporter = pkg_resources.ImpWrapper 175 176 namespace_path =[] 177 fp = None 178 for entry in path: 179 importer = pkg_resources.get_importer(entry) 180 if importer is None: 181 continue 182 183 if sys.version_info[:2] >= (3,3) and hasattr(importer, 'find_loader'): 184 loader, portions = importer.find_loader(name) 185 186 else: 187 loader = importer.find_module(name) 188 portions = [] 189 190 namespace_path.extend(portions) 191 192 if loader is None: continue 193 194 if isinstance(importer, ImpImporter): 195 filename = loader.filename 196 if filename.endswith('.pyc') or filename.endswith('.pyo'): 197 fp = open(filename, 'rb') 198 description = ('.pyc', 'rb', imp.PY_COMPILED) 199 return (fp, filename, description) 200 201 elif filename.endswith('.py'): 202 if sys.version_info[0] == 2: 203 fp = open(filename, _READ_MODE) 204 else: 205 with open(filename, 'rb') as fp: 206 encoding = util.guess_encoding(fp) 207 208 fp = open(filename, _READ_MODE, encoding=encoding) 209 description = ('.py', _READ_MODE, imp.PY_SOURCE) 210 return (fp, filename, description) 211 212 else: 213 for _sfx, _mode, _type in imp.get_suffixes(): 214 if _type == imp.C_EXTENSION and filename.endswith(_sfx): 215 description = (_sfx, 'rb', imp.C_EXTENSION) 216 break 217 else: 218 description = ('', '', imp.PKG_DIRECTORY) 219 220 return (None, filename, description) 221 222 if hasattr(loader, 'path'): 223 if loader.path.endswith('.pyc') or loader.path.endswith('.pyo'): 224 fp = open(loader.path, 'rb') 225 description = ('.pyc', 'rb', imp.PY_COMPILED) 226 return (fp, loader.path, description) 227 228 229 if hasattr(loader, 'get_source'): 230 source = loader.get_source(name) 231 fp = StringIO(source) 232 co = None 233 234 else: 235 source = None 236 237 if source is None: 238 if hasattr(loader, 'get_code'): 239 co = loader.get_code(name) 240 fp = _code_to_file(co) 241 242 else: 243 fp = None 244 co = None 245 246 pathname = os.path.join(entry, *name.split('.')) 247 248 if isinstance(loader, zipimport.zipimporter): 249 # Check if this happens to be a wrapper module introduced by 250 # setuptools, if it is we return the actual extension. 251 zn = '/'.join(name.split('.')) 252 for _sfx, _mode, _type in imp.get_suffixes(): 253 if _type == imp.C_EXTENSION: 254 p = loader.prefix + zn + _sfx 255 if loader._files is None: 256 loader_files = zipimport._zip_directory_cache[loader.archive] 257 else: 258 loader_files = loader._files 259 260 if p in loader_files: 261 description = (_sfx, 'rb', imp.C_EXTENSION) 262 return (None, pathname + _sfx, description) 263 264 if hasattr(loader, 'is_package') and loader.is_package(name): 265 return (None, pathname, ('', '', imp.PKG_DIRECTORY)) 266 267 if co is None: 268 if hasattr(loader, 'path'): 269 filename = loader.path 270 elif hasattr(loader, 'get_filename'): 271 filename = loader.get_filename(name) 272 if source is not None: 273 if filename.endswith(".pyc") or filename.endswith(".pyo"): 274 filename = filename[:-1] 275 else: 276 filename = None 277 278 if filename is not None and (filename.endswith('.py') or filename.endswith('.pyw')): 279 return (fp, filename, ('.py', 'rU', imp.PY_SOURCE)) 280 else: 281 if fp is not None: 282 fp.close() 283 return (None, filename, (os.path.splitext(filename)[-1], 'rb', imp.C_EXTENSION)) 284 285 else: 286 if hasattr(loader, 'path'): 287 return (fp, loader.path, ('.pyc', 'rb', imp.PY_COMPILED)) 288 else: 289 return (fp, pathname + '.pyc', ('.pyc', 'rb', imp.PY_COMPILED)) 290 291 if namespace_path: 292 if fp is not None: 293 fp.close() 294 return (None, namespace_path[0], ('', namespace_path, imp.PKG_DIRECTORY)) 295 296 raise ImportError(name) 297 298 def moduleInfoForPath(path): 299 for (ext, readmode, typ) in imp.get_suffixes(): 300 if path.endswith(ext): 301 return os.path.basename(path)[:-len(ext)], readmode, typ 302 return None 303 304 # A Public interface 305 import warnings 306 def AddPackagePath(packagename, path): 307 warnings.warn("Use addPackagePath instead of AddPackagePath", 308 DeprecationWarning) 309 310 addPackagePath(packagename, path) 311 312 def addPackagePath(packagename, path): 313 paths = _packagePathMap.get(packagename, []) 314 paths.append(path) 315 _packagePathMap[packagename] = paths 316 317 _replacePackageMap = {} 318 319 # This ReplacePackage mechanism allows modulefinder to work around the 320 # way the _xmlplus package injects itself under the name "xml" into 321 # sys.modules at runtime by calling ReplacePackage("_xmlplus", "xml") 322 # before running ModuleGraph. 323 def ReplacePackage(oldname, newname): 324 warnings.warn("use replacePackage instead of ReplacePackage", 325 DeprecationWarning) 326 replacePackage(oldname, newname) 327 328 def replacePackage(oldname, newname): 329 _replacePackageMap[oldname] = newname 330 331 332 class DependencyInfo (namedtuple("DependencyInfo", ["conditional", "function", "tryexcept", "fromlist"])): 333 __slots__ = () 334 335 def _merged(self, other): 336 if (not self.conditional and not self.function and not self.tryexcept) \ 337 or (not other.conditional and not other.function and not other.tryexcept): 338 return DependencyInfo(conditional=False, function=False, tryexcept=False, fromlist=self.fromlist and other.fromlist) 339 340 else: 341 return DependencyInfo( 342 conditional=self.conditional or other.conditional, 343 function=self.function or other.function, 344 tryexcept=self.tryexcept or other.tryexcept, 345 fromlist=self.fromlist and other.fromlist) 346 347 348 class Node(object): 349 def __init__(self, identifier): 350 self.debug = 0 351 self.graphident = identifier 352 self.identifier = identifier 353 self._namespace = {} 354 self.filename = None 355 self.packagepath = None 356 self.code = None 357 # The set of global names that are assigned to in the module. 358 # This includes those names imported through starimports of 359 # Python modules. 360 self.globalnames = set() 361 # The set of starimports this module did that could not be 362 # resolved, ie. a starimport from a non-Python module. 363 self.starimports = set() 364 365 def __contains__(self, name): 366 return name in self._namespace 367 368 def __getitem__(self, name): 369 return self._namespace[name] 370 371 def __setitem__(self, name, value): 372 self._namespace[name] = value 373 374 def get(self, *args): 375 return self._namespace.get(*args) 376 377 def __cmp__(self, other): 378 try: 379 otherIdent = getattr(other, 'graphident') 380 except AttributeError: 381 return NotImplemented 382 383 return cmp(self.graphident, otherIdent) 384 385 def __eq__(self, other): 386 try: 387 otherIdent = getattr(other, 'graphident') 388 except AttributeError: 389 return False 390 391 return self.graphident == otherIdent 392 393 def __ne__(self, other): 394 try: 395 otherIdent = getattr(other, 'graphident') 396 except AttributeError: 397 return True 398 399 return self.graphident != otherIdent 400 401 def __lt__(self, other): 402 try: 403 otherIdent = getattr(other, 'graphident') 404 except AttributeError: 405 return NotImplemented 406 407 return self.graphident < otherIdent 408 409 def __le__(self, other): 410 try: 411 otherIdent = getattr(other, 'graphident') 412 except AttributeError: 413 return NotImplemented 414 415 return self.graphident <= otherIdent 416 417 def __gt__(self, other): 418 try: 419 otherIdent = getattr(other, 'graphident') 420 except AttributeError: 421 return NotImplemented 422 423 return self.graphident > otherIdent 424 425 def __ge__(self, other): 426 try: 427 otherIdent = getattr(other, 'graphident') 428 except AttributeError: 429 return NotImplemented 430 431 return self.graphident >= otherIdent 432 433 434 def __hash__(self): 435 return hash(self.graphident) 436 437 def infoTuple(self): 438 return (self.identifier,) 439 440 def __repr__(self): 441 return '%s%r' % (type(self).__name__, self.infoTuple()) 442 443 class Alias(str): 444 pass 445 446 class AliasNode(Node): 447 def __init__(self, name, node): 448 super(AliasNode, self).__init__(name) 449 for k in 'identifier', 'packagepath', '_namespace', 'globalnames', 'starimports': 450 setattr(self, k, getattr(node, k, None)) 451 452 def infoTuple(self): 453 return (self.graphident, self.identifier) 454 455 class BadModule(Node): 456 pass 457 458 class ExcludedModule(BadModule): 459 pass 460 461 class MissingModule(BadModule): 462 pass 463 464 class Script(Node): 465 def __init__(self, filename): 466 super(Script, self).__init__(filename) 467 self.filename = filename 468 469 def infoTuple(self): 470 return (self.filename,) 471 472 class BaseModule(Node): 473 def __init__(self, name, filename=None, path=None): 474 super(BaseModule, self).__init__(name) 475 self.filename = filename 476 self.packagepath = path 477 478 def infoTuple(self): 479 return tuple(filter(None, (self.identifier, self.filename, self.packagepath))) 480 481 class BuiltinModule(BaseModule): 482 pass 483 484 class SourceModule(BaseModule): 485 pass 486 487 class InvalidSourceModule(SourceModule): 488 pass 489 490 class CompiledModule(BaseModule): 491 pass 492 493 class InvalidCompiledModule(BaseModule): 494 pass 495 496 class Package(BaseModule): 497 pass 498 499 class NamespacePackage(Package): 500 pass 501 502 class Extension(BaseModule): 503 pass 504 505 class FlatPackage(BaseModule): # nocoverage 506 def __init__(self, *args, **kwds): 507 warnings.warn("This class will be removed in a future version of modulegraph", 508 DeprecationWarning) 509 super(FlatPackage, *args, **kwds) 510 511 class ArchiveModule(BaseModule): # nocoverage 512 def __init__(self, *args, **kwds): 513 warnings.warn("This class will be removed in a future version of modulegraph", 514 DeprecationWarning) 515 super(FlatPackage, *args, **kwds) 516 517 # HTML templates for ModuleGraph generator 518 header = """\ 519 <html> 520 <head> 521 <title>%(TITLE)s</title> 522 <style> 523 .node { margin:1em 0; } 524 </style> 525 </head> 526 <body> 527 <h1>%(TITLE)s</h1>""" 528 entry = """ 529 <div class="node"> 530 <a name="%(NAME)s" /> 531 %(CONTENT)s 532 </div>""" 533 contpl = """<tt>%(NAME)s</tt> %(TYPE)s""" 534 contpl_linked = """\ 535 <a target="code" href="%(URL)s" type="text/plain"><tt>%(NAME)s</tt></a>""" 536 imports = """\ 537 <div class="import"> 538 %(HEAD)s: 539 %(LINKS)s 540 </div> 541 """ 542 footer = """ 543 </body> 544 </html>""" 545 546 def _ast_names(names): 547 result = [] 548 for nm in names: 549 if isinstance(nm, ast.alias): 550 result.append(nm.name) 551 else: 552 result.append(nm) 553 return result 554 555 556 if sys.version_info[0] == 2: 557 DEFAULT_IMPORT_LEVEL= -1 558 else: 559 DEFAULT_IMPORT_LEVEL= 0 560 561 class _Visitor (ast.NodeVisitor): 562 def __init__(self, graph, module): 563 self._graph = graph 564 self._module = module 565 self._level = DEFAULT_IMPORT_LEVEL 566 self._in_if = [False] 567 self._in_def = [False] 568 self._in_tryexcept = [False] 569 570 @property 571 def in_if(self): 572 return self._in_if[-1] 573 574 @property 575 def in_def(self): 576 return self._in_def[-1] 577 578 @property 579 def in_tryexcept(self): 580 return self._in_tryexcept[-1] 581 582 def _process_import(self, name, fromlist, level): 583 584 if sys.version_info[0] == 2: 585 if name == '__future__' and 'absolute_import' in (fromlist or ()): 586 self._level = 0 587 588 have_star = False 589 if fromlist is not None: 590 fromlist = set(fromlist) 591 if '*' in fromlist: 592 fromlist.remove('*') 593 have_star = True 594 595 imported_module = self._graph._safe_import_hook(name, 596 self._module, fromlist, level, attr=DependencyInfo( 597 conditional=self.in_if, 598 tryexcept=self.in_tryexcept, 599 function=self.in_def, 600 fromlist=False, 601 ))[0] 602 if have_star: 603 self._module.globalnames.update(imported_module.globalnames) 604 self._module.starimports.update(imported_module.starimports) 605 if imported_module.code is None: 606 self._module.starimports.add(name) 607 608 609 def visit_Import(self, node): 610 for nm in _ast_names(node.names): 611 self._process_import(nm, None, self._level) 612 613 def visit_ImportFrom(self, node): 614 level = node.level if node.level != 0 else self._level 615 self._process_import(node.module or '', _ast_names(node.names), level) 616 617 def visit_If(self, node): 618 self._in_if.append(True) 619 self.generic_visit(node) 620 self._in_if.pop() 621 622 def visit_FunctionDef(self, node): 623 self._in_def.append(True) 624 self.generic_visit(node) 625 self._in_def.pop() 626 627 def visit_Try(self, node): 628 self._in_tryexcept.append(True) 629 self.generic_visit(node) 630 self._in_tryexcept.pop() 631 632 def visit_ExceptHandler(self, node): 633 self._in_tryexcept.append(True) 634 self.generic_visit(node) 635 self._in_tryexcept.pop() 636 637 def visit_TryExcept(self, node): 638 self._in_tryexcept.append(True) 639 self.generic_visit(node) 640 self._in_tryexcept.pop() 641 642 def visit_ExceptHandler(self, node): 643 self._in_tryexcept.append(True) 644 self.generic_visit(node) 645 self._in_tryexcept.pop() 646 647 def visit_Expression(self, node): 648 # Expression node's cannot contain import statements or 649 # other nodes that are relevant for us. 650 pass 651 652 # Expression isn't actually used as such in AST trees, 653 # therefore define visitors for all kinds of expression nodes. 654 visit_BoolOp = visit_Expression 655 visit_BinOp = visit_Expression 656 visit_UnaryOp = visit_Expression 657 visit_Lambda = visit_Expression 658 visit_IfExp = visit_Expression 659 visit_Dict = visit_Expression 660 visit_Set = visit_Expression 661 visit_ListComp = visit_Expression 662 visit_SetComp = visit_Expression 663 visit_ListComp = visit_Expression 664 visit_GeneratorExp = visit_Expression 665 visit_Compare = visit_Expression 666 visit_Yield = visit_Expression 667 visit_YieldFrom = visit_Expression 668 visit_Await = visit_Expression 669 visit_Call = visit_Expression 670 671 672 673 class ModuleGraph(ObjectGraph): 674 def __init__(self, path=None, excludes=(), replace_paths=(), implies=(), graph=None, debug=0): 675 super(ModuleGraph, self).__init__(graph=graph, debug=debug) 676 if path is None: 677 path = sys.path 678 self.path = path 679 self.lazynodes = {} 680 # excludes is stronger than implies 681 self.lazynodes.update(dict(implies)) 682 for m in excludes: 683 self.lazynodes[m] = None 684 self.replace_paths = replace_paths 685 686 self.nspackages = self._calc_setuptools_nspackages() 687 688 def _calc_setuptools_nspackages(self): 689 # Setuptools has some magic handling for namespace 690 # packages when using 'install --single-version-externally-managed' 691 # (used by system packagers and also by pip) 692 # 693 # When this option is used namespace packages are writting to 694 # disk *without* an __init__.py file, which means the regular 695 # import machinery will not find them. 696 # 697 # We therefore explicitly look for the hack used by 698 # setuptools to get this kind of namespace packages to work. 699 700 pkgmap = {} 701 702 try: 703 from pkgutil import ImpImporter 704 except ImportError: 705 try: 706 from _pkgutil import ImpImporter 707 except ImportError: 708 ImpImporter = pkg_resources.ImpWrapper 709 710 if sys.version_info[:2] >= (3,3): 711 import importlib.machinery 712 ImpImporter = importlib.machinery.FileFinder 713 714 for entry in self.path: 715 importer = pkg_resources.get_importer(entry) 716 717 if isinstance(importer, ImpImporter): 718 try: 719 ldir = os.listdir(entry) 720 except os.error: 721 continue 722 723 for fn in ldir: 724 if fn.endswith('-nspkg.pth'): 725 fp = open(os.path.join(entry, fn), 'rU') 726 try: 727 for ln in fp: 728 for pfx in _SETUPTOOLS_NAMESPACEPKG_PTHs: 729 if ln.startswith(pfx): 730 try: 731 start = len(pfx)-2 732 stop = ln.index(')', start)+1 733 except ValueError: 734 continue 735 736 pkg = _eval_str_tuple(ln[start:stop]) 737 identifier = ".".join(pkg) 738 subdir = os.path.join(entry, *pkg) 739 if os.path.exists(os.path.join(subdir, '__init__.py')): 740 # There is a real __init__.py, ignore the setuptools hack 741 continue 742 743 if identifier in pkgmap: 744 pkgmap[identifier].append(subdir) 745 else: 746 pkgmap[identifier] = [subdir] 747 break 748 finally: 749 fp.close() 750 751 return pkgmap 752 753 def implyNodeReference(self, node, other, edge_data=None): 754 """ 755 Imply that one node depends on another. 756 other may be a module name or another node. 757 758 For use by extension modules and tricky import code 759 """ 760 if isinstance(other, Node): 761 self._updateReference(node, other, edge_data) 762 763 else: 764 if isinstance(other, tuple): 765 raise ValueError(other) 766 767 others = self._safe_import_hook(other, node, None) 768 for other in others: 769 self._updateReference(node, other, edge_data) 770 771 772 def getReferences(self, fromnode): 773 """ 774 Yield all nodes that 'fromnode' dependes on (that is, 775 all modules that 'fromnode' imports. 776 """ 777 node = self.findNode(fromnode) 778 out_edges, _ = self.get_edges(node) 779 return out_edges 780 781 def getReferers(self, tonode, collapse_missing_modules=True): 782 node = self.findNode(tonode) 783 _, in_edges = self.get_edges(node) 784 785 if collapse_missing_modules: 786 for n in in_edges: 787 if isinstance(n, MissingModule): 788 for n in self.getReferers(n, False): 789 yield n 790 791 else: 792 yield n 793 794 else: 795 for n in in_edges: 796 yield n 797 798 def hasEdge(self, fromnode, tonode): 799 """ Return True iff there is an edge from 'fromnode' to 'tonode' """ 800 fromnode = self.findNode(fromnode) 801 tonode = self.findNode(tonode) 802 803 return self.graph.edge_by_node(fromnode, tonode) is not None 804 805 806 def foldReferences(self, packagenode): 807 """ 808 Create edges to/from 'packagenode' based on the 809 edges to/from modules in package. The module nodes 810 are then hidden. 811 """ 812 pkg = self.findNode(packagenode) 813 814 for n in self.nodes(): 815 if not n.identifier.startswith(pkg.identifier + '.'): 816 continue 817 818 iter_out, iter_inc = n.get_edges() 819 for other in iter_out: 820 if other.identifier.startswith(pkg.identifier + '.'): 821 continue 822 823 if not self.hasEdge(pkg, other): 824 # Ignore circular dependencies 825 self._updateReference(pkg, other, 'pkg-internal-import') 826 827 for other in iter_in: 828 if other.identifier.startswith(pkg.identifier + '.'): 829 # Ignore circular dependencies 830 continue 831 832 if not self.hasEdge(other, pkg): 833 self._updateReference(other, pkg, 'pkg-import') 834 835 self.graph.hide_node(n) 836 837 # TODO: unfoldReferences(pkg) that restore the submodule nodes and 838 # removes 'pkg-import' and 'pkg-internal-import' edges. Care should 839 # be taken to ensure that references are correct if multiple packages 840 # are folded and then one of them in unfolded 841 842 843 def _updateReference(self, fromnode, tonode, edge_data): 844 try: 845 ed = self.edgeData(fromnode, tonode) 846 except (KeyError, GraphError): # XXX: Why 'GraphError' 847 return self.createReference(fromnode, tonode, edge_data) 848 849 if not (isinstance(ed, DependencyInfo) and isinstance(edge_data, DependencyInfo)): 850 self.updateEdgeData(fromnode, tonode, edge_data) 851 else: 852 self.updateEdgeData(fromnode, tonode, ed._merged(edge_data)) 853 854 855 def createReference(self, fromnode, tonode, edge_data='direct'): 856 """ 857 Create a reference from fromnode to tonode 858 """ 859 return super(ModuleGraph, self).createReference(fromnode, tonode, edge_data=edge_data) 860 861 def findNode(self, name): 862 """ 863 Find a node by identifier. If a node by that identifier exists, 864 it will be returned. 865 866 If a lazy node exists by that identifier with no dependencies (excluded), 867 it will be instantiated and returned. 868 869 If a lazy node exists by that identifier with dependencies, it and its 870 dependencies will be instantiated and scanned for additional dependencies. 871 """ 872 data = super(ModuleGraph, self).findNode(name) 873 if data is not None: 874 return data 875 if name in self.lazynodes: 876 deps = self.lazynodes.pop(name) 877 if deps is None: 878 # excluded module 879 m = self.createNode(ExcludedModule, name) 880 elif isinstance(deps, Alias): 881 other = self._safe_import_hook(deps, None, None).pop() 882 m = self.createNode(AliasNode, name, other) 883 self.implyNodeReference(m, other) 884 885 else: 886 m = self._safe_import_hook(name, None, None).pop() 887 for dep in deps: 888 self.implyNodeReference(m, dep) 889 return m 890 891 if name in self.nspackages: 892 # name is a --single-version-externally-managed 893 # namespace package (setuptools/distribute) 894 pathnames = self.nspackages.pop(name) 895 m = self.createNode(NamespacePackage, name) 896 897 # FIXME: The filename must be set to a string to ensure that py2app 898 # works, it is not clear yet why that is. Setting to None would be 899 # cleaner. 900 m.filename = '-' 901 m.packagepath = _namespace_package_path(name, pathnames, self.path) 902 903 # As per comment at top of file, simulate runtime packagepath additions. 904 m.packagepath = m.packagepath + _packagePathMap.get(name, []) 905 return m 906 907 return None 908 909 def run_script(self, pathname, caller=None): 910 """ 911 Create a node by path (not module name). It is expected to be a Python 912 source file, and will be scanned for dependencies. 913 """ 914 self.msg(2, "run_script", pathname) 915 pathname = os.path.realpath(pathname) 916 m = self.findNode(pathname) 917 if m is not None: 918 return m 919 920 if sys.version_info[0] != 2: 921 with open(pathname, 'rb') as fp: 922 encoding = util.guess_encoding(fp) 923 924 with open(pathname, _READ_MODE, encoding=encoding) as fp: 925 contents = fp.read() + '\n' 926 927 else: 928 with open(pathname, _READ_MODE) as fp: 929 contents = fp.read() + '\n' 930 931 co = compile(contents, pathname, 'exec', ast.PyCF_ONLY_AST, True) 932 m = self.createNode(Script, pathname) 933 self._updateReference(caller, m, None) 934 self._scan_code(co, m) 935 m.code = compile(co, pathname, 'exec', 0, True) 936 if self.replace_paths: 937 m.code = self._replace_paths_in_code(m.code) 938 return m 939 940 def import_hook(self, name, caller=None, fromlist=None, level=DEFAULT_IMPORT_LEVEL, attr=None): 941 """ 942 Import a module 943 944 Return the set of modules that are imported 945 """ 946 self.msg(3, "import_hook", name, caller, fromlist, level) 947 parent = self._determine_parent(caller) 948 q, tail = self._find_head_package(parent, name, level) 949 m = self._load_tail(q, tail) 950 modules = [m] 951 if fromlist and m.packagepath: 952 for s in self._ensure_fromlist(m, fromlist): 953 if s not in modules: 954 modules.append(s) 955 for m in modules: 956 self._updateReference(caller, m, edge_data=attr) 957 return modules 958 959 def _determine_parent(self, caller): 960 """ 961 Determine the package containing a node 962 """ 963 self.msgin(4, "determine_parent", caller) 964 parent = None 965 if caller: 966 pname = caller.identifier 967 968 if isinstance(caller, Package): 969 parent = caller 970 971 elif '.' in pname: 972 pname = pname[:pname.rfind('.')] 973 parent = self.findNode(pname) 974 975 elif caller.packagepath: 976 # XXX: I have no idea why this line 977 # is necessary. 978 parent = self.findNode(pname) 979 980 981 self.msgout(4, "determine_parent ->", parent) 982 return parent 983 984 def _find_head_package(self, parent, name, level=DEFAULT_IMPORT_LEVEL): 985 """ 986 Given a calling parent package and an import name determine the containing 987 package for the name 988 """ 989 self.msgin(4, "find_head_package", parent, name, level) 990 if '.' in name: 991 head, tail = name.split('.', 1) 992 else: 993 head, tail = name, '' 994 995 if level == -1: 996 if parent: 997 qname = parent.identifier + '.' + head 998 else: 999 qname = head 1000 1001 elif level == 0: 1002 qname = head 1003 1004 # Absolute import, ignore the parent 1005 parent = None 1006 1007 else: 1008 if parent is None: 1009 self.msg(2, "Relative import outside of package") 1010 raise ImportError("Relative import outside of package (name=%r, parent=%r, level=%r)"%(name, parent, level)) 1011 1012 for i in range(level-1): 1013 if '.' not in parent.identifier: 1014 self.msg(2, "Relative import outside of package") 1015 raise ImportError("Relative import outside of package (name=%r, parent=%r, level=%r)"%(name, parent, level)) 1016 1017 p_fqdn = parent.identifier.rsplit('.', 1)[0] 1018 new_parent = self.findNode(p_fqdn) 1019 if new_parent is None: 1020 self.msg(2, "Relative import outside of package") 1021 raise ImportError("Relative import outside of package (name=%r, parent=%r, level=%r)"%(name, parent, level)) 1022 1023 assert new_parent is not parent, (new_parent, parent) 1024 parent = new_parent 1025 1026 if head: 1027 qname = parent.identifier + '.' + head 1028 else: 1029 qname = parent.identifier 1030 1031 1032 q = self._import_module(head, qname, parent) 1033 if q: 1034 self.msgout(4, "find_head_package ->", (q, tail)) 1035 return q, tail 1036 if parent: 1037 qname = head 1038 parent = None 1039 q = self._import_module(head, qname, parent) 1040 if q: 1041 self.msgout(4, "find_head_package ->", (q, tail)) 1042 return q, tail 1043 self.msgout(4, "raise ImportError: No module named", qname) 1044 raise ImportError("No module named " + qname) 1045 1046 def _load_tail(self, mod, tail): 1047 self.msgin(4, "load_tail", mod, tail) 1048 result = mod 1049 while tail: 1050 i = tail.find('.') 1051 if i < 0: i = len(tail) 1052 head, tail = tail[:i], tail[i+1:] 1053 mname = "%s.%s" % (result.identifier, head) 1054 result = self._import_module(head, mname, result) 1055 if result is None: 1056 result = self.createNode(MissingModule, mname) 1057 #self.msgout(4, "raise ImportError: No module named", mname) 1058 #raise ImportError("No module named " + mname) 1059 self.msgout(4, "load_tail ->", result) 1060 return result 1061 1062 def _ensure_fromlist(self, m, fromlist): 1063 fromlist = set(fromlist) 1064 self.msg(4, "ensure_fromlist", m, fromlist) 1065 if '*' in fromlist: 1066 fromlist.update(self._find_all_submodules(m)) 1067 fromlist.remove('*') 1068 for sub in fromlist: 1069 submod = m.get(sub) 1070 if submod is None: 1071 if sub in m.globalnames: 1072 # Name is a global in the module 1073 continue 1074 # XXX: ^^^ need something simular for names imported 1075 # by 'm'. 1076 1077 fullname = m.identifier + '.' + sub 1078 submod = self._import_module(sub, fullname, m) 1079 if submod is None: 1080 raise ImportError("No module named " + fullname) 1081 yield submod 1082 1083 def _find_all_submodules(self, m): 1084 if not m.packagepath: 1085 return 1086 # 'suffixes' used to be a list hardcoded to [".py", ".pyc", ".pyo"]. 1087 # But we must also collect Python extension modules - although 1088 # we cannot separate normal dlls from Python extensions. 1089 suffixes = [triple[0] for triple in imp.get_suffixes()] 1090 for path in m.packagepath: 1091 try: 1092 names = zipio.listdir(path) 1093 except (os.error, IOError): 1094 self.msg(2, "can't list directory", path) 1095 continue 1096 for info in (moduleInfoForPath(p) for p in names): 1097 if info is None: continue 1098 if info[0] != '__init__': 1099 yield info[0] 1100 1101 def _import_module(self, partname, fqname, parent): 1102 # XXX: Review me for use with absolute imports. 1103 self.msgin(3, "import_module", partname, fqname, parent) 1104 m = self.findNode(fqname) 1105 if m is not None: 1106 self.msgout(3, "import_module ->", m) 1107 if parent: 1108 self._updateReference(m, parent, edge_data=DependencyInfo( 1109 conditional=False, fromlist=False, function=False, tryexcept=False 1110 )) 1111 return m 1112 1113 if parent and parent.packagepath is None: 1114 self.msgout(3, "import_module -> None") 1115 return None 1116 1117 try: 1118 searchpath = None 1119 if parent is not None and parent.packagepath: 1120 searchpath = parent.packagepath 1121 1122 fp, pathname, stuff = self._find_module(partname, 1123 searchpath, parent) 1124 1125 except ImportError: 1126 self.msgout(3, "import_module ->", None) 1127 return None 1128 1129 try: 1130 m = self._load_module(fqname, fp, pathname, stuff) 1131 1132 finally: 1133 if fp is not None: 1134 fp.close() 1135 1136 if parent: 1137 self.msgout(4, "create reference", m, "->", parent) 1138 self._updateReference(m, parent, edge_data=DependencyInfo( 1139 conditional=False, fromlist=False, function=False, tryexcept=False 1140 )) 1141 parent[partname] = m 1142 1143 self.msgout(3, "import_module ->", m) 1144 return m 1145 1146 def _load_module(self, fqname, fp, pathname, info): 1147 suffix, mode, typ = info 1148 self.msgin(2, "load_module", fqname, fp and "fp", pathname) 1149 1150 if typ == imp.PKG_DIRECTORY: 1151 if isinstance(mode, (list, tuple)): 1152 packagepath = mode 1153 else: 1154 packagepath = [] 1155 1156 m = self._load_package(fqname, pathname, packagepath) 1157 self.msgout(2, "load_module ->", m) 1158 return m 1159 1160 if typ == imp.PY_SOURCE: 1161 contents = fp.read() 1162 if isinstance(contents, bytes): 1163 contents += b'\n' 1164 else: 1165 contents += '\n' 1166 1167 try: 1168 co = compile(contents, pathname, 'exec', ast.PyCF_ONLY_AST, True) 1169 #co = compile(contents, pathname, 'exec', 0, True) 1170 except SyntaxError: 1171 co = None 1172 cls = InvalidSourceModule 1173 1174 else: 1175 cls = SourceModule 1176 1177 elif typ == imp.PY_COMPILED: 1178 if fp.read(4) != imp.get_magic(): 1179 self.msgout(2, "raise ImportError: Bad magic number", pathname) 1180 co = None 1181 cls = InvalidCompiledModule 1182 1183 else: 1184 fp.read(4) 1185 try: 1186 co = marshal.loads(fp.read()) 1187 cls = CompiledModule 1188 except Exception: 1189 co = None 1190 cls = InvalidCompiledModule 1191 1192 elif typ == imp.C_BUILTIN: 1193 cls = BuiltinModule 1194 co = None 1195 1196 else: 1197 cls = Extension 1198 co = None 1199 1200 m = self.createNode(cls, fqname) 1201 m.filename = pathname 1202 if co is not None: 1203 self._scan_code(co, m) 1204 1205 if isinstance(co, ast.AST): 1206 co = compile(co, pathname, 'exec', 0, True) 1207 if self.replace_paths: 1208 co = self._replace_paths_in_code(co) 1209 m.code = co 1210 1211 1212 self.msgout(2, "load_module ->", m) 1213 return m 1214 1215 def _safe_import_hook(self, name, caller, fromlist, level=DEFAULT_IMPORT_LEVEL, attr=None): 1216 # wrapper for self.import_hook() that won't raise ImportError 1217 try: 1218 mods = self.import_hook(name, caller, level=level, attr=attr) 1219 except ImportError as msg: 1220 self.msg(2, "ImportError:", str(msg)) 1221 m = self.createNode(MissingModule, _path_from_importerror(msg, name)) 1222 self._updateReference(caller, m, edge_data=attr) 1223 1224 else: 1225 assert len(mods) == 1 1226 m = list(mods)[0] 1227 1228 subs = [m] 1229 if isinstance(attr, DependencyInfo): 1230 attr = attr._replace(fromlist=True) 1231 for sub in (fromlist or ()): 1232 # If this name is in the module namespace already, 1233 # then add the entry to the list of substitutions 1234 if sub in m: 1235 sm = m[sub] 1236 if sm is not None: 1237 if sm not in subs: 1238 self._updateReference(caller, sm, edge_data=attr) 1239 subs.append(sm) 1240 continue 1241 1242 elif sub in m.globalnames: 1243 # Global variable in the module, ignore 1244 continue 1245 1246 1247 # See if we can load it 1248 # fullname = name + '.' + sub 1249 fullname = m.identifier + '.' + sub 1250 #else: 1251 # print("XXX", repr(name), repr(sub), repr(caller), repr(m)) 1252 sm = self.findNode(fullname) 1253 if sm is None: 1254 try: 1255 sm = self.import_hook(name, caller, fromlist=[sub], level=level, attr=attr) 1256 except ImportError as msg: 1257 self.msg(2, "ImportError:", str(msg)) 1258 #sm = self.createNode(MissingModule, _path_from_importerror(msg, fullname)) 1259 sm = self.createNode(MissingModule, fullname) 1260 else: 1261 sm = self.findNode(fullname) 1262 if sm is None: 1263 sm = self.createNode(MissingModule, fullname) 1264 1265 m[sub] = sm 1266 if sm is not None: 1267 self._updateReference(m, sm, edge_data=attr) 1268 self._updateReference(caller, sm, edge_data=attr) 1269 if sm not in subs: 1270 subs.append(sm) 1271 return subs 1272 1273 def _scan_code(self, co, m): 1274 if isinstance(co, ast.AST): 1275 #return self._scan_bytecode(compile(co, '-', 'exec', 0, True), m) 1276 self._scan_ast(co, m) 1277 self._scan_bytecode_stores( 1278 compile(co, '-', 'exec', 0, True), m) 1279 1280 else: 1281 self._scan_bytecode(co, m) 1282 1283 def _scan_ast(self, co, m): 1284 visitor = _Visitor(self, m) 1285 visitor.visit(co) 1286 1287 def _scan_bytecode_stores(self, co, m, 1288 STORE_NAME=_Bchr(dis.opname.index('STORE_NAME')), 1289 STORE_GLOBAL=_Bchr(dis.opname.index('STORE_GLOBAL')), 1290 HAVE_ARGUMENT=_Bchr(dis.HAVE_ARGUMENT), 1291 unpack=struct.unpack): 1292 1293 extended_import = bool(sys.version_info[:2] >= (2,5)) 1294 1295 code = co.co_code 1296 constants = co.co_consts 1297 n = len(code) 1298 i = 0 1299 1300 while i < n: 1301 c = code[i] 1302 i += 1 1303 if c >= HAVE_ARGUMENT: 1304 i = i+2 1305 1306 if c == STORE_NAME or c == STORE_GLOBAL: 1307 # keep track of all global names that are assigned to 1308 oparg = unpack('<H', code[i - 2:i])[0] 1309 name = co.co_names[oparg] 1310 m.globalnames.add(name) 1311 1312 cotype = type(co) 1313 for c in constants: 1314 if isinstance(c, cotype): 1315 self._scan_bytecode_stores(c, m) 1316 1317 def _scan_bytecode(self, co, m, 1318 HAVE_ARGUMENT=_Bchr(dis.HAVE_ARGUMENT), 1319 LOAD_CONST=_Bchr(dis.opname.index('LOAD_CONST')), 1320 IMPORT_NAME=_Bchr(dis.opname.index('IMPORT_NAME')), 1321 IMPORT_FROM=_Bchr(dis.opname.index('IMPORT_FROM')), 1322 STORE_NAME=_Bchr(dis.opname.index('STORE_NAME')), 1323 STORE_GLOBAL=_Bchr(dis.opname.index('STORE_GLOBAL')), 1324 unpack=struct.unpack): 1325 1326 # Python >=2.5: LOAD_CONST flags, LOAD_CONST names, IMPORT_NAME name 1327 # Python < 2.5: LOAD_CONST names, IMPORT_NAME name 1328 extended_import = bool(sys.version_info[:2] >= (2,5)) 1329 1330 code = co.co_code 1331 constants = co.co_consts 1332 n = len(code) 1333 i = 0 1334 1335 level = None 1336 fromlist = None 1337 1338 while i < n: 1339 c = code[i] 1340 i += 1 1341 if c >= HAVE_ARGUMENT: 1342 i = i+2 1343 1344 if c == IMPORT_NAME: 1345 if extended_import: 1346 assert code[i-9] == LOAD_CONST 1347 assert code[i-6] == LOAD_CONST 1348 arg1, arg2 = unpack('<xHxH', code[i-9:i-3]) 1349 level = co.co_consts[arg1] 1350 fromlist = co.co_consts[arg2] 1351 else: 1352 assert code[-6] == LOAD_CONST 1353 arg1, = unpack('<xH', code[i-6:i-3]) 1354 level = -1 1355 fromlist = co.co_consts[arg1] 1356 1357 assert fromlist is None or type(fromlist) is tuple 1358 oparg, = unpack('<H', code[i - 2:i]) 1359 name = co.co_names[oparg] 1360 have_star = False 1361 if fromlist is not None: 1362 fromlist = set(fromlist) 1363 if '*' in fromlist: 1364 fromlist.remove('*') 1365 have_star = True 1366 1367 #self.msgin(2, "Before import hook", repr(name), repr(m), repr(fromlist), repr(level)) 1368 1369 imported_module = self._safe_import_hook(name, m, fromlist, level)[0] 1370 1371 if have_star: 1372 m.globalnames.update(imported_module.globalnames) 1373 m.starimports.update(imported_module.starimports) 1374 if imported_module.code is None: 1375 m.starimports.add(name) 1376 1377 elif c == STORE_NAME or c == STORE_GLOBAL: 1378 # keep track of all global names that are assigned to 1379 oparg = unpack('<H', code[i - 2:i])[0] 1380 name = co.co_names[oparg] 1381 m.globalnames.add(name) 1382 1383 cotype = type(co) 1384 for c in constants: 1385 if isinstance(c, cotype): 1386 self._scan_bytecode(c, m) 1387 1388 def _load_package(self, fqname, pathname, pkgpath): 1389 """ 1390 Called only when an imp.PACKAGE_DIRECTORY is found 1391 """ 1392 self.msgin(2, "load_package", fqname, pathname, pkgpath) 1393 newname = _replacePackageMap.get(fqname) 1394 if newname: 1395 fqname = newname 1396 1397 ns_pkgpath = _namespace_package_path(fqname, pkgpath or [], self.path) 1398 if ns_pkgpath or pkgpath: 1399 # this is a namespace package 1400 m = self.createNode(NamespacePackage, fqname) 1401 m.filename = '-' 1402 m.packagepath = ns_pkgpath 1403 else: 1404 m = self.createNode(Package, fqname) 1405 m.filename = pathname 1406 m.packagepath = [pathname] + ns_pkgpath 1407 1408 # As per comment at top of file, simulate runtime packagepath additions. 1409 m.packagepath = m.packagepath + _packagePathMap.get(fqname, []) 1410 1411 1412 1413 try: 1414 self.msg(2, "find __init__ for %s"%(m.packagepath,)) 1415 fp, buf, stuff = self._find_module("__init__", m.packagepath, parent=m) 1416 except ImportError: 1417 pass 1418 1419 else: 1420 try: 1421 self.msg(2, "load __init__ for %s"%(m.packagepath,)) 1422 self._load_module(fqname, fp, buf, stuff) 1423 finally: 1424 if fp is not None: 1425 fp.close() 1426 self.msgout(2, "load_package ->", m) 1427 return m 1428 1429 def _find_module(self, name, path, parent=None): 1430 if parent is not None: 1431 # assert path is not None 1432 fullname = parent.identifier + '.' + name 1433 else: 1434 fullname = name 1435 1436 node = self.findNode(fullname) 1437 if node is not None: 1438 self.msgout(3, "find_module -> already included?", node) 1439 raise ImportError(name) 1440 1441 if path is None: 1442 if name in sys.builtin_module_names: 1443 return (None, None, ("", "", imp.C_BUILTIN)) 1444 1445 path = self.path 1446 1447 fp, buf, stuff = find_module(name, path) 1448 try: 1449 if buf: 1450 buf = os.path.realpath(buf) 1451 1452 return (fp, buf, stuff) 1453 except: 1454 fp.close() 1455 raise 1456 1457 def create_xref(self, out=None): 1458 global header, footer, entry, contpl, contpl_linked, imports 1459 if out is None: 1460 out = sys.stdout 1461 scripts = [] 1462 mods = [] 1463 for mod in self.flatten(): 1464 name = os.path.basename(mod.identifier) 1465 if isinstance(mod, Script): 1466 scripts.append((name, mod)) 1467 else: 1468 mods.append((name, mod)) 1469 scripts.sort() 1470 mods.sort() 1471 scriptnames = [name for name, m in scripts] 1472 scripts.extend(mods) 1473 mods = scripts 1474 1475 title = "modulegraph cross reference for " + ', '.join(scriptnames) 1476 print(header % {"TITLE": title}, file=out) 1477 1478 def sorted_namelist(mods): 1479 lst = [os.path.basename(mod.identifier) for mod in mods if mod] 1480 lst.sort() 1481 return lst 1482 for name, m in mods: 1483 content = "" 1484 if isinstance(m, BuiltinModule): 1485 content = contpl % {"NAME": name, 1486 "TYPE": "<i>(builtin module)</i>"} 1487 elif isinstance(m, Extension): 1488 content = contpl % {"NAME": name,\ 1489 "TYPE": "<tt>%s</tt>" % m.filename} 1490 else: 1491 url = pathname2url(m.filename or "") 1492 content = contpl_linked % {"NAME": name, "URL": url} 1493 oute, ince = map(sorted_namelist, self.get_edges(m)) 1494 if oute: 1495 links = "" 1496 for n in oute: 1497 links += """ <a href="#%s">%s</a>\n""" % (n, n) 1498 content += imports % {"HEAD": "imports", "LINKS": links} 1499 if ince: 1500 links = "" 1501 for n in ince: 1502 links += """ <a href="#%s">%s</a>\n""" % (n, n) 1503 content += imports % {"HEAD": "imported by", "LINKS": links} 1504 print(entry % {"NAME": name,"CONTENT": content}, file=out) 1505 print(footer, file=out) 1506 1507 1508 def itergraphreport(self, name='G', flatpackages=()): 1509 # XXX: Can this be implemented using Dot()? 1510 nodes = map(self.graph.describe_node, self.graph.iterdfs(self)) 1511 describe_edge = self.graph.describe_edge 1512 edges = deque() 1513 packagenodes = set() 1514 packageidents = {} 1515 nodetoident = {} 1516 inpackages = {} 1517 mainedges = set() 1518 1519 # XXX - implement 1520 flatpackages = dict(flatpackages) 1521 1522 def nodevisitor(node, data, outgoing, incoming): 1523 if not isinstance(data, Node): 1524 return {'label': str(node)} 1525 #if isinstance(d, (ExcludedModule, MissingModule, BadModule)): 1526 # return None 1527 s = '<f0> ' + type(data).__name__ 1528 for i,v in enumerate(data.infoTuple()[:1], 1): 1529 s += '| <f%d> %s' % (i,v) 1530 return {'label':s, 'shape':'record'} 1531 1532 1533 def edgevisitor(edge, data, head, tail): 1534 # XXX: This method nonsense, the edge 1535 # data is never initialized. 1536 if data == 'orphan': 1537 return {'style':'dashed'} 1538 elif data == 'pkgref': 1539 return {'style':'dotted'} 1540 return {} 1541 1542 yield 'digraph %s {\n' % (name,) 1543 attr = dict(rankdir='LR', concentrate='true') 1544 cpatt = '%s="%s"' 1545 for item in attr.items(): 1546 yield '\t%s;\n' % (cpatt % item,) 1547 1548 # find all packages (subgraphs) 1549 for (node, data, outgoing, incoming) in nodes: 1550 nodetoident[node] = getattr(data, 'identifier', None) 1551 if isinstance(data, Package): 1552 packageidents[data.identifier] = node 1553 inpackages[node] = set([node]) 1554 packagenodes.add(node) 1555 1556 1557 # create sets for subgraph, write out descriptions 1558 for (node, data, outgoing, incoming) in nodes: 1559 # update edges 1560 for edge in (describe_edge(e) for e in outgoing): 1561 edges.append(edge) 1562 1563 # describe node 1564 yield '\t"%s" [%s];\n' % ( 1565 node, 1566 ','.join([ 1567 (cpatt % item) for item in 1568 nodevisitor(node, data, outgoing, incoming).items() 1569 ]), 1570 ) 1571 1572 inside = inpackages.get(node) 1573 if inside is None: 1574 inside = inpackages[node] = set() 1575 ident = nodetoident[node] 1576 if ident is None: 1577 continue 1578 pkgnode = packageidents.get(ident[:ident.rfind('.')]) 1579 if pkgnode is not None: 1580 inside.add(pkgnode) 1581 1582 1583 graph = [] 1584 subgraphs = {} 1585 for key in packagenodes: 1586 subgraphs[key] = [] 1587 1588 while edges: 1589 edge, data, head, tail = edges.popleft() 1590 if ((head, tail)) in mainedges: 1591 continue 1592 mainedges.add((head, tail)) 1593 tailpkgs = inpackages[tail] 1594 common = inpackages[head] & tailpkgs 1595 if not common and tailpkgs: 1596 usepkgs = sorted(tailpkgs) 1597 if len(usepkgs) != 1 or usepkgs[0] != tail: 1598 edges.append((edge, data, head, usepkgs[0])) 1599 edges.append((edge, 'pkgref', usepkgs[-1], tail)) 1600 continue 1601 if common: 1602 common = common.pop() 1603 if tail == common: 1604 edges.append((edge, data, tail, head)) 1605 elif head == common: 1606 subgraphs[common].append((edge, 'pkgref', head, tail)) 1607 else: 1608 edges.append((edge, data, common, head)) 1609 edges.append((edge, data, common, tail)) 1610 1611 else: 1612 graph.append((edge, data, head, tail)) 1613 1614 def do_graph(edges, tabs): 1615 edgestr = tabs + '"%s" -> "%s" [%s];\n' 1616 # describe edge 1617 for (edge, data, head, tail) in edges: 1618 attribs = edgevisitor(edge, data, head, tail) 1619 yield edgestr % ( 1620 head, 1621 tail, 1622 ','.join([(cpatt % item) for item in attribs.items()]), 1623 ) 1624 1625 for g, edges in subgraphs.items(): 1626 yield '\tsubgraph "cluster_%s" {\n' % (g,) 1627 yield '\t\tlabel="%s";\n' % (nodetoident[g],) 1628 for s in do_graph(edges, '\t\t'): 1629 yield s 1630 yield '\t}\n' 1631 1632 for s in do_graph(graph, '\t'): 1633 yield s 1634 1635 yield '}\n' 1636 1637 1638 def graphreport(self, fileobj=None, flatpackages=()): 1639 if fileobj is None: 1640 fileobj = sys.stdout 1641 fileobj.writelines(self.itergraphreport(flatpackages=flatpackages)) 1642 1643 def report(self): 1644 """Print a report to stdout, listing the found modules with their 1645 paths, as well as modules that are missing, or seem to be missing. 1646 """ 1647 print() 1648 print("%-15s %-25s %s" % ("Class", "Name", "File")) 1649 print("%-15s %-25s %s" % ("-----", "----", "----")) 1650 # Print modules found 1651 sorted = [(os.path.basename(mod.identifier), mod) for mod in self.flatten()] 1652 sorted.sort() 1653 for (name, m) in sorted: 1654 print("%-15s %-25s %s" % (type(m).__name__, name, m.filename or "")) 1655 1656 def _replace_paths_in_code(self, co): 1657 new_filename = original_filename = os.path.normpath(co.co_filename) 1658 for f, r in self.replace_paths: 1659 f = os.path.join(f, '') 1660 r = os.path.join(r, '') 1661 if original_filename.startswith(f): 1662 new_filename = r + original_filename[len(f):] 1663 break 1664 1665 else: 1666 return co 1667 1668 consts = list(co.co_consts) 1669 for i in range(len(consts)): 1670 if isinstance(consts[i], type(co)): 1671 consts[i] = self._replace_paths_in_code(consts[i]) 1672 1673 code_func = type(co) 1674 1675 if hasattr(co, 'co_kwonlyargcount'): 1676 return code_func(co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, co.co_stacksize, 1677 co.co_flags, co.co_code, tuple(consts), co.co_names, 1678 co.co_varnames, new_filename, co.co_name, 1679 co.co_firstlineno, co.co_lnotab, 1680 co.co_freevars, co.co_cellvars) 1681 else: 1682 return code_func(co.co_argcount, co.co_nlocals, co.co_stacksize, 1683 co.co_flags, co.co_code, tuple(consts), co.co_names, 1684 co.co_varnames, new_filename, co.co_name, 1685 co.co_firstlineno, co.co_lnotab, 1686 co.co_freevars, co.co_cellvars) 1687
