1 """Utilities to support packages.""" 2 3 # NOTE: This module must remain compatible with Python 2.3, as it is shared 4 # by setuptools for distribution with Python 2.3 and up. 5 6 import os 7 import sys 8 import imp 9 import os.path 10 from types import ModuleType 11 12 __all__ = [ 13 'get_importer', 'iter_importers', 'get_loader', 'find_loader', 14 'walk_packages', 'iter_modules', 'get_data', 15 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', 16 ] 17 18 def read_code(stream): 19 # This helper is needed in order for the PEP 302 emulation to 20 # correctly handle compiled files 21 import marshal 22 23 magic = stream.read(4) 24 if magic != imp.get_magic(): 25 return None 26 27 stream.read(4) # Skip timestamp 28 return marshal.load(stream) 29 30 31 def simplegeneric(func): 32 """Make a trivial single-dispatch generic function""" 33 registry = {} 34 def wrapper(*args, **kw): 35 ob = args[0] 36 try: 37 cls = ob.__class__ 38 except AttributeError: 39 cls = type(ob) 40 try: 41 mro = cls.__mro__ 42 except AttributeError: 43 try: 44 class cls(cls, object): 45 pass 46 mro = cls.__mro__[1:] 47 except TypeError: 48 mro = object, # must be an ExtensionClass or some such :( 49 for t in mro: 50 if t in registry: 51 return registry[t](*args, **kw) 52 else: 53 return func(*args, **kw) 54 try: 55 wrapper.__name__ = func.__name__ 56 except (TypeError, AttributeError): 57 pass # Python 2.3 doesn't allow functions to be renamed 58 59 def register(typ, func=None): 60 if func is None: 61 return lambda f: register(typ, f) 62 registry[typ] = func 63 return func 64 65 wrapper.__dict__ = func.__dict__ 66 wrapper.__doc__ = func.__doc__ 67 wrapper.register = register 68 return wrapper 69 70 71 def walk_packages(path=None, prefix='', onerror=None): 72 """Yields (module_loader, name, ispkg) for all modules recursively 73 on path, or, if path is None, all accessible modules. 74 75 'path' should be either None or a list of paths to look for 76 modules in. 77 78 'prefix' is a string to output on the front of every module name 79 on output. 80 81 Note that this function must import all *packages* (NOT all 82 modules!) on the given path, in order to access the __path__ 83 attribute to find submodules. 84 85 'onerror' is a function which gets called with one argument (the 86 name of the package which was being imported) if any exception 87 occurs while trying to import a package. If no onerror function is 88 supplied, ImportErrors are caught and ignored, while all other 89 exceptions are propagated, terminating the search. 90 91 Examples: 92 93 # list all modules python can access 94 walk_packages() 95 96 # list all submodules of ctypes 97 walk_packages(ctypes.__path__, ctypes.__name__+'.') 98 """ 99 100 def seen(p, m={}): 101 if p in m: 102 return True 103 m[p] = True 104 105 for importer, name, ispkg in iter_modules(path, prefix): 106 yield importer, name, ispkg 107 108 if ispkg: 109 try: 110 __import__(name) 111 except ImportError: 112 if onerror is not None: 113 onerror(name) 114 except Exception: 115 if onerror is not None: 116 onerror(name) 117 else: 118 raise 119 else: 120 path = getattr(sys.modules[name], '__path__', None) or [] 121 122 # don't traverse path items we've seen before 123 path = [p for p in path if not seen(p)] 124 125 for item in walk_packages(path, name+'.', onerror): 126 yield item 127 128 129 def iter_modules(path=None, prefix=''): 130 """Yields (module_loader, name, ispkg) for all submodules on path, 131 or, if path is None, all top-level modules on sys.path. 132 133 'path' should be either None or a list of paths to look for 134 modules in. 135 136 'prefix' is a string to output on the front of every module name 137 on output. 138 """ 139 140 if path is None: 141 importers = iter_importers() 142 else: 143 importers = map(get_importer, path) 144 145 yielded = {} 146 for i in importers: 147 for name, ispkg in iter_importer_modules(i, prefix): 148 if name not in yielded: 149 yielded[name] = 1 150 yield i, name, ispkg 151 152 153 #@simplegeneric 154 def iter_importer_modules(importer, prefix=''): 155 if not hasattr(importer, 'iter_modules'): 156 return [] 157 return importer.iter_modules(prefix) 158 159 iter_importer_modules = simplegeneric(iter_importer_modules) 160 161 162 class ImpImporter: 163 """PEP 302 Importer that wraps Python's "classic" import algorithm 164 165 ImpImporter(dirname) produces a PEP 302 importer that searches that 166 directory. ImpImporter(None) produces a PEP 302 importer that searches 167 the current sys.path, plus any modules that are frozen or built-in. 168 169 Note that ImpImporter does not currently support being used by placement 170 on sys.meta_path. 171 """ 172 173 def __init__(self, path=None): 174 self.path = path 175 176 def find_module(self, fullname, path=None): 177 # Note: we ignore 'path' argument since it is only used via meta_path 178 subname = fullname.split(".")[-1] 179 if subname != fullname and self.path is None: 180 return None 181 if self.path is None: 182 path = None 183 else: 184 path = [os.path.realpath(self.path)] 185 try: 186 file, filename, etc = imp.find_module(subname, path) 187 except ImportError: 188 return None 189 return ImpLoader(fullname, file, filename, etc) 190 191 def iter_modules(self, prefix=''): 192 if self.path is None or not os.path.isdir(self.path): 193 return 194 195 yielded = {} 196 import inspect 197 198 filenames = os.listdir(self.path) 199 filenames.sort() # handle packages before same-named modules 200 201 for fn in filenames: 202 modname = inspect.getmodulename(fn) 203 if modname=='__init__' or modname in yielded: 204 continue 205 206 path = os.path.join(self.path, fn) 207 ispkg = False 208 209 if not modname and os.path.isdir(path) and '.' not in fn: 210 modname = fn 211 for fn in os.listdir(path): 212 subname = inspect.getmodulename(fn) 213 if subname=='__init__': 214 ispkg = True 215 break 216 else: 217 continue # not a package 218 219 if modname and '.' not in modname: 220 yielded[modname] = 1 221 yield prefix + modname, ispkg 222 223 224 class ImpLoader: 225 """PEP 302 Loader that wraps Python's "classic" import algorithm 226 """ 227 code = source = None 228 229 def __init__(self, fullname, file, filename, etc): 230 self.file = file 231 self.filename = filename 232 self.fullname = fullname 233 self.etc = etc 234 235 def load_module(self, fullname): 236 self._reopen() 237 try: 238 mod = imp.load_module(fullname, self.file, self.filename, self.etc) 239 finally: 240 if self.file: 241 self.file.close() 242 # Note: we don't set __loader__ because we want the module to look 243 # normal; i.e. this is just a wrapper for standard import machinery 244 return mod 245 246 def get_data(self, pathname): 247 return open(pathname, "rb").read() 248 249 def _reopen(self): 250 if self.file and self.file.closed: 251 mod_type = self.etc[2] 252 if mod_type==imp.PY_SOURCE: 253 self.file = open(self.filename, 'rU') 254 elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): 255 self.file = open(self.filename, 'rb') 256 257 def _fix_name(self, fullname): 258 if fullname is None: 259 fullname = self.fullname 260 elif fullname != self.fullname: 261 raise ImportError("Loader for module %s cannot handle " 262 "module %s" % (self.fullname, fullname)) 263 return fullname 264 265 def is_package(self, fullname): 266 fullname = self._fix_name(fullname) 267 return self.etc[2]==imp.PKG_DIRECTORY 268 269 def get_code(self, fullname=None): 270 fullname = self._fix_name(fullname) 271 if self.code is None: 272 mod_type = self.etc[2] 273 if mod_type==imp.PY_SOURCE: 274 source = self.get_source(fullname) 275 self.code = compile(source, self.filename, 'exec') 276 elif mod_type==imp.PY_COMPILED: 277 self._reopen() 278 try: 279 self.code = read_code(self.file) 280 finally: 281 self.file.close() 282 elif mod_type==imp.PKG_DIRECTORY: 283 self.code = self._get_delegate().get_code() 284 return self.code 285 286 def get_source(self, fullname=None): 287 fullname = self._fix_name(fullname) 288 if self.source is None: 289 mod_type = self.etc[2] 290 if mod_type==imp.PY_SOURCE: 291 self._reopen() 292 try: 293 self.source = self.file.read() 294 finally: 295 self.file.close() 296 elif mod_type==imp.PY_COMPILED: 297 if os.path.exists(self.filename[:-1]): 298 f = open(self.filename[:-1], 'rU') 299 self.source = f.read() 300 f.close() 301 elif mod_type==imp.PKG_DIRECTORY: 302 self.source = self._get_delegate().get_source() 303 return self.source 304 305 306 def _get_delegate(self): 307 return ImpImporter(self.filename).find_module('__init__') 308 309 def get_filename(self, fullname=None): 310 fullname = self._fix_name(fullname) 311 mod_type = self.etc[2] 312 if self.etc[2]==imp.PKG_DIRECTORY: 313 return self._get_delegate().get_filename() 314 elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): 315 return self.filename 316 return None 317 318 319 try: 320 import zipimport 321 from zipimport import zipimporter 322 323 def iter_zipimport_modules(importer, prefix=''): 324 dirlist = zipimport._zip_directory_cache[importer.archive].keys() 325 dirlist.sort() 326 _prefix = importer.prefix 327 plen = len(_prefix) 328 yielded = {} 329 import inspect 330 for fn in dirlist: 331 if not fn.startswith(_prefix): 332 continue 333 334 fn = fn[plen:].split(os.sep) 335 336 if len(fn)==2 and fn[1].startswith('__init__.py'): 337 if fn[0] not in yielded: 338 yielded[fn[0]] = 1 339 yield fn[0], True 340 341 if len(fn)!=1: 342 continue 343 344 modname = inspect.getmodulename(fn[0]) 345 if modname=='__init__': 346 continue 347 348 if modname and '.' not in modname and modname not in yielded: 349 yielded[modname] = 1 350 yield prefix + modname, False 351 352 iter_importer_modules.register(zipimporter, iter_zipimport_modules) 353 354 except ImportError: 355 pass 356 357 358 def get_importer(path_item): 359 """Retrieve a PEP 302 importer for the given path item 360 361 The returned importer is cached in sys.path_importer_cache 362 if it was newly created by a path hook. 363 364 If there is no importer, a wrapper around the basic import 365 machinery is returned. This wrapper is never inserted into 366 the importer cache (None is inserted instead). 367 368 The cache (or part of it) can be cleared manually if a 369 rescan of sys.path_hooks is necessary. 370 """ 371 try: 372 importer = sys.path_importer_cache[path_item] 373 except KeyError: 374 for path_hook in sys.path_hooks: 375 try: 376 importer = path_hook(path_item) 377 break 378 except ImportError: 379 pass 380 else: 381 importer = None 382 sys.path_importer_cache.setdefault(path_item, importer) 383 384 if importer is None: 385 try: 386 importer = ImpImporter(path_item) 387 except ImportError: 388 importer = None 389 return importer 390 391 392 def iter_importers(fullname=""): 393 """Yield PEP 302 importers for the given module name 394 395 If fullname contains a '.', the importers will be for the package 396 containing fullname, otherwise they will be importers for sys.meta_path, 397 sys.path, and Python's "classic" import machinery, in that order. If 398 the named module is in a package, that package is imported as a side 399 effect of invoking this function. 400 401 Non PEP 302 mechanisms (e.g. the Windows registry) used by the 402 standard import machinery to find files in alternative locations 403 are partially supported, but are searched AFTER sys.path. Normally, 404 these locations are searched BEFORE sys.path, preventing sys.path 405 entries from shadowing them. 406 407 For this to cause a visible difference in behaviour, there must 408 be a module or package name that is accessible via both sys.path 409 and one of the non PEP 302 file system mechanisms. In this case, 410 the emulation will find the former version, while the builtin 411 import mechanism will find the latter. 412 413 Items of the following types can be affected by this discrepancy: 414 imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY 415 """ 416 if fullname.startswith('.'): 417 raise ImportError("Relative module names not supported") 418 if '.' in fullname: 419 # Get the containing package's __path__ 420 pkg = '.'.join(fullname.split('.')[:-1]) 421 if pkg not in sys.modules: 422 __import__(pkg) 423 path = getattr(sys.modules[pkg], '__path__', None) or [] 424 else: 425 for importer in sys.meta_path: 426 yield importer 427 path = sys.path 428 for item in path: 429 yield get_importer(item) 430 if '.' not in fullname: 431 yield ImpImporter() 432 433 def get_loader(module_or_name): 434 """Get a PEP 302 "loader" object for module_or_name 435 436 If the module or package is accessible via the normal import 437 mechanism, a wrapper around the relevant part of that machinery 438 is returned. Returns None if the module cannot be found or imported. 439 If the named module is not already imported, its containing package 440 (if any) is imported, in order to establish the package __path__. 441 442 This function uses iter_importers(), and is thus subject to the same 443 limitations regarding platform-specific special import locations such 444 as the Windows registry. 445 """ 446 if module_or_name in sys.modules: 447 module_or_name = sys.modules[module_or_name] 448 if isinstance(module_or_name, ModuleType): 449 module = module_or_name 450 loader = getattr(module, '__loader__', None) 451 if loader is not None: 452 return loader 453 fullname = module.__name__ 454 else: 455 fullname = module_or_name 456 return find_loader(fullname) 457 458 def find_loader(fullname): 459 """Find a PEP 302 "loader" object for fullname 460 461 If fullname contains dots, path must be the containing package's __path__. 462 Returns None if the module cannot be found or imported. This function uses 463 iter_importers(), and is thus subject to the same limitations regarding 464 platform-specific special import locations such as the Windows registry. 465 """ 466 for importer in iter_importers(fullname): 467 loader = importer.find_module(fullname) 468 if loader is not None: 469 return loader 470 471 return None 472 473 474 def extend_path(path, name): 475 """Extend a package's path. 476 477 Intended use is to place the following code in a package's __init__.py: 478 479 from pkgutil import extend_path 480 __path__ = extend_path(__path__, __name__) 481 482 This will add to the package's __path__ all subdirectories of 483 directories on sys.path named after the package. This is useful 484 if one wants to distribute different parts of a single logical 485 package as multiple directories. 486 487 It also looks for *.pkg files beginning where * matches the name 488 argument. This feature is similar to *.pth files (see site.py), 489 except that it doesn't special-case lines starting with 'import'. 490 A *.pkg file is trusted at face value: apart from checking for 491 duplicates, all entries found in a *.pkg file are added to the 492 path, regardless of whether they are exist the filesystem. (This 493 is a feature.) 494 495 If the input path is not a list (as is the case for frozen 496 packages) it is returned unchanged. The input path is not 497 modified; an extended copy is returned. Items are only appended 498 to the copy at the end. 499 500 It is assumed that sys.path is a sequence. Items of sys.path that 501 are not (unicode or 8-bit) strings referring to existing 502 directories are ignored. Unicode items of sys.path that cause 503 errors when used as filenames may cause this function to raise an 504 exception (in line with os.path.isdir() behavior). 505 """ 506 507 if not isinstance(path, list): 508 # This could happen e.g. when this is called from inside a 509 # frozen package. Return the path unchanged in that case. 510 return path 511 512 pname = os.path.join(*name.split('.')) # Reconstitute as relative path 513 # Just in case os.extsep != '.' 514 sname = os.extsep.join(name.split('.')) 515 sname_pkg = sname + os.extsep + "pkg" 516 init_py = "__init__" + os.extsep + "py" 517 518 path = path[:] # Start with a copy of the existing path 519 520 for dir in sys.path: 521 if not isinstance(dir, basestring) or not os.path.isdir(dir): 522 continue 523 subdir = os.path.join(dir, pname) 524 # XXX This may still add duplicate entries to path on 525 # case-insensitive filesystems 526 initfile = os.path.join(subdir, init_py) 527 if subdir not in path and os.path.isfile(initfile): 528 path.append(subdir) 529 # XXX Is this the right thing for subpackages like zope.app? 530 # It looks for a file named "zope.app.pkg" 531 pkgfile = os.path.join(dir, sname_pkg) 532 if os.path.isfile(pkgfile): 533 try: 534 f = open(pkgfile) 535 except IOError, msg: 536 sys.stderr.write("Can't open %s: %s\n" % 537 (pkgfile, msg)) 538 else: 539 for line in f: 540 line = line.rstrip('\n') 541 if not line or line.startswith('#'): 542 continue 543 path.append(line) # Don't check for existence! 544 f.close() 545 546 return path 547 548 def get_data(package, resource): 549 """Get a resource from a package. 550 551 This is a wrapper round the PEP 302 loader get_data API. The package 552 argument should be the name of a package, in standard module format 553 (foo.bar). The resource argument should be in the form of a relative 554 filename, using '/' as the path separator. The parent directory name '..' 555 is not allowed, and nor is a rooted name (starting with a '/'). 556 557 The function returns a binary string, which is the contents of the 558 specified resource. 559 560 For packages located in the filesystem, which have already been imported, 561 this is the rough equivalent of 562 563 d = os.path.dirname(sys.modules[package].__file__) 564 data = open(os.path.join(d, resource), 'rb').read() 565 566 If the package cannot be located or loaded, or it uses a PEP 302 loader 567 which does not support get_data(), then None is returned. 568 """ 569 570 loader = get_loader(package) 571 if loader is None or not hasattr(loader, 'get_data'): 572 return None 573 mod = sys.modules.get(package) or loader.load_module(package) 574 if mod is None or not hasattr(mod, '__file__'): 575 return None 576 577 # Modify the resource name to be compatible with the loader.get_data 578 # signature - an os.path format "filename" starting with the dirname of 579 # the package's __file__ 580 parts = resource.split('/') 581 parts.insert(0, os.path.dirname(mod.__file__)) 582 resource_name = os.path.join(*parts) 583 return loader.get_data(resource_name) 584
