1 """ 2 Import hooks; when installed with the install() function, these hooks 3 allow importing .pyx files as if they were Python modules. 4 5 If you want the hook installed every time you run Python 6 you can add it to your Python version by adding these lines to 7 sitecustomize.py (which you can create from scratch in site-packages 8 if it doesn't exist there or somewhere else on your python path):: 9 10 import pyximport 11 pyximport.install() 12 13 For instance on the Mac with a non-system Python 2.3, you could create 14 sitecustomize.py with only those two lines at 15 /usr/local/lib/python2.3/site-packages/sitecustomize.py . 16 17 A custom distutils.core.Extension instance and setup() args 18 (Distribution) for for the build can be defined by a <modulename>.pyxbld 19 file like: 20 21 # examplemod.pyxbld 22 def make_ext(modname, pyxfilename): 23 from distutils.extension import Extension 24 return Extension(name = modname, 25 sources=[pyxfilename, 'hello.c'], 26 include_dirs=['/myinclude'] ) 27 def make_setup_args(): 28 return dict(script_args=["--compiler=mingw32"]) 29 30 Extra dependencies can be defined by a <modulename>.pyxdep . 31 See README. 32 33 Since Cython 0.11, the :mod:`pyximport` module also has experimental 34 compilation support for normal Python modules. This allows you to 35 automatically run Cython on every .pyx and .py module that Python 36 imports, including parts of the standard library and installed 37 packages. Cython will still fail to compile a lot of Python modules, 38 in which case the import mechanism will fall back to loading the 39 Python source modules instead. The .py import mechanism is installed 40 like this:: 41 42 pyximport.install(pyimport = True) 43 44 Running this module as a top-level script will run a test and then print 45 the documentation. 46 47 This code is based on the Py2.3+ import protocol as described in PEP 302. 48 """ 49 50 import sys 51 import os 52 import glob 53 import imp 54 55 mod_name = "pyximport" 56 57 assert sys.hexversion >= 0x2030000, "need Python 2.3 or later" 58 59 PYX_EXT = ".pyx" 60 PYXDEP_EXT = ".pyxdep" 61 PYXBLD_EXT = ".pyxbld" 62 63 DEBUG_IMPORT = False 64 65 def _print(message, args): 66 if args: 67 message = message % args 68 print(message) 69 70 def _debug(message, *args): 71 if DEBUG_IMPORT: 72 _print(message, args) 73 74 def _info(message, *args): 75 _print(message, args) 76 77 # Performance problem: for every PYX file that is imported, we will 78 # invoke the whole distutils infrastructure even if the module is 79 # already built. It might be more efficient to only do it when the 80 # mod time of the .pyx is newer than the mod time of the .so but 81 # the question is how to get distutils to tell me the name of the .so 82 # before it builds it. Maybe it is easy...but maybe the peformance 83 # issue isn't real. 84 def _load_pyrex(name, filename): 85 "Load a pyrex file given a name and filename." 86 87 def get_distutils_extension(modname, pyxfilename, language_level=None): 88 # try: 89 # import hashlib 90 # except ImportError: 91 # import md5 as hashlib 92 # extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest() 93 # modname = modname + extra 94 extension_mod,setup_args = handle_special_build(modname, pyxfilename) 95 if not extension_mod: 96 if not isinstance(pyxfilename, str): 97 # distutils is stupid in Py2 and requires exactly 'str' 98 # => encode accidentally coerced unicode strings back to str 99 pyxfilename = pyxfilename.encode(sys.getfilesystemencoding()) 100 from distutils.extension import Extension 101 extension_mod = Extension(name = modname, sources=[pyxfilename]) 102 if language_level is not None: 103 extension_mod.cython_directives = {'language_level': language_level} 104 return extension_mod,setup_args 105 106 def handle_special_build(modname, pyxfilename): 107 special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT 108 ext = None 109 setup_args={} 110 if os.path.exists(special_build): 111 # globls = {} 112 # locs = {} 113 # execfile(special_build, globls, locs) 114 # ext = locs["make_ext"](modname, pyxfilename) 115 mod = imp.load_source("XXXX", special_build, open(special_build)) 116 make_ext = getattr(mod,'make_ext',None) 117 if make_ext: 118 ext = make_ext(modname, pyxfilename) 119 assert ext and ext.sources, ("make_ext in %s did not return Extension" 120 % special_build) 121 make_setup_args = getattr(mod,'make_setup_args',None) 122 if make_setup_args: 123 setup_args = make_setup_args() 124 assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict" 125 % special_build) 126 assert set or setup_args, ("neither make_ext nor make_setup_args %s" 127 % special_build) 128 ext.sources = [os.path.join(os.path.dirname(special_build), source) 129 for source in ext.sources] 130 return ext, setup_args 131 132 def handle_dependencies(pyxfilename): 133 testing = '_test_files' in globals() 134 dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT 135 136 # by default let distutils decide whether to rebuild on its own 137 # (it has a better idea of what the output file will be) 138 139 # but we know more about dependencies so force a rebuild if 140 # some of the dependencies are newer than the pyxfile. 141 if os.path.exists(dependfile): 142 depends = open(dependfile).readlines() 143 depends = [depend.strip() for depend in depends] 144 145 # gather dependencies in the "files" variable 146 # the dependency file is itself a dependency 147 files = [dependfile] 148 for depend in depends: 149 fullpath = os.path.join(os.path.dirname(dependfile), 150 depend) 151 files.extend(glob.glob(fullpath)) 152 153 # only for unit testing to see we did the right thing 154 if testing: 155 _test_files[:] = [] #$pycheck_no 156 157 # if any file that the pyxfile depends upon is newer than 158 # the pyx file, 'touch' the pyx file so that distutils will 159 # be tricked into rebuilding it. 160 for file in files: 161 from distutils.dep_util import newer 162 if newer(file, pyxfilename): 163 _debug("Rebuilding %s because of %s", pyxfilename, file) 164 filetime = os.path.getmtime(file) 165 os.utime(pyxfilename, (filetime, filetime)) 166 if testing: 167 _test_files.append(file) 168 169 def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None): 170 assert os.path.exists(pyxfilename), ( 171 "Path does not exist: %s" % pyxfilename) 172 handle_dependencies(pyxfilename) 173 174 extension_mod,setup_args = get_distutils_extension(name, pyxfilename, language_level) 175 build_in_temp=pyxargs.build_in_temp 176 sargs=pyxargs.setup_args.copy() 177 sargs.update(setup_args) 178 build_in_temp=sargs.pop('build_in_temp',build_in_temp) 179 180 import pyxbuild 181 so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod, 182 build_in_temp=build_in_temp, 183 pyxbuild_dir=pyxbuild_dir, 184 setup_args=sargs, 185 inplace=inplace, 186 reload_support=pyxargs.reload_support) 187 assert os.path.exists(so_path), "Cannot find: %s" % so_path 188 189 junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) 190 junkstuff = glob.glob(junkpath) 191 for path in junkstuff: 192 if path!=so_path: 193 try: 194 os.remove(path) 195 except IOError: 196 _info("Couldn't remove %s", path) 197 198 return so_path 199 200 def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False, 201 build_inplace=False, language_level=None, so_path=None): 202 try: 203 if so_path is None: 204 if is_package: 205 module_name = name + '.__init__' 206 else: 207 module_name = name 208 so_path = build_module(module_name, pyxfilename, pyxbuild_dir, 209 inplace=build_inplace, language_level=language_level) 210 mod = imp.load_dynamic(name, so_path) 211 if is_package and not hasattr(mod, '__path__'): 212 mod.__path__ = [os.path.dirname(so_path)] 213 assert mod.__file__ == so_path, (mod.__file__, so_path) 214 except Exception: 215 if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'): 216 # try to fall back to normal import 217 mod = imp.load_source(name, pyxfilename) 218 assert mod.__file__ in (pyxfilename, pyxfilename+'c', pyxfilename+'o'), (mod.__file__, pyxfilename) 219 else: 220 import traceback 221 raise ImportError("Building module %s failed: %s" % 222 (name, 223 traceback.format_exception_only(*sys.exc_info()[:2]))), None, sys.exc_info()[2] 224 return mod 225 226 227 # import hooks 228 229 class PyxImporter(object): 230 """A meta-path importer for .pyx files. 231 """ 232 def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False, 233 language_level=None): 234 self.extension = extension 235 self.pyxbuild_dir = pyxbuild_dir 236 self.inplace = inplace 237 self.language_level = language_level 238 239 def find_module(self, fullname, package_path=None): 240 if fullname in sys.modules and not pyxargs.reload_support: 241 return None # only here when reload() 242 try: 243 fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path) 244 if fp: fp.close() # Python should offer a Default-Loader to avoid this double find/open! 245 if pathname and ty == imp.PKG_DIRECTORY: 246 pkg_file = os.path.join(pathname, '__init__'+self.extension) 247 if os.path.isfile(pkg_file): 248 return PyxLoader(fullname, pathname, 249 init_path=pkg_file, 250 pyxbuild_dir=self.pyxbuild_dir, 251 inplace=self.inplace, 252 language_level=self.language_level) 253 if pathname and pathname.endswith(self.extension): 254 return PyxLoader(fullname, pathname, 255 pyxbuild_dir=self.pyxbuild_dir, 256 inplace=self.inplace, 257 language_level=self.language_level) 258 if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! 259 return None 260 261 # find .pyx fast, when .so/.pyd exist --inplace 262 pyxpath = os.path.splitext(pathname)[0]+self.extension 263 if os.path.isfile(pyxpath): 264 return PyxLoader(fullname, pyxpath, 265 pyxbuild_dir=self.pyxbuild_dir, 266 inplace=self.inplace, 267 language_level=self.language_level) 268 269 # .so/.pyd's on PATH should not be remote from .pyx's 270 # think no need to implement PyxArgs.importer_search_remote here? 271 272 except ImportError: 273 pass 274 275 # searching sys.path ... 276 277 #if DEBUG_IMPORT: print "SEARCHING", fullname, package_path 278 if '.' in fullname: # only when package_path anyway? 279 mod_parts = fullname.split('.') 280 module_name = mod_parts[-1] 281 else: 282 module_name = fullname 283 pyx_module_name = module_name + self.extension 284 # this may work, but it returns the file content, not its path 285 #import pkgutil 286 #pyx_source = pkgutil.get_data(package, pyx_module_name) 287 288 if package_path: 289 paths = package_path 290 else: 291 paths = sys.path 292 join_path = os.path.join 293 is_file = os.path.isfile 294 is_abs = os.path.isabs 295 abspath = os.path.abspath 296 #is_dir = os.path.isdir 297 sep = os.path.sep 298 for path in paths: 299 if not path: 300 path = os.getcwd() 301 elif not is_abs(path): 302 path = abspath(path) 303 if is_file(path+sep+pyx_module_name): 304 return PyxLoader(fullname, join_path(path, pyx_module_name), 305 pyxbuild_dir=self.pyxbuild_dir, 306 inplace=self.inplace, 307 language_level=self.language_level) 308 309 # not found, normal package, not a .pyx file, none of our business 310 _debug("%s not found" % fullname) 311 return None 312 313 class PyImporter(PyxImporter): 314 """A meta-path importer for normal .py files. 315 """ 316 def __init__(self, pyxbuild_dir=None, inplace=False, language_level=None): 317 if language_level is None: 318 language_level = sys.version_info[0] 319 self.super = super(PyImporter, self) 320 self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir, inplace=inplace, 321 language_level=language_level) 322 self.uncompilable_modules = {} 323 self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild', 324 'distutils.extension', 'distutils.sysconfig'] 325 326 def find_module(self, fullname, package_path=None): 327 if fullname in sys.modules: 328 return None 329 if fullname.startswith('Cython.'): 330 return None 331 if fullname in self.blocked_modules: 332 # prevent infinite recursion 333 return None 334 if _lib_loader.knows(fullname): 335 return _lib_loader 336 _debug("trying import of module '%s'", fullname) 337 if fullname in self.uncompilable_modules: 338 path, last_modified = self.uncompilable_modules[fullname] 339 try: 340 new_last_modified = os.stat(path).st_mtime 341 if new_last_modified > last_modified: 342 # import would fail again 343 return None 344 except OSError: 345 # module is no longer where we found it, retry the import 346 pass 347 348 self.blocked_modules.append(fullname) 349 try: 350 importer = self.super.find_module(fullname, package_path) 351 if importer is not None: 352 if importer.init_path: 353 path = importer.init_path 354 real_name = fullname + '.__init__' 355 else: 356 path = importer.path 357 real_name = fullname 358 _debug("importer found path %s for module %s", path, real_name) 359 try: 360 so_path = build_module( 361 real_name, path, 362 pyxbuild_dir=self.pyxbuild_dir, 363 language_level=self.language_level, 364 inplace=self.inplace) 365 _lib_loader.add_lib(fullname, path, so_path, 366 is_package=bool(importer.init_path)) 367 return _lib_loader 368 except Exception: 369 if DEBUG_IMPORT: 370 import traceback 371 traceback.print_exc() 372 # build failed, not a compilable Python module 373 try: 374 last_modified = os.stat(path).st_mtime 375 except OSError: 376 last_modified = 0 377 self.uncompilable_modules[fullname] = (path, last_modified) 378 importer = None 379 finally: 380 self.blocked_modules.pop() 381 return importer 382 383 class LibLoader(object): 384 def __init__(self): 385 self._libs = {} 386 387 def load_module(self, fullname): 388 try: 389 source_path, so_path, is_package = self._libs[fullname] 390 except KeyError: 391 raise ValueError("invalid module %s" % fullname) 392 _debug("Loading shared library module '%s' from %s", fullname, so_path) 393 return load_module(fullname, source_path, so_path=so_path, is_package=is_package) 394 395 def add_lib(self, fullname, path, so_path, is_package): 396 self._libs[fullname] = (path, so_path, is_package) 397 398 def knows(self, fullname): 399 return fullname in self._libs 400 401 _lib_loader = LibLoader() 402 403 class PyxLoader(object): 404 def __init__(self, fullname, path, init_path=None, pyxbuild_dir=None, 405 inplace=False, language_level=None): 406 _debug("PyxLoader created for loading %s from %s (init path: %s)", 407 fullname, path, init_path) 408 self.fullname = fullname 409 self.path, self.init_path = path, init_path 410 self.pyxbuild_dir = pyxbuild_dir 411 self.inplace = inplace 412 self.language_level = language_level 413 414 def load_module(self, fullname): 415 assert self.fullname == fullname, ( 416 "invalid module, expected %s, got %s" % ( 417 self.fullname, fullname)) 418 if self.init_path: 419 # package 420 #print "PACKAGE", fullname 421 module = load_module(fullname, self.init_path, 422 self.pyxbuild_dir, is_package=True, 423 build_inplace=self.inplace, 424 language_level=self.language_level) 425 module.__path__ = [self.path] 426 else: 427 #print "MODULE", fullname 428 module = load_module(fullname, self.path, 429 self.pyxbuild_dir, 430 build_inplace=self.inplace, 431 language_level=self.language_level) 432 return module 433 434 435 #install args 436 class PyxArgs(object): 437 build_dir=True 438 build_in_temp=True 439 setup_args={} #None 440 441 ##pyxargs=None 442 443 def _have_importers(): 444 has_py_importer = False 445 has_pyx_importer = False 446 for importer in sys.meta_path: 447 if isinstance(importer, PyxImporter): 448 if isinstance(importer, PyImporter): 449 has_py_importer = True 450 else: 451 has_pyx_importer = True 452 453 return has_py_importer, has_pyx_importer 454 455 def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True, 456 setup_args={}, reload_support=False, 457 load_py_module_on_import_failure=False, inplace=False, 458 language_level=None): 459 """Main entry point. Call this to install the .pyx import hook in 460 your meta-path for a single Python process. If you want it to be 461 installed whenever you use Python, add it to your sitecustomize 462 (as described above). 463 464 You can pass ``pyimport=True`` to also install the .py import hook 465 in your meta-path. Note, however, that it is highly experimental, 466 will not work for most .py files, and will therefore only slow 467 down your imports. Use at your own risk. 468 469 By default, compiled modules will end up in a ``.pyxbld`` 470 directory in the user's home directory. Passing a different path 471 as ``build_dir`` will override this. 472 473 ``build_in_temp=False`` will produce the C files locally. Working 474 with complex dependencies and debugging becomes more easy. This 475 can principally interfere with existing files of the same name. 476 build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args() 477 by a dict item of 'build_in_temp' 478 479 ``setup_args``: dict of arguments for Distribution - see 480 distutils.core.setup() . They are extended/overriden by those of 481 <modulename>.pyxbld/make_setup_args() 482 483 ``reload_support``: Enables support for dynamic 484 reload(<pyxmodulename>), e.g. after a change in the Cython code. 485 Additional files <so_path>.reloadNN may arise on that account, when 486 the previously loaded module file cannot be overwritten. 487 488 ``load_py_module_on_import_failure``: If the compilation of a .py 489 file succeeds, but the subsequent import fails for some reason, 490 retry the import with the normal .py module instead of the 491 compiled module. Note that this may lead to unpredictable results 492 for modules that change the system state during their import, as 493 the second import will rerun these modifications in whatever state 494 the system was left after the import of the compiled module 495 failed. 496 497 ``inplace``: Install the compiled module next to the source file. 498 499 ``language_level``: The source language level to use: 2 or 3. 500 The default is to use the language level of the current Python 501 runtime for .py files and Py2 for .pyx files. 502 """ 503 if not build_dir: 504 build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld') 505 506 global pyxargs 507 pyxargs = PyxArgs() #$pycheck_no 508 pyxargs.build_dir = build_dir 509 pyxargs.build_in_temp = build_in_temp 510 pyxargs.setup_args = (setup_args or {}).copy() 511 pyxargs.reload_support = reload_support 512 pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure 513 514 has_py_importer, has_pyx_importer = _have_importers() 515 py_importer, pyx_importer = None, None 516 517 if pyimport and not has_py_importer: 518 py_importer = PyImporter(pyxbuild_dir=build_dir, inplace=inplace, 519 language_level=language_level) 520 # make sure we import Cython before we install the import hook 521 import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize 522 sys.meta_path.insert(0, py_importer) 523 524 if pyximport and not has_pyx_importer: 525 pyx_importer = PyxImporter(pyxbuild_dir=build_dir, inplace=inplace, 526 language_level=language_level) 527 sys.meta_path.append(pyx_importer) 528 529 return py_importer, pyx_importer 530 531 def uninstall(py_importer, pyx_importer): 532 """ 533 Uninstall an import hook. 534 """ 535 try: 536 sys.meta_path.remove(py_importer) 537 except ValueError: 538 pass 539 540 try: 541 sys.meta_path.remove(pyx_importer) 542 except ValueError: 543 pass 544 545 # MAIN 546 547 def show_docs(): 548 import __main__ 549 __main__.__name__ = mod_name 550 for name in dir(__main__): 551 item = getattr(__main__, name) 552 try: 553 setattr(item, "__module__", mod_name) 554 except (AttributeError, TypeError): 555 pass 556 help(__main__) 557 558 if __name__ == '__main__': 559 show_docs() 560