Home | History | Annotate | Download | only in setuptools 
      1 import os
      2 import sys
      3 import tempfile
      4 import operator
      5 import functools
      6 import itertools
      7 import re
      8 import contextlib
      9 import pickle
     10 import textwrap
     11 
     12 from setuptools.extern import six
     13 from setuptools.extern.six.moves import builtins, map
     14 
     15 import pkg_resources.py31compat
     16 
     17 if sys.platform.startswith('java'):
     18     import org.python.modules.posix.PosixModule as _os
     19 else:
     20     _os = sys.modules[os.name]
     21 try:
     22     _file = file
     23 except NameError:
     24     _file = None
     25 _open = open
     26 from distutils.errors import DistutilsError
     27 from pkg_resources import working_set
     28 
     29 
     30 __all__ = [
     31     "AbstractSandbox", "DirectorySandbox", "SandboxViolation", "run_setup",
     32 ]
     33 
     34 
     35 def _execfile(filename, globals, locals=None):
     36     """
     37     Python 3 implementation of execfile.
     38     """
     39     mode = 'rb'
     40     with open(filename, mode) as stream:
     41         script = stream.read()
     42     if locals is None:
     43         locals = globals
     44     code = compile(script, filename, 'exec')
     45     exec(code, globals, locals)
     46 
     47 
     48 @contextlib.contextmanager
     49 def save_argv(repl=None):
     50     saved = sys.argv[:]
     51     if repl is not None:
     52         sys.argv[:] = repl
     53     try:
     54         yield saved
     55     finally:
     56         sys.argv[:] = saved
     57 
     58 
     59 @contextlib.contextmanager
     60 def save_path():
     61     saved = sys.path[:]
     62     try:
     63         yield saved
     64     finally:
     65         sys.path[:] = saved
     66 
     67 
     68 @contextlib.contextmanager
     69 def override_temp(replacement):
     70     """
     71     Monkey-patch tempfile.tempdir with replacement, ensuring it exists
     72     """
     73     pkg_resources.py31compat.makedirs(replacement, exist_ok=True)
     74 
     75     saved = tempfile.tempdir
     76 
     77     tempfile.tempdir = replacement
     78 
     79     try:
     80         yield
     81     finally:
     82         tempfile.tempdir = saved
     83 
     84 
     85 @contextlib.contextmanager
     86 def pushd(target):
     87     saved = os.getcwd()
     88     os.chdir(target)
     89     try:
     90         yield saved
     91     finally:
     92         os.chdir(saved)
     93 
     94 
     95 class UnpickleableException(Exception):
     96     """
     97     An exception representing another Exception that could not be pickled.
     98     """
     99 
    100     @staticmethod
    101     def dump(type, exc):
    102         """
    103         Always return a dumped (pickled) type and exc. If exc can't be pickled,
    104         wrap it in UnpickleableException first.
    105         """
    106         try:
    107             return pickle.dumps(type), pickle.dumps(exc)
    108         except Exception:
    109             # get UnpickleableException inside the sandbox
    110             from setuptools.sandbox import UnpickleableException as cls
    111             return cls.dump(cls, cls(repr(exc)))
    112 
    113 
    114 class ExceptionSaver:
    115     """
    116     A Context Manager that will save an exception, serialized, and restore it
    117     later.
    118     """
    119 
    120     def __enter__(self):
    121         return self
    122 
    123     def __exit__(self, type, exc, tb):
    124         if not exc:
    125             return
    126 
    127         # dump the exception
    128         self._saved = UnpickleableException.dump(type, exc)
    129         self._tb = tb
    130 
    131         # suppress the exception
    132         return True
    133 
    134     def resume(self):
    135         "restore and re-raise any exception"
    136 
    137         if '_saved' not in vars(self):
    138             return
    139 
    140         type, exc = map(pickle.loads, self._saved)
    141         six.reraise(type, exc, self._tb)
    142 
    143 
    144 @contextlib.contextmanager
    145 def save_modules():
    146     """
    147     Context in which imported modules are saved.
    148 
    149     Translates exceptions internal to the context into the equivalent exception
    150     outside the context.
    151     """
    152     saved = sys.modules.copy()
    153     with ExceptionSaver() as saved_exc:
    154         yield saved
    155 
    156     sys.modules.update(saved)
    157     # remove any modules imported since
    158     del_modules = (
    159         mod_name for mod_name in sys.modules
    160         if mod_name not in saved
    161         # exclude any encodings modules. See #285
    162         and not mod_name.startswith('encodings.')
    163     )
    164     _clear_modules(del_modules)
    165 
    166     saved_exc.resume()
    167 
    168 
    169 def _clear_modules(module_names):
    170     for mod_name in list(module_names):
    171         del sys.modules[mod_name]
    172 
    173 
    174 @contextlib.contextmanager
    175 def save_pkg_resources_state():
    176     saved = pkg_resources.__getstate__()
    177     try:
    178         yield saved
    179     finally:
    180         pkg_resources.__setstate__(saved)
    181 
    182 
    183 @contextlib.contextmanager
    184 def setup_context(setup_dir):
    185     temp_dir = os.path.join(setup_dir, 'temp')
    186     with save_pkg_resources_state():
    187         with save_modules():
    188             hide_setuptools()
    189             with save_path():
    190                 with save_argv():
    191                     with override_temp(temp_dir):
    192                         with pushd(setup_dir):
    193                             # ensure setuptools commands are available
    194                             __import__('setuptools')
    195                             yield
    196 
    197 
    198 def _needs_hiding(mod_name):
    199     """
    200     >>> _needs_hiding('setuptools')
    201     True
    202     >>> _needs_hiding('pkg_resources')
    203     True
    204     >>> _needs_hiding('setuptools_plugin')
    205     False
    206     >>> _needs_hiding('setuptools.__init__')
    207     True
    208     >>> _needs_hiding('distutils')
    209     True
    210     >>> _needs_hiding('os')
    211     False
    212     >>> _needs_hiding('Cython')
    213     True
    214     """
    215     pattern = re.compile(r'(setuptools|pkg_resources|distutils|Cython)(\.|$)')
    216     return bool(pattern.match(mod_name))
    217 
    218 
    219 def hide_setuptools():
    220     """
    221     Remove references to setuptools' modules from sys.modules to allow the
    222     invocation to import the most appropriate setuptools. This technique is
    223     necessary to avoid issues such as #315 where setuptools upgrading itself
    224     would fail to find a function declared in the metadata.
    225     """
    226     modules = filter(_needs_hiding, sys.modules)
    227     _clear_modules(modules)
    228 
    229 
    230 def run_setup(setup_script, args):
    231     """Run a distutils setup script, sandboxed in its directory"""
    232     setup_dir = os.path.abspath(os.path.dirname(setup_script))
    233     with setup_context(setup_dir):
    234         try:
    235             sys.argv[:] = [setup_script] + list(args)
    236             sys.path.insert(0, setup_dir)
    237             # reset to include setup dir, w/clean callback list
    238             working_set.__init__()
    239             working_set.callbacks.append(lambda dist: dist.activate())
    240 
    241             # __file__ should be a byte string on Python 2 (#712)
    242             dunder_file = (
    243                 setup_script
    244                 if isinstance(setup_script, str) else
    245                 setup_script.encode(sys.getfilesystemencoding())
    246             )
    247 
    248             with DirectorySandbox(setup_dir):
    249                 ns = dict(__file__=dunder_file, __name__='__main__')
    250                 _execfile(setup_script, ns)
    251         except SystemExit as v:
    252             if v.args and v.args[0]:
    253                 raise
    254             # Normal exit, just return
    255 
    256 
    257 class AbstractSandbox:
    258     """Wrap 'os' module and 'open()' builtin for virtualizing setup scripts"""
    259 
    260     _active = False
    261 
    262     def __init__(self):
    263         self._attrs = [
    264             name for name in dir(_os)
    265             if not name.startswith('_') and hasattr(self, name)
    266         ]
    267 
    268     def _copy(self, source):
    269         for name in self._attrs:
    270             setattr(os, name, getattr(source, name))
    271 
    272     def __enter__(self):
    273         self._copy(self)
    274         if _file:
    275             builtins.file = self._file
    276         builtins.open = self._open
    277         self._active = True
    278 
    279     def __exit__(self, exc_type, exc_value, traceback):
    280         self._active = False
    281         if _file:
    282             builtins.file = _file
    283         builtins.open = _open
    284         self._copy(_os)
    285 
    286     def run(self, func):
    287         """Run 'func' under os sandboxing"""
    288         with self:
    289             return func()
    290 
    291     def _mk_dual_path_wrapper(name):
    292         original = getattr(_os, name)
    293 
    294         def wrap(self, src, dst, *args, **kw):
    295             if self._active:
    296                 src, dst = self._remap_pair(name, src, dst, *args, **kw)
    297             return original(src, dst, *args, **kw)
    298 
    299         return wrap
    300 
    301     for name in ["rename", "link", "symlink"]:
    302         if hasattr(_os, name):
    303             locals()[name] = _mk_dual_path_wrapper(name)
    304 
    305     def _mk_single_path_wrapper(name, original=None):
    306         original = original or getattr(_os, name)
    307 
    308         def wrap(self, path, *args, **kw):
    309             if self._active:
    310                 path = self._remap_input(name, path, *args, **kw)
    311             return original(path, *args, **kw)
    312 
    313         return wrap
    314 
    315     if _file:
    316         _file = _mk_single_path_wrapper('file', _file)
    317     _open = _mk_single_path_wrapper('open', _open)
    318     for name in [
    319         "stat", "listdir", "chdir", "open", "chmod", "chown", "mkdir",
    320         "remove", "unlink", "rmdir", "utime", "lchown", "chroot", "lstat",
    321         "startfile", "mkfifo", "mknod", "pathconf", "access"
    322     ]:
    323         if hasattr(_os, name):
    324             locals()[name] = _mk_single_path_wrapper(name)
    325 
    326     def _mk_single_with_return(name):
    327         original = getattr(_os, name)
    328 
    329         def wrap(self, path, *args, **kw):
    330             if self._active:
    331                 path = self._remap_input(name, path, *args, **kw)
    332                 return self._remap_output(name, original(path, *args, **kw))
    333             return original(path, *args, **kw)
    334 
    335         return wrap
    336 
    337     for name in ['readlink', 'tempnam']:
    338         if hasattr(_os, name):
    339             locals()[name] = _mk_single_with_return(name)
    340 
    341     def _mk_query(name):
    342         original = getattr(_os, name)
    343 
    344         def wrap(self, *args, **kw):
    345             retval = original(*args, **kw)
    346             if self._active:
    347                 return self._remap_output(name, retval)
    348             return retval
    349 
    350         return wrap
    351 
    352     for name in ['getcwd', 'tmpnam']:
    353         if hasattr(_os, name):
    354             locals()[name] = _mk_query(name)
    355 
    356     def _validate_path(self, path):
    357         """Called to remap or validate any path, whether input or output"""
    358         return path
    359 
    360     def _remap_input(self, operation, path, *args, **kw):
    361         """Called for path inputs"""
    362         return self._validate_path(path)
    363 
    364     def _remap_output(self, operation, path):
    365         """Called for path outputs"""
    366         return self._validate_path(path)
    367 
    368     def _remap_pair(self, operation, src, dst, *args, **kw):
    369         """Called for path pairs like rename, link, and symlink operations"""
    370         return (
    371             self._remap_input(operation + '-from', src, *args, **kw),
    372             self._remap_input(operation + '-to', dst, *args, **kw)
    373         )
    374 
    375 
    376 if hasattr(os, 'devnull'):
    377     _EXCEPTIONS = [os.devnull,]
    378 else:
    379     _EXCEPTIONS = []
    380 
    381 
    382 class DirectorySandbox(AbstractSandbox):
    383     """Restrict operations to a single subdirectory - pseudo-chroot"""
    384 
    385     write_ops = dict.fromkeys([
    386         "open", "chmod", "chown", "mkdir", "remove", "unlink", "rmdir",
    387         "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
    388     ])
    389 
    390     _exception_patterns = [
    391         # Allow lib2to3 to attempt to save a pickled grammar object (#121)
    392         r'.*lib2to3.*\.pickle$',
    393     ]
    394     "exempt writing to paths that match the pattern"
    395 
    396     def __init__(self, sandbox, exceptions=_EXCEPTIONS):
    397         self._sandbox = os.path.normcase(os.path.realpath(sandbox))
    398         self._prefix = os.path.join(self._sandbox, '')
    399         self._exceptions = [
    400             os.path.normcase(os.path.realpath(path))
    401             for path in exceptions
    402         ]
    403         AbstractSandbox.__init__(self)
    404 
    405     def _violation(self, operation, *args, **kw):
    406         from setuptools.sandbox import SandboxViolation
    407         raise SandboxViolation(operation, args, kw)
    408 
    409     if _file:
    410 
    411         def _file(self, path, mode='r', *args, **kw):
    412             if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
    413                 self._violation("file", path, mode, *args, **kw)
    414             return _file(path, mode, *args, **kw)
    415 
    416     def _open(self, path, mode='r', *args, **kw):
    417         if mode not in ('r', 'rt', 'rb', 'rU', 'U') and not self._ok(path):
    418             self._violation("open", path, mode, *args, **kw)
    419         return _open(path, mode, *args, **kw)
    420 
    421     def tmpnam(self):
    422         self._violation("tmpnam")
    423 
    424     def _ok(self, path):
    425         active = self._active
    426         try:
    427             self._active = False
    428             realpath = os.path.normcase(os.path.realpath(path))
    429             return (
    430                 self._exempted(realpath)
    431                 or realpath == self._sandbox
    432                 or realpath.startswith(self._prefix)
    433             )
    434         finally:
    435             self._active = active
    436 
    437     def _exempted(self, filepath):
    438         start_matches = (
    439             filepath.startswith(exception)
    440             for exception in self._exceptions
    441         )
    442         pattern_matches = (
    443             re.match(pattern, filepath)
    444             for pattern in self._exception_patterns
    445         )
    446         candidates = itertools.chain(start_matches, pattern_matches)
    447         return any(candidates)
    448 
    449     def _remap_input(self, operation, path, *args, **kw):
    450         """Called for path inputs"""
    451         if operation in self.write_ops and not self._ok(path):
    452             self._violation(operation, os.path.realpath(path), *args, **kw)
    453         return path
    454 
    455     def _remap_pair(self, operation, src, dst, *args, **kw):
    456         """Called for path pairs like rename, link, and symlink operations"""
    457         if not self._ok(src) or not self._ok(dst):
    458             self._violation(operation, src, dst, *args, **kw)
    459         return (src, dst)
    460 
    461     def open(self, file, flags, mode=0o777, *args, **kw):
    462         """Called for low-level os.open()"""
    463         if flags & WRITE_FLAGS and not self._ok(file):
    464             self._violation("os.open", file, flags, mode, *args, **kw)
    465         return _os.open(file, flags, mode, *args, **kw)
    466 
    467 
    468 WRITE_FLAGS = functools.reduce(
    469     operator.or_, [getattr(_os, a, 0) for a in
    470         "O_WRONLY O_RDWR O_APPEND O_CREAT O_TRUNC O_TEMPORARY".split()]
    471 )
    472 
    473 
    474 class SandboxViolation(DistutilsError):
    475     """A setup script attempted to modify the filesystem outside the sandbox"""
    476 
    477     tmpl = textwrap.dedent("""
    478         SandboxViolation: {cmd}{args!r} {kwargs}
    479 
    480         The package setup script has attempted to modify files on your system
    481         that are not within the EasyInstall build area, and has been aborted.
    482 
    483         This package cannot be safely installed by EasyInstall, and may not
    484         support alternate installation locations even if you run its setup
    485         script by hand.  Please inform the package's author and the EasyInstall
    486         maintainers to find out if a fix or workaround is available.
    487         """).lstrip()
    488 
    489     def __str__(self):
    490         cmd, args, kwargs = self.args
    491         return self.tmpl.format(**locals())
    492