Home | History | Annotate | Download | only in ssl 
      1 #./python
      2 """Run Python tests with multiple installations of OpenSSL
      3 
      4 The script
      5 
      6   (1) downloads OpenSSL tar bundle
      7   (2) extracts it to ../openssl/src/openssl-VERSION/
      8   (3) compiles OpenSSL
      9   (4) installs OpenSSL into ../openssl/VERSION/
     10   (5) forces a recompilation of Python modules using the
     11       header and library files from ../openssl/VERSION/
     12   (6) runs Python's test suite
     13 
     14 The script must be run with Python's build directory as current working
     15 directory:
     16 
     17     ./python Tools/ssl/test_multiple_versions.py
     18 
     19 The script uses LD_RUN_PATH, LD_LIBRARY_PATH, CPPFLAGS and LDFLAGS to bend
     20 search paths for header files and shared libraries. It's known to work on
     21 Linux with GCC 4.x.
     22 
     23 (c) 2013 Christian Heimes <christian (at] python.org>
     24 """
     25 import logging
     26 import os
     27 import tarfile
     28 import shutil
     29 import subprocess
     30 import sys
     31 from urllib import urlopen
     32 
     33 log = logging.getLogger("multissl")
     34 
     35 OPENSSL_VERSIONS = [
     36     "0.9.7m", "0.9.8i", "0.9.8l", "0.9.8m", "0.9.8y", "1.0.0k", "1.0.1e"
     37 ]
     38 FULL_TESTS = [
     39     "test_asyncio", "test_ftplib", "test_hashlib", "test_httplib",
     40     "test_imaplib", "test_nntplib", "test_poplib", "test_smtplib",
     41     "test_smtpnet", "test_urllib2_localnet", "test_venv"
     42 ]
     43 MINIMAL_TESTS = ["test_ssl", "test_hashlib"]
     44 CADEFAULT = True
     45 HERE = os.path.abspath(os.getcwd())
     46 DEST_DIR = os.path.abspath(os.path.join(HERE, os.pardir, "openssl"))
     47 
     48 
     49 class BuildSSL(object):
     50     url_template = "https://www.openssl.org/source/openssl-{}.tar.gz"
     51 
     52     module_files = ["Modules/_ssl.c",
     53                     "Modules/socketmodule.c",
     54                     "Modules/_hashopenssl.c"]
     55 
     56     def __init__(self, version, openssl_compile_args=(), destdir=DEST_DIR):
     57         self._check_python_builddir()
     58         self.version = version
     59         self.openssl_compile_args = openssl_compile_args
     60         # installation directory
     61         self.install_dir = os.path.join(destdir, version)
     62         # source file
     63         self.src_file = os.path.join(destdir, "src",
     64                                      "openssl-{}.tar.gz".format(version))
     65         # build directory (removed after install)
     66         self.build_dir = os.path.join(destdir, "src",
     67                                       "openssl-{}".format(version))
     68 
     69     @property
     70     def openssl_cli(self):
     71         """openssl CLI binary"""
     72         return os.path.join(self.install_dir, "bin", "openssl")
     73 
     74     @property
     75     def openssl_version(self):
     76         """output of 'bin/openssl version'"""
     77         env = os.environ.copy()
     78         env["LD_LIBRARY_PATH"] = self.lib_dir
     79         cmd = [self.openssl_cli, "version"]
     80         return self._subprocess_output(cmd, env=env)
     81 
     82     @property
     83     def pyssl_version(self):
     84         """Value of ssl.OPENSSL_VERSION"""
     85         env = os.environ.copy()
     86         env["LD_LIBRARY_PATH"] = self.lib_dir
     87         cmd = ["./python", "-c", "import ssl; print(ssl.OPENSSL_VERSION)"]
     88         return self._subprocess_output(cmd, env=env)
     89 
     90     @property
     91     def include_dir(self):
     92         return os.path.join(self.install_dir, "include")
     93 
     94     @property
     95     def lib_dir(self):
     96         return os.path.join(self.install_dir, "lib")
     97 
     98     @property
     99     def has_openssl(self):
    100         return os.path.isfile(self.openssl_cli)
    101 
    102     @property
    103     def has_src(self):
    104         return os.path.isfile(self.src_file)
    105 
    106     def _subprocess_call(self, cmd, stdout=subprocess.DEVNULL, env=None,
    107                          **kwargs):
    108         log.debug("Call '{}'".format(" ".join(cmd)))
    109         return subprocess.check_call(cmd, stdout=stdout, env=env, **kwargs)
    110 
    111     def _subprocess_output(self, cmd, env=None, **kwargs):
    112         log.debug("Call '{}'".format(" ".join(cmd)))
    113         out = subprocess.check_output(cmd, env=env)
    114         return out.strip().decode("utf-8")
    115 
    116     def _check_python_builddir(self):
    117         if not os.path.isfile("python") or not os.path.isfile("setup.py"):
    118             raise ValueError("Script must be run in Python build directory")
    119 
    120     def _download_openssl(self):
    121         """Download OpenSSL source dist"""
    122         src_dir = os.path.dirname(self.src_file)
    123         if not os.path.isdir(src_dir):
    124             os.makedirs(src_dir)
    125         url = self.url_template.format(self.version)
    126         log.info("Downloading OpenSSL from {}".format(url))
    127         req = urlopen(url, cadefault=CADEFAULT)
    128         # KISS, read all, write all
    129         data = req.read()
    130         log.info("Storing {}".format(self.src_file))
    131         with open(self.src_file, "wb") as f:
    132             f.write(data)
    133 
    134     def _unpack_openssl(self):
    135         """Unpack tar.gz bundle"""
    136         # cleanup
    137         if os.path.isdir(self.build_dir):
    138             shutil.rmtree(self.build_dir)
    139         os.makedirs(self.build_dir)
    140 
    141         tf = tarfile.open(self.src_file)
    142         base = "openssl-{}/".format(self.version)
    143         # force extraction into build dir
    144         members = tf.getmembers()
    145         for member in members:
    146             if not member.name.startswith(base):
    147                 raise ValueError(member.name)
    148             member.name = member.name[len(base):]
    149         log.info("Unpacking files to {}".format(self.build_dir))
    150         tf.extractall(self.build_dir, members)
    151 
    152     def _build_openssl(self):
    153         """Now build openssl"""
    154         log.info("Running build in {}".format(self.install_dir))
    155         cwd = self.build_dir
    156         cmd = ["./config", "shared", "--prefix={}".format(self.install_dir)]
    157         cmd.extend(self.openssl_compile_args)
    158         self._subprocess_call(cmd, cwd=cwd)
    159         self._subprocess_call(["make"], cwd=cwd)
    160 
    161     def _install_openssl(self, remove=True):
    162         self._subprocess_call(["make", "install"], cwd=self.build_dir)
    163         if remove:
    164             shutil.rmtree(self.build_dir)
    165 
    166     def install_openssl(self):
    167         if not self.has_openssl:
    168             if not self.has_src:
    169                 self._download_openssl()
    170             else:
    171                 log.debug("Already has src {}".format(self.src_file))
    172             self._unpack_openssl()
    173             self._build_openssl()
    174             self._install_openssl()
    175         else:
    176             log.info("Already has installation {}".format(self.install_dir))
    177         # validate installation
    178         version = self.openssl_version
    179         if self.version not in version:
    180             raise ValueError(version)
    181 
    182     def touch_pymods(self):
    183         # force a rebuild of all modules that use OpenSSL APIs
    184         for fname in self.module_files:
    185             os.utime(fname)
    186 
    187     def recompile_pymods(self):
    188         log.info("Using OpenSSL build from {}".format(self.build_dir))
    189         # overwrite header and library search paths
    190         env = os.environ.copy()
    191         env["CPPFLAGS"] = "-I{}".format(self.include_dir)
    192         env["LDFLAGS"] = "-L{}".format(self.lib_dir)
    193         # set rpath
    194         env["LD_RUN_PATH"] = self.lib_dir
    195 
    196         log.info("Rebuilding Python modules")
    197         self.touch_pymods()
    198         cmd = ["./python", "setup.py", "build"]
    199         self._subprocess_call(cmd, env=env)
    200 
    201     def check_pyssl(self):
    202         version = self.pyssl_version
    203         if self.version not in version:
    204             raise ValueError(version)
    205 
    206     def run_pytests(self, *args):
    207         cmd = ["./python", "-m", "test"]
    208         cmd.extend(args)
    209         self._subprocess_call(cmd, stdout=None)
    210 
    211     def run_python_tests(self, *args):
    212         self.recompile_pymods()
    213         self.check_pyssl()
    214         self.run_pytests(*args)
    215 
    216 
    217 def main(*args):
    218     builders = []
    219     for version in OPENSSL_VERSIONS:
    220         if version in ("0.9.8i", "0.9.8l"):
    221             openssl_compile_args = ("no-asm",)
    222         else:
    223             openssl_compile_args = ()
    224         builder = BuildSSL(version, openssl_compile_args)
    225         builder.install_openssl()
    226         builders.append(builder)
    227 
    228     for builder in builders:
    229         builder.run_python_tests(*args)
    230     # final touch
    231     builder.touch_pymods()
    232 
    233 
    234 if __name__ == "__main__":
    235     logging.basicConfig(level=logging.INFO,
    236                         format="*** %(levelname)s %(message)s")
    237     args = sys.argv[1:]
    238     if not args:
    239         args = ["-unetwork", "-v"]
    240         args.extend(FULL_TESTS)
    241     main(*args)
    242