Home | History | Annotate | Download | only in command
      1 """distutils.command.build_py
      2 
      3 Implements the Distutils 'build_py' command."""
      4 
      5 import os
      6 import importlib.util
      7 import sys
      8 from glob import glob
      9 
     10 from distutils.core import Command
     11 from distutils.errors import *
     12 from distutils.util import convert_path, Mixin2to3
     13 from distutils import log
     14 
     15 class build_py (Command):
     16 
     17     description = "\"build\" pure Python modules (copy to build directory)"
     18 
     19     user_options = [
     20         ('build-lib=', 'd', "directory to \"build\" (copy) to"),
     21         ('compile', 'c', "compile .py to .pyc"),
     22         ('no-compile', None, "don't compile .py files [default]"),
     23         ('optimize=', 'O',
     24          "also compile with optimization: -O1 for \"python -O\", "
     25          "-O2 for \"python -OO\", and -O0 to disable [default: -O0]"),
     26         ('force', 'f', "forcibly build everything (ignore file timestamps)"),
     27         ]
     28 
     29     boolean_options = ['compile', 'force']
     30     negative_opt = {'no-compile' : 'compile'}
     31 
     32     def initialize_options(self):
     33         self.build_lib = None
     34         self.py_modules = None
     35         self.package = None
     36         self.package_data = None
     37         self.package_dir = None
     38         self.compile = 0
     39         self.optimize = 0
     40         self.force = None
     41 
     42     def finalize_options(self):
     43         self.set_undefined_options('build',
     44                                    ('build_lib', 'build_lib'),
     45                                    ('force', 'force'))
     46 
     47         # Get the distribution options that are aliases for build_py
     48         # options -- list of packages and list of modules.
     49         self.packages = self.distribution.packages
     50         self.py_modules = self.distribution.py_modules
     51         self.package_data = self.distribution.package_data
     52         self.package_dir = {}
     53         if self.distribution.package_dir:
     54             for name, path in self.distribution.package_dir.items():
     55                 self.package_dir[name] = convert_path(path)
     56         self.data_files = self.get_data_files()
     57 
     58         # Ick, copied straight from install_lib.py (fancy_getopt needs a
     59         # type system!  Hell, *everything* needs a type system!!!)
     60         if not isinstance(self.optimize, int):
     61             try:
     62                 self.optimize = int(self.optimize)
     63                 assert 0 <= self.optimize <= 2
     64             except (ValueError, AssertionError):
     65                 raise DistutilsOptionError("optimize must be 0, 1, or 2")
     66 
     67     def run(self):
     68         # XXX copy_file by default preserves atime and mtime.  IMHO this is
     69         # the right thing to do, but perhaps it should be an option -- in
     70         # particular, a site administrator might want installed files to
     71         # reflect the time of installation rather than the last
     72         # modification time before the installed release.
     73 
     74         # XXX copy_file by default preserves mode, which appears to be the
     75         # wrong thing to do: if a file is read-only in the working
     76         # directory, we want it to be installed read/write so that the next
     77         # installation of the same module distribution can overwrite it
     78         # without problems.  (This might be a Unix-specific issue.)  Thus
     79         # we turn off 'preserve_mode' when copying to the build directory,
     80         # since the build directory is supposed to be exactly what the
     81         # installation will look like (ie. we preserve mode when
     82         # installing).
     83 
     84         # Two options control which modules will be installed: 'packages'
     85         # and 'py_modules'.  The former lets us work with whole packages, not
     86         # specifying individual modules at all; the latter is for
     87         # specifying modules one-at-a-time.
     88 
     89         if self.py_modules:
     90             self.build_modules()
     91         if self.packages:
     92             self.build_packages()
     93             self.build_package_data()
     94 
     95         self.byte_compile(self.get_outputs(include_bytecode=0))
     96 
     97     def get_data_files(self):
     98         """Generate list of '(package,src_dir,build_dir,filenames)' tuples"""
     99         data = []
    100         if not self.packages:
    101             return data
    102         for package in self.packages:
    103             # Locate package source directory
    104             src_dir = self.get_package_dir(package)
    105 
    106             # Compute package build directory
    107             build_dir = os.path.join(*([self.build_lib] + package.split('.')))
    108 
    109             # Length of path to strip from found files
    110             plen = 0
    111             if src_dir:
    112                 plen = len(src_dir)+1
    113 
    114             # Strip directory from globbed filenames
    115             filenames = [
    116                 file[plen:] for file in self.find_data_files(package, src_dir)
    117                 ]
    118             data.append((package, src_dir, build_dir, filenames))
    119         return data
    120 
    121     def find_data_files(self, package, src_dir):
    122         """Return filenames for package's data files in 'src_dir'"""
    123         globs = (self.package_data.get('', [])
    124                  + self.package_data.get(package, []))
    125         files = []
    126         for pattern in globs:
    127             # Each pattern has to be converted to a platform-specific path
    128             filelist = glob(os.path.join(src_dir, convert_path(pattern)))
    129             # Files that match more than one pattern are only added once
    130             files.extend([fn for fn in filelist if fn not in files
    131                 and os.path.isfile(fn)])
    132         return files
    133 
    134     def build_package_data(self):
    135         """Copy data files into build directory"""
    136         lastdir = None
    137         for package, src_dir, build_dir, filenames in self.data_files:
    138             for filename in filenames:
    139                 target = os.path.join(build_dir, filename)
    140                 self.mkpath(os.path.dirname(target))
    141                 self.copy_file(os.path.join(src_dir, filename), target,
    142                                preserve_mode=False)
    143 
    144     def get_package_dir(self, package):
    145         """Return the directory, relative to the top of the source
    146            distribution, where package 'package' should be found
    147            (at least according to the 'package_dir' option, if any)."""
    148         path = package.split('.')
    149 
    150         if not self.package_dir:
    151             if path:
    152                 return os.path.join(*path)
    153             else:
    154                 return ''
    155         else:
    156             tail = []
    157             while path:
    158                 try:
    159                     pdir = self.package_dir['.'.join(path)]
    160                 except KeyError:
    161                     tail.insert(0, path[-1])
    162                     del path[-1]
    163                 else:
    164                     tail.insert(0, pdir)
    165                     return os.path.join(*tail)
    166             else:
    167                 # Oops, got all the way through 'path' without finding a
    168                 # match in package_dir.  If package_dir defines a directory
    169                 # for the root (nameless) package, then fallback on it;
    170                 # otherwise, we might as well have not consulted
    171                 # package_dir at all, as we just use the directory implied
    172                 # by 'tail' (which should be the same as the original value
    173                 # of 'path' at this point).
    174                 pdir = self.package_dir.get('')
    175                 if pdir is not None:
    176                     tail.insert(0, pdir)
    177 
    178                 if tail:
    179                     return os.path.join(*tail)
    180                 else:
    181                     return ''
    182 
    183     def check_package(self, package, package_dir):
    184         # Empty dir name means current directory, which we can probably
    185         # assume exists.  Also, os.path.exists and isdir don't know about
    186         # my "empty string means current dir" convention, so we have to
    187         # circumvent them.
    188         if package_dir != "":
    189             if not os.path.exists(package_dir):
    190                 raise DistutilsFileError(
    191                       "package directory '%s' does not exist" % package_dir)
    192             if not os.path.isdir(package_dir):
    193                 raise DistutilsFileError(
    194                        "supposed package directory '%s' exists, "
    195                        "but is not a directory" % package_dir)
    196 
    197         # Require __init__.py for all but the "root package"
    198         if package:
    199             init_py = os.path.join(package_dir, "__init__.py")
    200             if os.path.isfile(init_py):
    201                 return init_py
    202             else:
    203                 log.warn(("package init file '%s' not found " +
    204                           "(or not a regular file)"), init_py)
    205 
    206         # Either not in a package at all (__init__.py not expected), or
    207         # __init__.py doesn't exist -- so don't return the filename.
    208         return None
    209 
    210     def check_module(self, module, module_file):
    211         if not os.path.isfile(module_file):
    212             log.warn("file %s (for module %s) not found", module_file, module)
    213             return False
    214         else:
    215             return True
    216 
    217     def find_package_modules(self, package, package_dir):
    218         self.check_package(package, package_dir)
    219         module_files = glob(os.path.join(package_dir, "*.py"))
    220         modules = []
    221         setup_script = os.path.abspath(self.distribution.script_name)
    222 
    223         for f in module_files:
    224             abs_f = os.path.abspath(f)
    225             if abs_f != setup_script:
    226                 module = os.path.splitext(os.path.basename(f))[0]
    227                 modules.append((package, module, f))
    228             else:
    229                 self.debug_print("excluding %s" % setup_script)
    230         return modules
    231 
    232     def find_modules(self):
    233         """Finds individually-specified Python modules, ie. those listed by
    234         module name in 'self.py_modules'.  Returns a list of tuples (package,
    235         module_base, filename): 'package' is a tuple of the path through
    236         package-space to the module; 'module_base' is the bare (no
    237         packages, no dots) module name, and 'filename' is the path to the
    238         ".py" file (relative to the distribution root) that implements the
    239         module.
    240         """
    241         # Map package names to tuples of useful info about the package:
    242         #    (package_dir, checked)
    243         # package_dir - the directory where we'll find source files for
    244         #   this package
    245         # checked - true if we have checked that the package directory
    246         #   is valid (exists, contains __init__.py, ... ?)
    247         packages = {}
    248 
    249         # List of (package, module, filename) tuples to return
    250         modules = []
    251 
    252         # We treat modules-in-packages almost the same as toplevel modules,
    253         # just the "package" for a toplevel is empty (either an empty
    254         # string or empty list, depending on context).  Differences:
    255         #   - don't check for __init__.py in directory for empty package
    256         for module in self.py_modules:
    257             path = module.split('.')
    258             package = '.'.join(path[0:-1])
    259             module_base = path[-1]
    260 
    261             try:
    262                 (package_dir, checked) = packages[package]
    263             except KeyError:
    264                 package_dir = self.get_package_dir(package)
    265                 checked = 0
    266 
    267             if not checked:
    268                 init_py = self.check_package(package, package_dir)
    269                 packages[package] = (package_dir, 1)
    270                 if init_py:
    271                     modules.append((package, "__init__", init_py))
    272 
    273             # XXX perhaps we should also check for just .pyc files
    274             # (so greedy closed-source bastards can distribute Python
    275             # modules too)
    276             module_file = os.path.join(package_dir, module_base + ".py")
    277             if not self.check_module(module, module_file):
    278                 continue
    279 
    280             modules.append((package, module_base, module_file))
    281 
    282         return modules
    283 
    284     def find_all_modules(self):
    285         """Compute the list of all modules that will be built, whether
    286         they are specified one-module-at-a-time ('self.py_modules') or
    287         by whole packages ('self.packages').  Return a list of tuples
    288         (package, module, module_file), just like 'find_modules()' and
    289         'find_package_modules()' do."""
    290         modules = []
    291         if self.py_modules:
    292             modules.extend(self.find_modules())
    293         if self.packages:
    294             for package in self.packages:
    295                 package_dir = self.get_package_dir(package)
    296                 m = self.find_package_modules(package, package_dir)
    297                 modules.extend(m)
    298         return modules
    299 
    300     def get_source_files(self):
    301         return [module[-1] for module in self.find_all_modules()]
    302 
    303     def get_module_outfile(self, build_dir, package, module):
    304         outfile_path = [build_dir] + list(package) + [module + ".py"]
    305         return os.path.join(*outfile_path)
    306 
    307     def get_outputs(self, include_bytecode=1):
    308         modules = self.find_all_modules()
    309         outputs = []
    310         for (package, module, module_file) in modules:
    311             package = package.split('.')
    312             filename = self.get_module_outfile(self.build_lib, package, module)
    313             outputs.append(filename)
    314             if include_bytecode:
    315                 if self.compile:
    316                     outputs.append(importlib.util.cache_from_source(
    317                         filename, optimization=''))
    318                 if self.optimize > 0:
    319                     outputs.append(importlib.util.cache_from_source(
    320                         filename, optimization=self.optimize))
    321 
    322         outputs += [
    323             os.path.join(build_dir, filename)
    324             for package, src_dir, build_dir, filenames in self.data_files
    325             for filename in filenames
    326             ]
    327 
    328         return outputs
    329 
    330     def build_module(self, module, module_file, package):
    331         if isinstance(package, str):
    332             package = package.split('.')
    333         elif not isinstance(package, (list, tuple)):
    334             raise TypeError(
    335                   "'package' must be a string (dot-separated), list, or tuple")
    336 
    337         # Now put the module source file into the "build" area -- this is
    338         # easy, we just copy it somewhere under self.build_lib (the build
    339         # directory for Python source).
    340         outfile = self.get_module_outfile(self.build_lib, package, module)
    341         dir = os.path.dirname(outfile)
    342         self.mkpath(dir)
    343         return self.copy_file(module_file, outfile, preserve_mode=0)
    344 
    345     def build_modules(self):
    346         modules = self.find_modules()
    347         for (package, module, module_file) in modules:
    348             # Now "build" the module -- ie. copy the source file to
    349             # self.build_lib (the build directory for Python source).
    350             # (Actually, it gets copied to the directory for this package
    351             # under self.build_lib.)
    352             self.build_module(module, module_file, package)
    353 
    354     def build_packages(self):
    355         for package in self.packages:
    356             # Get list of (package, module, module_file) tuples based on
    357             # scanning the package directory.  'package' is only included
    358             # in the tuple so that 'find_modules()' and
    359             # 'find_package_tuples()' have a consistent interface; it's
    360             # ignored here (apart from a sanity check).  Also, 'module' is
    361             # the *unqualified* module name (ie. no dots, no package -- we
    362             # already know its package!), and 'module_file' is the path to
    363             # the .py file, relative to the current directory
    364             # (ie. including 'package_dir').
    365             package_dir = self.get_package_dir(package)
    366             modules = self.find_package_modules(package, package_dir)
    367 
    368             # Now loop over the modules we found, "building" each one (just
    369             # copy it to self.build_lib).
    370             for (package_, module, module_file) in modules:
    371                 assert package == package_
    372                 self.build_module(module, module_file, package)
    373 
    374     def byte_compile(self, files):
    375         if sys.dont_write_bytecode:
    376             self.warn('byte-compiling is disabled, skipping.')
    377             return
    378 
    379         from distutils.util import byte_compile
    380         prefix = self.build_lib
    381         if prefix[-1] != os.sep:
    382             prefix = prefix + os.sep
    383 
    384         # XXX this code is essentially the same as the 'byte_compile()
    385         # method of the "install_lib" command, except for the determination
    386         # of the 'prefix' string.  Hmmm.
    387         if self.compile:
    388             byte_compile(files, optimize=0,
    389                          force=self.force, prefix=prefix, dry_run=self.dry_run)
    390         if self.optimize > 0:
    391             byte_compile(files, optimize=self.optimize,
    392                          force=self.force, prefix=prefix, dry_run=self.dry_run)
    393 
    394 class build_py_2to3(build_py, Mixin2to3):
    395     def run(self):
    396         self.updated_files = []
    397 
    398         # Base class code
    399         if self.py_modules:
    400             self.build_modules()
    401         if self.packages:
    402             self.build_packages()
    403             self.build_package_data()
    404 
    405         # 2to3
    406         self.run_2to3(self.updated_files)
    407 
    408         # Remaining base class code
    409         self.byte_compile(self.get_outputs(include_bytecode=0))
    410 
    411     def build_module(self, module, module_file, package):
    412         res = build_py.build_module(self, module, module_file, package)
    413         if res[1]:
    414             # file was copied
    415             self.updated_files.append(res[0])
    416         return res
    417