1 """distutils.command.sdist 2 3 Implements the Distutils 'sdist' command (create a source distribution).""" 4 5 __revision__ = "$Id$" 6 7 import os 8 import string 9 import sys 10 from glob import glob 11 from warnings import warn 12 13 from distutils.core import Command 14 from distutils import dir_util, dep_util, file_util, archive_util 15 from distutils.text_file import TextFile 16 from distutils.errors import (DistutilsPlatformError, DistutilsOptionError, 17 DistutilsTemplateError) 18 from distutils.filelist import FileList 19 from distutils import log 20 from distutils.util import convert_path 21 22 def show_formats(): 23 """Print all possible values for the 'formats' option (used by 24 the "--help-formats" command-line option). 25 """ 26 from distutils.fancy_getopt import FancyGetopt 27 from distutils.archive_util import ARCHIVE_FORMATS 28 formats = [] 29 for format in ARCHIVE_FORMATS.keys(): 30 formats.append(("formats=" + format, None, 31 ARCHIVE_FORMATS[format][2])) 32 formats.sort() 33 FancyGetopt(formats).print_help( 34 "List of available source distribution formats:") 35 36 class sdist(Command): 37 38 description = "create a source distribution (tarball, zip file, etc.)" 39 40 def checking_metadata(self): 41 """Callable used for the check sub-command. 42 43 Placed here so user_options can view it""" 44 return self.metadata_check 45 46 user_options = [ 47 ('template=', 't', 48 "name of manifest template file [default: MANIFEST.in]"), 49 ('manifest=', 'm', 50 "name of manifest file [default: MANIFEST]"), 51 ('use-defaults', None, 52 "include the default file set in the manifest " 53 "[default; disable with --no-defaults]"), 54 ('no-defaults', None, 55 "don't include the default file set"), 56 ('prune', None, 57 "specifically exclude files/directories that should not be " 58 "distributed (build tree, RCS/CVS dirs, etc.) " 59 "[default; disable with --no-prune]"), 60 ('no-prune', None, 61 "don't automatically exclude anything"), 62 ('manifest-only', 'o', 63 "just regenerate the manifest and then stop " 64 "(implies --force-manifest)"), 65 ('force-manifest', 'f', 66 "forcibly regenerate the manifest and carry on as usual. " 67 "Deprecated: now the manifest is always regenerated."), 68 ('formats=', None, 69 "formats for source distribution (comma-separated list)"), 70 ('keep-temp', 'k', 71 "keep the distribution tree around after creating " + 72 "archive file(s)"), 73 ('dist-dir=', 'd', 74 "directory to put the source distribution archive(s) in " 75 "[default: dist]"), 76 ('metadata-check', None, 77 "Ensure that all required elements of meta-data " 78 "are supplied. Warn if any missing. [default]"), 79 ('owner=', 'u', 80 "Owner name used when creating a tar file [default: current user]"), 81 ('group=', 'g', 82 "Group name used when creating a tar file [default: current group]"), 83 ] 84 85 boolean_options = ['use-defaults', 'prune', 86 'manifest-only', 'force-manifest', 87 'keep-temp', 'metadata-check'] 88 89 help_options = [ 90 ('help-formats', None, 91 "list available distribution formats", show_formats), 92 ] 93 94 negative_opt = {'no-defaults': 'use-defaults', 95 'no-prune': 'prune' } 96 97 default_format = {'posix': 'gztar', 98 'nt': 'zip' } 99 100 sub_commands = [('check', checking_metadata)] 101 102 def initialize_options(self): 103 # 'template' and 'manifest' are, respectively, the names of 104 # the manifest template and manifest file. 105 self.template = None 106 self.manifest = None 107 108 # 'use_defaults': if true, we will include the default file set 109 # in the manifest 110 self.use_defaults = 1 111 self.prune = 1 112 113 self.manifest_only = 0 114 self.force_manifest = 0 115 116 self.formats = None 117 self.keep_temp = 0 118 self.dist_dir = None 119 120 self.archive_files = None 121 self.metadata_check = 1 122 self.owner = None 123 self.group = None 124 125 def finalize_options(self): 126 if self.manifest is None: 127 self.manifest = "MANIFEST" 128 if self.template is None: 129 self.template = "MANIFEST.in" 130 131 self.ensure_string_list('formats') 132 if self.formats is None: 133 try: 134 self.formats = [self.default_format[os.name]] 135 except KeyError: 136 raise DistutilsPlatformError, \ 137 "don't know how to create source distributions " + \ 138 "on platform %s" % os.name 139 140 bad_format = archive_util.check_archive_formats(self.formats) 141 if bad_format: 142 raise DistutilsOptionError, \ 143 "unknown archive format '%s'" % bad_format 144 145 if self.dist_dir is None: 146 self.dist_dir = "dist" 147 148 def run(self): 149 # 'filelist' contains the list of files that will make up the 150 # manifest 151 self.filelist = FileList() 152 153 # Run sub commands 154 for cmd_name in self.get_sub_commands(): 155 self.run_command(cmd_name) 156 157 # Do whatever it takes to get the list of files to process 158 # (process the manifest template, read an existing manifest, 159 # whatever). File list is accumulated in 'self.filelist'. 160 self.get_file_list() 161 162 # If user just wanted us to regenerate the manifest, stop now. 163 if self.manifest_only: 164 return 165 166 # Otherwise, go ahead and create the source distribution tarball, 167 # or zipfile, or whatever. 168 self.make_distribution() 169 170 def check_metadata(self): 171 """Deprecated API.""" 172 warn("distutils.command.sdist.check_metadata is deprecated, \ 173 use the check command instead", PendingDeprecationWarning) 174 check = self.distribution.get_command_obj('check') 175 check.ensure_finalized() 176 check.run() 177 178 def get_file_list(self): 179 """Figure out the list of files to include in the source 180 distribution, and put it in 'self.filelist'. This might involve 181 reading the manifest template (and writing the manifest), or just 182 reading the manifest, or just using the default file set -- it all 183 depends on the user's options. 184 """ 185 # new behavior when using a template: 186 # the file list is recalculated everytime because 187 # even if MANIFEST.in or setup.py are not changed 188 # the user might have added some files in the tree that 189 # need to be included. 190 # 191 # This makes --force the default and only behavior with templates. 192 template_exists = os.path.isfile(self.template) 193 if not template_exists and self._manifest_is_not_generated(): 194 self.read_manifest() 195 self.filelist.sort() 196 self.filelist.remove_duplicates() 197 return 198 199 if not template_exists: 200 self.warn(("manifest template '%s' does not exist " + 201 "(using default file list)") % 202 self.template) 203 self.filelist.findall() 204 205 if self.use_defaults: 206 self.add_defaults() 207 208 if template_exists: 209 self.read_template() 210 211 if self.prune: 212 self.prune_file_list() 213 214 self.filelist.sort() 215 self.filelist.remove_duplicates() 216 self.write_manifest() 217 218 def add_defaults(self): 219 """Add all the default files to self.filelist: 220 - README or README.txt 221 - setup.py 222 - test/test*.py 223 - all pure Python modules mentioned in setup script 224 - all files pointed by package_data (build_py) 225 - all files defined in data_files. 226 - all files defined as scripts. 227 - all C sources listed as part of extensions or C libraries 228 in the setup script (doesn't catch C headers!) 229 Warns if (README or README.txt) or setup.py are missing; everything 230 else is optional. 231 """ 232 233 standards = [('README', 'README.txt'), self.distribution.script_name] 234 for fn in standards: 235 if isinstance(fn, tuple): 236 alts = fn 237 got_it = 0 238 for fn in alts: 239 if os.path.exists(fn): 240 got_it = 1 241 self.filelist.append(fn) 242 break 243 244 if not got_it: 245 self.warn("standard file not found: should have one of " + 246 string.join(alts, ', ')) 247 else: 248 if os.path.exists(fn): 249 self.filelist.append(fn) 250 else: 251 self.warn("standard file '%s' not found" % fn) 252 253 optional = ['test/test*.py', 'setup.cfg'] 254 for pattern in optional: 255 files = filter(os.path.isfile, glob(pattern)) 256 if files: 257 self.filelist.extend(files) 258 259 # build_py is used to get: 260 # - python modules 261 # - files defined in package_data 262 build_py = self.get_finalized_command('build_py') 263 264 # getting python files 265 if self.distribution.has_pure_modules(): 266 self.filelist.extend(build_py.get_source_files()) 267 268 # getting package_data files 269 # (computed in build_py.data_files by build_py.finalize_options) 270 for pkg, src_dir, build_dir, filenames in build_py.data_files: 271 for filename in filenames: 272 self.filelist.append(os.path.join(src_dir, filename)) 273 274 # getting distribution.data_files 275 if self.distribution.has_data_files(): 276 for item in self.distribution.data_files: 277 if isinstance(item, str): # plain file 278 item = convert_path(item) 279 if os.path.isfile(item): 280 self.filelist.append(item) 281 else: # a (dirname, filenames) tuple 282 dirname, filenames = item 283 for f in filenames: 284 f = convert_path(f) 285 if os.path.isfile(f): 286 self.filelist.append(f) 287 288 if self.distribution.has_ext_modules(): 289 build_ext = self.get_finalized_command('build_ext') 290 self.filelist.extend(build_ext.get_source_files()) 291 292 if self.distribution.has_c_libraries(): 293 build_clib = self.get_finalized_command('build_clib') 294 self.filelist.extend(build_clib.get_source_files()) 295 296 if self.distribution.has_scripts(): 297 build_scripts = self.get_finalized_command('build_scripts') 298 self.filelist.extend(build_scripts.get_source_files()) 299 300 def read_template(self): 301 """Read and parse manifest template file named by self.template. 302 303 (usually "MANIFEST.in") The parsing and processing is done by 304 'self.filelist', which updates itself accordingly. 305 """ 306 log.info("reading manifest template '%s'", self.template) 307 template = TextFile(self.template, 308 strip_comments=1, 309 skip_blanks=1, 310 join_lines=1, 311 lstrip_ws=1, 312 rstrip_ws=1, 313 collapse_join=1) 314 315 try: 316 while 1: 317 line = template.readline() 318 if line is None: # end of file 319 break 320 321 try: 322 self.filelist.process_template_line(line) 323 # the call above can raise a DistutilsTemplateError for 324 # malformed lines, or a ValueError from the lower-level 325 # convert_path function 326 except (DistutilsTemplateError, ValueError) as msg: 327 self.warn("%s, line %d: %s" % (template.filename, 328 template.current_line, 329 msg)) 330 finally: 331 template.close() 332 333 def prune_file_list(self): 334 """Prune off branches that might slip into the file list as created 335 by 'read_template()', but really don't belong there: 336 * the build tree (typically "build") 337 * the release tree itself (only an issue if we ran "sdist" 338 previously with --keep-temp, or it aborted) 339 * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories 340 """ 341 build = self.get_finalized_command('build') 342 base_dir = self.distribution.get_fullname() 343 344 self.filelist.exclude_pattern(None, prefix=build.build_base) 345 self.filelist.exclude_pattern(None, prefix=base_dir) 346 347 # pruning out vcs directories 348 # both separators are used under win32 349 if sys.platform == 'win32': 350 seps = r'/|\\' 351 else: 352 seps = '/' 353 354 vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr', 355 '_darcs'] 356 vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps) 357 self.filelist.exclude_pattern(vcs_ptrn, is_regex=1) 358 359 def write_manifest(self): 360 """Write the file list in 'self.filelist' (presumably as filled in 361 by 'add_defaults()' and 'read_template()') to the manifest file 362 named by 'self.manifest'. 363 """ 364 if self._manifest_is_not_generated(): 365 log.info("not writing to manually maintained " 366 "manifest file '%s'" % self.manifest) 367 return 368 369 content = self.filelist.files[:] 370 content.insert(0, '# file GENERATED by distutils, do NOT edit') 371 self.execute(file_util.write_file, (self.manifest, content), 372 "writing manifest file '%s'" % self.manifest) 373 374 def _manifest_is_not_generated(self): 375 # check for special comment used in 2.7.1 and higher 376 if not os.path.isfile(self.manifest): 377 return False 378 379 fp = open(self.manifest, 'rU') 380 try: 381 first_line = fp.readline() 382 finally: 383 fp.close() 384 return first_line != '# file GENERATED by distutils, do NOT edit\n' 385 386 def read_manifest(self): 387 """Read the manifest file (named by 'self.manifest') and use it to 388 fill in 'self.filelist', the list of files to include in the source 389 distribution. 390 """ 391 log.info("reading manifest file '%s'", self.manifest) 392 manifest = open(self.manifest) 393 for line in manifest: 394 # ignore comments and blank lines 395 line = line.strip() 396 if line.startswith('#') or not line: 397 continue 398 self.filelist.append(line) 399 manifest.close() 400 401 def make_release_tree(self, base_dir, files): 402 """Create the directory tree that will become the source 403 distribution archive. All directories implied by the filenames in 404 'files' are created under 'base_dir', and then we hard link or copy 405 (if hard linking is unavailable) those files into place. 406 Essentially, this duplicates the developer's source tree, but in a 407 directory named after the distribution, containing only the files 408 to be distributed. 409 """ 410 # Create all the directories under 'base_dir' necessary to 411 # put 'files' there; the 'mkpath()' is just so we don't die 412 # if the manifest happens to be empty. 413 self.mkpath(base_dir) 414 dir_util.create_tree(base_dir, files, dry_run=self.dry_run) 415 416 # And walk over the list of files, either making a hard link (if 417 # os.link exists) to each one that doesn't already exist in its 418 # corresponding location under 'base_dir', or copying each file 419 # that's out-of-date in 'base_dir'. (Usually, all files will be 420 # out-of-date, because by default we blow away 'base_dir' when 421 # we're done making the distribution archives.) 422 423 if hasattr(os, 'link'): # can make hard links on this system 424 link = 'hard' 425 msg = "making hard links in %s..." % base_dir 426 else: # nope, have to copy 427 link = None 428 msg = "copying files to %s..." % base_dir 429 430 if not files: 431 log.warn("no files to distribute -- empty manifest?") 432 else: 433 log.info(msg) 434 for file in files: 435 if not os.path.isfile(file): 436 log.warn("'%s' not a regular file -- skipping" % file) 437 else: 438 dest = os.path.join(base_dir, file) 439 self.copy_file(file, dest, link=link) 440 441 self.distribution.metadata.write_pkg_info(base_dir) 442 443 def make_distribution(self): 444 """Create the source distribution(s). First, we create the release 445 tree with 'make_release_tree()'; then, we create all required 446 archive files (according to 'self.formats') from the release tree. 447 Finally, we clean up by blowing away the release tree (unless 448 'self.keep_temp' is true). The list of archive files created is 449 stored so it can be retrieved later by 'get_archive_files()'. 450 """ 451 # Don't warn about missing meta-data here -- should be (and is!) 452 # done elsewhere. 453 base_dir = self.distribution.get_fullname() 454 base_name = os.path.join(self.dist_dir, base_dir) 455 456 self.make_release_tree(base_dir, self.filelist.files) 457 archive_files = [] # remember names of files we create 458 # tar archive must be created last to avoid overwrite and remove 459 if 'tar' in self.formats: 460 self.formats.append(self.formats.pop(self.formats.index('tar'))) 461 462 for fmt in self.formats: 463 file = self.make_archive(base_name, fmt, base_dir=base_dir, 464 owner=self.owner, group=self.group) 465 archive_files.append(file) 466 self.distribution.dist_files.append(('sdist', '', file)) 467 468 self.archive_files = archive_files 469 470 if not self.keep_temp: 471 dir_util.remove_tree(base_dir, dry_run=self.dry_run) 472 473 def get_archive_files(self): 474 """Return the list of archive files created when the command 475 was run, or None if the command hasn't run yet. 476 """ 477 return self.archive_files 478
