1 """distutils.command.sdist 2 3 Implements the Distutils 'sdist' command (create a source distribution).""" 4 5 import os 6 import sys 7 from types import * 8 from glob import glob 9 from warnings import warn 10 11 from distutils.core import Command 12 from distutils import dir_util, dep_util, file_util, archive_util 13 from distutils.text_file import TextFile 14 from distutils.errors import * 15 from distutils.filelist import FileList 16 from distutils import log 17 from distutils.util import convert_path 18 19 def show_formats(): 20 """Print all possible values for the 'formats' option (used by 21 the "--help-formats" command-line option). 22 """ 23 from distutils.fancy_getopt import FancyGetopt 24 from distutils.archive_util import ARCHIVE_FORMATS 25 formats = [] 26 for format in ARCHIVE_FORMATS.keys(): 27 formats.append(("formats=" + format, None, 28 ARCHIVE_FORMATS[format][2])) 29 formats.sort() 30 FancyGetopt(formats).print_help( 31 "List of available source distribution formats:") 32 33 class sdist(Command): 34 35 description = "create a source distribution (tarball, zip file, etc.)" 36 37 def checking_metadata(self): 38 """Callable used for the check sub-command. 39 40 Placed here so user_options can view it""" 41 return self.metadata_check 42 43 user_options = [ 44 ('template=', 't', 45 "name of manifest template file [default: MANIFEST.in]"), 46 ('manifest=', 'm', 47 "name of manifest file [default: MANIFEST]"), 48 ('use-defaults', None, 49 "include the default file set in the manifest " 50 "[default; disable with --no-defaults]"), 51 ('no-defaults', None, 52 "don't include the default file set"), 53 ('prune', None, 54 "specifically exclude files/directories that should not be " 55 "distributed (build tree, RCS/CVS dirs, etc.) " 56 "[default; disable with --no-prune]"), 57 ('no-prune', None, 58 "don't automatically exclude anything"), 59 ('manifest-only', 'o', 60 "just regenerate the manifest and then stop " 61 "(implies --force-manifest)"), 62 ('force-manifest', 'f', 63 "forcibly regenerate the manifest and carry on as usual. " 64 "Deprecated: now the manifest is always regenerated."), 65 ('formats=', None, 66 "formats for source distribution (comma-separated list)"), 67 ('keep-temp', 'k', 68 "keep the distribution tree around after creating " + 69 "archive file(s)"), 70 ('dist-dir=', 'd', 71 "directory to put the source distribution archive(s) in " 72 "[default: dist]"), 73 ('metadata-check', None, 74 "Ensure that all required elements of meta-data " 75 "are supplied. Warn if any missing. [default]"), 76 ('owner=', 'u', 77 "Owner name used when creating a tar file [default: current user]"), 78 ('group=', 'g', 79 "Group name used when creating a tar file [default: current group]"), 80 ] 81 82 boolean_options = ['use-defaults', 'prune', 83 'manifest-only', 'force-manifest', 84 'keep-temp', 'metadata-check'] 85 86 help_options = [ 87 ('help-formats', None, 88 "list available distribution formats", show_formats), 89 ] 90 91 negative_opt = {'no-defaults': 'use-defaults', 92 'no-prune': 'prune' } 93 94 sub_commands = [('check', checking_metadata)] 95 96 def initialize_options(self): 97 # 'template' and 'manifest' are, respectively, the names of 98 # the manifest template and manifest file. 99 self.template = None 100 self.manifest = None 101 102 # 'use_defaults': if true, we will include the default file set 103 # in the manifest 104 self.use_defaults = 1 105 self.prune = 1 106 107 self.manifest_only = 0 108 self.force_manifest = 0 109 110 self.formats = ['gztar'] 111 self.keep_temp = 0 112 self.dist_dir = None 113 114 self.archive_files = None 115 self.metadata_check = 1 116 self.owner = None 117 self.group = None 118 119 def finalize_options(self): 120 if self.manifest is None: 121 self.manifest = "MANIFEST" 122 if self.template is None: 123 self.template = "MANIFEST.in" 124 125 self.ensure_string_list('formats') 126 127 bad_format = archive_util.check_archive_formats(self.formats) 128 if bad_format: 129 raise DistutilsOptionError( 130 "unknown archive format '%s'" % bad_format) 131 132 if self.dist_dir is None: 133 self.dist_dir = "dist" 134 135 def run(self): 136 # 'filelist' contains the list of files that will make up the 137 # manifest 138 self.filelist = FileList() 139 140 # Run sub commands 141 for cmd_name in self.get_sub_commands(): 142 self.run_command(cmd_name) 143 144 # Do whatever it takes to get the list of files to process 145 # (process the manifest template, read an existing manifest, 146 # whatever). File list is accumulated in 'self.filelist'. 147 self.get_file_list() 148 149 # If user just wanted us to regenerate the manifest, stop now. 150 if self.manifest_only: 151 return 152 153 # Otherwise, go ahead and create the source distribution tarball, 154 # or zipfile, or whatever. 155 self.make_distribution() 156 157 def check_metadata(self): 158 """Deprecated API.""" 159 warn("distutils.command.sdist.check_metadata is deprecated, \ 160 use the check command instead", PendingDeprecationWarning) 161 check = self.distribution.get_command_obj('check') 162 check.ensure_finalized() 163 check.run() 164 165 def get_file_list(self): 166 """Figure out the list of files to include in the source 167 distribution, and put it in 'self.filelist'. This might involve 168 reading the manifest template (and writing the manifest), or just 169 reading the manifest, or just using the default file set -- it all 170 depends on the user's options. 171 """ 172 # new behavior when using a template: 173 # the file list is recalculated every time because 174 # even if MANIFEST.in or setup.py are not changed 175 # the user might have added some files in the tree that 176 # need to be included. 177 # 178 # This makes --force the default and only behavior with templates. 179 template_exists = os.path.isfile(self.template) 180 if not template_exists and self._manifest_is_not_generated(): 181 self.read_manifest() 182 self.filelist.sort() 183 self.filelist.remove_duplicates() 184 return 185 186 if not template_exists: 187 self.warn(("manifest template '%s' does not exist " + 188 "(using default file list)") % 189 self.template) 190 self.filelist.findall() 191 192 if self.use_defaults: 193 self.add_defaults() 194 195 if template_exists: 196 self.read_template() 197 198 if self.prune: 199 self.prune_file_list() 200 201 self.filelist.sort() 202 self.filelist.remove_duplicates() 203 self.write_manifest() 204 205 def add_defaults(self): 206 """Add all the default files to self.filelist: 207 - README or README.txt 208 - setup.py 209 - test/test*.py 210 - all pure Python modules mentioned in setup script 211 - all files pointed by package_data (build_py) 212 - all files defined in data_files. 213 - all files defined as scripts. 214 - all C sources listed as part of extensions or C libraries 215 in the setup script (doesn't catch C headers!) 216 Warns if (README or README.txt) or setup.py are missing; everything 217 else is optional. 218 """ 219 standards = [('README', 'README.txt'), self.distribution.script_name] 220 for fn in standards: 221 if isinstance(fn, tuple): 222 alts = fn 223 got_it = False 224 for fn in alts: 225 if os.path.exists(fn): 226 got_it = True 227 self.filelist.append(fn) 228 break 229 230 if not got_it: 231 self.warn("standard file not found: should have one of " + 232 ', '.join(alts)) 233 else: 234 if os.path.exists(fn): 235 self.filelist.append(fn) 236 else: 237 self.warn("standard file '%s' not found" % fn) 238 239 optional = ['test/test*.py', 'setup.cfg'] 240 for pattern in optional: 241 files = filter(os.path.isfile, glob(pattern)) 242 self.filelist.extend(files) 243 244 # build_py is used to get: 245 # - python modules 246 # - files defined in package_data 247 build_py = self.get_finalized_command('build_py') 248 249 # getting python files 250 if self.distribution.has_pure_modules(): 251 self.filelist.extend(build_py.get_source_files()) 252 253 # getting package_data files 254 # (computed in build_py.data_files by build_py.finalize_options) 255 for pkg, src_dir, build_dir, filenames in build_py.data_files: 256 for filename in filenames: 257 self.filelist.append(os.path.join(src_dir, filename)) 258 259 # getting distribution.data_files 260 if self.distribution.has_data_files(): 261 for item in self.distribution.data_files: 262 if isinstance(item, str): # plain file 263 item = convert_path(item) 264 if os.path.isfile(item): 265 self.filelist.append(item) 266 else: # a (dirname, filenames) tuple 267 dirname, filenames = item 268 for f in filenames: 269 f = convert_path(f) 270 if os.path.isfile(f): 271 self.filelist.append(f) 272 273 if self.distribution.has_ext_modules(): 274 build_ext = self.get_finalized_command('build_ext') 275 self.filelist.extend(build_ext.get_source_files()) 276 277 if self.distribution.has_c_libraries(): 278 build_clib = self.get_finalized_command('build_clib') 279 self.filelist.extend(build_clib.get_source_files()) 280 281 if self.distribution.has_scripts(): 282 build_scripts = self.get_finalized_command('build_scripts') 283 self.filelist.extend(build_scripts.get_source_files()) 284 285 def read_template(self): 286 """Read and parse manifest template file named by self.template. 287 288 (usually "MANIFEST.in") The parsing and processing is done by 289 'self.filelist', which updates itself accordingly. 290 """ 291 log.info("reading manifest template '%s'", self.template) 292 template = TextFile(self.template, strip_comments=1, skip_blanks=1, 293 join_lines=1, lstrip_ws=1, rstrip_ws=1, 294 collapse_join=1) 295 296 try: 297 while True: 298 line = template.readline() 299 if line is None: # end of file 300 break 301 302 try: 303 self.filelist.process_template_line(line) 304 # the call above can raise a DistutilsTemplateError for 305 # malformed lines, or a ValueError from the lower-level 306 # convert_path function 307 except (DistutilsTemplateError, ValueError) as msg: 308 self.warn("%s, line %d: %s" % (template.filename, 309 template.current_line, 310 msg)) 311 finally: 312 template.close() 313 314 def prune_file_list(self): 315 """Prune off branches that might slip into the file list as created 316 by 'read_template()', but really don't belong there: 317 * the build tree (typically "build") 318 * the release tree itself (only an issue if we ran "sdist" 319 previously with --keep-temp, or it aborted) 320 * any RCS, CVS, .svn, .hg, .git, .bzr, _darcs directories 321 """ 322 build = self.get_finalized_command('build') 323 base_dir = self.distribution.get_fullname() 324 325 self.filelist.exclude_pattern(None, prefix=build.build_base) 326 self.filelist.exclude_pattern(None, prefix=base_dir) 327 328 if sys.platform == 'win32': 329 seps = r'/|\\' 330 else: 331 seps = '/' 332 333 vcs_dirs = ['RCS', 'CVS', r'\.svn', r'\.hg', r'\.git', r'\.bzr', 334 '_darcs'] 335 vcs_ptrn = r'(^|%s)(%s)(%s).*' % (seps, '|'.join(vcs_dirs), seps) 336 self.filelist.exclude_pattern(vcs_ptrn, is_regex=1) 337 338 def write_manifest(self): 339 """Write the file list in 'self.filelist' (presumably as filled in 340 by 'add_defaults()' and 'read_template()') to the manifest file 341 named by 'self.manifest'. 342 """ 343 if self._manifest_is_not_generated(): 344 log.info("not writing to manually maintained " 345 "manifest file '%s'" % self.manifest) 346 return 347 348 content = self.filelist.files[:] 349 content.insert(0, '# file GENERATED by distutils, do NOT edit') 350 self.execute(file_util.write_file, (self.manifest, content), 351 "writing manifest file '%s'" % self.manifest) 352 353 def _manifest_is_not_generated(self): 354 # check for special comment used in 3.1.3 and higher 355 if not os.path.isfile(self.manifest): 356 return False 357 358 fp = open(self.manifest) 359 try: 360 first_line = fp.readline() 361 finally: 362 fp.close() 363 return first_line != '# file GENERATED by distutils, do NOT edit\n' 364 365 def read_manifest(self): 366 """Read the manifest file (named by 'self.manifest') and use it to 367 fill in 'self.filelist', the list of files to include in the source 368 distribution. 369 """ 370 log.info("reading manifest file '%s'", self.manifest) 371 manifest = open(self.manifest) 372 for line in manifest: 373 # ignore comments and blank lines 374 line = line.strip() 375 if line.startswith('#') or not line: 376 continue 377 self.filelist.append(line) 378 manifest.close() 379 380 def make_release_tree(self, base_dir, files): 381 """Create the directory tree that will become the source 382 distribution archive. All directories implied by the filenames in 383 'files' are created under 'base_dir', and then we hard link or copy 384 (if hard linking is unavailable) those files into place. 385 Essentially, this duplicates the developer's source tree, but in a 386 directory named after the distribution, containing only the files 387 to be distributed. 388 """ 389 # Create all the directories under 'base_dir' necessary to 390 # put 'files' there; the 'mkpath()' is just so we don't die 391 # if the manifest happens to be empty. 392 self.mkpath(base_dir) 393 dir_util.create_tree(base_dir, files, dry_run=self.dry_run) 394 395 # And walk over the list of files, either making a hard link (if 396 # os.link exists) to each one that doesn't already exist in its 397 # corresponding location under 'base_dir', or copying each file 398 # that's out-of-date in 'base_dir'. (Usually, all files will be 399 # out-of-date, because by default we blow away 'base_dir' when 400 # we're done making the distribution archives.) 401 402 if hasattr(os, 'link'): # can make hard links on this system 403 link = 'hard' 404 msg = "making hard links in %s..." % base_dir 405 else: # nope, have to copy 406 link = None 407 msg = "copying files to %s..." % base_dir 408 409 if not files: 410 log.warn("no files to distribute -- empty manifest?") 411 else: 412 log.info(msg) 413 for file in files: 414 if not os.path.isfile(file): 415 log.warn("'%s' not a regular file -- skipping", file) 416 else: 417 dest = os.path.join(base_dir, file) 418 self.copy_file(file, dest, link=link) 419 420 self.distribution.metadata.write_pkg_info(base_dir) 421 422 def make_distribution(self): 423 """Create the source distribution(s). First, we create the release 424 tree with 'make_release_tree()'; then, we create all required 425 archive files (according to 'self.formats') from the release tree. 426 Finally, we clean up by blowing away the release tree (unless 427 'self.keep_temp' is true). The list of archive files created is 428 stored so it can be retrieved later by 'get_archive_files()'. 429 """ 430 # Don't warn about missing meta-data here -- should be (and is!) 431 # done elsewhere. 432 base_dir = self.distribution.get_fullname() 433 base_name = os.path.join(self.dist_dir, base_dir) 434 435 self.make_release_tree(base_dir, self.filelist.files) 436 archive_files = [] # remember names of files we create 437 # tar archive must be created last to avoid overwrite and remove 438 if 'tar' in self.formats: 439 self.formats.append(self.formats.pop(self.formats.index('tar'))) 440 441 for fmt in self.formats: 442 file = self.make_archive(base_name, fmt, base_dir=base_dir, 443 owner=self.owner, group=self.group) 444 archive_files.append(file) 445 self.distribution.dist_files.append(('sdist', '', file)) 446 447 self.archive_files = archive_files 448 449 if not self.keep_temp: 450 dir_util.remove_tree(base_dir, dry_run=self.dry_run) 451 452 def get_archive_files(self): 453 """Return the list of archive files created when the command 454 was run, or None if the command hasn't run yet. 455 """ 456 return self.archive_files 457