1 """distutils.filelist 2 3 Provides the FileList class, used for poking about the filesystem 4 and building lists of files. 5 """ 6 7 __revision__ = "$Id$" 8 9 import os, re 10 import fnmatch 11 from distutils.util import convert_path 12 from distutils.errors import DistutilsTemplateError, DistutilsInternalError 13 from distutils import log 14 15 class FileList: 16 """A list of files built by on exploring the filesystem and filtered by 17 applying various patterns to what we find there. 18 19 Instance attributes: 20 dir 21 directory from which files will be taken -- only used if 22 'allfiles' not supplied to constructor 23 files 24 list of filenames currently being built/filtered/manipulated 25 allfiles 26 complete list of files under consideration (ie. without any 27 filtering applied) 28 """ 29 30 def __init__(self, warn=None, debug_print=None): 31 # ignore argument to FileList, but keep them for backwards 32 # compatibility 33 self.allfiles = None 34 self.files = [] 35 36 def set_allfiles(self, allfiles): 37 self.allfiles = allfiles 38 39 def findall(self, dir=os.curdir): 40 self.allfiles = findall(dir) 41 42 def debug_print(self, msg): 43 """Print 'msg' to stdout if the global DEBUG (taken from the 44 DISTUTILS_DEBUG environment variable) flag is true. 45 """ 46 from distutils.debug import DEBUG 47 if DEBUG: 48 print msg 49 50 # -- List-like methods --------------------------------------------- 51 52 def append(self, item): 53 self.files.append(item) 54 55 def extend(self, items): 56 self.files.extend(items) 57 58 def sort(self): 59 # Not a strict lexical sort! 60 sortable_files = map(os.path.split, self.files) 61 sortable_files.sort() 62 self.files = [] 63 for sort_tuple in sortable_files: 64 self.files.append(os.path.join(*sort_tuple)) 65 66 67 # -- Other miscellaneous utility methods --------------------------- 68 69 def remove_duplicates(self): 70 # Assumes list has been sorted! 71 for i in range(len(self.files) - 1, 0, -1): 72 if self.files[i] == self.files[i - 1]: 73 del self.files[i] 74 75 76 # -- "File template" methods --------------------------------------- 77 78 def _parse_template_line(self, line): 79 words = line.split() 80 action = words[0] 81 82 patterns = dir = dir_pattern = None 83 84 if action in ('include', 'exclude', 85 'global-include', 'global-exclude'): 86 if len(words) < 2: 87 raise DistutilsTemplateError, \ 88 "'%s' expects <pattern1> <pattern2> ..." % action 89 90 patterns = map(convert_path, words[1:]) 91 92 elif action in ('recursive-include', 'recursive-exclude'): 93 if len(words) < 3: 94 raise DistutilsTemplateError, \ 95 "'%s' expects <dir> <pattern1> <pattern2> ..." % action 96 97 dir = convert_path(words[1]) 98 patterns = map(convert_path, words[2:]) 99 100 elif action in ('graft', 'prune'): 101 if len(words) != 2: 102 raise DistutilsTemplateError, \ 103 "'%s' expects a single <dir_pattern>" % action 104 105 dir_pattern = convert_path(words[1]) 106 107 else: 108 raise DistutilsTemplateError, "unknown action '%s'" % action 109 110 return (action, patterns, dir, dir_pattern) 111 112 def process_template_line(self, line): 113 # Parse the line: split it up, make sure the right number of words 114 # is there, and return the relevant words. 'action' is always 115 # defined: it's the first word of the line. Which of the other 116 # three are defined depends on the action; it'll be either 117 # patterns, (dir and patterns), or (dir_pattern). 118 action, patterns, dir, dir_pattern = self._parse_template_line(line) 119 120 # OK, now we know that the action is valid and we have the 121 # right number of words on the line for that action -- so we 122 # can proceed with minimal error-checking. 123 if action == 'include': 124 self.debug_print("include " + ' '.join(patterns)) 125 for pattern in patterns: 126 if not self.include_pattern(pattern, anchor=1): 127 log.warn("warning: no files found matching '%s'", 128 pattern) 129 130 elif action == 'exclude': 131 self.debug_print("exclude " + ' '.join(patterns)) 132 for pattern in patterns: 133 if not self.exclude_pattern(pattern, anchor=1): 134 log.warn(("warning: no previously-included files " 135 "found matching '%s'"), pattern) 136 137 elif action == 'global-include': 138 self.debug_print("global-include " + ' '.join(patterns)) 139 for pattern in patterns: 140 if not self.include_pattern(pattern, anchor=0): 141 log.warn(("warning: no files found matching '%s' " + 142 "anywhere in distribution"), pattern) 143 144 elif action == 'global-exclude': 145 self.debug_print("global-exclude " + ' '.join(patterns)) 146 for pattern in patterns: 147 if not self.exclude_pattern(pattern, anchor=0): 148 log.warn(("warning: no previously-included files matching " 149 "'%s' found anywhere in distribution"), 150 pattern) 151 152 elif action == 'recursive-include': 153 self.debug_print("recursive-include %s %s" % 154 (dir, ' '.join(patterns))) 155 for pattern in patterns: 156 if not self.include_pattern(pattern, prefix=dir): 157 log.warn(("warning: no files found matching '%s' " + 158 "under directory '%s'"), 159 pattern, dir) 160 161 elif action == 'recursive-exclude': 162 self.debug_print("recursive-exclude %s %s" % 163 (dir, ' '.join(patterns))) 164 for pattern in patterns: 165 if not self.exclude_pattern(pattern, prefix=dir): 166 log.warn(("warning: no previously-included files matching " 167 "'%s' found under directory '%s'"), 168 pattern, dir) 169 170 elif action == 'graft': 171 self.debug_print("graft " + dir_pattern) 172 if not self.include_pattern(None, prefix=dir_pattern): 173 log.warn("warning: no directories found matching '%s'", 174 dir_pattern) 175 176 elif action == 'prune': 177 self.debug_print("prune " + dir_pattern) 178 if not self.exclude_pattern(None, prefix=dir_pattern): 179 log.warn(("no previously-included directories found " + 180 "matching '%s'"), dir_pattern) 181 else: 182 raise DistutilsInternalError, \ 183 "this cannot happen: invalid action '%s'" % action 184 185 # -- Filtering/selection methods ----------------------------------- 186 187 def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): 188 """Select strings (presumably filenames) from 'self.files' that 189 match 'pattern', a Unix-style wildcard (glob) pattern. 190 191 Patterns are not quite the same as implemented by the 'fnmatch' 192 module: '*' and '?' match non-special characters, where "special" 193 is platform-dependent: slash on Unix; colon, slash, and backslash on 194 DOS/Windows; and colon on Mac OS. 195 196 If 'anchor' is true (the default), then the pattern match is more 197 stringent: "*.py" will match "foo.py" but not "foo/bar.py". If 198 'anchor' is false, both of these will match. 199 200 If 'prefix' is supplied, then only filenames starting with 'prefix' 201 (itself a pattern) and ending with 'pattern', with anything in between 202 them, will match. 'anchor' is ignored in this case. 203 204 If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and 205 'pattern' is assumed to be either a string containing a regex or a 206 regex object -- no translation is done, the regex is just compiled 207 and used as-is. 208 209 Selected strings will be added to self.files. 210 211 Return 1 if files are found. 212 """ 213 # XXX docstring lying about what the special chars are? 214 files_found = 0 215 pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) 216 self.debug_print("include_pattern: applying regex r'%s'" % 217 pattern_re.pattern) 218 219 # delayed loading of allfiles list 220 if self.allfiles is None: 221 self.findall() 222 223 for name in self.allfiles: 224 if pattern_re.search(name): 225 self.debug_print(" adding " + name) 226 self.files.append(name) 227 files_found = 1 228 229 return files_found 230 231 232 def exclude_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): 233 """Remove strings (presumably filenames) from 'files' that match 234 'pattern'. 235 236 Other parameters are the same as for 'include_pattern()', above. 237 The list 'self.files' is modified in place. Return 1 if files are 238 found. 239 """ 240 files_found = 0 241 pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) 242 self.debug_print("exclude_pattern: applying regex r'%s'" % 243 pattern_re.pattern) 244 for i in range(len(self.files)-1, -1, -1): 245 if pattern_re.search(self.files[i]): 246 self.debug_print(" removing " + self.files[i]) 247 del self.files[i] 248 files_found = 1 249 250 return files_found 251 252 253 # ---------------------------------------------------------------------- 254 # Utility functions 255 256 def findall(dir = os.curdir): 257 """Find all files under 'dir' and return the list of full filenames 258 (relative to 'dir'). 259 """ 260 from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK 261 262 list = [] 263 stack = [dir] 264 pop = stack.pop 265 push = stack.append 266 267 while stack: 268 dir = pop() 269 names = os.listdir(dir) 270 271 for name in names: 272 if dir != os.curdir: # avoid the dreaded "./" syndrome 273 fullname = os.path.join(dir, name) 274 else: 275 fullname = name 276 277 # Avoid excess stat calls -- just one will do, thank you! 278 stat = os.stat(fullname) 279 mode = stat[ST_MODE] 280 if S_ISREG(mode): 281 list.append(fullname) 282 elif S_ISDIR(mode) and not S_ISLNK(mode): 283 push(fullname) 284 285 return list 286 287 288 def glob_to_re(pattern): 289 """Translate a shell-like glob pattern to a regular expression. 290 291 Return a string containing the regex. Differs from 292 'fnmatch.translate()' in that '*' does not match "special characters" 293 (which are platform-specific). 294 """ 295 pattern_re = fnmatch.translate(pattern) 296 297 # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which 298 # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, 299 # and by extension they shouldn't match such "special characters" under 300 # any OS. So change all non-escaped dots in the RE to match any 301 # character except the special characters (currently: just os.sep). 302 sep = os.sep 303 if os.sep == '\\': 304 # we're using a regex to manipulate a regex, so we need 305 # to escape the backslash twice 306 sep = r'\\\\' 307 escaped = r'\1[^%s]' % sep 308 pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', escaped, pattern_re) 309 return pattern_re 310 311 312 def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0): 313 """Translate a shell-like wildcard pattern to a compiled regular 314 expression. 315 316 Return the compiled regex. If 'is_regex' true, 317 then 'pattern' is directly compiled to a regex (if it's a string) 318 or just returned as-is (assumes it's a regex object). 319 """ 320 if is_regex: 321 if isinstance(pattern, str): 322 return re.compile(pattern) 323 else: 324 return pattern 325 326 if pattern: 327 pattern_re = glob_to_re(pattern) 328 else: 329 pattern_re = '' 330 331 if prefix is not None: 332 # ditch end of pattern character 333 empty_pattern = glob_to_re('') 334 prefix_re = glob_to_re(prefix)[:-len(empty_pattern)] 335 sep = os.sep 336 if os.sep == '\\': 337 sep = r'\\' 338 pattern_re = "^" + sep.join((prefix_re, ".*" + pattern_re)) 339 else: # no prefix -- respect anchor flag 340 if anchor: 341 pattern_re = "^" + pattern_re 342 343 return re.compile(pattern_re) 344
