1 """distutils.filelist 2 3 Provides the FileList class, used for poking about the filesystem 4 and building lists of files. 5 """ 6 7 __revision__ = "$Id$" 8 9 import os, re 10 import fnmatch 11 from distutils.util import convert_path 12 from distutils.errors import DistutilsTemplateError, DistutilsInternalError 13 from distutils import log 14 15 class FileList: 16 """A list of files built by on exploring the filesystem and filtered by 17 applying various patterns to what we find there. 18 19 Instance attributes: 20 dir 21 directory from which files will be taken -- only used if 22 'allfiles' not supplied to constructor 23 files 24 list of filenames currently being built/filtered/manipulated 25 allfiles 26 complete list of files under consideration (ie. without any 27 filtering applied) 28 """ 29 30 def __init__(self, warn=None, debug_print=None): 31 # ignore argument to FileList, but keep them for backwards 32 # compatibility 33 self.allfiles = None 34 self.files = [] 35 36 def set_allfiles(self, allfiles): 37 self.allfiles = allfiles 38 39 def findall(self, dir=os.curdir): 40 self.allfiles = findall(dir) 41 42 def debug_print(self, msg): 43 """Print 'msg' to stdout if the global DEBUG (taken from the 44 DISTUTILS_DEBUG environment variable) flag is true. 45 """ 46 from distutils.debug import DEBUG 47 if DEBUG: 48 print msg 49 50 # -- List-like methods --------------------------------------------- 51 52 def append(self, item): 53 self.files.append(item) 54 55 def extend(self, items): 56 self.files.extend(items) 57 58 def sort(self): 59 # Not a strict lexical sort! 60 sortable_files = map(os.path.split, self.files) 61 sortable_files.sort() 62 self.files = [] 63 for sort_tuple in sortable_files: 64 self.files.append(os.path.join(*sort_tuple)) 65 66 67 # -- Other miscellaneous utility methods --------------------------- 68 69 def remove_duplicates(self): 70 # Assumes list has been sorted! 71 for i in range(len(self.files) - 1, 0, -1): 72 if self.files[i] == self.files[i - 1]: 73 del self.files[i] 74 75 76 # -- "File template" methods --------------------------------------- 77 78 def _parse_template_line(self, line): 79 words = line.split() 80 action = words[0] 81 82 patterns = dir = dir_pattern = None 83 84 if action in ('include', 'exclude', 85 'global-include', 'global-exclude'): 86 if len(words) < 2: 87 raise DistutilsTemplateError, \ 88 "'%s' expects <pattern1> <pattern2> ..." % action 89 90 patterns = map(convert_path, words[1:]) 91 92 elif action in ('recursive-include', 'recursive-exclude'): 93 if len(words) < 3: 94 raise DistutilsTemplateError, \ 95 "'%s' expects <dir> <pattern1> <pattern2> ..." % action 96 97 dir = convert_path(words[1]) 98 patterns = map(convert_path, words[2:]) 99 100 elif action in ('graft', 'prune'): 101 if len(words) != 2: 102 raise DistutilsTemplateError, \ 103 "'%s' expects a single <dir_pattern>" % action 104 105 dir_pattern = convert_path(words[1]) 106 107 else: 108 raise DistutilsTemplateError, "unknown action '%s'" % action 109 110 return (action, patterns, dir, dir_pattern) 111 112 def process_template_line(self, line): 113 # Parse the line: split it up, make sure the right number of words 114 # is there, and return the relevant words. 'action' is always 115 # defined: it's the first word of the line. Which of the other 116 # three are defined depends on the action; it'll be either 117 # patterns, (dir and patterns), or (dir_pattern). 118 action, patterns, dir, dir_pattern = self._parse_template_line(line) 119 120 # OK, now we know that the action is valid and we have the 121 # right number of words on the line for that action -- so we 122 # can proceed with minimal error-checking. 123 if action == 'include': 124 self.debug_print("include " + ' '.join(patterns)) 125 for pattern in patterns: 126 if not self.include_pattern(pattern, anchor=1): 127 log.warn("warning: no files found matching '%s'", 128 pattern) 129 130 elif action == 'exclude': 131 self.debug_print("exclude " + ' '.join(patterns)) 132 for pattern in patterns: 133 if not self.exclude_pattern(pattern, anchor=1): 134 log.warn(("warning: no previously-included files " 135 "found matching '%s'"), pattern) 136 137 elif action == 'global-include': 138 self.debug_print("global-include " + ' '.join(patterns)) 139 for pattern in patterns: 140 if not self.include_pattern(pattern, anchor=0): 141 log.warn(("warning: no files found matching '%s' " + 142 "anywhere in distribution"), pattern) 143 144 elif action == 'global-exclude': 145 self.debug_print("global-exclude " + ' '.join(patterns)) 146 for pattern in patterns: 147 if not self.exclude_pattern(pattern, anchor=0): 148 log.warn(("warning: no previously-included files matching " 149 "'%s' found anywhere in distribution"), 150 pattern) 151 152 elif action == 'recursive-include': 153 self.debug_print("recursive-include %s %s" % 154 (dir, ' '.join(patterns))) 155 for pattern in patterns: 156 if not self.include_pattern(pattern, prefix=dir): 157 log.warn(("warning: no files found matching '%s' " + 158 "under directory '%s'"), 159 pattern, dir) 160 161 elif action == 'recursive-exclude': 162 self.debug_print("recursive-exclude %s %s" % 163 (dir, ' '.join(patterns))) 164 for pattern in patterns: 165 if not self.exclude_pattern(pattern, prefix=dir): 166 log.warn(("warning: no previously-included files matching " 167 "'%s' found under directory '%s'"), 168 pattern, dir) 169 170 elif action == 'graft': 171 self.debug_print("graft " + dir_pattern) 172 if not self.include_pattern(None, prefix=dir_pattern): 173 log.warn("warning: no directories found matching '%s'", 174 dir_pattern) 175 176 elif action == 'prune': 177 self.debug_print("prune " + dir_pattern) 178 if not self.exclude_pattern(None, prefix=dir_pattern): 179 log.warn(("no previously-included directories found " + 180 "matching '%s'"), dir_pattern) 181 else: 182 raise DistutilsInternalError, \ 183 "this cannot happen: invalid action '%s'" % action 184 185 # -- Filtering/selection methods ----------------------------------- 186 187 def include_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): 188 """Select strings (presumably filenames) from 'self.files' that 189 match 'pattern', a Unix-style wildcard (glob) pattern. 190 191 Patterns are not quite the same as implemented by the 'fnmatch' 192 module: '*' and '?' match non-special characters, where "special" 193 is platform-dependent: slash on Unix; colon, slash, and backslash on 194 DOS/Windows; and colon on Mac OS. 195 196 If 'anchor' is true (the default), then the pattern match is more 197 stringent: "*.py" will match "foo.py" but not "foo/bar.py". If 198 'anchor' is false, both of these will match. 199 200 If 'prefix' is supplied, then only filenames starting with 'prefix' 201 (itself a pattern) and ending with 'pattern', with anything in between 202 them, will match. 'anchor' is ignored in this case. 203 204 If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and 205 'pattern' is assumed to be either a string containing a regex or a 206 regex object -- no translation is done, the regex is just compiled 207 and used as-is. 208 209 Selected strings will be added to self.files. 210 211 Return 1 if files are found. 212 """ 213 files_found = 0 214 pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) 215 self.debug_print("include_pattern: applying regex r'%s'" % 216 pattern_re.pattern) 217 218 # delayed loading of allfiles list 219 if self.allfiles is None: 220 self.findall() 221 222 for name in self.allfiles: 223 if pattern_re.search(name): 224 self.debug_print(" adding " + name) 225 self.files.append(name) 226 files_found = 1 227 228 return files_found 229 230 231 def exclude_pattern(self, pattern, anchor=1, prefix=None, is_regex=0): 232 """Remove strings (presumably filenames) from 'files' that match 233 'pattern'. 234 235 Other parameters are the same as for 'include_pattern()', above. 236 The list 'self.files' is modified in place. Return 1 if files are 237 found. 238 """ 239 files_found = 0 240 pattern_re = translate_pattern(pattern, anchor, prefix, is_regex) 241 self.debug_print("exclude_pattern: applying regex r'%s'" % 242 pattern_re.pattern) 243 for i in range(len(self.files)-1, -1, -1): 244 if pattern_re.search(self.files[i]): 245 self.debug_print(" removing " + self.files[i]) 246 del self.files[i] 247 files_found = 1 248 249 return files_found 250 251 252 # ---------------------------------------------------------------------- 253 # Utility functions 254 255 def findall(dir = os.curdir): 256 """Find all files under 'dir' and return the list of full filenames 257 (relative to 'dir'). 258 """ 259 from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK 260 261 list = [] 262 stack = [dir] 263 pop = stack.pop 264 push = stack.append 265 266 while stack: 267 dir = pop() 268 names = os.listdir(dir) 269 270 for name in names: 271 if dir != os.curdir: # avoid the dreaded "./" syndrome 272 fullname = os.path.join(dir, name) 273 else: 274 fullname = name 275 276 # Avoid excess stat calls -- just one will do, thank you! 277 stat = os.stat(fullname) 278 mode = stat[ST_MODE] 279 if S_ISREG(mode): 280 list.append(fullname) 281 elif S_ISDIR(mode) and not S_ISLNK(mode): 282 push(fullname) 283 284 return list 285 286 287 def glob_to_re(pattern): 288 """Translate a shell-like glob pattern to a regular expression. 289 290 Return a string containing the regex. Differs from 291 'fnmatch.translate()' in that '*' does not match "special characters" 292 (which are platform-specific). 293 """ 294 pattern_re = fnmatch.translate(pattern) 295 296 # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which 297 # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix, 298 # and by extension they shouldn't match such "special characters" under 299 # any OS. So change all non-escaped dots in the RE to match any 300 # character except the special characters. 301 # XXX currently the "special characters" are just slash -- i.e. this is 302 # Unix-only. 303 pattern_re = re.sub(r'((?<!\\)(\\\\)*)\.', r'\1[^/]', pattern_re) 304 305 return pattern_re 306 307 308 def translate_pattern(pattern, anchor=1, prefix=None, is_regex=0): 309 """Translate a shell-like wildcard pattern to a compiled regular 310 expression. 311 312 Return the compiled regex. If 'is_regex' true, 313 then 'pattern' is directly compiled to a regex (if it's a string) 314 or just returned as-is (assumes it's a regex object). 315 """ 316 if is_regex: 317 if isinstance(pattern, str): 318 return re.compile(pattern) 319 else: 320 return pattern 321 322 if pattern: 323 pattern_re = glob_to_re(pattern) 324 else: 325 pattern_re = '' 326 327 if prefix is not None: 328 # ditch end of pattern character 329 empty_pattern = glob_to_re('') 330 prefix_re = glob_to_re(prefix)[:-len(empty_pattern)] 331 pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re) 332 else: # no prefix -- respect anchor flag 333 if anchor: 334 pattern_re = "^" + pattern_re 335 336 return re.compile(pattern_re) 337
