1 #! /usr/bin/env python 2 3 """Mirror a remote ftp subtree into a local directory tree. 4 5 usage: ftpmirror [-v] [-q] [-i] [-m] [-n] [-r] [-s pat] 6 [-l username [-p passwd [-a account]]] 7 hostname[:port] [remotedir [localdir]] 8 -v: verbose 9 -q: quiet 10 -i: interactive mode 11 -m: macintosh server (NCSA telnet 2.4) (implies -n -s '*.o') 12 -n: don't log in 13 -r: remove local files/directories no longer pertinent 14 -l username [-p passwd [-a account]]: login info (default .netrc or anonymous) 15 -s pat: skip files matching pattern 16 hostname: remote host w/ optional port separated by ':' 17 remotedir: remote directory (default initial) 18 localdir: local directory (default current) 19 """ 20 21 import os 22 import sys 23 import time 24 import getopt 25 import ftplib 26 import netrc 27 from fnmatch import fnmatch 28 29 # Print usage message and exit 30 def usage(*args): 31 sys.stdout = sys.stderr 32 for msg in args: print msg 33 print __doc__ 34 sys.exit(2) 35 36 verbose = 1 # 0 for -q, 2 for -v 37 interactive = 0 38 mac = 0 39 rmok = 0 40 nologin = 0 41 skippats = ['.', '..', '.mirrorinfo'] 42 43 # Main program: parse command line and start processing 44 def main(): 45 global verbose, interactive, mac, rmok, nologin 46 try: 47 opts, args = getopt.getopt(sys.argv[1:], 'a:bil:mnp:qrs:v') 48 except getopt.error, msg: 49 usage(msg) 50 login = '' 51 passwd = '' 52 account = '' 53 if not args: usage('hostname missing') 54 host = args[0] 55 port = 0 56 if ':' in host: 57 host, port = host.split(':', 1) 58 port = int(port) 59 try: 60 auth = netrc.netrc().authenticators(host) 61 if auth is not None: 62 login, account, passwd = auth 63 except (netrc.NetrcParseError, IOError): 64 pass 65 for o, a in opts: 66 if o == '-l': login = a 67 if o == '-p': passwd = a 68 if o == '-a': account = a 69 if o == '-v': verbose = verbose + 1 70 if o == '-q': verbose = 0 71 if o == '-i': interactive = 1 72 if o == '-m': mac = 1; nologin = 1; skippats.append('*.o') 73 if o == '-n': nologin = 1 74 if o == '-r': rmok = 1 75 if o == '-s': skippats.append(a) 76 remotedir = '' 77 localdir = '' 78 if args[1:]: 79 remotedir = args[1] 80 if args[2:]: 81 localdir = args[2] 82 if args[3:]: usage('too many arguments') 83 # 84 f = ftplib.FTP() 85 if verbose: print "Connecting to '%s%s'..." % (host, 86 (port and ":%d"%port or "")) 87 f.connect(host,port) 88 if not nologin: 89 if verbose: 90 print 'Logging in as %r...' % (login or 'anonymous') 91 f.login(login, passwd, account) 92 if verbose: print 'OK.' 93 pwd = f.pwd() 94 if verbose > 1: print 'PWD =', repr(pwd) 95 if remotedir: 96 if verbose > 1: print 'cwd(%s)' % repr(remotedir) 97 f.cwd(remotedir) 98 if verbose > 1: print 'OK.' 99 pwd = f.pwd() 100 if verbose > 1: print 'PWD =', repr(pwd) 101 # 102 mirrorsubdir(f, localdir) 103 104 # Core logic: mirror one subdirectory (recursively) 105 def mirrorsubdir(f, localdir): 106 pwd = f.pwd() 107 if localdir and not os.path.isdir(localdir): 108 if verbose: print 'Creating local directory', repr(localdir) 109 try: 110 makedir(localdir) 111 except os.error, msg: 112 print "Failed to establish local directory", repr(localdir) 113 return 114 infofilename = os.path.join(localdir, '.mirrorinfo') 115 try: 116 text = open(infofilename, 'r').read() 117 except IOError, msg: 118 text = '{}' 119 try: 120 info = eval(text) 121 except (SyntaxError, NameError): 122 print 'Bad mirror info in', repr(infofilename) 123 info = {} 124 subdirs = [] 125 listing = [] 126 if verbose: print 'Listing remote directory %r...' % (pwd,) 127 f.retrlines('LIST', listing.append) 128 filesfound = [] 129 for line in listing: 130 if verbose > 1: print '-->', repr(line) 131 if mac: 132 # Mac listing has just filenames; 133 # trailing / means subdirectory 134 filename = line.strip() 135 mode = '-' 136 if filename[-1:] == '/': 137 filename = filename[:-1] 138 mode = 'd' 139 infostuff = '' 140 else: 141 # Parse, assuming a UNIX listing 142 words = line.split(None, 8) 143 if len(words) < 6: 144 if verbose > 1: print 'Skipping short line' 145 continue 146 filename = words[-1].lstrip() 147 i = filename.find(" -> ") 148 if i >= 0: 149 # words[0] had better start with 'l'... 150 if verbose > 1: 151 print 'Found symbolic link %r' % (filename,) 152 linkto = filename[i+4:] 153 filename = filename[:i] 154 infostuff = words[-5:-1] 155 mode = words[0] 156 skip = 0 157 for pat in skippats: 158 if fnmatch(filename, pat): 159 if verbose > 1: 160 print 'Skip pattern', repr(pat), 161 print 'matches', repr(filename) 162 skip = 1 163 break 164 if skip: 165 continue 166 if mode[0] == 'd': 167 if verbose > 1: 168 print 'Remembering subdirectory', repr(filename) 169 subdirs.append(filename) 170 continue 171 filesfound.append(filename) 172 if info.has_key(filename) and info[filename] == infostuff: 173 if verbose > 1: 174 print 'Already have this version of',repr(filename) 175 continue 176 fullname = os.path.join(localdir, filename) 177 tempname = os.path.join(localdir, '@'+filename) 178 if interactive: 179 doit = askabout('file', filename, pwd) 180 if not doit: 181 if not info.has_key(filename): 182 info[filename] = 'Not retrieved' 183 continue 184 try: 185 os.unlink(tempname) 186 except os.error: 187 pass 188 if mode[0] == 'l': 189 if verbose: 190 print "Creating symlink %r -> %r" % (filename, linkto) 191 try: 192 os.symlink(linkto, tempname) 193 except IOError, msg: 194 print "Can't create %r: %s" % (tempname, msg) 195 continue 196 else: 197 try: 198 fp = open(tempname, 'wb') 199 except IOError, msg: 200 print "Can't create %r: %s" % (tempname, msg) 201 continue 202 if verbose: 203 print 'Retrieving %r from %r as %r...' % (filename, pwd, fullname) 204 if verbose: 205 fp1 = LoggingFile(fp, 1024, sys.stdout) 206 else: 207 fp1 = fp 208 t0 = time.time() 209 try: 210 f.retrbinary('RETR ' + filename, 211 fp1.write, 8*1024) 212 except ftplib.error_perm, msg: 213 print msg 214 t1 = time.time() 215 bytes = fp.tell() 216 fp.close() 217 if fp1 != fp: 218 fp1.close() 219 try: 220 os.unlink(fullname) 221 except os.error: 222 pass # Ignore the error 223 try: 224 os.rename(tempname, fullname) 225 except os.error, msg: 226 print "Can't rename %r to %r: %s" % (tempname, fullname, msg) 227 continue 228 info[filename] = infostuff 229 writedict(info, infofilename) 230 if verbose and mode[0] != 'l': 231 dt = t1 - t0 232 kbytes = bytes / 1024.0 233 print int(round(kbytes)), 234 print 'Kbytes in', 235 print int(round(dt)), 236 print 'seconds', 237 if t1 > t0: 238 print '(~%d Kbytes/sec)' % \ 239 int(round(kbytes/dt),) 240 print 241 # 242 # Remove files from info that are no longer remote 243 deletions = 0 244 for filename in info.keys(): 245 if filename not in filesfound: 246 if verbose: 247 print "Removing obsolete info entry for", 248 print repr(filename), "in", repr(localdir or ".") 249 del info[filename] 250 deletions = deletions + 1 251 if deletions: 252 writedict(info, infofilename) 253 # 254 # Remove local files that are no longer in the remote directory 255 try: 256 if not localdir: names = os.listdir(os.curdir) 257 else: names = os.listdir(localdir) 258 except os.error: 259 names = [] 260 for name in names: 261 if name[0] == '.' or info.has_key(name) or name in subdirs: 262 continue 263 skip = 0 264 for pat in skippats: 265 if fnmatch(name, pat): 266 if verbose > 1: 267 print 'Skip pattern', repr(pat), 268 print 'matches', repr(name) 269 skip = 1 270 break 271 if skip: 272 continue 273 fullname = os.path.join(localdir, name) 274 if not rmok: 275 if verbose: 276 print 'Local file', repr(fullname), 277 print 'is no longer pertinent' 278 continue 279 if verbose: print 'Removing local file/dir', repr(fullname) 280 remove(fullname) 281 # 282 # Recursively mirror subdirectories 283 for subdir in subdirs: 284 if interactive: 285 doit = askabout('subdirectory', subdir, pwd) 286 if not doit: continue 287 if verbose: print 'Processing subdirectory', repr(subdir) 288 localsubdir = os.path.join(localdir, subdir) 289 pwd = f.pwd() 290 if verbose > 1: 291 print 'Remote directory now:', repr(pwd) 292 print 'Remote cwd', repr(subdir) 293 try: 294 f.cwd(subdir) 295 except ftplib.error_perm, msg: 296 print "Can't chdir to", repr(subdir), ":", repr(msg) 297 else: 298 if verbose: print 'Mirroring as', repr(localsubdir) 299 mirrorsubdir(f, localsubdir) 300 if verbose > 1: print 'Remote cwd ..' 301 f.cwd('..') 302 newpwd = f.pwd() 303 if newpwd != pwd: 304 print 'Ended up in wrong directory after cd + cd ..' 305 print 'Giving up now.' 306 break 307 else: 308 if verbose > 1: print 'OK.' 309 310 # Helper to remove a file or directory tree 311 def remove(fullname): 312 if os.path.isdir(fullname) and not os.path.islink(fullname): 313 try: 314 names = os.listdir(fullname) 315 except os.error: 316 names = [] 317 ok = 1 318 for name in names: 319 if not remove(os.path.join(fullname, name)): 320 ok = 0 321 if not ok: 322 return 0 323 try: 324 os.rmdir(fullname) 325 except os.error, msg: 326 print "Can't remove local directory %r: %s" % (fullname, msg) 327 return 0 328 else: 329 try: 330 os.unlink(fullname) 331 except os.error, msg: 332 print "Can't remove local file %r: %s" % (fullname, msg) 333 return 0 334 return 1 335 336 # Wrapper around a file for writing to write a hash sign every block. 337 class LoggingFile: 338 def __init__(self, fp, blocksize, outfp): 339 self.fp = fp 340 self.bytes = 0 341 self.hashes = 0 342 self.blocksize = blocksize 343 self.outfp = outfp 344 def write(self, data): 345 self.bytes = self.bytes + len(data) 346 hashes = int(self.bytes) / self.blocksize 347 while hashes > self.hashes: 348 self.outfp.write('#') 349 self.outfp.flush() 350 self.hashes = self.hashes + 1 351 self.fp.write(data) 352 def close(self): 353 self.outfp.write('\n') 354 355 # Ask permission to download a file. 356 def askabout(filetype, filename, pwd): 357 prompt = 'Retrieve %s %s from %s ? [ny] ' % (filetype, filename, pwd) 358 while 1: 359 reply = raw_input(prompt).strip().lower() 360 if reply in ['y', 'ye', 'yes']: 361 return 1 362 if reply in ['', 'n', 'no', 'nop', 'nope']: 363 return 0 364 print 'Please answer yes or no.' 365 366 # Create a directory if it doesn't exist. Recursively create the 367 # parent directory as well if needed. 368 def makedir(pathname): 369 if os.path.isdir(pathname): 370 return 371 dirname = os.path.dirname(pathname) 372 if dirname: makedir(dirname) 373 os.mkdir(pathname, 0777) 374 375 # Write a dictionary to a file in a way that can be read back using 376 # rval() but is still somewhat readable (i.e. not a single long line). 377 # Also creates a backup file. 378 def writedict(dict, filename): 379 dir, fname = os.path.split(filename) 380 tempname = os.path.join(dir, '@' + fname) 381 backup = os.path.join(dir, fname + '~') 382 try: 383 os.unlink(backup) 384 except os.error: 385 pass 386 fp = open(tempname, 'w') 387 fp.write('{\n') 388 for key, value in dict.items(): 389 fp.write('%r: %r,\n' % (key, value)) 390 fp.write('}\n') 391 fp.close() 392 try: 393 os.rename(filename, backup) 394 except os.error: 395 pass 396 os.rename(tempname, filename) 397 398 399 if __name__ == '__main__': 400 main() 401
