1 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes.""" 2 3 # Notes for authors of new mailbox subclasses: 4 # 5 # Remember to fsync() changes to disk before closing a modified file 6 # or returning from a flush() method. See functions _sync_flush() and 7 # _sync_close(). 8 9 import os 10 import time 11 import calendar 12 import socket 13 import errno 14 import copy 15 import warnings 16 import email 17 import email.message 18 import email.generator 19 import io 20 import contextlib 21 try: 22 import fcntl 23 except ImportError: 24 fcntl = None 25 26 __all__ = ['Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF', 27 'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage', 28 'BabylMessage', 'MMDFMessage', 'Error', 'NoSuchMailboxError', 29 'NotEmptyError', 'ExternalClashError', 'FormatError'] 30 31 linesep = os.linesep.encode('ascii') 32 33 class Mailbox: 34 """A group of messages in a particular place.""" 35 36 def __init__(self, path, factory=None, create=True): 37 """Initialize a Mailbox instance.""" 38 self._path = os.path.abspath(os.path.expanduser(path)) 39 self._factory = factory 40 41 def add(self, message): 42 """Add message and return assigned key.""" 43 raise NotImplementedError('Method must be implemented by subclass') 44 45 def remove(self, key): 46 """Remove the keyed message; raise KeyError if it doesn't exist.""" 47 raise NotImplementedError('Method must be implemented by subclass') 48 49 def __delitem__(self, key): 50 self.remove(key) 51 52 def discard(self, key): 53 """If the keyed message exists, remove it.""" 54 try: 55 self.remove(key) 56 except KeyError: 57 pass 58 59 def __setitem__(self, key, message): 60 """Replace the keyed message; raise KeyError if it doesn't exist.""" 61 raise NotImplementedError('Method must be implemented by subclass') 62 63 def get(self, key, default=None): 64 """Return the keyed message, or default if it doesn't exist.""" 65 try: 66 return self.__getitem__(key) 67 except KeyError: 68 return default 69 70 def __getitem__(self, key): 71 """Return the keyed message; raise KeyError if it doesn't exist.""" 72 if not self._factory: 73 return self.get_message(key) 74 else: 75 with contextlib.closing(self.get_file(key)) as file: 76 return self._factory(file) 77 78 def get_message(self, key): 79 """Return a Message representation or raise a KeyError.""" 80 raise NotImplementedError('Method must be implemented by subclass') 81 82 def get_string(self, key): 83 """Return a string representation or raise a KeyError. 84 85 Uses email.message.Message to create a 7bit clean string 86 representation of the message.""" 87 return email.message_from_bytes(self.get_bytes(key)).as_string() 88 89 def get_bytes(self, key): 90 """Return a byte string representation or raise a KeyError.""" 91 raise NotImplementedError('Method must be implemented by subclass') 92 93 def get_file(self, key): 94 """Return a file-like representation or raise a KeyError.""" 95 raise NotImplementedError('Method must be implemented by subclass') 96 97 def iterkeys(self): 98 """Return an iterator over keys.""" 99 raise NotImplementedError('Method must be implemented by subclass') 100 101 def keys(self): 102 """Return a list of keys.""" 103 return list(self.iterkeys()) 104 105 def itervalues(self): 106 """Return an iterator over all messages.""" 107 for key in self.iterkeys(): 108 try: 109 value = self[key] 110 except KeyError: 111 continue 112 yield value 113 114 def __iter__(self): 115 return self.itervalues() 116 117 def values(self): 118 """Return a list of messages. Memory intensive.""" 119 return list(self.itervalues()) 120 121 def iteritems(self): 122 """Return an iterator over (key, message) tuples.""" 123 for key in self.iterkeys(): 124 try: 125 value = self[key] 126 except KeyError: 127 continue 128 yield (key, value) 129 130 def items(self): 131 """Return a list of (key, message) tuples. Memory intensive.""" 132 return list(self.iteritems()) 133 134 def __contains__(self, key): 135 """Return True if the keyed message exists, False otherwise.""" 136 raise NotImplementedError('Method must be implemented by subclass') 137 138 def __len__(self): 139 """Return a count of messages in the mailbox.""" 140 raise NotImplementedError('Method must be implemented by subclass') 141 142 def clear(self): 143 """Delete all messages.""" 144 for key in self.keys(): 145 self.discard(key) 146 147 def pop(self, key, default=None): 148 """Delete the keyed message and return it, or default.""" 149 try: 150 result = self[key] 151 except KeyError: 152 return default 153 self.discard(key) 154 return result 155 156 def popitem(self): 157 """Delete an arbitrary (key, message) pair and return it.""" 158 for key in self.iterkeys(): 159 return (key, self.pop(key)) # This is only run once. 160 else: 161 raise KeyError('No messages in mailbox') 162 163 def update(self, arg=None): 164 """Change the messages that correspond to certain keys.""" 165 if hasattr(arg, 'iteritems'): 166 source = arg.iteritems() 167 elif hasattr(arg, 'items'): 168 source = arg.items() 169 else: 170 source = arg 171 bad_key = False 172 for key, message in source: 173 try: 174 self[key] = message 175 except KeyError: 176 bad_key = True 177 if bad_key: 178 raise KeyError('No message with key(s)') 179 180 def flush(self): 181 """Write any pending changes to the disk.""" 182 raise NotImplementedError('Method must be implemented by subclass') 183 184 def lock(self): 185 """Lock the mailbox.""" 186 raise NotImplementedError('Method must be implemented by subclass') 187 188 def unlock(self): 189 """Unlock the mailbox if it is locked.""" 190 raise NotImplementedError('Method must be implemented by subclass') 191 192 def close(self): 193 """Flush and close the mailbox.""" 194 raise NotImplementedError('Method must be implemented by subclass') 195 196 def _string_to_bytes(self, message): 197 # If a message is not 7bit clean, we refuse to handle it since it 198 # likely came from reading invalid messages in text mode, and that way 199 # lies mojibake. 200 try: 201 return message.encode('ascii') 202 except UnicodeError: 203 raise ValueError("String input must be ASCII-only; " 204 "use bytes or a Message instead") 205 206 # Whether each message must end in a newline 207 _append_newline = False 208 209 def _dump_message(self, message, target, mangle_from_=False): 210 # This assumes the target file is open in binary mode. 211 """Dump message contents to target file.""" 212 if isinstance(message, email.message.Message): 213 buffer = io.BytesIO() 214 gen = email.generator.BytesGenerator(buffer, mangle_from_, 0) 215 gen.flatten(message) 216 buffer.seek(0) 217 data = buffer.read() 218 data = data.replace(b'\n', linesep) 219 target.write(data) 220 if self._append_newline and not data.endswith(linesep): 221 # Make sure the message ends with a newline 222 target.write(linesep) 223 elif isinstance(message, (str, bytes, io.StringIO)): 224 if isinstance(message, io.StringIO): 225 warnings.warn("Use of StringIO input is deprecated, " 226 "use BytesIO instead", DeprecationWarning, 3) 227 message = message.getvalue() 228 if isinstance(message, str): 229 message = self._string_to_bytes(message) 230 if mangle_from_: 231 message = message.replace(b'\nFrom ', b'\n>From ') 232 message = message.replace(b'\n', linesep) 233 target.write(message) 234 if self._append_newline and not message.endswith(linesep): 235 # Make sure the message ends with a newline 236 target.write(linesep) 237 elif hasattr(message, 'read'): 238 if hasattr(message, 'buffer'): 239 warnings.warn("Use of text mode files is deprecated, " 240 "use a binary mode file instead", DeprecationWarning, 3) 241 message = message.buffer 242 lastline = None 243 while True: 244 line = message.readline() 245 # Universal newline support. 246 if line.endswith(b'\r\n'): 247 line = line[:-2] + b'\n' 248 elif line.endswith(b'\r'): 249 line = line[:-1] + b'\n' 250 if not line: 251 break 252 if mangle_from_ and line.startswith(b'From '): 253 line = b'>From ' + line[5:] 254 line = line.replace(b'\n', linesep) 255 target.write(line) 256 lastline = line 257 if self._append_newline and lastline and not lastline.endswith(linesep): 258 # Make sure the message ends with a newline 259 target.write(linesep) 260 else: 261 raise TypeError('Invalid message type: %s' % type(message)) 262 263 264 class Maildir(Mailbox): 265 """A qmail-style Maildir mailbox.""" 266 267 colon = ':' 268 269 def __init__(self, dirname, factory=None, create=True): 270 """Initialize a Maildir instance.""" 271 Mailbox.__init__(self, dirname, factory, create) 272 self._paths = { 273 'tmp': os.path.join(self._path, 'tmp'), 274 'new': os.path.join(self._path, 'new'), 275 'cur': os.path.join(self._path, 'cur'), 276 } 277 if not os.path.exists(self._path): 278 if create: 279 os.mkdir(self._path, 0o700) 280 for path in self._paths.values(): 281 os.mkdir(path, 0o700) 282 else: 283 raise NoSuchMailboxError(self._path) 284 self._toc = {} 285 self._toc_mtimes = {'cur': 0, 'new': 0} 286 self._last_read = 0 # Records last time we read cur/new 287 self._skewfactor = 0.1 # Adjust if os/fs clocks are skewing 288 289 def add(self, message): 290 """Add message and return assigned key.""" 291 tmp_file = self._create_tmp() 292 try: 293 self._dump_message(message, tmp_file) 294 except BaseException: 295 tmp_file.close() 296 os.remove(tmp_file.name) 297 raise 298 _sync_close(tmp_file) 299 if isinstance(message, MaildirMessage): 300 subdir = message.get_subdir() 301 suffix = self.colon + message.get_info() 302 if suffix == self.colon: 303 suffix = '' 304 else: 305 subdir = 'new' 306 suffix = '' 307 uniq = os.path.basename(tmp_file.name).split(self.colon)[0] 308 dest = os.path.join(self._path, subdir, uniq + suffix) 309 if isinstance(message, MaildirMessage): 310 os.utime(tmp_file.name, 311 (os.path.getatime(tmp_file.name), message.get_date())) 312 # No file modification should be done after the file is moved to its 313 # final position in order to prevent race conditions with changes 314 # from other programs 315 try: 316 try: 317 os.link(tmp_file.name, dest) 318 except (AttributeError, PermissionError): 319 os.rename(tmp_file.name, dest) 320 else: 321 os.remove(tmp_file.name) 322 except OSError as e: 323 os.remove(tmp_file.name) 324 if e.errno == errno.EEXIST: 325 raise ExternalClashError('Name clash with existing message: %s' 326 % dest) 327 else: 328 raise 329 return uniq 330 331 def remove(self, key): 332 """Remove the keyed message; raise KeyError if it doesn't exist.""" 333 os.remove(os.path.join(self._path, self._lookup(key))) 334 335 def discard(self, key): 336 """If the keyed message exists, remove it.""" 337 # This overrides an inapplicable implementation in the superclass. 338 try: 339 self.remove(key) 340 except (KeyError, FileNotFoundError): 341 pass 342 343 def __setitem__(self, key, message): 344 """Replace the keyed message; raise KeyError if it doesn't exist.""" 345 old_subpath = self._lookup(key) 346 temp_key = self.add(message) 347 temp_subpath = self._lookup(temp_key) 348 if isinstance(message, MaildirMessage): 349 # temp's subdir and suffix were specified by message. 350 dominant_subpath = temp_subpath 351 else: 352 # temp's subdir and suffix were defaults from add(). 353 dominant_subpath = old_subpath 354 subdir = os.path.dirname(dominant_subpath) 355 if self.colon in dominant_subpath: 356 suffix = self.colon + dominant_subpath.split(self.colon)[-1] 357 else: 358 suffix = '' 359 self.discard(key) 360 tmp_path = os.path.join(self._path, temp_subpath) 361 new_path = os.path.join(self._path, subdir, key + suffix) 362 if isinstance(message, MaildirMessage): 363 os.utime(tmp_path, 364 (os.path.getatime(tmp_path), message.get_date())) 365 # No file modification should be done after the file is moved to its 366 # final position in order to prevent race conditions with changes 367 # from other programs 368 os.rename(tmp_path, new_path) 369 370 def get_message(self, key): 371 """Return a Message representation or raise a KeyError.""" 372 subpath = self._lookup(key) 373 with open(os.path.join(self._path, subpath), 'rb') as f: 374 if self._factory: 375 msg = self._factory(f) 376 else: 377 msg = MaildirMessage(f) 378 subdir, name = os.path.split(subpath) 379 msg.set_subdir(subdir) 380 if self.colon in name: 381 msg.set_info(name.split(self.colon)[-1]) 382 msg.set_date(os.path.getmtime(os.path.join(self._path, subpath))) 383 return msg 384 385 def get_bytes(self, key): 386 """Return a bytes representation or raise a KeyError.""" 387 with open(os.path.join(self._path, self._lookup(key)), 'rb') as f: 388 return f.read().replace(linesep, b'\n') 389 390 def get_file(self, key): 391 """Return a file-like representation or raise a KeyError.""" 392 f = open(os.path.join(self._path, self._lookup(key)), 'rb') 393 return _ProxyFile(f) 394 395 def iterkeys(self): 396 """Return an iterator over keys.""" 397 self._refresh() 398 for key in self._toc: 399 try: 400 self._lookup(key) 401 except KeyError: 402 continue 403 yield key 404 405 def __contains__(self, key): 406 """Return True if the keyed message exists, False otherwise.""" 407 self._refresh() 408 return key in self._toc 409 410 def __len__(self): 411 """Return a count of messages in the mailbox.""" 412 self._refresh() 413 return len(self._toc) 414 415 def flush(self): 416 """Write any pending changes to disk.""" 417 # Maildir changes are always written immediately, so there's nothing 418 # to do. 419 pass 420 421 def lock(self): 422 """Lock the mailbox.""" 423 return 424 425 def unlock(self): 426 """Unlock the mailbox if it is locked.""" 427 return 428 429 def close(self): 430 """Flush and close the mailbox.""" 431 return 432 433 def list_folders(self): 434 """Return a list of folder names.""" 435 result = [] 436 for entry in os.listdir(self._path): 437 if len(entry) > 1 and entry[0] == '.' and \ 438 os.path.isdir(os.path.join(self._path, entry)): 439 result.append(entry[1:]) 440 return result 441 442 def get_folder(self, folder): 443 """Return a Maildir instance for the named folder.""" 444 return Maildir(os.path.join(self._path, '.' + folder), 445 factory=self._factory, 446 create=False) 447 448 def add_folder(self, folder): 449 """Create a folder and return a Maildir instance representing it.""" 450 path = os.path.join(self._path, '.' + folder) 451 result = Maildir(path, factory=self._factory) 452 maildirfolder_path = os.path.join(path, 'maildirfolder') 453 if not os.path.exists(maildirfolder_path): 454 os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY, 455 0o666)) 456 return result 457 458 def remove_folder(self, folder): 459 """Delete the named folder, which must be empty.""" 460 path = os.path.join(self._path, '.' + folder) 461 for entry in os.listdir(os.path.join(path, 'new')) + \ 462 os.listdir(os.path.join(path, 'cur')): 463 if len(entry) < 1 or entry[0] != '.': 464 raise NotEmptyError('Folder contains message(s): %s' % folder) 465 for entry in os.listdir(path): 466 if entry != 'new' and entry != 'cur' and entry != 'tmp' and \ 467 os.path.isdir(os.path.join(path, entry)): 468 raise NotEmptyError("Folder contains subdirectory '%s': %s" % 469 (folder, entry)) 470 for root, dirs, files in os.walk(path, topdown=False): 471 for entry in files: 472 os.remove(os.path.join(root, entry)) 473 for entry in dirs: 474 os.rmdir(os.path.join(root, entry)) 475 os.rmdir(path) 476 477 def clean(self): 478 """Delete old files in "tmp".""" 479 now = time.time() 480 for entry in os.listdir(os.path.join(self._path, 'tmp')): 481 path = os.path.join(self._path, 'tmp', entry) 482 if now - os.path.getatime(path) > 129600: # 60 * 60 * 36 483 os.remove(path) 484 485 _count = 1 # This is used to generate unique file names. 486 487 def _create_tmp(self): 488 """Create a file in the tmp subdirectory and open and return it.""" 489 now = time.time() 490 hostname = socket.gethostname() 491 if '/' in hostname: 492 hostname = hostname.replace('/', r'\057') 493 if ':' in hostname: 494 hostname = hostname.replace(':', r'\072') 495 uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(), 496 Maildir._count, hostname) 497 path = os.path.join(self._path, 'tmp', uniq) 498 try: 499 os.stat(path) 500 except FileNotFoundError: 501 Maildir._count += 1 502 try: 503 return _create_carefully(path) 504 except FileExistsError: 505 pass 506 507 # Fall through to here if stat succeeded or open raised EEXIST. 508 raise ExternalClashError('Name clash prevented file creation: %s' % 509 path) 510 511 def _refresh(self): 512 """Update table of contents mapping.""" 513 # If it has been less than two seconds since the last _refresh() call, 514 # we have to unconditionally re-read the mailbox just in case it has 515 # been modified, because os.path.mtime() has a 2 sec resolution in the 516 # most common worst case (FAT) and a 1 sec resolution typically. This 517 # results in a few unnecessary re-reads when _refresh() is called 518 # multiple times in that interval, but once the clock ticks over, we 519 # will only re-read as needed. Because the filesystem might be being 520 # served by an independent system with its own clock, we record and 521 # compare with the mtimes from the filesystem. Because the other 522 # system's clock might be skewing relative to our clock, we add an 523 # extra delta to our wait. The default is one tenth second, but is an 524 # instance variable and so can be adjusted if dealing with a 525 # particularly skewed or irregular system. 526 if time.time() - self._last_read > 2 + self._skewfactor: 527 refresh = False 528 for subdir in self._toc_mtimes: 529 mtime = os.path.getmtime(self._paths[subdir]) 530 if mtime > self._toc_mtimes[subdir]: 531 refresh = True 532 self._toc_mtimes[subdir] = mtime 533 if not refresh: 534 return 535 # Refresh toc 536 self._toc = {} 537 for subdir in self._toc_mtimes: 538 path = self._paths[subdir] 539 for entry in os.listdir(path): 540 p = os.path.join(path, entry) 541 if os.path.isdir(p): 542 continue 543 uniq = entry.split(self.colon)[0] 544 self._toc[uniq] = os.path.join(subdir, entry) 545 self._last_read = time.time() 546 547 def _lookup(self, key): 548 """Use TOC to return subpath for given key, or raise a KeyError.""" 549 try: 550 if os.path.exists(os.path.join(self._path, self._toc[key])): 551 return self._toc[key] 552 except KeyError: 553 pass 554 self._refresh() 555 try: 556 return self._toc[key] 557 except KeyError: 558 raise KeyError('No message with key: %s' % key) 559 560 # This method is for backward compatibility only. 561 def next(self): 562 """Return the next message in a one-time iteration.""" 563 if not hasattr(self, '_onetime_keys'): 564 self._onetime_keys = self.iterkeys() 565 while True: 566 try: 567 return self[next(self._onetime_keys)] 568 except StopIteration: 569 return None 570 except KeyError: 571 continue 572 573 574 class _singlefileMailbox(Mailbox): 575 """A single-file mailbox.""" 576 577 def __init__(self, path, factory=None, create=True): 578 """Initialize a single-file mailbox.""" 579 Mailbox.__init__(self, path, factory, create) 580 try: 581 f = open(self._path, 'rb+') 582 except OSError as e: 583 if e.errno == errno.ENOENT: 584 if create: 585 f = open(self._path, 'wb+') 586 else: 587 raise NoSuchMailboxError(self._path) 588 elif e.errno in (errno.EACCES, errno.EROFS): 589 f = open(self._path, 'rb') 590 else: 591 raise 592 self._file = f 593 self._toc = None 594 self._next_key = 0 595 self._pending = False # No changes require rewriting the file. 596 self._pending_sync = False # No need to sync the file 597 self._locked = False 598 self._file_length = None # Used to record mailbox size 599 600 def add(self, message): 601 """Add message and return assigned key.""" 602 self._lookup() 603 self._toc[self._next_key] = self._append_message(message) 604 self._next_key += 1 605 # _append_message appends the message to the mailbox file. We 606 # don't need a full rewrite + rename, sync is enough. 607 self._pending_sync = True 608 return self._next_key - 1 609 610 def remove(self, key): 611 """Remove the keyed message; raise KeyError if it doesn't exist.""" 612 self._lookup(key) 613 del self._toc[key] 614 self._pending = True 615 616 def __setitem__(self, key, message): 617 """Replace the keyed message; raise KeyError if it doesn't exist.""" 618 self._lookup(key) 619 self._toc[key] = self._append_message(message) 620 self._pending = True 621 622 def iterkeys(self): 623 """Return an iterator over keys.""" 624 self._lookup() 625 yield from self._toc.keys() 626 627 def __contains__(self, key): 628 """Return True if the keyed message exists, False otherwise.""" 629 self._lookup() 630 return key in self._toc 631 632 def __len__(self): 633 """Return a count of messages in the mailbox.""" 634 self._lookup() 635 return len(self._toc) 636 637 def lock(self): 638 """Lock the mailbox.""" 639 if not self._locked: 640 _lock_file(self._file) 641 self._locked = True 642 643 def unlock(self): 644 """Unlock the mailbox if it is locked.""" 645 if self._locked: 646 _unlock_file(self._file) 647 self._locked = False 648 649 def flush(self): 650 """Write any pending changes to disk.""" 651 if not self._pending: 652 if self._pending_sync: 653 # Messages have only been added, so syncing the file 654 # is enough. 655 _sync_flush(self._file) 656 self._pending_sync = False 657 return 658 659 # In order to be writing anything out at all, self._toc must 660 # already have been generated (and presumably has been modified 661 # by adding or deleting an item). 662 assert self._toc is not None 663 664 # Check length of self._file; if it's changed, some other process 665 # has modified the mailbox since we scanned it. 666 self._file.seek(0, 2) 667 cur_len = self._file.tell() 668 if cur_len != self._file_length: 669 raise ExternalClashError('Size of mailbox file changed ' 670 '(expected %i, found %i)' % 671 (self._file_length, cur_len)) 672 673 new_file = _create_temporary(self._path) 674 try: 675 new_toc = {} 676 self._pre_mailbox_hook(new_file) 677 for key in sorted(self._toc.keys()): 678 start, stop = self._toc[key] 679 self._file.seek(start) 680 self._pre_message_hook(new_file) 681 new_start = new_file.tell() 682 while True: 683 buffer = self._file.read(min(4096, 684 stop - self._file.tell())) 685 if not buffer: 686 break 687 new_file.write(buffer) 688 new_toc[key] = (new_start, new_file.tell()) 689 self._post_message_hook(new_file) 690 self._file_length = new_file.tell() 691 except: 692 new_file.close() 693 os.remove(new_file.name) 694 raise 695 _sync_close(new_file) 696 # self._file is about to get replaced, so no need to sync. 697 self._file.close() 698 # Make sure the new file's mode is the same as the old file's 699 mode = os.stat(self._path).st_mode 700 os.chmod(new_file.name, mode) 701 try: 702 os.rename(new_file.name, self._path) 703 except FileExistsError: 704 os.remove(self._path) 705 os.rename(new_file.name, self._path) 706 self._file = open(self._path, 'rb+') 707 self._toc = new_toc 708 self._pending = False 709 self._pending_sync = False 710 if self._locked: 711 _lock_file(self._file, dotlock=False) 712 713 def _pre_mailbox_hook(self, f): 714 """Called before writing the mailbox to file f.""" 715 return 716 717 def _pre_message_hook(self, f): 718 """Called before writing each message to file f.""" 719 return 720 721 def _post_message_hook(self, f): 722 """Called after writing each message to file f.""" 723 return 724 725 def close(self): 726 """Flush and close the mailbox.""" 727 try: 728 self.flush() 729 finally: 730 try: 731 if self._locked: 732 self.unlock() 733 finally: 734 self._file.close() # Sync has been done by self.flush() above. 735 736 def _lookup(self, key=None): 737 """Return (start, stop) or raise KeyError.""" 738 if self._toc is None: 739 self._generate_toc() 740 if key is not None: 741 try: 742 return self._toc[key] 743 except KeyError: 744 raise KeyError('No message with key: %s' % key) 745 746 def _append_message(self, message): 747 """Append message to mailbox and return (start, stop) offsets.""" 748 self._file.seek(0, 2) 749 before = self._file.tell() 750 if len(self._toc) == 0 and not self._pending: 751 # This is the first message, and the _pre_mailbox_hook 752 # hasn't yet been called. If self._pending is True, 753 # messages have been removed, so _pre_mailbox_hook must 754 # have been called already. 755 self._pre_mailbox_hook(self._file) 756 try: 757 self._pre_message_hook(self._file) 758 offsets = self._install_message(message) 759 self._post_message_hook(self._file) 760 except BaseException: 761 self._file.truncate(before) 762 raise 763 self._file.flush() 764 self._file_length = self._file.tell() # Record current length of mailbox 765 return offsets 766 767 768 769 class _mboxMMDF(_singlefileMailbox): 770 """An mbox or MMDF mailbox.""" 771 772 _mangle_from_ = True 773 774 def get_message(self, key): 775 """Return a Message representation or raise a KeyError.""" 776 start, stop = self._lookup(key) 777 self._file.seek(start) 778 from_line = self._file.readline().replace(linesep, b'') 779 string = self._file.read(stop - self._file.tell()) 780 msg = self._message_factory(string.replace(linesep, b'\n')) 781 msg.set_from(from_line[5:].decode('ascii')) 782 return msg 783 784 def get_string(self, key, from_=False): 785 """Return a string representation or raise a KeyError.""" 786 return email.message_from_bytes( 787 self.get_bytes(key)).as_string(unixfrom=from_) 788 789 def get_bytes(self, key, from_=False): 790 """Return a string representation or raise a KeyError.""" 791 start, stop = self._lookup(key) 792 self._file.seek(start) 793 if not from_: 794 self._file.readline() 795 string = self._file.read(stop - self._file.tell()) 796 return string.replace(linesep, b'\n') 797 798 def get_file(self, key, from_=False): 799 """Return a file-like representation or raise a KeyError.""" 800 start, stop = self._lookup(key) 801 self._file.seek(start) 802 if not from_: 803 self._file.readline() 804 return _PartialFile(self._file, self._file.tell(), stop) 805 806 def _install_message(self, message): 807 """Format a message and blindly write to self._file.""" 808 from_line = None 809 if isinstance(message, str): 810 message = self._string_to_bytes(message) 811 if isinstance(message, bytes) and message.startswith(b'From '): 812 newline = message.find(b'\n') 813 if newline != -1: 814 from_line = message[:newline] 815 message = message[newline + 1:] 816 else: 817 from_line = message 818 message = b'' 819 elif isinstance(message, _mboxMMDFMessage): 820 author = message.get_from().encode('ascii') 821 from_line = b'From ' + author 822 elif isinstance(message, email.message.Message): 823 from_line = message.get_unixfrom() # May be None. 824 if from_line is not None: 825 from_line = from_line.encode('ascii') 826 if from_line is None: 827 from_line = b'From MAILER-DAEMON ' + time.asctime(time.gmtime()).encode() 828 start = self._file.tell() 829 self._file.write(from_line + linesep) 830 self._dump_message(message, self._file, self._mangle_from_) 831 stop = self._file.tell() 832 return (start, stop) 833 834 835 class mbox(_mboxMMDF): 836 """A classic mbox mailbox.""" 837 838 _mangle_from_ = True 839 840 # All messages must end in a newline character, and 841 # _post_message_hooks outputs an empty line between messages. 842 _append_newline = True 843 844 def __init__(self, path, factory=None, create=True): 845 """Initialize an mbox mailbox.""" 846 self._message_factory = mboxMessage 847 _mboxMMDF.__init__(self, path, factory, create) 848 849 def _post_message_hook(self, f): 850 """Called after writing each message to file f.""" 851 f.write(linesep) 852 853 def _generate_toc(self): 854 """Generate key-to-(start, stop) table of contents.""" 855 starts, stops = [], [] 856 last_was_empty = False 857 self._file.seek(0) 858 while True: 859 line_pos = self._file.tell() 860 line = self._file.readline() 861 if line.startswith(b'From '): 862 if len(stops) < len(starts): 863 if last_was_empty: 864 stops.append(line_pos - len(linesep)) 865 else: 866 # The last line before the "From " line wasn't 867 # blank, but we consider it a start of a 868 # message anyway. 869 stops.append(line_pos) 870 starts.append(line_pos) 871 last_was_empty = False 872 elif not line: 873 if last_was_empty: 874 stops.append(line_pos - len(linesep)) 875 else: 876 stops.append(line_pos) 877 break 878 elif line == linesep: 879 last_was_empty = True 880 else: 881 last_was_empty = False 882 self._toc = dict(enumerate(zip(starts, stops))) 883 self._next_key = len(self._toc) 884 self._file_length = self._file.tell() 885 886 887 class MMDF(_mboxMMDF): 888 """An MMDF mailbox.""" 889 890 def __init__(self, path, factory=None, create=True): 891 """Initialize an MMDF mailbox.""" 892 self._message_factory = MMDFMessage 893 _mboxMMDF.__init__(self, path, factory, create) 894 895 def _pre_message_hook(self, f): 896 """Called before writing each message to file f.""" 897 f.write(b'\001\001\001\001' + linesep) 898 899 def _post_message_hook(self, f): 900 """Called after writing each message to file f.""" 901 f.write(linesep + b'\001\001\001\001' + linesep) 902 903 def _generate_toc(self): 904 """Generate key-to-(start, stop) table of contents.""" 905 starts, stops = [], [] 906 self._file.seek(0) 907 next_pos = 0 908 while True: 909 line_pos = next_pos 910 line = self._file.readline() 911 next_pos = self._file.tell() 912 if line.startswith(b'\001\001\001\001' + linesep): 913 starts.append(next_pos) 914 while True: 915 line_pos = next_pos 916 line = self._file.readline() 917 next_pos = self._file.tell() 918 if line == b'\001\001\001\001' + linesep: 919 stops.append(line_pos - len(linesep)) 920 break 921 elif not line: 922 stops.append(line_pos) 923 break 924 elif not line: 925 break 926 self._toc = dict(enumerate(zip(starts, stops))) 927 self._next_key = len(self._toc) 928 self._file.seek(0, 2) 929 self._file_length = self._file.tell() 930 931 932 class MH(Mailbox): 933 """An MH mailbox.""" 934 935 def __init__(self, path, factory=None, create=True): 936 """Initialize an MH instance.""" 937 Mailbox.__init__(self, path, factory, create) 938 if not os.path.exists(self._path): 939 if create: 940 os.mkdir(self._path, 0o700) 941 os.close(os.open(os.path.join(self._path, '.mh_sequences'), 942 os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)) 943 else: 944 raise NoSuchMailboxError(self._path) 945 self._locked = False 946 947 def add(self, message): 948 """Add message and return assigned key.""" 949 keys = self.keys() 950 if len(keys) == 0: 951 new_key = 1 952 else: 953 new_key = max(keys) + 1 954 new_path = os.path.join(self._path, str(new_key)) 955 f = _create_carefully(new_path) 956 closed = False 957 try: 958 if self._locked: 959 _lock_file(f) 960 try: 961 try: 962 self._dump_message(message, f) 963 except BaseException: 964 # Unlock and close so it can be deleted on Windows 965 if self._locked: 966 _unlock_file(f) 967 _sync_close(f) 968 closed = True 969 os.remove(new_path) 970 raise 971 if isinstance(message, MHMessage): 972 self._dump_sequences(message, new_key) 973 finally: 974 if self._locked: 975 _unlock_file(f) 976 finally: 977 if not closed: 978 _sync_close(f) 979 return new_key 980 981 def remove(self, key): 982 """Remove the keyed message; raise KeyError if it doesn't exist.""" 983 path = os.path.join(self._path, str(key)) 984 try: 985 f = open(path, 'rb+') 986 except OSError as e: 987 if e.errno == errno.ENOENT: 988 raise KeyError('No message with key: %s' % key) 989 else: 990 raise 991 else: 992 f.close() 993 os.remove(path) 994 995 def __setitem__(self, key, message): 996 """Replace the keyed message; raise KeyError if it doesn't exist.""" 997 path = os.path.join(self._path, str(key)) 998 try: 999 f = open(path, 'rb+') 1000 except OSError as e: 1001 if e.errno == errno.ENOENT: 1002 raise KeyError('No message with key: %s' % key) 1003 else: 1004 raise 1005 try: 1006 if self._locked: 1007 _lock_file(f) 1008 try: 1009 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC)) 1010 self._dump_message(message, f) 1011 if isinstance(message, MHMessage): 1012 self._dump_sequences(message, key) 1013 finally: 1014 if self._locked: 1015 _unlock_file(f) 1016 finally: 1017 _sync_close(f) 1018 1019 def get_message(self, key): 1020 """Return a Message representation or raise a KeyError.""" 1021 try: 1022 if self._locked: 1023 f = open(os.path.join(self._path, str(key)), 'rb+') 1024 else: 1025 f = open(os.path.join(self._path, str(key)), 'rb') 1026 except OSError as e: 1027 if e.errno == errno.ENOENT: 1028 raise KeyError('No message with key: %s' % key) 1029 else: 1030 raise 1031 with f: 1032 if self._locked: 1033 _lock_file(f) 1034 try: 1035 msg = MHMessage(f) 1036 finally: 1037 if self._locked: 1038 _unlock_file(f) 1039 for name, key_list in self.get_sequences().items(): 1040 if key in key_list: 1041 msg.add_sequence(name) 1042 return msg 1043 1044 def get_bytes(self, key): 1045 """Return a bytes representation or raise a KeyError.""" 1046 try: 1047 if self._locked: 1048 f = open(os.path.join(self._path, str(key)), 'rb+') 1049 else: 1050 f = open(os.path.join(self._path, str(key)), 'rb') 1051 except OSError as e: 1052 if e.errno == errno.ENOENT: 1053 raise KeyError('No message with key: %s' % key) 1054 else: 1055 raise 1056 with f: 1057 if self._locked: 1058 _lock_file(f) 1059 try: 1060 return f.read().replace(linesep, b'\n') 1061 finally: 1062 if self._locked: 1063 _unlock_file(f) 1064 1065 def get_file(self, key): 1066 """Return a file-like representation or raise a KeyError.""" 1067 try: 1068 f = open(os.path.join(self._path, str(key)), 'rb') 1069 except OSError as e: 1070 if e.errno == errno.ENOENT: 1071 raise KeyError('No message with key: %s' % key) 1072 else: 1073 raise 1074 return _ProxyFile(f) 1075 1076 def iterkeys(self): 1077 """Return an iterator over keys.""" 1078 return iter(sorted(int(entry) for entry in os.listdir(self._path) 1079 if entry.isdigit())) 1080 1081 def __contains__(self, key): 1082 """Return True if the keyed message exists, False otherwise.""" 1083 return os.path.exists(os.path.join(self._path, str(key))) 1084 1085 def __len__(self): 1086 """Return a count of messages in the mailbox.""" 1087 return len(list(self.iterkeys())) 1088 1089 def lock(self): 1090 """Lock the mailbox.""" 1091 if not self._locked: 1092 self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+') 1093 _lock_file(self._file) 1094 self._locked = True 1095 1096 def unlock(self): 1097 """Unlock the mailbox if it is locked.""" 1098 if self._locked: 1099 _unlock_file(self._file) 1100 _sync_close(self._file) 1101 del self._file 1102 self._locked = False 1103 1104 def flush(self): 1105 """Write any pending changes to the disk.""" 1106 return 1107 1108 def close(self): 1109 """Flush and close the mailbox.""" 1110 if self._locked: 1111 self.unlock() 1112 1113 def list_folders(self): 1114 """Return a list of folder names.""" 1115 result = [] 1116 for entry in os.listdir(self._path): 1117 if os.path.isdir(os.path.join(self._path, entry)): 1118 result.append(entry) 1119 return result 1120 1121 def get_folder(self, folder): 1122 """Return an MH instance for the named folder.""" 1123 return MH(os.path.join(self._path, folder), 1124 factory=self._factory, create=False) 1125 1126 def add_folder(self, folder): 1127 """Create a folder and return an MH instance representing it.""" 1128 return MH(os.path.join(self._path, folder), 1129 factory=self._factory) 1130 1131 def remove_folder(self, folder): 1132 """Delete the named folder, which must be empty.""" 1133 path = os.path.join(self._path, folder) 1134 entries = os.listdir(path) 1135 if entries == ['.mh_sequences']: 1136 os.remove(os.path.join(path, '.mh_sequences')) 1137 elif entries == []: 1138 pass 1139 else: 1140 raise NotEmptyError('Folder not empty: %s' % self._path) 1141 os.rmdir(path) 1142 1143 def get_sequences(self): 1144 """Return a name-to-key-list dictionary to define each sequence.""" 1145 results = {} 1146 with open(os.path.join(self._path, '.mh_sequences'), 'r', encoding='ASCII') as f: 1147 all_keys = set(self.keys()) 1148 for line in f: 1149 try: 1150 name, contents = line.split(':') 1151 keys = set() 1152 for spec in contents.split(): 1153 if spec.isdigit(): 1154 keys.add(int(spec)) 1155 else: 1156 start, stop = (int(x) for x in spec.split('-')) 1157 keys.update(range(start, stop + 1)) 1158 results[name] = [key for key in sorted(keys) \ 1159 if key in all_keys] 1160 if len(results[name]) == 0: 1161 del results[name] 1162 except ValueError: 1163 raise FormatError('Invalid sequence specification: %s' % 1164 line.rstrip()) 1165 return results 1166 1167 def set_sequences(self, sequences): 1168 """Set sequences using the given name-to-key-list dictionary.""" 1169 f = open(os.path.join(self._path, '.mh_sequences'), 'r+', encoding='ASCII') 1170 try: 1171 os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC)) 1172 for name, keys in sequences.items(): 1173 if len(keys) == 0: 1174 continue 1175 f.write(name + ':') 1176 prev = None 1177 completing = False 1178 for key in sorted(set(keys)): 1179 if key - 1 == prev: 1180 if not completing: 1181 completing = True 1182 f.write('-') 1183 elif completing: 1184 completing = False 1185 f.write('%s %s' % (prev, key)) 1186 else: 1187 f.write(' %s' % key) 1188 prev = key 1189 if completing: 1190 f.write(str(prev) + '\n') 1191 else: 1192 f.write('\n') 1193 finally: 1194 _sync_close(f) 1195 1196 def pack(self): 1197 """Re-name messages to eliminate numbering gaps. Invalidates keys.""" 1198 sequences = self.get_sequences() 1199 prev = 0 1200 changes = [] 1201 for key in self.iterkeys(): 1202 if key - 1 != prev: 1203 changes.append((key, prev + 1)) 1204 try: 1205 os.link(os.path.join(self._path, str(key)), 1206 os.path.join(self._path, str(prev + 1))) 1207 except (AttributeError, PermissionError): 1208 os.rename(os.path.join(self._path, str(key)), 1209 os.path.join(self._path, str(prev + 1))) 1210 else: 1211 os.unlink(os.path.join(self._path, str(key))) 1212 prev += 1 1213 self._next_key = prev + 1 1214 if len(changes) == 0: 1215 return 1216 for name, key_list in sequences.items(): 1217 for old, new in changes: 1218 if old in key_list: 1219 key_list[key_list.index(old)] = new 1220 self.set_sequences(sequences) 1221 1222 def _dump_sequences(self, message, key): 1223 """Inspect a new MHMessage and update sequences appropriately.""" 1224 pending_sequences = message.get_sequences() 1225 all_sequences = self.get_sequences() 1226 for name, key_list in all_sequences.items(): 1227 if name in pending_sequences: 1228 key_list.append(key) 1229 elif key in key_list: 1230 del key_list[key_list.index(key)] 1231 for sequence in pending_sequences: 1232 if sequence not in all_sequences: 1233 all_sequences[sequence] = [key] 1234 self.set_sequences(all_sequences) 1235 1236 1237 class Babyl(_singlefileMailbox): 1238 """An Rmail-style Babyl mailbox.""" 1239 1240 _special_labels = frozenset({'unseen', 'deleted', 'filed', 'answered', 1241 'forwarded', 'edited', 'resent'}) 1242 1243 def __init__(self, path, factory=None, create=True): 1244 """Initialize a Babyl mailbox.""" 1245 _singlefileMailbox.__init__(self, path, factory, create) 1246 self._labels = {} 1247 1248 def add(self, message): 1249 """Add message and return assigned key.""" 1250 key = _singlefileMailbox.add(self, message) 1251 if isinstance(message, BabylMessage): 1252 self._labels[key] = message.get_labels() 1253 return key 1254 1255 def remove(self, key): 1256 """Remove the keyed message; raise KeyError if it doesn't exist.""" 1257 _singlefileMailbox.remove(self, key) 1258 if key in self._labels: 1259 del self._labels[key] 1260 1261 def __setitem__(self, key, message): 1262 """Replace the keyed message; raise KeyError if it doesn't exist.""" 1263 _singlefileMailbox.__setitem__(self, key, message) 1264 if isinstance(message, BabylMessage): 1265 self._labels[key] = message.get_labels() 1266 1267 def get_message(self, key): 1268 """Return a Message representation or raise a KeyError.""" 1269 start, stop = self._lookup(key) 1270 self._file.seek(start) 1271 self._file.readline() # Skip b'1,' line specifying labels. 1272 original_headers = io.BytesIO() 1273 while True: 1274 line = self._file.readline() 1275 if line == b'*** EOOH ***' + linesep or not line: 1276 break 1277 original_headers.write(line.replace(linesep, b'\n')) 1278 visible_headers = io.BytesIO() 1279 while True: 1280 line = self._file.readline() 1281 if line == linesep or not line: 1282 break 1283 visible_headers.write(line.replace(linesep, b'\n')) 1284 # Read up to the stop, or to the end 1285 n = stop - self._file.tell() 1286 assert n >= 0 1287 body = self._file.read(n) 1288 body = body.replace(linesep, b'\n') 1289 msg = BabylMessage(original_headers.getvalue() + body) 1290 msg.set_visible(visible_headers.getvalue()) 1291 if key in self._labels: 1292 msg.set_labels(self._labels[key]) 1293 return msg 1294 1295 def get_bytes(self, key): 1296 """Return a string representation or raise a KeyError.""" 1297 start, stop = self._lookup(key) 1298 self._file.seek(start) 1299 self._file.readline() # Skip b'1,' line specifying labels. 1300 original_headers = io.BytesIO() 1301 while True: 1302 line = self._file.readline() 1303 if line == b'*** EOOH ***' + linesep or not line: 1304 break 1305 original_headers.write(line.replace(linesep, b'\n')) 1306 while True: 1307 line = self._file.readline() 1308 if line == linesep or not line: 1309 break 1310 headers = original_headers.getvalue() 1311 n = stop - self._file.tell() 1312 assert n >= 0 1313 data = self._file.read(n) 1314 data = data.replace(linesep, b'\n') 1315 return headers + data 1316 1317 def get_file(self, key): 1318 """Return a file-like representation or raise a KeyError.""" 1319 return io.BytesIO(self.get_bytes(key).replace(b'\n', linesep)) 1320 1321 def get_labels(self): 1322 """Return a list of user-defined labels in the mailbox.""" 1323 self._lookup() 1324 labels = set() 1325 for label_list in self._labels.values(): 1326 labels.update(label_list) 1327 labels.difference_update(self._special_labels) 1328 return list(labels) 1329 1330 def _generate_toc(self): 1331 """Generate key-to-(start, stop) table of contents.""" 1332 starts, stops = [], [] 1333 self._file.seek(0) 1334 next_pos = 0 1335 label_lists = [] 1336 while True: 1337 line_pos = next_pos 1338 line = self._file.readline() 1339 next_pos = self._file.tell() 1340 if line == b'\037\014' + linesep: 1341 if len(stops) < len(starts): 1342 stops.append(line_pos - len(linesep)) 1343 starts.append(next_pos) 1344 labels = [label.strip() for label 1345 in self._file.readline()[1:].split(b',') 1346 if label.strip()] 1347 label_lists.append(labels) 1348 elif line == b'\037' or line == b'\037' + linesep: 1349 if len(stops) < len(starts): 1350 stops.append(line_pos - len(linesep)) 1351 elif not line: 1352 stops.append(line_pos - len(linesep)) 1353 break 1354 self._toc = dict(enumerate(zip(starts, stops))) 1355 self._labels = dict(enumerate(label_lists)) 1356 self._next_key = len(self._toc) 1357 self._file.seek(0, 2) 1358 self._file_length = self._file.tell() 1359 1360 def _pre_mailbox_hook(self, f): 1361 """Called before writing the mailbox to file f.""" 1362 babyl = b'BABYL OPTIONS:' + linesep 1363 babyl += b'Version: 5' + linesep 1364 labels = self.get_labels() 1365 labels = (label.encode() for label in labels) 1366 babyl += b'Labels:' + b','.join(labels) + linesep 1367 babyl += b'\037' 1368 f.write(babyl) 1369 1370 def _pre_message_hook(self, f): 1371 """Called before writing each message to file f.""" 1372 f.write(b'\014' + linesep) 1373 1374 def _post_message_hook(self, f): 1375 """Called after writing each message to file f.""" 1376 f.write(linesep + b'\037') 1377 1378 def _install_message(self, message): 1379 """Write message contents and return (start, stop).""" 1380 start = self._file.tell() 1381 if isinstance(message, BabylMessage): 1382 special_labels = [] 1383 labels = [] 1384 for label in message.get_labels(): 1385 if label in self._special_labels: 1386 special_labels.append(label) 1387 else: 1388 labels.append(label) 1389 self._file.write(b'1') 1390 for label in special_labels: 1391 self._file.write(b', ' + label.encode()) 1392 self._file.write(b',,') 1393 for label in labels: 1394 self._file.write(b' ' + label.encode() + b',') 1395 self._file.write(linesep) 1396 else: 1397 self._file.write(b'1,,' + linesep) 1398 if isinstance(message, email.message.Message): 1399 orig_buffer = io.BytesIO() 1400 orig_generator = email.generator.BytesGenerator(orig_buffer, False, 0) 1401 orig_generator.flatten(message) 1402 orig_buffer.seek(0) 1403 while True: 1404 line = orig_buffer.readline() 1405 self._file.write(line.replace(b'\n', linesep)) 1406 if line == b'\n' or not line: 1407 break 1408 self._file.write(b'*** EOOH ***' + linesep) 1409 if isinstance(message, BabylMessage): 1410 vis_buffer = io.BytesIO() 1411 vis_generator = email.generator.BytesGenerator(vis_buffer, False, 0) 1412 vis_generator.flatten(message.get_visible()) 1413 while True: 1414 line = vis_buffer.readline() 1415 self._file.write(line.replace(b'\n', linesep)) 1416 if line == b'\n' or not line: 1417 break 1418 else: 1419 orig_buffer.seek(0) 1420 while True: 1421 line = orig_buffer.readline() 1422 self._file.write(line.replace(b'\n', linesep)) 1423 if line == b'\n' or not line: 1424 break 1425 while True: 1426 buffer = orig_buffer.read(4096) # Buffer size is arbitrary. 1427 if not buffer: 1428 break 1429 self._file.write(buffer.replace(b'\n', linesep)) 1430 elif isinstance(message, (bytes, str, io.StringIO)): 1431 if isinstance(message, io.StringIO): 1432 warnings.warn("Use of StringIO input is deprecated, " 1433 "use BytesIO instead", DeprecationWarning, 3) 1434 message = message.getvalue() 1435 if isinstance(message, str): 1436 message = self._string_to_bytes(message) 1437 body_start = message.find(b'\n\n') + 2 1438 if body_start - 2 != -1: 1439 self._file.write(message[:body_start].replace(b'\n', linesep)) 1440 self._file.write(b'*** EOOH ***' + linesep) 1441 self._file.write(message[:body_start].replace(b'\n', linesep)) 1442 self._file.write(message[body_start:].replace(b'\n', linesep)) 1443 else: 1444 self._file.write(b'*** EOOH ***' + linesep + linesep) 1445 self._file.write(message.replace(b'\n', linesep)) 1446 elif hasattr(message, 'readline'): 1447 if hasattr(message, 'buffer'): 1448 warnings.warn("Use of text mode files is deprecated, " 1449 "use a binary mode file instead", DeprecationWarning, 3) 1450 message = message.buffer 1451 original_pos = message.tell() 1452 first_pass = True 1453 while True: 1454 line = message.readline() 1455 # Universal newline support. 1456 if line.endswith(b'\r\n'): 1457 line = line[:-2] + b'\n' 1458 elif line.endswith(b'\r'): 1459 line = line[:-1] + b'\n' 1460 self._file.write(line.replace(b'\n', linesep)) 1461 if line == b'\n' or not line: 1462 if first_pass: 1463 first_pass = False 1464 self._file.write(b'*** EOOH ***' + linesep) 1465 message.seek(original_pos) 1466 else: 1467 break 1468 while True: 1469 line = message.readline() 1470 if not line: 1471 break 1472 # Universal newline support. 1473 if line.endswith(b'\r\n'): 1474 line = line[:-2] + linesep 1475 elif line.endswith(b'\r'): 1476 line = line[:-1] + linesep 1477 elif line.endswith(b'\n'): 1478 line = line[:-1] + linesep 1479 self._file.write(line) 1480 else: 1481 raise TypeError('Invalid message type: %s' % type(message)) 1482 stop = self._file.tell() 1483 return (start, stop) 1484 1485 1486 class Message(email.message.Message): 1487 """Message with mailbox-format-specific properties.""" 1488 1489 def __init__(self, message=None): 1490 """Initialize a Message instance.""" 1491 if isinstance(message, email.message.Message): 1492 self._become_message(copy.deepcopy(message)) 1493 if isinstance(message, Message): 1494 message._explain_to(self) 1495 elif isinstance(message, bytes): 1496 self._become_message(email.message_from_bytes(message)) 1497 elif isinstance(message, str): 1498 self._become_message(email.message_from_string(message)) 1499 elif isinstance(message, io.TextIOWrapper): 1500 self._become_message(email.message_from_file(message)) 1501 elif hasattr(message, "read"): 1502 self._become_message(email.message_from_binary_file(message)) 1503 elif message is None: 1504 email.message.Message.__init__(self) 1505 else: 1506 raise TypeError('Invalid message type: %s' % type(message)) 1507 1508 def _become_message(self, message): 1509 """Assume the non-format-specific state of message.""" 1510 type_specific = getattr(message, '_type_specific_attributes', []) 1511 for name in message.__dict__: 1512 if name not in type_specific: 1513 self.__dict__[name] = message.__dict__[name] 1514 1515 def _explain_to(self, message): 1516 """Copy format-specific state to message insofar as possible.""" 1517 if isinstance(message, Message): 1518 return # There's nothing format-specific to explain. 1519 else: 1520 raise TypeError('Cannot convert to specified type') 1521 1522 1523 class MaildirMessage(Message): 1524 """Message with Maildir-specific properties.""" 1525 1526 _type_specific_attributes = ['_subdir', '_info', '_date'] 1527 1528 def __init__(self, message=None): 1529 """Initialize a MaildirMessage instance.""" 1530 self._subdir = 'new' 1531 self._info = '' 1532 self._date = time.time() 1533 Message.__init__(self, message) 1534 1535 def get_subdir(self): 1536 """Return 'new' or 'cur'.""" 1537 return self._subdir 1538 1539 def set_subdir(self, subdir): 1540 """Set subdir to 'new' or 'cur'.""" 1541 if subdir == 'new' or subdir == 'cur': 1542 self._subdir = subdir 1543 else: 1544 raise ValueError("subdir must be 'new' or 'cur': %s" % subdir) 1545 1546 def get_flags(self): 1547 """Return as a string the flags that are set.""" 1548 if self._info.startswith('2,'): 1549 return self._info[2:] 1550 else: 1551 return '' 1552 1553 def set_flags(self, flags): 1554 """Set the given flags and unset all others.""" 1555 self._info = '2,' + ''.join(sorted(flags)) 1556 1557 def add_flag(self, flag): 1558 """Set the given flag(s) without changing others.""" 1559 self.set_flags(''.join(set(self.get_flags()) | set(flag))) 1560 1561 def remove_flag(self, flag): 1562 """Unset the given string flag(s) without changing others.""" 1563 if self.get_flags(): 1564 self.set_flags(''.join(set(self.get_flags()) - set(flag))) 1565 1566 def get_date(self): 1567 """Return delivery date of message, in seconds since the epoch.""" 1568 return self._date 1569 1570 def set_date(self, date): 1571 """Set delivery date of message, in seconds since the epoch.""" 1572 try: 1573 self._date = float(date) 1574 except ValueError: 1575 raise TypeError("can't convert to float: %s" % date) 1576 1577 def get_info(self): 1578 """Get the message's "info" as a string.""" 1579 return self._info 1580 1581 def set_info(self, info): 1582 """Set the message's "info" string.""" 1583 if isinstance(info, str): 1584 self._info = info 1585 else: 1586 raise TypeError('info must be a string: %s' % type(info)) 1587 1588 def _explain_to(self, message): 1589 """Copy Maildir-specific state to message insofar as possible.""" 1590 if isinstance(message, MaildirMessage): 1591 message.set_flags(self.get_flags()) 1592 message.set_subdir(self.get_subdir()) 1593 message.set_date(self.get_date()) 1594 elif isinstance(message, _mboxMMDFMessage): 1595 flags = set(self.get_flags()) 1596 if 'S' in flags: 1597 message.add_flag('R') 1598 if self.get_subdir() == 'cur': 1599 message.add_flag('O') 1600 if 'T' in flags: 1601 message.add_flag('D') 1602 if 'F' in flags: 1603 message.add_flag('F') 1604 if 'R' in flags: 1605 message.add_flag('A') 1606 message.set_from('MAILER-DAEMON', time.gmtime(self.get_date())) 1607 elif isinstance(message, MHMessage): 1608 flags = set(self.get_flags()) 1609 if 'S' not in flags: 1610 message.add_sequence('unseen') 1611 if 'R' in flags: 1612 message.add_sequence('replied') 1613 if 'F' in flags: 1614 message.add_sequence('flagged') 1615 elif isinstance(message, BabylMessage): 1616 flags = set(self.get_flags()) 1617 if 'S' not in flags: 1618 message.add_label('unseen') 1619 if 'T' in flags: 1620 message.add_label('deleted') 1621 if 'R' in flags: 1622 message.add_label('answered') 1623 if 'P' in flags: 1624 message.add_label('forwarded') 1625 elif isinstance(message, Message): 1626 pass 1627 else: 1628 raise TypeError('Cannot convert to specified type: %s' % 1629 type(message)) 1630 1631 1632 class _mboxMMDFMessage(Message): 1633 """Message with mbox- or MMDF-specific properties.""" 1634 1635 _type_specific_attributes = ['_from'] 1636 1637 def __init__(self, message=None): 1638 """Initialize an mboxMMDFMessage instance.""" 1639 self.set_from('MAILER-DAEMON', True) 1640 if isinstance(message, email.message.Message): 1641 unixfrom = message.get_unixfrom() 1642 if unixfrom is not None and unixfrom.startswith('From '): 1643 self.set_from(unixfrom[5:]) 1644 Message.__init__(self, message) 1645 1646 def get_from(self): 1647 """Return contents of "From " line.""" 1648 return self._from 1649 1650 def set_from(self, from_, time_=None): 1651 """Set "From " line, formatting and appending time_ if specified.""" 1652 if time_ is not None: 1653 if time_ is True: 1654 time_ = time.gmtime() 1655 from_ += ' ' + time.asctime(time_) 1656 self._from = from_ 1657 1658 def get_flags(self): 1659 """Return as a string the flags that are set.""" 1660 return self.get('Status', '') + self.get('X-Status', '') 1661 1662 def set_flags(self, flags): 1663 """Set the given flags and unset all others.""" 1664 flags = set(flags) 1665 status_flags, xstatus_flags = '', '' 1666 for flag in ('R', 'O'): 1667 if flag in flags: 1668 status_flags += flag 1669 flags.remove(flag) 1670 for flag in ('D', 'F', 'A'): 1671 if flag in flags: 1672 xstatus_flags += flag 1673 flags.remove(flag) 1674 xstatus_flags += ''.join(sorted(flags)) 1675 try: 1676 self.replace_header('Status', status_flags) 1677 except KeyError: 1678 self.add_header('Status', status_flags) 1679 try: 1680 self.replace_header('X-Status', xstatus_flags) 1681 except KeyError: 1682 self.add_header('X-Status', xstatus_flags) 1683 1684 def add_flag(self, flag): 1685 """Set the given flag(s) without changing others.""" 1686 self.set_flags(''.join(set(self.get_flags()) | set(flag))) 1687 1688 def remove_flag(self, flag): 1689 """Unset the given string flag(s) without changing others.""" 1690 if 'Status' in self or 'X-Status' in self: 1691 self.set_flags(''.join(set(self.get_flags()) - set(flag))) 1692 1693 def _explain_to(self, message): 1694 """Copy mbox- or MMDF-specific state to message insofar as possible.""" 1695 if isinstance(message, MaildirMessage): 1696 flags = set(self.get_flags()) 1697 if 'O' in flags: 1698 message.set_subdir('cur') 1699 if 'F' in flags: 1700 message.add_flag('F') 1701 if 'A' in flags: 1702 message.add_flag('R') 1703 if 'R' in flags: 1704 message.add_flag('S') 1705 if 'D' in flags: 1706 message.add_flag('T') 1707 del message['status'] 1708 del message['x-status'] 1709 maybe_date = ' '.join(self.get_from().split()[-5:]) 1710 try: 1711 message.set_date(calendar.timegm(time.strptime(maybe_date, 1712 '%a %b %d %H:%M:%S %Y'))) 1713 except (ValueError, OverflowError): 1714 pass 1715 elif isinstance(message, _mboxMMDFMessage): 1716 message.set_flags(self.get_flags()) 1717 message.set_from(self.get_from()) 1718 elif isinstance(message, MHMessage): 1719 flags = set(self.get_flags()) 1720 if 'R' not in flags: 1721 message.add_sequence('unseen') 1722 if 'A' in flags: 1723 message.add_sequence('replied') 1724 if 'F' in flags: 1725 message.add_sequence('flagged') 1726 del message['status'] 1727 del message['x-status'] 1728 elif isinstance(message, BabylMessage): 1729 flags = set(self.get_flags()) 1730 if 'R' not in flags: 1731 message.add_label('unseen') 1732 if 'D' in flags: 1733 message.add_label('deleted') 1734 if 'A' in flags: 1735 message.add_label('answered') 1736 del message['status'] 1737 del message['x-status'] 1738 elif isinstance(message, Message): 1739 pass 1740 else: 1741 raise TypeError('Cannot convert to specified type: %s' % 1742 type(message)) 1743 1744 1745 class mboxMessage(_mboxMMDFMessage): 1746 """Message with mbox-specific properties.""" 1747 1748 1749 class MHMessage(Message): 1750 """Message with MH-specific properties.""" 1751 1752 _type_specific_attributes = ['_sequences'] 1753 1754 def __init__(self, message=None): 1755 """Initialize an MHMessage instance.""" 1756 self._sequences = [] 1757 Message.__init__(self, message) 1758 1759 def get_sequences(self): 1760 """Return a list of sequences that include the message.""" 1761 return self._sequences[:] 1762 1763 def set_sequences(self, sequences): 1764 """Set the list of sequences that include the message.""" 1765 self._sequences = list(sequences) 1766 1767 def add_sequence(self, sequence): 1768 """Add sequence to list of sequences including the message.""" 1769 if isinstance(sequence, str): 1770 if not sequence in self._sequences: 1771 self._sequences.append(sequence) 1772 else: 1773 raise TypeError('sequence type must be str: %s' % type(sequence)) 1774 1775 def remove_sequence(self, sequence): 1776 """Remove sequence from the list of sequences including the message.""" 1777 try: 1778 self._sequences.remove(sequence) 1779 except ValueError: 1780 pass 1781 1782 def _explain_to(self, message): 1783 """Copy MH-specific state to message insofar as possible.""" 1784 if isinstance(message, MaildirMessage): 1785 sequences = set(self.get_sequences()) 1786 if 'unseen' in sequences: 1787 message.set_subdir('cur') 1788 else: 1789 message.set_subdir('cur') 1790 message.add_flag('S') 1791 if 'flagged' in sequences: 1792 message.add_flag('F') 1793 if 'replied' in sequences: 1794 message.add_flag('R') 1795 elif isinstance(message, _mboxMMDFMessage): 1796 sequences = set(self.get_sequences()) 1797 if 'unseen' not in sequences: 1798 message.add_flag('RO') 1799 else: 1800 message.add_flag('O') 1801 if 'flagged' in sequences: 1802 message.add_flag('F') 1803 if 'replied' in sequences: 1804 message.add_flag('A') 1805 elif isinstance(message, MHMessage): 1806 for sequence in self.get_sequences(): 1807 message.add_sequence(sequence) 1808 elif isinstance(message, BabylMessage): 1809 sequences = set(self.get_sequences()) 1810 if 'unseen' in sequences: 1811 message.add_label('unseen') 1812 if 'replied' in sequences: 1813 message.add_label('answered') 1814 elif isinstance(message, Message): 1815 pass 1816 else: 1817 raise TypeError('Cannot convert to specified type: %s' % 1818 type(message)) 1819 1820 1821 class BabylMessage(Message): 1822 """Message with Babyl-specific properties.""" 1823 1824 _type_specific_attributes = ['_labels', '_visible'] 1825 1826 def __init__(self, message=None): 1827 """Initialize a BabylMessage instance.""" 1828 self._labels = [] 1829 self._visible = Message() 1830 Message.__init__(self, message) 1831 1832 def get_labels(self): 1833 """Return a list of labels on the message.""" 1834 return self._labels[:] 1835 1836 def set_labels(self, labels): 1837 """Set the list of labels on the message.""" 1838 self._labels = list(labels) 1839 1840 def add_label(self, label): 1841 """Add label to list of labels on the message.""" 1842 if isinstance(label, str): 1843 if label not in self._labels: 1844 self._labels.append(label) 1845 else: 1846 raise TypeError('label must be a string: %s' % type(label)) 1847 1848 def remove_label(self, label): 1849 """Remove label from the list of labels on the message.""" 1850 try: 1851 self._labels.remove(label) 1852 except ValueError: 1853 pass 1854 1855 def get_visible(self): 1856 """Return a Message representation of visible headers.""" 1857 return Message(self._visible) 1858 1859 def set_visible(self, visible): 1860 """Set the Message representation of visible headers.""" 1861 self._visible = Message(visible) 1862 1863 def update_visible(self): 1864 """Update and/or sensibly generate a set of visible headers.""" 1865 for header in self._visible.keys(): 1866 if header in self: 1867 self._visible.replace_header(header, self[header]) 1868 else: 1869 del self._visible[header] 1870 for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'): 1871 if header in self and header not in self._visible: 1872 self._visible[header] = self[header] 1873 1874 def _explain_to(self, message): 1875 """Copy Babyl-specific state to message insofar as possible.""" 1876 if isinstance(message, MaildirMessage): 1877 labels = set(self.get_labels()) 1878 if 'unseen' in labels: 1879 message.set_subdir('cur') 1880 else: 1881 message.set_subdir('cur') 1882 message.add_flag('S') 1883 if 'forwarded' in labels or 'resent' in labels: 1884 message.add_flag('P') 1885 if 'answered' in labels: 1886 message.add_flag('R') 1887 if 'deleted' in labels: 1888 message.add_flag('T') 1889 elif isinstance(message, _mboxMMDFMessage): 1890 labels = set(self.get_labels()) 1891 if 'unseen' not in labels: 1892 message.add_flag('RO') 1893 else: 1894 message.add_flag('O') 1895 if 'deleted' in labels: 1896 message.add_flag('D') 1897 if 'answered' in labels: 1898 message.add_flag('A') 1899 elif isinstance(message, MHMessage): 1900 labels = set(self.get_labels()) 1901 if 'unseen' in labels: 1902 message.add_sequence('unseen') 1903 if 'answered' in labels: 1904 message.add_sequence('replied') 1905 elif isinstance(message, BabylMessage): 1906 message.set_visible(self.get_visible()) 1907 for label in self.get_labels(): 1908 message.add_label(label) 1909 elif isinstance(message, Message): 1910 pass 1911 else: 1912 raise TypeError('Cannot convert to specified type: %s' % 1913 type(message)) 1914 1915 1916 class MMDFMessage(_mboxMMDFMessage): 1917 """Message with MMDF-specific properties.""" 1918 1919 1920 class _ProxyFile: 1921 """A read-only wrapper of a file.""" 1922 1923 def __init__(self, f, pos=None): 1924 """Initialize a _ProxyFile.""" 1925 self._file = f 1926 if pos is None: 1927 self._pos = f.tell() 1928 else: 1929 self._pos = pos 1930 1931 def read(self, size=None): 1932 """Read bytes.""" 1933 return self._read(size, self._file.read) 1934 1935 def read1(self, size=None): 1936 """Read bytes.""" 1937 return self._read(size, self._file.read1) 1938 1939 def readline(self, size=None): 1940 """Read a line.""" 1941 return self._read(size, self._file.readline) 1942 1943 def readlines(self, sizehint=None): 1944 """Read multiple lines.""" 1945 result = [] 1946 for line in self: 1947 result.append(line) 1948 if sizehint is not None: 1949 sizehint -= len(line) 1950 if sizehint <= 0: 1951 break 1952 return result 1953 1954 def __iter__(self): 1955 """Iterate over lines.""" 1956 while True: 1957 line = self.readline() 1958 if not line: 1959 return 1960 yield line 1961 1962 def tell(self): 1963 """Return the position.""" 1964 return self._pos 1965 1966 def seek(self, offset, whence=0): 1967 """Change position.""" 1968 if whence == 1: 1969 self._file.seek(self._pos) 1970 self._file.seek(offset, whence) 1971 self._pos = self._file.tell() 1972 1973 def close(self): 1974 """Close the file.""" 1975 if hasattr(self, '_file'): 1976 try: 1977 if hasattr(self._file, 'close'): 1978 self._file.close() 1979 finally: 1980 del self._file 1981 1982 def _read(self, size, read_method): 1983 """Read size bytes using read_method.""" 1984 if size is None: 1985 size = -1 1986 self._file.seek(self._pos) 1987 result = read_method(size) 1988 self._pos = self._file.tell() 1989 return result 1990 1991 def __enter__(self): 1992 """Context management protocol support.""" 1993 return self 1994 1995 def __exit__(self, *exc): 1996 self.close() 1997 1998 def readable(self): 1999 return self._file.readable() 2000 2001 def writable(self): 2002 return self._file.writable() 2003 2004 def seekable(self): 2005 return self._file.seekable() 2006 2007 def flush(self): 2008 return self._file.flush() 2009 2010 @property 2011 def closed(self): 2012 if not hasattr(self, '_file'): 2013 return True 2014 if not hasattr(self._file, 'closed'): 2015 return False 2016 return self._file.closed 2017 2018 2019 class _PartialFile(_ProxyFile): 2020 """A read-only wrapper of part of a file.""" 2021 2022 def __init__(self, f, start=None, stop=None): 2023 """Initialize a _PartialFile.""" 2024 _ProxyFile.__init__(self, f, start) 2025 self._start = start 2026 self._stop = stop 2027 2028 def tell(self): 2029 """Return the position with respect to start.""" 2030 return _ProxyFile.tell(self) - self._start 2031 2032 def seek(self, offset, whence=0): 2033 """Change position, possibly with respect to start or stop.""" 2034 if whence == 0: 2035 self._pos = self._start 2036 whence = 1 2037 elif whence == 2: 2038 self._pos = self._stop 2039 whence = 1 2040 _ProxyFile.seek(self, offset, whence) 2041 2042 def _read(self, size, read_method): 2043 """Read size bytes using read_method, honoring start and stop.""" 2044 remaining = self._stop - self._pos 2045 if remaining <= 0: 2046 return b'' 2047 if size is None or size < 0 or size > remaining: 2048 size = remaining 2049 return _ProxyFile._read(self, size, read_method) 2050 2051 def close(self): 2052 # do *not* close the underlying file object for partial files, 2053 # since it's global to the mailbox object 2054 if hasattr(self, '_file'): 2055 del self._file 2056 2057 2058 def _lock_file(f, dotlock=True): 2059 """Lock file f using lockf and dot locking.""" 2060 dotlock_done = False 2061 try: 2062 if fcntl: 2063 try: 2064 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB) 2065 except OSError as e: 2066 if e.errno in (errno.EAGAIN, errno.EACCES, errno.EROFS): 2067 raise ExternalClashError('lockf: lock unavailable: %s' % 2068 f.name) 2069 else: 2070 raise 2071 if dotlock: 2072 try: 2073 pre_lock = _create_temporary(f.name + '.lock') 2074 pre_lock.close() 2075 except OSError as e: 2076 if e.errno in (errno.EACCES, errno.EROFS): 2077 return # Without write access, just skip dotlocking. 2078 else: 2079 raise 2080 try: 2081 try: 2082 os.link(pre_lock.name, f.name + '.lock') 2083 dotlock_done = True 2084 except (AttributeError, PermissionError): 2085 os.rename(pre_lock.name, f.name + '.lock') 2086 dotlock_done = True 2087 else: 2088 os.unlink(pre_lock.name) 2089 except FileExistsError: 2090 os.remove(pre_lock.name) 2091 raise ExternalClashError('dot lock unavailable: %s' % 2092 f.name) 2093 except: 2094 if fcntl: 2095 fcntl.lockf(f, fcntl.LOCK_UN) 2096 if dotlock_done: 2097 os.remove(f.name + '.lock') 2098 raise 2099 2100 def _unlock_file(f): 2101 """Unlock file f using lockf and dot locking.""" 2102 if fcntl: 2103 fcntl.lockf(f, fcntl.LOCK_UN) 2104 if os.path.exists(f.name + '.lock'): 2105 os.remove(f.name + '.lock') 2106 2107 def _create_carefully(path): 2108 """Create a file if it doesn't exist and open for reading and writing.""" 2109 fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666) 2110 try: 2111 return open(path, 'rb+') 2112 finally: 2113 os.close(fd) 2114 2115 def _create_temporary(path): 2116 """Create a temp file based on path and open for reading and writing.""" 2117 return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()), 2118 socket.gethostname(), 2119 os.getpid())) 2120 2121 def _sync_flush(f): 2122 """Ensure changes to file f are physically on disk.""" 2123 f.flush() 2124 if hasattr(os, 'fsync'): 2125 os.fsync(f.fileno()) 2126 2127 def _sync_close(f): 2128 """Close file f, ensuring all changes are physically on disk.""" 2129 _sync_flush(f) 2130 f.close() 2131 2132 2133 class Error(Exception): 2134 """Raised for module-specific errors.""" 2135 2136 class NoSuchMailboxError(Error): 2137 """The specified mailbox does not exist and won't be created.""" 2138 2139 class NotEmptyError(Error): 2140 """The specified mailbox is not empty and deletion was requested.""" 2141 2142 class ExternalClashError(Error): 2143 """Another process caused an action to fail.""" 2144 2145 class FormatError(Error): 2146 """A file appears to have an invalid format.""" 2147