Home | History | Annotate | Download | only in python2.7
      1 """A dumb and slow but simple dbm clone.
      2 
      3 For database spam, spam.dir contains the index (a text file),
      4 spam.bak *may* contain a backup of the index (also a text file),
      5 while spam.dat contains the data (a binary file).
      6 
      7 XXX TO DO:
      8 
      9 - seems to contain a bug when updating...
     10 
     11 - reclaim free space (currently, space once occupied by deleted or expanded
     12 items is never reused)
     13 
     14 - support concurrent access (currently, if two processes take turns making
     15 updates, they can mess up the index)
     16 
     17 - support efficient access to large databases (currently, the whole index
     18 is read when the database is opened, and some updates rewrite the whole index)
     19 
     20 - support opening for read-only (flag = 'm')
     21 
     22 """
     23 
     24 import os as _os
     25 import __builtin__
     26 import UserDict
     27 
     28 _open = __builtin__.open
     29 
     30 _BLOCKSIZE = 512
     31 
     32 error = IOError                         # For anydbm
     33 
     34 class _Database(UserDict.DictMixin):
     35 
     36     # The on-disk directory and data files can remain in mutually
     37     # inconsistent states for an arbitrarily long time (see comments
     38     # at the end of __setitem__).  This is only repaired when _commit()
     39     # gets called.  One place _commit() gets called is from __del__(),
     40     # and if that occurs at program shutdown time, module globals may
     41     # already have gotten rebound to None.  Since it's crucial that
     42     # _commit() finish successfully, we can't ignore shutdown races
     43     # here, and _commit() must not reference any globals.
     44     _os = _os       # for _commit()
     45     _open = _open   # for _commit()
     46 
     47     def __init__(self, filebasename, mode):
     48         self._mode = mode
     49 
     50         # The directory file is a text file.  Each line looks like
     51         #    "%r, (%d, %d)\n" % (key, pos, siz)
     52         # where key is the string key, pos is the offset into the dat
     53         # file of the associated value's first byte, and siz is the number
     54         # of bytes in the associated value.
     55         self._dirfile = filebasename + _os.extsep + 'dir'
     56 
     57         # The data file is a binary file pointed into by the directory
     58         # file, and holds the values associated with keys.  Each value
     59         # begins at a _BLOCKSIZE-aligned byte offset, and is a raw
     60         # binary 8-bit string value.
     61         self._datfile = filebasename + _os.extsep + 'dat'
     62         self._bakfile = filebasename + _os.extsep + 'bak'
     63 
     64         # The index is an in-memory dict, mirroring the directory file.
     65         self._index = None  # maps keys to (pos, siz) pairs
     66 
     67         # Mod by Jack: create data file if needed
     68         try:
     69             f = _open(self._datfile, 'r')
     70         except IOError:
     71             f = _open(self._datfile, 'w')
     72             self._chmod(self._datfile)
     73         f.close()
     74         self._update()
     75 
     76     # Read directory file into the in-memory index dict.
     77     def _update(self):
     78         self._index = {}
     79         try:
     80             f = _open(self._dirfile)
     81         except IOError:
     82             pass
     83         else:
     84             for line in f:
     85                 line = line.rstrip()
     86                 key, pos_and_siz_pair = eval(line)
     87                 self._index[key] = pos_and_siz_pair
     88             f.close()
     89 
     90     # Write the index dict to the directory file.  The original directory
     91     # file (if any) is renamed with a .bak extension first.  If a .bak
     92     # file currently exists, it's deleted.
     93     def _commit(self):
     94         # CAUTION:  It's vital that _commit() succeed, and _commit() can
     95         # be called from __del__().  Therefore we must never reference a
     96         # global in this routine.
     97         if self._index is None:
     98             return  # nothing to do
     99 
    100         try:
    101             self._os.unlink(self._bakfile)
    102         except self._os.error:
    103             pass
    104 
    105         try:
    106             self._os.rename(self._dirfile, self._bakfile)
    107         except self._os.error:
    108             pass
    109 
    110         f = self._open(self._dirfile, 'w')
    111         self._chmod(self._dirfile)
    112         for key, pos_and_siz_pair in self._index.iteritems():
    113             f.write("%r, %r\n" % (key, pos_and_siz_pair))
    114         f.close()
    115 
    116     sync = _commit
    117 
    118     def __getitem__(self, key):
    119         pos, siz = self._index[key]     # may raise KeyError
    120         f = _open(self._datfile, 'rb')
    121         f.seek(pos)
    122         dat = f.read(siz)
    123         f.close()
    124         return dat
    125 
    126     # Append val to the data file, starting at a _BLOCKSIZE-aligned
    127     # offset.  The data file is first padded with NUL bytes (if needed)
    128     # to get to an aligned offset.  Return pair
    129     #     (starting offset of val, len(val))
    130     def _addval(self, val):
    131         f = _open(self._datfile, 'rb+')
    132         f.seek(0, 2)
    133         pos = int(f.tell())
    134         npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
    135         f.write('\0'*(npos-pos))
    136         pos = npos
    137         f.write(val)
    138         f.close()
    139         return (pos, len(val))
    140 
    141     # Write val to the data file, starting at offset pos.  The caller
    142     # is responsible for ensuring that there's enough room starting at
    143     # pos to hold val, without overwriting some other value.  Return
    144     # pair (pos, len(val)).
    145     def _setval(self, pos, val):
    146         f = _open(self._datfile, 'rb+')
    147         f.seek(pos)
    148         f.write(val)
    149         f.close()
    150         return (pos, len(val))
    151 
    152     # key is a new key whose associated value starts in the data file
    153     # at offset pos and with length siz.  Add an index record to
    154     # the in-memory index dict, and append one to the directory file.
    155     def _addkey(self, key, pos_and_siz_pair):
    156         self._index[key] = pos_and_siz_pair
    157         f = _open(self._dirfile, 'a')
    158         self._chmod(self._dirfile)
    159         f.write("%r, %r\n" % (key, pos_and_siz_pair))
    160         f.close()
    161 
    162     def __setitem__(self, key, val):
    163         if not type(key) == type('') == type(val):
    164             raise TypeError, "keys and values must be strings"
    165         if key not in self._index:
    166             self._addkey(key, self._addval(val))
    167         else:
    168             # See whether the new value is small enough to fit in the
    169             # (padded) space currently occupied by the old value.
    170             pos, siz = self._index[key]
    171             oldblocks = (siz + _BLOCKSIZE - 1) // _BLOCKSIZE
    172             newblocks = (len(val) + _BLOCKSIZE - 1) // _BLOCKSIZE
    173             if newblocks <= oldblocks:
    174                 self._index[key] = self._setval(pos, val)
    175             else:
    176                 # The new value doesn't fit in the (padded) space used
    177                 # by the old value.  The blocks used by the old value are
    178                 # forever lost.
    179                 self._index[key] = self._addval(val)
    180 
    181             # Note that _index may be out of synch with the directory
    182             # file now:  _setval() and _addval() don't update the directory
    183             # file.  This also means that the on-disk directory and data
    184             # files are in a mutually inconsistent state, and they'll
    185             # remain that way until _commit() is called.  Note that this
    186             # is a disaster (for the database) if the program crashes
    187             # (so that _commit() never gets called).
    188 
    189     def __delitem__(self, key):
    190         # The blocks used by the associated value are lost.
    191         del self._index[key]
    192         # XXX It's unclear why we do a _commit() here (the code always
    193         # XXX has, so I'm not changing it).  _setitem__ doesn't try to
    194         # XXX keep the directory file in synch.  Why should we?  Or
    195         # XXX why shouldn't __setitem__?
    196         self._commit()
    197 
    198     def keys(self):
    199         return self._index.keys()
    200 
    201     def has_key(self, key):
    202         return key in self._index
    203 
    204     def __contains__(self, key):
    205         return key in self._index
    206 
    207     def iterkeys(self):
    208         return self._index.iterkeys()
    209     __iter__ = iterkeys
    210 
    211     def __len__(self):
    212         return len(self._index)
    213 
    214     def close(self):
    215         self._commit()
    216         self._index = self._datfile = self._dirfile = self._bakfile = None
    217 
    218     __del__ = close
    219 
    220     def _chmod (self, file):
    221         if hasattr(self._os, 'chmod'):
    222             self._os.chmod(file, self._mode)
    223 
    224 
    225 def open(file, flag=None, mode=0666):
    226     """Open the database file, filename, and return corresponding object.
    227 
    228     The flag argument, used to control how the database is opened in the
    229     other DBM implementations, is ignored in the dumbdbm module; the
    230     database is always opened for update, and will be created if it does
    231     not exist.
    232 
    233     The optional mode argument is the UNIX mode of the file, used only when
    234     the database has to be created.  It defaults to octal code 0666 (and
    235     will be modified by the prevailing umask).
    236 
    237     """
    238     # flag argument is currently ignored
    239 
    240     # Modify mode depending on the umask
    241     try:
    242         um = _os.umask(0)
    243         _os.umask(um)
    244     except AttributeError:
    245         pass
    246     else:
    247         # Turn off any bits that are set in the umask
    248         mode = mode & (~um)
    249 
    250     return _Database(file, mode)
    251