1 #---------------------------------------------------------------------- 2 # Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA 3 # and Andrew Kuchling. All rights reserved. 4 # 5 # Redistribution and use in source and binary forms, with or without 6 # modification, are permitted provided that the following conditions are 7 # met: 8 # 9 # o Redistributions of source code must retain the above copyright 10 # notice, this list of conditions, and the disclaimer that follows. 11 # 12 # o Redistributions in binary form must reproduce the above copyright 13 # notice, this list of conditions, and the following disclaimer in 14 # the documentation and/or other materials provided with the 15 # distribution. 16 # 17 # o Neither the name of Digital Creations nor the names of its 18 # contributors may be used to endorse or promote products derived 19 # from this software without specific prior written permission. 20 # 21 # THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS 22 # IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 24 # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DIGITAL 25 # CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 28 # OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 29 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 30 # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 31 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 32 # DAMAGE. 33 #---------------------------------------------------------------------- 34 35 36 """Support for Berkeley DB 4.3 through 5.3 with a simple interface. 37 38 For the full featured object oriented interface use the bsddb.db module 39 instead. It mirrors the Oracle Berkeley DB C API. 40 """ 41 42 import sys 43 absolute_import = (sys.version_info[0] >= 3) 44 45 if (sys.version_info >= (2, 6)) and (sys.version_info < (3, 0)) : 46 import warnings 47 if sys.py3kwarning and (__name__ != 'bsddb3') : 48 warnings.warnpy3k("in 3.x, the bsddb module has been removed; " 49 "please use the pybsddb project instead", 50 DeprecationWarning, 2) 51 warnings.filterwarnings("ignore", ".*CObject.*", DeprecationWarning, 52 "bsddb.__init__") 53 54 try: 55 if __name__ == 'bsddb3': 56 # import _pybsddb binary as it should be the more recent version from 57 # a standalone pybsddb addon package than the version included with 58 # python as bsddb._bsddb. 59 if absolute_import : 60 # Because this syntaxis is not valid before Python 2.5 61 exec("from . import _pybsddb") 62 else : 63 import _pybsddb 64 _bsddb = _pybsddb 65 from bsddb3.dbutils import DeadlockWrap as _DeadlockWrap 66 else: 67 import _bsddb 68 from bsddb.dbutils import DeadlockWrap as _DeadlockWrap 69 except ImportError: 70 # Remove ourselves from sys.modules 71 import sys 72 del sys.modules[__name__] 73 raise 74 75 # bsddb3 calls it db, but provide _db for backwards compatibility 76 db = _db = _bsddb 77 __version__ = db.__version__ 78 79 error = db.DBError # So bsddb.error will mean something... 80 81 #---------------------------------------------------------------------- 82 83 import sys, os 84 85 from weakref import ref 86 87 if sys.version_info < (2, 6) : 88 import UserDict 89 MutableMapping = UserDict.DictMixin 90 else : 91 import collections 92 MutableMapping = collections.MutableMapping 93 94 class _iter_mixin(MutableMapping): 95 def _make_iter_cursor(self): 96 cur = _DeadlockWrap(self.db.cursor) 97 key = id(cur) 98 self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key)) 99 return cur 100 101 def _gen_cref_cleaner(self, key): 102 # use generate the function for the weakref callback here 103 # to ensure that we do not hold a strict reference to cur 104 # in the callback. 105 return lambda ref: self._cursor_refs.pop(key, None) 106 107 def __iter__(self): 108 self._kill_iteration = False 109 self._in_iter += 1 110 try: 111 try: 112 cur = self._make_iter_cursor() 113 114 # FIXME-20031102-greg: race condition. cursor could 115 # be closed by another thread before this call. 116 117 # since we're only returning keys, we call the cursor 118 # methods with flags=0, dlen=0, dofs=0 119 key = _DeadlockWrap(cur.first, 0,0,0)[0] 120 yield key 121 122 next = getattr(cur, "next") 123 while 1: 124 try: 125 key = _DeadlockWrap(next, 0,0,0)[0] 126 yield key 127 except _bsddb.DBCursorClosedError: 128 if self._kill_iteration: 129 raise RuntimeError('Database changed size ' 130 'during iteration.') 131 cur = self._make_iter_cursor() 132 # FIXME-20031101-greg: race condition. cursor could 133 # be closed by another thread before this call. 134 _DeadlockWrap(cur.set, key,0,0,0) 135 next = getattr(cur, "next") 136 except _bsddb.DBNotFoundError: 137 pass 138 except _bsddb.DBCursorClosedError: 139 # the database was modified during iteration. abort. 140 pass 141 # When Python 2.4 not supported in bsddb3, we can change this to "finally" 142 except : 143 self._in_iter -= 1 144 raise 145 146 self._in_iter -= 1 147 148 def iteritems(self): 149 if not self.db: 150 return 151 self._kill_iteration = False 152 self._in_iter += 1 153 try: 154 try: 155 cur = self._make_iter_cursor() 156 157 # FIXME-20031102-greg: race condition. cursor could 158 # be closed by another thread before this call. 159 160 kv = _DeadlockWrap(cur.first) 161 key = kv[0] 162 yield kv 163 164 next = getattr(cur, "next") 165 while 1: 166 try: 167 kv = _DeadlockWrap(next) 168 key = kv[0] 169 yield kv 170 except _bsddb.DBCursorClosedError: 171 if self._kill_iteration: 172 raise RuntimeError('Database changed size ' 173 'during iteration.') 174 cur = self._make_iter_cursor() 175 # FIXME-20031101-greg: race condition. cursor could 176 # be closed by another thread before this call. 177 _DeadlockWrap(cur.set, key,0,0,0) 178 next = getattr(cur, "next") 179 except _bsddb.DBNotFoundError: 180 pass 181 except _bsddb.DBCursorClosedError: 182 # the database was modified during iteration. abort. 183 pass 184 # When Python 2.4 not supported in bsddb3, we can change this to "finally" 185 except : 186 self._in_iter -= 1 187 raise 188 189 self._in_iter -= 1 190 191 192 class _DBWithCursor(_iter_mixin): 193 """ 194 A simple wrapper around DB that makes it look like the bsddbobject in 195 the old module. It uses a cursor as needed to provide DB traversal. 196 """ 197 def __init__(self, db): 198 self.db = db 199 self.db.set_get_returns_none(0) 200 201 # FIXME-20031101-greg: I believe there is still the potential 202 # for deadlocks in a multithreaded environment if someone 203 # attempts to use the any of the cursor interfaces in one 204 # thread while doing a put or delete in another thread. The 205 # reason is that _checkCursor and _closeCursors are not atomic 206 # operations. Doing our own locking around self.dbc, 207 # self.saved_dbc_key and self._cursor_refs could prevent this. 208 # TODO: A test case demonstrating the problem needs to be written. 209 210 # self.dbc is a DBCursor object used to implement the 211 # first/next/previous/last/set_location methods. 212 self.dbc = None 213 self.saved_dbc_key = None 214 215 # a collection of all DBCursor objects currently allocated 216 # by the _iter_mixin interface. 217 self._cursor_refs = {} 218 self._in_iter = 0 219 self._kill_iteration = False 220 221 def __del__(self): 222 self.close() 223 224 def _checkCursor(self): 225 if self.dbc is None: 226 self.dbc = _DeadlockWrap(self.db.cursor) 227 if self.saved_dbc_key is not None: 228 _DeadlockWrap(self.dbc.set, self.saved_dbc_key) 229 self.saved_dbc_key = None 230 231 # This method is needed for all non-cursor DB calls to avoid 232 # Berkeley DB deadlocks (due to being opened with DB_INIT_LOCK 233 # and DB_THREAD to be thread safe) when intermixing database 234 # operations that use the cursor internally with those that don't. 235 def _closeCursors(self, save=1): 236 if self.dbc: 237 c = self.dbc 238 self.dbc = None 239 if save: 240 try: 241 self.saved_dbc_key = _DeadlockWrap(c.current, 0,0,0)[0] 242 except db.DBError: 243 pass 244 _DeadlockWrap(c.close) 245 del c 246 for cref in self._cursor_refs.values(): 247 c = cref() 248 if c is not None: 249 _DeadlockWrap(c.close) 250 251 def _checkOpen(self): 252 if self.db is None: 253 raise error, "BSDDB object has already been closed" 254 255 def isOpen(self): 256 return self.db is not None 257 258 def __len__(self): 259 self._checkOpen() 260 return _DeadlockWrap(lambda: len(self.db)) # len(self.db) 261 262 if sys.version_info >= (2, 6) : 263 def __repr__(self) : 264 if self.isOpen() : 265 return repr(dict(_DeadlockWrap(self.db.items))) 266 return repr(dict()) 267 268 def __getitem__(self, key): 269 self._checkOpen() 270 return _DeadlockWrap(lambda: self.db[key]) # self.db[key] 271 272 def __setitem__(self, key, value): 273 self._checkOpen() 274 self._closeCursors() 275 if self._in_iter and key not in self: 276 self._kill_iteration = True 277 def wrapF(): 278 self.db[key] = value 279 _DeadlockWrap(wrapF) # self.db[key] = value 280 281 def __delitem__(self, key): 282 self._checkOpen() 283 self._closeCursors() 284 if self._in_iter and key in self: 285 self._kill_iteration = True 286 def wrapF(): 287 del self.db[key] 288 _DeadlockWrap(wrapF) # del self.db[key] 289 290 def close(self): 291 self._closeCursors(save=0) 292 if self.dbc is not None: 293 _DeadlockWrap(self.dbc.close) 294 v = 0 295 if self.db is not None: 296 v = _DeadlockWrap(self.db.close) 297 self.dbc = None 298 self.db = None 299 return v 300 301 def keys(self): 302 self._checkOpen() 303 return _DeadlockWrap(self.db.keys) 304 305 def has_key(self, key): 306 self._checkOpen() 307 return _DeadlockWrap(self.db.has_key, key) 308 309 def set_location(self, key): 310 self._checkOpen() 311 self._checkCursor() 312 return _DeadlockWrap(self.dbc.set_range, key) 313 314 def next(self): # Renamed by "2to3" 315 self._checkOpen() 316 self._checkCursor() 317 rv = _DeadlockWrap(getattr(self.dbc, "next")) 318 return rv 319 320 if sys.version_info[0] >= 3 : # For "2to3" conversion 321 next = __next__ 322 323 def previous(self): 324 self._checkOpen() 325 self._checkCursor() 326 rv = _DeadlockWrap(self.dbc.prev) 327 return rv 328 329 def first(self): 330 self._checkOpen() 331 # fix 1725856: don't needlessly try to restore our cursor position 332 self.saved_dbc_key = None 333 self._checkCursor() 334 rv = _DeadlockWrap(self.dbc.first) 335 return rv 336 337 def last(self): 338 self._checkOpen() 339 # fix 1725856: don't needlessly try to restore our cursor position 340 self.saved_dbc_key = None 341 self._checkCursor() 342 rv = _DeadlockWrap(self.dbc.last) 343 return rv 344 345 def sync(self): 346 self._checkOpen() 347 return _DeadlockWrap(self.db.sync) 348 349 350 #---------------------------------------------------------------------- 351 # Compatibility object factory functions 352 353 def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None, 354 cachesize=None, lorder=None, hflags=0): 355 356 flags = _checkflag(flag, file) 357 e = _openDBEnv(cachesize) 358 d = db.DB(e) 359 d.set_flags(hflags) 360 if pgsize is not None: d.set_pagesize(pgsize) 361 if lorder is not None: d.set_lorder(lorder) 362 if ffactor is not None: d.set_h_ffactor(ffactor) 363 if nelem is not None: d.set_h_nelem(nelem) 364 d.open(file, db.DB_HASH, flags, mode) 365 return _DBWithCursor(d) 366 367 #---------------------------------------------------------------------- 368 369 def btopen(file, flag='c', mode=0666, 370 btflags=0, cachesize=None, maxkeypage=None, minkeypage=None, 371 pgsize=None, lorder=None): 372 373 flags = _checkflag(flag, file) 374 e = _openDBEnv(cachesize) 375 d = db.DB(e) 376 if pgsize is not None: d.set_pagesize(pgsize) 377 if lorder is not None: d.set_lorder(lorder) 378 d.set_flags(btflags) 379 if minkeypage is not None: d.set_bt_minkey(minkeypage) 380 if maxkeypage is not None: d.set_bt_maxkey(maxkeypage) 381 d.open(file, db.DB_BTREE, flags, mode) 382 return _DBWithCursor(d) 383 384 #---------------------------------------------------------------------- 385 386 387 def rnopen(file, flag='c', mode=0666, 388 rnflags=0, cachesize=None, pgsize=None, lorder=None, 389 rlen=None, delim=None, source=None, pad=None): 390 391 flags = _checkflag(flag, file) 392 e = _openDBEnv(cachesize) 393 d = db.DB(e) 394 if pgsize is not None: d.set_pagesize(pgsize) 395 if lorder is not None: d.set_lorder(lorder) 396 d.set_flags(rnflags) 397 if delim is not None: d.set_re_delim(delim) 398 if rlen is not None: d.set_re_len(rlen) 399 if source is not None: d.set_re_source(source) 400 if pad is not None: d.set_re_pad(pad) 401 d.open(file, db.DB_RECNO, flags, mode) 402 return _DBWithCursor(d) 403 404 #---------------------------------------------------------------------- 405 406 def _openDBEnv(cachesize): 407 e = db.DBEnv() 408 if cachesize is not None: 409 if cachesize >= 20480: 410 e.set_cachesize(0, cachesize) 411 else: 412 raise error, "cachesize must be >= 20480" 413 e.set_lk_detect(db.DB_LOCK_DEFAULT) 414 e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL) 415 return e 416 417 def _checkflag(flag, file): 418 if flag == 'r': 419 flags = db.DB_RDONLY 420 elif flag == 'rw': 421 flags = 0 422 elif flag == 'w': 423 flags = db.DB_CREATE 424 elif flag == 'c': 425 flags = db.DB_CREATE 426 elif flag == 'n': 427 flags = db.DB_CREATE 428 #flags = db.DB_CREATE | db.DB_TRUNCATE 429 # we used db.DB_TRUNCATE flag for this before but Berkeley DB 430 # 4.2.52 changed to disallowed truncate with txn environments. 431 if file is not None and os.path.isfile(file): 432 os.unlink(file) 433 else: 434 raise error, "flags should be one of 'r', 'w', 'c' or 'n'" 435 return flags | db.DB_THREAD 436 437 #---------------------------------------------------------------------- 438 439 440 # This is a silly little hack that allows apps to continue to use the 441 # DB_THREAD flag even on systems without threads without freaking out 442 # Berkeley DB. 443 # 444 # This assumes that if Python was built with thread support then 445 # Berkeley DB was too. 446 447 try: 448 # 2to3 automatically changes "import thread" to "import _thread" 449 import thread as T 450 del T 451 452 except ImportError: 453 db.DB_THREAD = 0 454 455 #---------------------------------------------------------------------- 456
