Removed the Berkeley DB storage from the Zope (and ZODB 3) repository.

The storage was added to these repositories hoping that that would
encourage people to test it. The storage is not (yet) usable in
production due to Berkeley DB log handling problems.  The storage is
a constant source of irritation due to errors in implicit algorithms
to decide whether to build BDB extensions when building Python and
due to implicit algorithms used to decide whether the BDB extension
was suitable for use by the storage when running tests.  The BDB
storage tests are also rather time consuming.

The final straw was the recent ZODB change that caused many BDB
storage tests to fail. :(

lib/python/BDBStorage/__init__.py

-##############################################################################
-#
-# Copyright (c) 2001 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Requirements:
-#
-# All: BerkeleyDB 4.1.25, available from www.sleepycat.com
-# Python 2.2: PyBSDDB 4.1.3 or better, from pybsddb.sf.net
-# Python 2.3: nothing extra
-
-try:
-    from bsddb import db
-except ImportError:
-    try:
-        from bsddb3 import db
-    except ImportError:
-        db = None
-
-# This flag tells other components whether Berkeley storage is available
-is_available = bool(db)
-
-# Useful constants
-ZERO = '\0'*8

lib/python/BDBStorage/_helper.c

-/*****************************************************************************
-
-  Copyright (c) 2002 Zope Corporation and Contributors.
-  All Rights Reserved.
-
-  This software is subject to the provisions of the Zope Public License,
-  Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-  THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-  WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-  WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-  FOR A PARTICULAR PURPOSE
-
- ****************************************************************************/
-
-#include <Python.h>
-
-/* This helper only works for Python 2.2 and beyond.  If we're using an
- * older version of Python, stop out now so we don't leave a broken, but
- * compiled and importable module laying about.  BDBFullStorage.py has a
- * workaround for when this extension isn't available.
- */
-#if PY_VERSION_HEX < 0x020200F0
-#error "Must be using at least Python 2.2"
-#endif
-
-/* Increment an 8-byte unsigned integer (represented as an 8-byte raw string),
- * by a Python integer.
- * The arguments are an 8-byte Python string, and a Python int or long.
- * The result is an 8-byte Python string, representing their sum.
- * XXX It's unclear what this intends to do if the sum overflows an 8-byte
- * XXX unsigned integer.  _PyLong_AsByteArray should raise OverflowError then.
- */
-static PyObject*
-helper_incr(PyObject* self, PyObject* args)
-{
-    PyObject *pylong = NULL, *incr, *sum = NULL, *result = NULL;
-    char *s, x[8];
-    int len, res;
-
-    if (!PyArg_ParseTuple(args, "s#O:incr", &s, &len, &incr))
-        return NULL;
-
-    assert(len == 8);
-
-    /* There seems to be no direct route from byte array to long long, so
-     * first convert it to a PyLongObject*, then convert /that/ thing to a
-     * long long.
-     */
-    pylong = _PyLong_FromByteArray(s, len,
-                                   0 /* big endian */,
-                                   0 /* unsigned */);
-
-    if (!pylong)
-        return NULL;
-
-    sum = PyNumber_Add(pylong, incr);
-    if (!sum)
-	goto err;
-
-    res = _PyLong_AsByteArray((PyLongObject*)sum, x, 8,
-                              0 /* big endian */,
-                              0 /* unsigned */);
-    if (res < 0)
-	goto err;
-
-    result = PyString_FromStringAndSize(x, 8);
- err:
-    Py_XDECREF(pylong);
-    Py_XDECREF(sum);
-    return result;
-}
-
-
-static PyMethodDef helper_methods[] = {
-    {"incr", helper_incr, METH_VARARGS},
-    {NULL, NULL}                             /* sentinel */
-};
-
-
-DL_EXPORT(void)
-init_helper(void)
-{
-    (void)Py_InitModule("_helper", helper_methods);
-}

lib/python/BDBStorage/profout.py

-import os
-import pprint
-from hotshot.log import LogReader
-
-prevloc = None
-byline = {}
-log = LogReader('profile.dat')
-
-for what, place, tdelta in log:
-    byline[prevloc] = tdelta + byline.get(prevloc, 0)
-    byline.setdefault(place,0)
-    prevloc = place
-
-# Sort
-results = [(v, k) for k, v in byline.items()]
-results.sort()
-
-for usecs, place in results:
-    if not place:
-        print 'Bad unpack:', usecs, place
-        continue
-    filename, line, funcname = place
-    print '%08d' % usecs, os.path.split(filename)[1], line

lib/python/BDBStorage/tests/BerkeleyTestBase.py

-##############################################################################
-#
-# Copyright (c) 2001 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Basic test framework class for both the BDBFullStorage and BDBMinimalStorage
-# Berkeley storages
-
-import os
-import shutil
-import sys
-import unittest
-
-import BDBStorage
-from BDBStorage.BerkeleyBase import BerkeleyConfig
-from ZODB.tests.StorageTestBase import StorageTestBase
-
-DBHOME = 'test-db'
-
-
-
-class BerkeleyTestBase(StorageTestBase):
-    def _config(self, read_only=False):
-        # Checkpointing just slows the tests down because we have to wait for
-        # the thread to properly shutdown.  This can take up to 10 seconds, so
-        # for the purposes of the test suite we shut off this thread.
-        config = BerkeleyConfig()
-        config.interval = 0
-        config.read_only = read_only
-        return config
-
-    def _envdir(self):
-        return DBHOME
-
-    def open(self, read_only=False):
-        self._storage = self.ConcreteStorage(
-            self._envdir(), config=self._config(read_only))
-
-    def _zap_dbhome(self, dir=None):
-        if dir is None:
-            dir = self._envdir()
-        if os.path.isdir(dir):
-            shutil.rmtree(dir)
-
-    def _mk_dbhome(self, dir=None):
-        if dir is None:
-            dir = self._get_envdir()
-        os.mkdir(dir)
-        try:
-            return self.ConcreteStorage(dir, config=self._config())
-        except:
-            self._zap_dbhome()
-            raise
-
-    def setUp(self):
-        StorageTestBase.setUp(self)
-        self._zap_dbhome()
-        self.open()
-
-    def tearDown(self):
-        StorageTestBase.tearDown(self)
-        self._zap_dbhome()
-
-
-
-class MinimalTestBase(BerkeleyTestBase):
-    from BDBStorage import BDBMinimalStorage
-    ConcreteStorage = BDBMinimalStorage.BDBMinimalStorage
-
-
-class FullTestBase(BerkeleyTestBase):
-    from BDBStorage import BDBFullStorage
-    ConcreteStorage = BDBFullStorage.BDBFullStorage
-
-def makeSuite(*args,**kw):
-    prefix = kw.get('prefix','check')
-    level = kw.get('level')
-    suite = unittest.TestSuite()
-    if level:
-        suite.level = level
-    if BDBStorage.is_available:
-        for klass in args:
-            suite.addTest(unittest.makeSuite(klass, prefix))
-    else:
-        sys.stderr.write("BDBStorage not available, tests disabled\n")
-
-    return suite

lib/python/BDBStorage/tests/ZODBTestBase.py

-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Base class for unit tests at the ZODB layer
-
-import os
-
-from ZODB import DB
-from BDBStorage.tests.BerkeleyTestBase import BerkeleyTestBase
-
-DBHOME = 'test-db'
-
-
-
-class ZODBTestBase(BerkeleyTestBase):
-    def setUp(self):
-        BerkeleyTestBase.setUp(self)
-        self._db = None
-        try:
-            self._db = DB(self._storage)
-            self._conn = self._db.open()
-            self._root = self._conn.root()
-        except:
-            self.tearDown()
-            raise
-
-    def _close(self):
-        if self._db is not None:
-            self._db.close()
-            self._db = self._storage = self._conn = self._root = None
-
-    def tearDown(self):
-        # If the tests exited with any uncommitted objects, they'll blow up
-        # subsequent tests because the next transaction commit will try to
-        # commit those object.  But they're tied to closed databases, so
-        # that's broken.  Aborting the transaction now saves us the headache.
-        try:
-            get_transaction().abort()
-            self._close()
-        finally:
-            BerkeleyTestBase.tearDown(self)

lib/python/BDBStorage/tests/__init__.py

-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################

lib/python/BDBStorage/tests/test_create.py

-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Unit test for database creation
-
-import os
-import time
-import unittest
-
-import BDBStorage
-if BDBStorage.is_available:
-    from BDBStorage.BerkeleyBase import BerkeleyConfig
-    from BDBStorage.BDBFullStorage import BDBFullStorage
-
-from BDBStorage.tests import BerkeleyTestBase
-
-
-
-class TestMixin:
-    def testDBHomeExists(self):
-        self.failUnless(os.path.isdir(BerkeleyTestBase.DBHOME))
-
-
-class MinimalCreateTest(BerkeleyTestBase.MinimalTestBase, TestMixin):
-    pass
-
-
-class FullCreateTest(BerkeleyTestBase.FullTestBase, TestMixin):
-    pass
-
-
-
-class FullOpenExistingTest(BerkeleyTestBase.FullTestBase):
-    def testOpenWithExistingVersions(self):
-        version = 'test-version'
-        oid = self._storage.new_oid()
-        revid = self._dostore(oid, data=7, version=version)
-        # Now close the current storage and re-open it
-        self._storage.close()
-        self._storage = self.ConcreteStorage(BerkeleyTestBase.DBHOME)
-        self.assertEqual(self._storage.modifiedInVersion(oid), version)
-
-    def testOpenAddVersion(self):
-        eq = self.assertEqual
-        version1 = 'test-version'
-        oid1 = self._storage.new_oid()
-        revid = self._dostore(oid1, data=7, version=version1)
-        # Now close the current storage and re-open it
-        self._storage.close()
-        self._storage = self.ConcreteStorage(BerkeleyTestBase.DBHOME)
-        eq(self._storage.modifiedInVersion(oid1), version1)
-        # Now create a 2nd version string, then close/reopen
-        version2 = 'new-version'
-        oid2 = self._storage.new_oid()
-        revid = self._dostore(oid2, data=8, version=version2)
-        # Now close the current storage and re-open it
-        self._storage.close()
-        self._storage = self.ConcreteStorage(BerkeleyTestBase.DBHOME)
-        eq(self._storage.modifiedInVersion(oid1), version1)
-        # Now create a 2nd version string, then close/reopen
-        eq(self._storage.modifiedInVersion(oid2), version2)
-
-
-
-class FullOpenCloseTest(BerkeleyTestBase.FullTestBase):
-    level = 2
-
-    def _mk_dbhome(self, dir):
-        config = BerkeleyConfig
-        config.interval = 10
-        os.mkdir(dir)
-        try:
-            return self.ConcreteStorage(dir, config=config)
-        except:
-            self._zap_dbhome(dir)
-            raise
-
-    def testCloseWithCheckpointingThread(self):
-        # All the interesting stuff happens in the setUp and tearDown
-        time.sleep(20)
-
-
-
-class OpenRecoveryTest(BerkeleyTestBase.FullTestBase):
-    def open(self):
-        self._storage = None
-
-    def testOpenWithBogusConfig(self):
-        class C: pass
-        c = C()
-        # This instance won't have the necessary attributes, so the creation
-        # will fail.  We want to be sure that everything gets cleaned up
-        # enough to fix that and create a proper storage.
-        dir = self._envdir()
-        self.assertRaises(AttributeError, BDBFullStorage, dir, config=c)
-        c = BerkeleyConfig()
-        s = BDBFullStorage(dir, config=c)
-        s.close()
-
-
-
-def test_suite():
-    return BDBStorage.tests.BerkeleyTestBase.makeSuite(
-        MinimalCreateTest,
-        FullCreateTest,
-        FullOpenExistingTest,
-        FullOpenCloseTest,
-        OpenRecoveryTest,
-        prefix='test'
-        )
-    return suite
-
-
-
-if __name__ == '__main__':
-    unittest.main(defaultTest='test_suite')

lib/python/BDBStorage/tests/test_storage_api.py

-##############################################################################
-#
-# Copyright (c) 2001 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Unit tests for basic storage functionality
-
-import unittest
-from ZODB import POSException
-
-import BDBStorage
-from BDBStorage.tests import BerkeleyTestBase
-
-from ZODB.tests.BasicStorage import BasicStorage
-from ZODB.tests.HistoryStorage import HistoryStorage
-from ZODB.tests.IteratorStorage import IteratorStorage, ExtendedIteratorStorage
-from ZODB.tests.MTStorage import MTStorage
-from ZODB.tests.PackableStorage import PackableStorage, PackableUndoStorage
-from ZODB.tests.PersistentStorage import PersistentStorage
-from ZODB.tests.ReadOnlyStorage import ReadOnlyStorage
-from ZODB.tests.RecoveryStorage import RecoveryStorage
-from ZODB.tests.RevisionStorage import RevisionStorage
-from ZODB.tests.Synchronization import SynchronizedStorage
-from ZODB.tests.TransactionalUndoStorage import TransactionalUndoStorage
-from ZODB.tests.TransactionalUndoVersionStorage import \
-     TransactionalUndoVersionStorage
-from ZODB.tests.VersionStorage import VersionStorage
-
-from ZODB.tests import ConflictResolution
-
-
-
-class MinimalTest(BerkeleyTestBase.MinimalTestBase,
-                  BasicStorage,
-                  MTStorage,
-                  PackableStorage,
-                  ReadOnlyStorage,
-                  SynchronizedStorage,
-                  ):
-
-    def checkVersionedStoreAndLoad(self):
-        # This storage doesn't support versions, so we should get an exception
-        oid = self._storage.new_oid()
-        self.assertRaises(POSException.Unsupported,
-                          self._dostore,
-                          oid, data=11, version='a version')
-
-
-class FullTest(BerkeleyTestBase.FullTestBase, BasicStorage,
-               RevisionStorage, VersionStorage,
-               TransactionalUndoStorage,
-               TransactionalUndoVersionStorage,
-               PackableStorage, PackableUndoStorage,
-               HistoryStorage,
-               IteratorStorage, ExtendedIteratorStorage,
-               ConflictResolution.ConflictResolvingStorage,
-               ConflictResolution.ConflictResolvingTransUndoStorage,
-               SynchronizedStorage,
-               PersistentStorage,
-               MTStorage,
-               ReadOnlyStorage):
-    pass
-
-
-
-DST_DBHOME = 'test-dst'
-
-class FullRecoveryTest(BerkeleyTestBase.FullTestBase,
-                       RecoveryStorage):
-    def setUp(self):
-        BerkeleyTestBase.FullTestBase.setUp(self)
-        self._zap_dbhome(DST_DBHOME)
-        self._dst = self._mk_dbhome(DST_DBHOME)
-
-    def tearDown(self):
-        BerkeleyTestBase.FullTestBase.tearDown(self)
-        self._dst.close()
-        self._zap_dbhome(DST_DBHOME)
-
-    def new_dest(self):
-        self._zap_dbhome(DST_DBHOME)
-        return self._mk_dbhome(DST_DBHOME)
-
-
-
-def test_suite():
-    return BDBStorage.tests.BerkeleyTestBase.makeSuite(
-        FullTest,
-        FullRecoveryTest,
-        MinimalTest,
-        )
-
-
-if __name__ == '__main__':
-    unittest.main(defaultTest='test_suite')

lib/python/BDBStorage/tests/test_virgin.py

-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Test creation of a brand new database, and insertion of root objects.
-
-import unittest
-
-import BDBStorage
-from BDBStorage.tests.ZODBTestBase import ZODBTestBase
-from persistent.mapping import PersistentMapping
-
-
-class InsertMixin:
-    def checkIsEmpty(self):
-        self.failUnless(not self._root.has_key('names'))
-
-    def checkNewInserts(self):
-        self._root['names'] = names = PersistentMapping()
-        names['Warsaw'] = 'Barry'
-        names['Hylton'] = 'Jeremy'
-        get_transaction().commit()
-
-
-
-class FullNewInsertsTest(ZODBTestBase, InsertMixin):
-    from BDBStorage import BDBFullStorage
-    ConcreteStorage = BDBFullStorage.BDBFullStorage
-
-
-class MinimalNewInsertsTest(ZODBTestBase, InsertMixin):
-    from BDBStorage import BDBMinimalStorage
-    ConcreteStorage = BDBMinimalStorage.BDBMinimalStorage
-
-
-
-def test_suite():
-    return BDBStorage.tests.BerkeleyTestBase.makeSuite(
-        MinimalNewInsertsTest,
-        FullNewInsertsTest,
-        )
-
-
-if __name__ == '__main__':
-    unittest.main(defaultTest='test_suite')

lib/python/BDBStorage/tests/test_whitebox.py

-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Whitebox testing of storage implementation details.
-
-import unittest
-
-from ZODB.utils import U64
-from ZODB.tests.MinPO import MinPO
-from ZODB.tests.StorageTestBase import zodb_unpickle
-
-import BDBStorage
-if BDBStorage.is_available:
-    from BDBStorage.BDBMinimalStorage import BDBMinimalStorage
-    from BDBStorage.BDBFullStorage import BDBFullStorage
-else:
-    # Sigh
-    class FakeBaseClass: pass
-    BDBFullStorage = BDBMinimalStorage = FakeBaseClass
-
-from BDBStorage.tests.ZODBTestBase import ZODBTestBase
-from BDBStorage.tests.BerkeleyTestBase import BerkeleyTestBase
-
-from persistent import Persistent
-
-ZERO = '\0'*8
-
-
-
-class Object(Persistent):
-    pass
-
-
-
-class WhiteboxLowLevelMinimal(BerkeleyTestBase):
-    ConcreteStorage = BDBMinimalStorage
-
-    def checkTableConsistencyAfterCommit(self):
-        unless = self.failIf
-        eq = self.assertEqual
-        oid = self._storage.new_oid()
-        revid1 = self._dostore(oid, data=11)
-        revid2 = self._dostore(oid, revid=revid1, data=12)
-        revid3 = self._dostore(oid, revid=revid2, data=13)
-        # First off, there should be no entries in the pending table
-        unless(self._storage._pending.keys())
-        # Also, there should be no entries in the oids table
-        unless(self._storage._oids.keys())
-        # Now, there should be exactly one oid in the serials table, and
-        # exactly one record for that oid in the table too.
-        oids = {}
-        c = self._storage._serials.cursor()
-        try:
-            rec = c.first()
-            while rec:
-                oid, serial = rec
-                oids.setdefault(oid, []).append(serial)
-                rec = c.next()
-        finally:
-            c.close()
-        eq(len(oids), 1)
-        eq(len(oids[oids.keys()[0]]), 1)
-        # There should now be exactly one entry in the pickles table.
-        pickles = self._storage._pickles.items()
-        eq(len(pickles), 1)
-        key, data = pickles[0]
-        poid = key[:8]
-        pserial = key[8:]
-        eq(oid, poid)
-        eq(revid3, pserial)
-        obj = zodb_unpickle(data)
-        eq(obj.value, 13)
-        # Now verify the refcounts table, which should be empty because the
-        # stored object isn't referenced by any other objects.
-        eq(len(self._storage._refcounts.keys()), 0)
-
-    def checkStorageVersionAfterCreation(self):
-        from BDBStorage.BDBMinimalStorage import BDBMINIMAL_SCHEMA_VERSION
-        eq = self.assertEqual
-        eq(self._storage._info['version'], BDBMINIMAL_SCHEMA_VERSION)
-        self._storage.close()
-        self.open()
-        eq(self._storage._info['version'], BDBMINIMAL_SCHEMA_VERSION)
-
-
-
-class WhiteboxHighLevelMinimal(ZODBTestBase):
-    ConcreteStorage = BDBMinimalStorage
-
-    def checkReferenceCounting(self):
-        eq = self.assertEqual
-        obj = MinPO(11)
-        self._root.obj = obj
-        get_transaction().commit()
-        obj.value = 12
-        get_transaction().commit()
-        obj.value = 13
-        get_transaction().commit()
-        # Make sure the databases have what we expect
-        eq(len(self._storage._serials.items()), 2)
-        eq(len(self._storage._pickles.items()), 2)
-        # And now refcount out the object
-        del self._root.obj
-        get_transaction().commit()
-        # Verification stage.  Our serials table should have exactly one
-        # entry, oid == 0
-        keys = self._storage._serials.keys()
-        eq(len(keys), 1)
-        eq(len(self._storage._serials.items()), 1)
-        eq(keys[0], ZERO)
-        # The pickles table now should have exactly one revision of the root
-        # object, and no revisions of the MinPO object, which should have been
-        # collected away.
-        pickles = self._storage._pickles.items()
-        eq(len(pickles), 1)
-        rec = pickles[0]
-        key = rec[0]
-        data = rec[1]
-        eq(key[:8], ZERO)
-        # And that pickle should have no 'obj' attribute.
-        unobj = zodb_unpickle(data)
-        self.failIf(hasattr(unobj, 'obj'))
-        # Our refcounts table should have no entries in it, because the root
-        # object is an island.
-        eq(len(self._storage._refcounts.keys()), 0)
-        # And of course, oids and pendings should be empty too
-        eq(len(self._storage._oids.keys()), 0)
-        eq(len(self._storage._pending.keys()), 0)
-
-    def checkRecursiveReferenceCounting(self):
-        eq = self.assertEqual
-        obj1 = Object()
-        obj2 = Object()
-        obj3 = Object()
-        obj4 = Object()
-        self._root.obj = obj1
-        obj1.obj = obj2
-        obj2.obj = obj3
-        obj3.obj = obj4
-        get_transaction().commit()
-        # Make sure the databases have what we expect
-        eq(len(self._storage._serials.items()), 5)
-        eq(len(self._storage._pickles.items()), 5)
-        # And now refcount out the object
-        del self._root.obj
-        get_transaction().commit()
-        # Verification stage.  Our serials table should have exactly one
-        # entry, oid == 0
-        keys = self._storage._serials.keys()
-        eq(len(keys), 1)
-        eq(len(self._storage._serials.items()), 1)
-        eq(keys[0], ZERO)
-        # The pickles table now should have exactly one revision of the root
-        # object, and no revisions of any other objects, which should have
-        # been collected away.
-        pickles = self._storage._pickles.items()
-        eq(len(pickles), 1)
-        rec = pickles[0]
-        key = rec[0]
-        data = rec[1]
-        eq(key[:8], ZERO)
-        # And that pickle should have no 'obj' attribute.
-        unobj = zodb_unpickle(data)
-        self.failIf(hasattr(unobj, 'obj'))
-        # Our refcounts table should have no entries in it, because the root
-        # object is an island.
-        eq(len(self._storage._refcounts.keys()), 0)
-        # And of course, oids and pendings should be empty too
-        eq(len(self._storage._oids.keys()), 0)
-        eq(len(self._storage._pending.keys()), 0)
-
-
-
-class WhiteboxLowLevelFull(BerkeleyTestBase):
-    ConcreteStorage = BDBFullStorage
-
-    def checkStorageVersionAfterCreation(self):
-        from BDBStorage.BDBFullStorage import BDBFULL_SCHEMA_VERSION
-        eq = self.assertEqual
-        eq(self._storage._info['version'], BDBFULL_SCHEMA_VERSION)
-        self._storage.close()
-        self.open()
-        eq(self._storage._info['version'], BDBFULL_SCHEMA_VERSION)
-
-
-
-class WhiteboxHighLevelFull(ZODBTestBase):
-    ConcreteStorage = BDBFullStorage
-
-    def checkReferenceCounting(self):
-        eq = self.assertEqual
-        # Make sure the databases have what we expect
-        eq(len(self._storage._serials.items()), 1)
-        eq(len(self._storage._pickles.items()), 1)
-        # Now store an object
-        obj = MinPO(11)
-        self._root.obj = obj
-        get_transaction().commit()
-        # Make sure the databases have what we expect
-        eq(len(self._storage._serials.items()), 2)
-        eq(len(self._storage._pickles.items()), 3)
-        obj.value = 12
-        get_transaction().commit()
-        # Make sure the databases have what we expect
-        eq(len(self._storage._serials.items()), 2)
-        eq(len(self._storage._pickles.items()), 4)
-        obj.value = 13
-        get_transaction().commit()
-        # Make sure the databases have what we expect
-        eq(len(self._storage._serials.items()), 2)
-        eq(len(self._storage._pickles.items()), 5)
-        # And now refcount out the object
-        del self._root.obj
-        get_transaction().commit()
-        # Verification stage.  Our serials tabl should still have 2 entries,
-        # one for the root object and one for the now unlinked MinPO obj.
-        keys = self._storage._serials.keys()
-        eq(len(keys), 2)
-        eq(len(self._storage._serials.items()), 2)
-        eq(keys[0], ZERO)
-        # The pickles table should now have 6 entries, broken down like so:
-        # - 3 revisions of the root object: the initial database-open
-        #   revision, the revision that got its obj attribute set, and the
-        #   revision that got its obj attribute deleted.
-        # - 3 Three revisions of obj, corresponding to values 11, 12, and 13
-        pickles = self._storage._pickles.items()
-        eq(len(pickles), 6)
-        # Our refcounts table should have one entry in it for the MinPO that's
-        # referenced in an earlier revision of the root object
-        eq(len(self._storage._refcounts.keys()), 1)
-        # And of course, oids and pendings should be empty too
-        eq(len(self._storage._oids.keys()), 0)
-        eq(len(self._storage._pending.keys()), 0)
-
-
-
-def test_suite():
-    return BDBStorage.tests.BerkeleyTestBase.makeSuite(
-        WhiteboxLowLevelMinimal,
-        WhiteboxHighLevelMinimal,
-        WhiteboxLowLevelFull,
-        WhiteboxHighLevelFull,
-        )
-
-
-if __name__ == '__main__':
-    unittest.main(defaultTest='test_suite')

lib/python/BDBStorage/tests/test_zodb_simple.py

-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Test some simple ZODB level stuff common to both the BDBMinimalStorage and
-# BDBFullStorage storages, like transaction aborts and commits, changing
-# objects, etc.  Doesn't test undo, versions, or packing.
-
-import time
-import unittest
-
-import BDBStorage
-from BDBStorage.tests.ZODBTestBase import ZODBTestBase
-from persistent.mapping import PersistentMapping
-
-
-
-class CommitAndRead:
-    def checkCommit(self):
-        self.failUnless(not self._root)
-        names = self._root['names'] = PersistentMapping()
-        names['Warsaw'] = 'Barry'
-        names['Hylton'] = 'Jeremy'
-        get_transaction().commit()
-
-    def checkReadAfterCommit(self):
-        eq = self.assertEqual
-        self.checkCommit()
-        names = self._root['names']
-        eq(names['Warsaw'], 'Barry')
-        eq(names['Hylton'], 'Jeremy')
-        self.failUnless(names.get('Drake') is None)
-
-    def checkAbortAfterRead(self):
-        self.checkReadAfterCommit()
-        names = self._root['names']
-        names['Drake'] = 'Fred'
-        get_transaction().abort()
-
-    def checkReadAfterAbort(self):
-        self.checkAbortAfterRead()
-        names = self._root['names']
-        self.failUnless(names.get('Drake') is None)
-
-    def checkChangingCommits(self):
-        self.checkReadAfterAbort()
-        now = time.time()
-        # Make sure the last timestamp was more than 3 seconds ago
-        timestamp = self._root.get('timestamp')
-        if timestamp is None:
-            timestamp = self._root['timestamp'] = 0
-            get_transaction().commit()
-        self.failUnless(now > timestamp + 3)
-        self._root['timestamp'] = now
-        time.sleep(3)
-
-
-
-class MinimalCommitAndRead(ZODBTestBase, CommitAndRead):
-    from BDBStorage import BDBMinimalStorage
-    ConcreteStorage = BDBMinimalStorage.BDBMinimalStorage
-
-
-class FullCommitAndRead(ZODBTestBase, CommitAndRead):
-    from BDBStorage import BDBFullStorage
-    ConcreteStorage = BDBFullStorage.BDBFullStorage
-
-
-
-def test_suite():
-    return BDBStorage.tests.BerkeleyTestBase.makeSuite(
-        MinimalCommitAndRead,
-        FullCommitAndRead
-        )
-
-if __name__ == '__main__':
-    unittest.main(defaultTest='test_suite')

lib/python/BDBStorage/tests/testall.py

-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-# Framework for running Unit tests
-
-import unittest
-
-MODULES = ('commitlog', 'create', 'virgin', 'zodb_simple', 'storage_api')
-
-
-
-def suite():
-    alltests = unittest.TestSuite()
-    for modname in MODULES:
-        mod = __import__('test_'+modname)
-        alltests.addTest(mod.test_suite())
-    return alltests
-
-def test_suite():
-    # Just to silence the top-level test.py
-    return None
-
-
-
-if __name__ == '__main__':
-    unittest.main(defaultTest='suite')

lib/python/BDBStorage/tests/timeiter.py

-#! /usr/bin/env python
-
-##############################################################################
-#
-# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-# All Rights Reserved.
-#
-# This software is subject to the provisions of the Zope Public License,
-# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-# FOR A PARTICULAR PURPOSE
-#
-##############################################################################
-
-"""Time transaction commits and normalize vs. pickle size and #objects.
-
-Actually just counts the size of pickles in the transaction via the iterator
-protocol, so storage overheads aren't counted.
-
-Usage: %(PROGRAM)s [options]
-Options:
-    -h/--help
-        Print this message and exit.
-
-    -s filename
-    --source=filename
-        Use database in filename as the source (must be a FileStorage)
-
-    -d filename
-    --dest=filename
-        Use database in filename as the destination (must be a BDB storage)
-
-    -o filename
-    --output=filename
-        Print results in filename, otherwise stdout.
-
-    -m txncount
-    --max=txncount
-        Stop after committing txncount transactions.
-
-    -k txncount
-    --skip=txncount
-        Skip the first txncount transactions.
-
-    -p/--profile
-        Turn on specialized profiling.
-
-    -q/--quiet
-        Be quite.
-"""
-
-import sys
-import os
-import getopt
-import time
-import errno
-import profile
-import traceback
-import marshal
-
-from bsddb3 import db
-
-from ZODB import utils
-from persistent.TimeStamp import TimeStamp
-from ZODB.FileStorage import FileStorage
-from BDBStorage.BDBFullStorage import BDBFullStorage
-
-PROGRAM = sys.argv[0]
-ZERO = '\0'*8
-
-
-
-def usage(code, msg=''):
-    print >> sys.stderr, __doc__ % globals()
-    if msg:
-        print >> sys.stderr, msg
-    sys.exit(code)
-
-
-
-def main():
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hs:d:qo:l:pm:k:',
-                                   ['help', 'source=', 'dest=', 'quiet',
-                                    'output=', 'logfile=', 'profile',
-                                    'max=', 'skip='])
-    except getopt.error, msg:
-        usage(1, msg)
-
-    class Options:
-        source = None
-        dest = None
-        verbose = 1
-        outfile = None
-        logfile = None
-        profilep = 0
-        maxtxn = -1
-        skiptxn = -1
-
-    options = Options()
-
-    for opt, arg in opts:
-        if opt in ('-h', '--help'):
-            usage(0)
-        elif opt in ('-s', '--source'):
-            options.source = arg
-        elif opt in ('-d', '--dest'):
-            options.dest = arg
-        elif opt in ('-q', '--quiet'):
-            options.verbose = 0
-        elif opt in ('-o', '--output'):
-            options.outfile = arg
-        elif opt in ('-l', '--logfile'):
-            options.logfile = arg
-        elif opt in ('-p', '--profile'):
-            options.profilep = 1
-        elif opt in ('-m', '--max'):
-            options.maxtxn = int(arg)
-        elif opt in ('-k', '--skip'):
-            options.skiptxn = int(arg)
-
-    if args:
-        usage(1)
-
-    if not options.source or not options.dest:
-        usage(1, 'Source and destination databases must be provided')
-
-    # Open the output file
-    if options.outfile is None:
-        options.outfp = sys.stdout
-        options.outclosep = 0
-    else:
-        options.outfp = open(options.outfile, 'w')
-        options.outclosep = 1
-
-    # Open the logfile
-    if options.logfile is None:
-        options.logfp = sys.stdout
-        options.logclosep = 0
-    else:
-        options.logfp = open(options.logfile, 'w')
-        options.logclosep = 1
-
-    # Print a comment, this is a hack
-    print >> options.outfp, '# FS->BDB 3.3.11'
-    print >> options.outfp, '#', time.ctime()
-
-    print >>sys.stderr, 'Opening source FileStorage...'
-    t0 = time.time()
-    srcdb = FileStorage(options.source, read_only=1)
-    t1 = time.time()
-    print >>sys.stderr, 'Opening source FileStorage done. %s seconds' % (t1-t0)
-#
-# Uncomment this section to do a FS->BDB migration
-#
-    print >>sys.stderr, 'Opening destination BDB...'
-    t0 = time.time()
-    dstdb = BDBFullStorage(options.dest)
-    t1 = time.time()
-    print >>sys.stderr, 'Opening destination BDB done. %s seconds' % (t1-t0)
-
-#
-# Uncomment this section to do a FS->FS migration
-#
-##    print >>sys.stderr, 'Opening destination FileStorage...'
-##    t0 = time.time()
-##    dstdb = FileStorage(dest)
-##    t1 = time.time()
-##    print >>sys.stderr, 'Opening destination FileStorage done. %s seconds' % (
-##        t1-t0)
-
-    try:
-        t0 = time.time()
-        doit(srcdb, dstdb, options)
-        t1 = time.time()
-        print 'Total time:', t1-t0
-    finally:
-        # Done
-        srcdb.close()
-        dstdb.close()
-        if options.outclosep:
-            options.outfp.close()
-        if options.logclosep:
-            options.logfp.close()
-
-
-
-def doit(srcdb, dstdb, options):
-    outfp = options.outfp
-    logfp = options.logfp
-    profilep = options.profilep
-    verbose = options.verbose
-    # some global information
-    largest_pickle = 0
-    largest_txn_in_size = 0
-    largest_txn_in_objects = 0
-    # Ripped from BaseStorage.copyTransactionsFrom()
-    ts = None
-    ok = 1
-    prevrevids = {}
-    counter = 0
-    skipper = 0
-    for txn in srcdb.iterator():
-        skipper += 1
-        if skipper <= options.skiptxn:
-            continue
-        counter += 1
-        if counter > options.maxtxn > 0:
-            break
-        tid = txn.tid
-        if ts is None:
-            ts = TimeStamp(tid)
-        else:
-            t = TimeStamp(tid)
-            if t <= ts:
-                if ok:
-                    print 'Time stamps are out of order %s, %s' % (ts, t)
-                    ok = 0
-                    ts = t.laterThan(ts)
-                    tid = `ts`
-                else:
-                    ts = t
-                    if not ok:
-                        print 'Time stamps are back in order %s' % t
-                        ok = 1
-        if verbose:
-            print ts
-
-        prof = None
-        if profilep and (counter % 100) == 0:
-            prof = profile.Profile()
-        objects = 0
-        size = 0
-        t0 = time.time()
-        dstdb.tpc_begin(txn, tid, txn.status)
-        t1 = time.time()
-        try:
-            for r in txn:
-                oid = r.oid
-                objects += 1
-                thissize = len(r.data)
-                size += thissize
-                if thissize > largest_pickle:
-                    largest_pickle = thissize
-                if verbose:
-                    if not r.version:
-                        vstr = 'norev'
-                    else:
-                        vstr = r.version
-                    print utils.U64(oid), vstr, len(r.data)
-                oldrevid = prevrevids.get(oid, ZERO)
-                newrevid = dstdb.store(oid, oldrevid, r.data, r.version, txn)
-                prevrevids[oid] = newrevid
-            t2 = time.time()
-            dstdb.tpc_vote(txn)
-            t3 = time.time()
-            # Profile every 100 transactions
-            if prof:
-                prof.runcall(dstdb.tpc_finish, txn)
-            else:
-                dstdb.tpc_finish(txn)
-            t4 = time.time()
-        except KeyError, e:
-            traceback.print_exc(file=logfp)
-
-        # record the results
-        if objects > largest_txn_in_objects:
-            largest_txn_in_objects = objects
-        if size > largest_txn_in_size:
-            largest_txn_in_size = size
-        print >> outfp, utils.U64(tid), objects, size, t4-t0, \
-              t1-t0, t2-t1, t3-t2, t4-t3
-
-        if prof:
-            prof.create_stats()
-            fp = open('profile-%02d.txt' % (counter / 100), 'wb')
-            marshal.dump(prof.stats, fp)
-            fp.close()
-    print >> outfp, largest_pickle, largest_txn_in_size, largest_txn_in_objects
-
-
-
-if __name__ == '__main__':
-    main()

setup.py

@@ -215,18 +215,6 @@ setup(
                 ['App/www', ['App/www/*']]],
     )
 
-# BDBStorage
-setup(
-    name='BDBStorage',
-    author=AUTHOR,
-
-    packages=['BDBStorage', 'BDBStorage.tests'],
-    ext_modules = [
-        Extension(name='BDBStorage._helper',
-        sources=['BDBStorage/_helper.c']),
-        ]
-    )
-
 # BTrees
 setup(
     name='BTrees',