Merge zope3-zodb3-devel-branch to the Zope head (Zope 2 head).

Add support for persistent weak references.  The implementation is in
pure Python right now; coptimizations.c was disabled.  We need to
restore the C code before going to beta.

The persistent reference format has evolved a little, but the code on
the branch doesn't document it.  The ref is usually a an oid-class
pair, where the class is an actual class object.  It can also be a
list, for weak references, or a bare oid.

Add support for data managers written against the ZODB4 transaction
API.  The transaction implementation provides an adapter between the
two interfaces.  There's a substantial doctest test suite for this
feature.

Add add() method to Connection to explicitly add an object and get it
an assigned _p_oid.  It's possible to add an unconnected object; this
isn't recommended, because it will be lost at pack time.

src/ZODB/ConflictResolution.py

@@ -19,25 +19,39 @@ from pickle import PicklingError
 from ZODB.POSException import ConflictError
 import zLOG
 
-bad_classes = {}
+ResolvedSerial = 'rs'
 
-def bad_class(class_tuple):
-    if bad_classes.has_key(class_tuple) or class_tuple[0][0] == '*':
-        # if we've seen the class before or if it's a ZClass, we know that
-        # we can't resolve the conflict
-        return 1
+class BadClassName(Exception):
+    pass
 
-ResolvedSerial = 'rs'
+_class_cache = {}
+_class_cache_get = _class_cache.get
+def find_global(*args):
+    cls = _class_cache_get(args, 0)
+    if cls == 0:
+        # Not cached. Try to import
+        try:
+            module = __import__(args[0], {}, {}, ['cluck'])
+        except ImportError:
+            cls = 1
+        else:
+            cls = getattr(module, args[1], 1)
+        _class_cache[args] = cls
+
+        if cls == 1:
+            zLOG.LOG("Conflict Resolution", zLOG.BLATHER,
+                     "Unable to load class", error=sys.exc_info())
 
-def _classFactory(location, name,
-                  _silly=('__doc__',), _globals={}):
-    return getattr(__import__(location, _globals, _globals, _silly),
-                   name)
+    if cls == 1:
+        # Not importable
+        raise BadClassName(*args)
+    return cls
 
 def state(self, oid, serial, prfactory, p=''):
     p = p or self.loadSerial(oid, serial)
     file = StringIO(p)
     unpickler = Unpickler(file)
+    unpickler.find_global = find_global
     unpickler.persistent_load = prfactory.persistent_load
     unpickler.load() # skip the class tuple
     return unpickler.load()
@@ -71,16 +85,7 @@ def persistent_id(object):
         return None
     return object.data
     
-def load_class(class_tuple):
-    try:
-        klass = _classFactory(class_tuple[0], class_tuple[1])
-    except (ImportError, AttributeError):
-        zLOG.LOG("Conflict Resolution", zLOG.BLATHER,
-                 "Unable to load class", error=sys.exc_info())
-        bad_classes[class_tuple] = 1
-        return None
-    return klass
-
+_unresolvable = {}
 def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
                          committedData=''):
     # class_tuple, old, committed, newstate = ('',''), 0, 0, 0
@@ -88,21 +93,28 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         prfactory = PersistentReferenceFactory()
         file = StringIO(newpickle)
         unpickler = Unpickler(file)
+        unpickler.find_global = find_global
         unpickler.persistent_load = prfactory.persistent_load
         meta = unpickler.load()
-        class_tuple = meta[0]
-        if bad_class(class_tuple):
+        if isinstance(meta, tuple):
+            klass = meta[0]
+            newargs = meta[1] or ()
+            if isinstance(klass, tuple):
+                klass = find_global(*klass)
+        else:
+            klass = meta
+            newargs = ()
+            
+        if klass in _unresolvable:
             return None
+
         newstate = unpickler.load()
-        klass = load_class(class_tuple)
-        if klass is None:
-            return None
-        inst = klass.__new__(klass)
+        inst = klass.__new__(klass, *newargs)
 
         try:
             resolve = inst._p_resolveConflict
         except AttributeError:
-            bad_classes[class_tuple] = 1
+            _unresolvable[klass] = 1
             return None
 
         old = state(self, oid, oldSerial, prfactory)
@@ -116,7 +128,7 @@ def tryToResolveConflict(self, oid, committedSerial, oldSerial, newpickle,
         pickler.dump(meta)
         pickler.dump(resolved)
         return file.getvalue(1)
-    except ConflictError:
+    except (ConflictError, BadClassName):
         return None
     except:
         # If anything else went wrong, catch it here and avoid passing an

src/ZODB/Connection.py

@@ -13,34 +13,26 @@
 ##############################################################################
 """Database connection support
 
-$Id: Connection.py,v 1.117 2004/01/14 18:58:08 jeremy Exp $"""
+$Id: Connection.py,v 1.118 2004/02/19 02:59:06 jeremy Exp $"""
 
 import logging
 import sys
 import threading
+import itertools
 from time import time
-from types import ClassType
 from utils import u64
 
-_marker = object()
-
-def myhasattr(obj, attr):
-    # builtin hasattr() swallows exceptions
-    return getattr(obj, attr, _marker) is not _marker
-
 from persistent import PickleCache
 from zLOG import LOG, ERROR, BLATHER, WARNING
 
 from ZODB.ConflictResolution import ResolvedSerial
-from ZODB.coptimizations import new_persistent_id
 from ZODB.ExportImport import ExportImport
 from ZODB.POSException \
-     import ConflictError, ReadConflictError, TransactionError
+     import ConflictError, ReadConflictError, InvalidObjectReference
 from ZODB.TmpStore import TmpStore
 from ZODB.Transaction import Transaction, get_transaction
 from ZODB.utils import oid_repr, z64
-from ZODB.serialize \
-     import ObjectWriter, getClassMetadata, ConnectionObjectReader
+from ZODB.serialize import ObjectWriter, ConnectionObjectReader, myhasattr
 
 global_reset_counter = 0
 
@@ -70,6 +62,7 @@ class Connection(ExportImport, object):
     _opened = None
     _code_timestamp = 0
     _transaction = None
+    _added_during_commit = None
 
     def __init__(self, version='', cache_size=400,
                  cache_deactivate_after=60, mvcc=True):
@@ -88,6 +81,8 @@ class Connection(ExportImport, object):
 
             self._cache.cache_drain_resistance = 100
         self._incrgc = self.cacheGC = cache.incrgc
+        self._committed = []
+        self._added = {}
         self._reset_counter = global_reset_counter
         self._load_count = 0   # Number of objects unghosted
         self._store_count = 0  # Number of objects stored
@@ -158,6 +153,9 @@ class Connection(ExportImport, object):
 
     def __getitem__(self, oid):
         obj = self._cache.get(oid, None)
+        if obj is not None:
+            return obj
+        obj = self._added.get(oid, None)
         if obj is not None:
             return obj
 
@@ -172,6 +170,21 @@ class Connection(ExportImport, object):
         self._cache[oid] = obj
         return obj
 
+    def add(self, obj):
+        marker = object()
+        oid = getattr(obj, "_p_oid", marker)
+        if oid is marker:
+            raise TypeError("Only first-class persistent objects may be"
+                            " added to a Connection.", obj)
+        elif obj._p_jar is None:
+            oid = obj._p_oid = self._storage.new_oid()
+            obj._p_jar = self
+            self._added[oid] = obj
+            if self._added_during_commit is not None:
+                self._added_during_commit.append(obj)
+        elif obj._p_jar is not self:
+            raise InvalidObjectReference(obj, obj._p_jar)
+
     def sortKey(self):
         # XXX will raise an exception if the DB hasn't been set
         storage_key = self._sortKey()
@@ -219,7 +232,13 @@ class Connection(ExportImport, object):
         if object is self:
             self._flush_invalidations()
         else:
-            assert object._p_oid is not None
+            oid = object._p_oid
+            assert oid is not None
+            if oid in self._added:
+                del self._added[oid]
+                del object._p_jar
+                del object._p_oid
+            else:
                 self._cache.invalidate(object._p_oid)
 
     def cacheFullSweep(self, dt=0):
@@ -269,38 +288,56 @@ class Connection(ExportImport, object):
             raise ReadConflictError(object=object)
 
         invalid = self._invalid
+
+        # XXX In the case of a new object or an object added using add(),
+        #     the oid is appended to _creating.
+        #     However, this ought to be unnecessary because the _p_serial
+        #     of the object will be z64 or None, so it will be appended
+        #     to _creating about 30 lines down. The removal from _added
+        #     ought likewise to be unnecessary.
         if oid is None or object._p_jar is not self:
             # new object
             oid = self.new_oid()
             object._p_jar = self
             object._p_oid = oid
+            self._creating.append(oid) # maybe don't need this
+        elif oid in self._added:
+            # maybe don't need these
             self._creating.append(oid)
-
+            del self._added[oid]
         elif object._p_changed:
             if invalid(oid):
                 resolve = getattr(object, "_p_resolveConflict", None)
                 if resolve is None:
                     raise ConflictError(object=object)
             self._modified.append(oid)
-
         else:
             # Nothing to do
             return
 
         w = ObjectWriter(object)
-        for obj in w:
+        self._added_during_commit = []
+        try:
+            for obj in itertools.chain(w, self._added_during_commit):
                 oid = obj._p_oid
                 serial = getattr(obj, '_p_serial', z64)
-            if serial == z64:
+
+                # XXX which one? z64 or None? Why do I have to check both?
+                if serial == z64 or serial is None:
                     # new object
                     self._creating.append(oid)
+                    # If this object was added, it is now in _creating, so can
+                    # be removed from _added.
+                    self._added.pop(oid, None)
                 else:
-                if invalid(oid) and not hasattr(object, '_p_resolveConflict'):
+                    if (invalid(oid)
+                        and not hasattr(object, '_p_resolveConflict')):
                         raise ConflictError(object=obj)
                     self._modified.append(oid)
+                p = w.serialize(obj)  # This calls __getstate__ of obj
                 
-            p = w.serialize(obj)
-            s = self._storage.store(oid, serial, p, self._version, transaction)
+                s = self._storage.store(oid, serial, p, self._version,
+                                        transaction)
                 self._store_count = self._store_count + 1
                 # Put the object in the cache before handling the
                 # response, just in case the response contains the
@@ -315,9 +352,12 @@ class Connection(ExportImport, object):
                         raise
 
                 self._handle_serial(s, oid)
+        finally:
+            del self._added_during_commit
+       
 
     def commit_sub(self, t):
-        """Commit all work done in subtransactions"""
+        """Commit all work done in all subtransactions for this transaction"""
         tmp=self._tmp
         if tmp is None: return
         src=self._storage
@@ -347,7 +387,7 @@ class Connection(ExportImport, object):
             self._handle_serial(s, oid, change=0)
 
     def abort_sub(self, t):
-        """Abort work done in subtransactions"""
+        """Abort work done in all subtransactions for this transaction"""
         tmp=self._tmp
         if tmp is None: return
         src=self._storage
@@ -586,6 +626,10 @@ class Connection(ExportImport, object):
         self._flush_invalidations()
         self._conflicts.clear()
         self._invalidate_creating()
+        while self._added:
+            oid, obj = self._added.popitem()
+            del obj._p_oid
+            del obj._p_jar
 
     def tpc_begin(self, transaction, sub=None):
         self._modified = []

src/ZODB/DB.py

@@ -13,8 +13,8 @@
 ##############################################################################
 """Database objects
 
-$Id: DB.py,v 1.60 2004/02/18 01:13:00 jeremy Exp $"""
-__version__='$Revision: 1.60 $'[11:-2]
+$Id: DB.py,v 1.61 2004/02/19 02:59:06 jeremy Exp $"""
+__version__='$Revision: 1.61 $'[11:-2]
 
 import cPickle, cStringIO, sys, POSException, UndoLogCompatible
 from Connection import Connection
@@ -228,8 +228,8 @@ class DB(UndoLogCompatible.UndoLogCompatible, object):
         self._connectionMap(f)
         return detail
 
-    def cacheFullSweep(self, value):
-        self._connectionMap(lambda c, v=value: c._cache.full_sweep(v))
+    def cacheFullSweep(self):
+        self._connectionMap(lambda c: c._cache.full_sweep())
 
     def cacheLastGCTime(self):
         m=[0]
@@ -240,8 +240,8 @@ class DB(UndoLogCompatible.UndoLogCompatible, object):
         self._connectionMap(f)
         return m[0]
 
-    def cacheMinimize(self, value):
-        self._connectionMap(lambda c, v=value: c._cache.minimize(v))
+    def cacheMinimize(self):
+        self._connectionMap(lambda c: c._cache.minimize())
 
     def cacheMeanAge(self): return self._cacheMean('cache_mean_age')
     def cacheMeanDeac(self): return self._cacheMean('cache_mean_deac')

src/ZODB/DemoStorage.py

@@ -20,7 +20,7 @@ The Demo storage serves two purposes:
 
   - Provide a volatile storage that is useful for giving demonstrations.
 
-The demo strorage can have a "base" storage that is used in a
+The demo storage can have a "base" storage that is used in a
 read-only fashion. The base storage must not not to contain version
 data.
 
@@ -79,7 +79,7 @@ method::
 and call it to monitor the storage.
 
 """
-__version__='$Revision: 1.24 $'[11:-2]
+__version__='$Revision: 1.25 $'[11:-2]
 
 import base64, time, string
 from ZODB import POSException, BaseStorage, utils
@@ -403,6 +403,11 @@ class DemoStorage(BaseStorage.BaseStorage):
             # Build indexes up to the pack time:
             index, vindex = self._build_indexes(stop)
 
+
+            # XXX This packing algorithm is flawed. It ignores
+            # references from non-current records after the pack
+            # time.
+
             # Now build an index of *only* those objects reachable
             # from the root.
             rootl = ['\0\0\0\0\0\0\0\0']

src/ZODB/Transaction.py

@@ -13,7 +13,7 @@
 ##############################################################################
 """Transaction management
 
-$Id: Transaction.py,v 1.56 2003/12/29 22:40:48 tim_one Exp $
+$Id: Transaction.py,v 1.57 2004/02/19 02:59:06 jeremy Exp $
 """
 import sys
 from thread import get_ident as _get_ident
@@ -90,6 +90,9 @@ class Transaction:
         else:
             return "Transaction thread=%s user=%s" % (self._id, `self.user`)
 
+    # XXX This whole freeme business is lame.
+    # As a separate task, we really need to revisit transaction management.
+
     def __del__(self):
         if self._objects:
             self.abort(freeme=0)
@@ -428,6 +431,14 @@ class Transaction:
         'Register the given object for transaction control.'
         self._append(object)
 
+    def join(self, zodb4datamanager):
+        """Join a transaction.interfaces.IDataManager with the transaction
+
+        This method is provided for "backward-compatability" with ZODB 4
+        data managers.
+        """
+        self._append(DataManagerAdapter(zodb4datamanager))
+
     def note(self, text):
         if self.description:
             self.description = "%s\n\n%s" % (self.description, text.strip())
@@ -455,6 +466,69 @@ the system problem.  See your application log for
 information on the error that lead to this problem.
 """
 
+
+
+class DataManagerAdapter(object):
+    """Adapt zodb 4-style data managers to zodb3 style
+
+    Adapt transaction.interfaces.IDataManager to
+    ZODB.interfaces.IPureDatamanager
+
+    """
+
+    # Note that it is pretty important that this does not have a _p_jar
+    # attribute. This object will be registered with a zodb3 TM, which
+    # will then try to get a _p_jar from it, using it as the default.
+    # (Objects without a _p_jar are their own data managers.)
+
+    def __init__(self, datamanager):
+        self._datamanager = datamanager
+        self._rollback = None
+
+    def commit(self, ob, transaction):
+        assert ob is self
+
+    def abort(self, ob, transaction):
+        assert ob is self
+
+        # We need to discard any changes since the last save point, or all
+        # changes
+
+        if self._rollback is None:
+            # No previous savepoint, so just abort
+            self._datamanager.abort(transaction)
+        else:
+            self._rollback()
+
+    def abort_sub(self, transaction):
+        self._datamanager.abort(transaction)
+
+    def commit_sub(self, transaction):
+        # Nothing to do wrt data, be we begin 2pc for the top-level
+        # trans
+        self._sub = False
+        
+    def tpc_begin(self, transaction, subtransaction=False):
+        self._sub = subtransaction
+
+    def tpc_abort(self, transaction):
+        if self._sub:
+            self.abort(self, transaction)
+        else:
+            self._datamanager.abort(transaction)
+
+    def tpc_finish(self, transaction):
+        if self._sub:
+            self._rollback = self._datamanager.savepoint(transaction).rollback
+        else:
+            self._datamanager.commit(transaction)
+
+    def tpc_vote(self, transaction):
+        if not self._sub:
+            self._datamanager.prepare(transaction)
+        
+
+
 ############################################################################
 # install get_transaction:
 

src/ZODB/coptimizations.c

-/*****************************************************************************
-
-  Copyright (c) 2001, 2002 Zope Corporation and Contributors.
-  All Rights Reserved.
-  
-  This software is subject to the provisions of the Zope Public License,
-  Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
-  THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
-  WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-  WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
-  FOR A PARTICULAR PURPOSE
-  
- ****************************************************************************/
-static char coptimizations_doc_string[] = 
-"C optimization for new_persistent_id().\n"
-"\n"
-"$Id: coptimizations.c,v 1.26 2003/12/11 16:02:56 jeremy Exp $\n";
-
-#include "cPersistence.h"
-
-static PyObject *py__p_oid, *py__p_jar, *py___getinitargs__, *py___module__;
-static PyObject *py_new_oid, *py___class__, *py___name__;
-
-static PyObject *InvalidObjectReference;
-
-typedef struct {
-    PyObject_HEAD
-    PyObject *jar, *stack, *new_oid;
-} persistent_id;
-
-static PyTypeObject persistent_idType;
-
-static persistent_id *
-newpersistent_id(PyObject *ignored, PyObject *args)
-{
-    persistent_id *self;
-    PyObject *jar, *stack;
-
-    if (!PyArg_ParseTuple(args, "OO!", &jar, &PyList_Type, &stack)) 
-	return NULL;
-    self = PyObject_NEW(persistent_id, &persistent_idType);
-    if (!self)
-	return NULL;
-    Py_INCREF(jar);
-    self->jar = jar;
-    Py_INCREF(stack);
-    self->stack = stack;
-    self->new_oid = NULL;
-    return self;
-}
-
-static void
-persistent_id_dealloc(persistent_id *self)
-{
-    Py_DECREF(self->jar);
-    Py_DECREF(self->stack);
-    Py_XDECREF(self->new_oid);
-    PyObject_DEL(self);
-}
-
-/* Returns the klass of a persistent object.
-   Returns NULL for other objects.
-*/
-int
-get_class(PyObject *object, PyObject **out_class)
-{
-    PyObject *class = NULL;
-
-    if (!PyType_Check(object)) {
-	if (!PER_TypeCheck(object)) 
-	    return 0;
-
-	class = PyObject_GetAttr(object, py___class__);
-	if (!class) {
-	    PyErr_Clear();
-	    return 0;
-	}
-    }
-    *out_class = class;
-    return 1;
-}
-
-/* Return a two-tuple of the class's module and name.
- */
-static PyObject *
-get_class_tuple(PyObject *class, PyObject *oid)
-{
-    PyObject *module = NULL, *name = NULL, *tuple;
-
-    module = PyObject_GetAttr(class, py___module__);
-    if (!module)
-	goto err;
-    if (!PyObject_IsTrue(module)) {
-	Py_DECREF(module);
-	/* If the class has no __module__, it must be a degnerate ZClass. */
-	return oid;
-    }
-
-    name = PyObject_GetAttr(class, py___name__);
-    if (!name)
-	goto err;
-
-    tuple = PyTuple_New(2);
-    if (!tuple)
-	goto err;
-    PyTuple_SET_ITEM(tuple, 0, module);
-    PyTuple_SET_ITEM(tuple, 1, name);
-
-    return tuple;
- err:
-    Py_XDECREF(module);
-    Py_XDECREF(name);
-    return NULL;
-}
-
-static PyObject *
-set_oid(persistent_id *self, PyObject *object)
-{
-    PyObject *oid;
-
-    if (!self->new_oid) {
-	self->new_oid = PyObject_GetAttr(self->jar, py_new_oid);
-	if (!self->new_oid)
-	    return NULL;
-    }
-    oid = PyObject_CallObject(self->new_oid, NULL);
-    if (!oid)
-	return NULL;
-    if (PyObject_SetAttr(object, py__p_oid, oid) < 0) 
-	goto err;
-    if (PyObject_SetAttr(object, py__p_jar, self->jar) < 0) 
-	goto err;
-    if (PyList_Append(self->stack, object) < 0)
-	goto err;
-    return oid;
- err:
-    Py_DECREF(oid);
-    return NULL;
-}
-
-/* persistent_id_call()
-
-   Returns a reference to a persistent object, appending it to the the
-   persistent_id's list of objects.  If a non-persistent object is
-   found, return None.
-
-   The returned reference can be either class info, oid pair or a
-   plain old oid.  If it is a pair, the class info is the module and
-   the name of the class.  The class info can be used to create a
-   ghost without loading the class.
-
-   For unusual objects, e.g. ZClasses, return just the oid.  An object
-   is unusual if it isn't an ExtensionClass, because that means it
-   doesn't inherit from Persistence, or if it has __getinitargs__().
-*/
-
-static PyObject *
-persistent_id_call(persistent_id *self, PyObject *args, PyObject *kwargs)
-{
-    PyObject *object, *oid=NULL, *klass=NULL;
-    PyObject *t1, *t2;
-    int setjar = 0;
-
-    if (!PyArg_ParseTuple(args, "O", &object))
-	return NULL;
-
-    /* If it is not an extension class, get the object's class. */
-    if (!get_class(object, &klass))
-	goto return_none;
-
-    oid = PyObject_GetAttr(object, py__p_oid);
-    if (!oid) {
-	PyErr_Clear();
-	goto return_none;
-    }
-    if (oid != Py_None) {
-	PyObject *jar;
-
-	if (!PyString_Check(oid)) {
-	    /* If the object is a class, then asking for _p_oid or
-	       _p_jar will return a descriptor.  There is no API to
-	       ask whether something is a descriptor; the best you
-	       can do is call anything with an __get__ a descriptor.
-
-	       The getattr check is potentially expensive so do the
-	       cheap PyString_Check() first, assuming that most oids
-	       that aren't None are real oids.  ZODB always uses
-	       strings, although some other user of Persistent could
-	       use something else.
-	    */
-	    static PyObject *__get__;
-	    PyObject *descr;
-	    if (!__get__) {
-		__get__ = PyString_InternFromString("__get__");
-		if (!__get__)
-		    goto err;
-	    }
-	    descr = PyObject_GetAttr(oid, __get__);
-	    if (descr) {
-		Py_DECREF(descr);
-		goto return_none;
-	    }
-	    /* Otherwise it's not a descriptor and it's just some
-	       weird value.  Maybe we'll get an error later.
-	    */
-
-	    /* XXX should check that this was an AttributeError */
-	    PyErr_Clear();
-	}
-	jar = PyObject_GetAttr(object, py__p_jar);
-	if (!jar)
-	    PyErr_Clear();
-	else {
-	    if (jar != Py_None && jar != self->jar) {
-		PyErr_SetString(InvalidObjectReference, 
-				"Attempt to store an object from a foreign "
-				"database connection");
-		goto err;
-	    }
-	    /* Ignore the oid of the unknown jar and assign a new one. */
-	    if (jar == Py_None)
-		setjar = 1;
-	    Py_DECREF(jar);
-	}
-    }
-
-    if (oid == Py_None || setjar) {
-	Py_DECREF(oid);
-	oid = set_oid(self, object);
-	if (!oid)
-	    goto err;
-    }
-
-    if (PyType_Check(object) || PyObject_HasAttr(klass, py___getinitargs__))
-	goto return_oid;
-
-    t2 = get_class_tuple(klass, oid);
-    if (!t2)
-	goto err;
-    if (t2 == oid) /* Couldn't find class info, just used oid. */
-	goto return_oid;
-    t1 = PyTuple_New(2);
-    if (!t1) {
-	Py_DECREF(t2);
-	goto err;
-    }
-    /* use borrowed references to oid and t2 */
-    PyTuple_SET_ITEM(t1, 0, oid);
-    PyTuple_SET_ITEM(t1, 1, t2);
-
-    Py_DECREF(klass);
-
-    return t1;
-
- err:
-    Py_XDECREF(oid);
-    oid = NULL;
-
- return_oid:
-    Py_XDECREF(klass);
-    return oid;
-
- return_none:
-    Py_XDECREF(oid);
-    Py_XDECREF(klass);
-    Py_INCREF(Py_None);
-    return Py_None;
-}
-
-
-static PyTypeObject persistent_idType = {
-    PyObject_HEAD_INIT(NULL)
-    0,					/*ob_size*/
-    "persistent_id",			/*tp_name*/
-    sizeof(persistent_id),		/*tp_basicsize*/
-    0,					/*tp_itemsize*/
-    (destructor)persistent_id_dealloc,	/*tp_dealloc*/
-    0,					/*tp_print*/
-    0,					/*tp_getattr*/
-    0,					/*tp_setattr*/
-    0,					/*tp_compare*/
-    0,					/*tp_repr*/
-    0,					/*tp_as_number*/
-    0,					/*tp_as_sequence*/
-    0,					/*tp_as_mapping*/
-    0,					/*tp_hash*/
-    (ternaryfunc)persistent_id_call,	/*tp_call*/
-    0,					/*tp_str*/
-    0,					/*tp_getattro*/
-    0,					/*tp_setattro*/
-    0,					/* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,			/* tp_flags */
-    "C implementation of the persistent_id function defined in Connection.py"
-    					/* tp_doc */
-};
-
-/* End of code for persistent_id objects */
-/* -------------------------------------------------------- */
-
-
-/* List of methods defined in the module */
-
-static struct PyMethodDef Module_Level__methods[] = {
-    {"new_persistent_id", (PyCFunction)newpersistent_id, METH_VARARGS,
-     "new_persistent_id(jar, stack) -- get a new persistent_id function"},
-    {NULL, NULL}		/* sentinel */
-};
-
-void
-initcoptimizations(void)
-{
-    PyObject *m;
-
-#define make_string(S) if (! (py_ ## S=PyString_FromString(#S))) return
-    make_string(_p_oid);
-    make_string(_p_jar);
-    make_string(__getinitargs__);
-    make_string(__module__);
-    make_string(__class__);
-    make_string(__name__);
-    make_string(new_oid);
-			
-    /* Get InvalidObjectReference error */
-    m = PyImport_ImportModule("ZODB.POSException");
-    if (!m)
-	return;
-    InvalidObjectReference = PyObject_GetAttrString(m, 
-						    "InvalidObjectReference");
-    Py_DECREF(m);
-    if (!InvalidObjectReference)
-	return;
-
-    cPersistenceCAPI = PyCObject_Import("persistent.cPersistence", "CAPI");
-    if (!cPersistenceCAPI)
-	return;
-
-    m = Py_InitModule3("coptimizations", Module_Level__methods,
-		       coptimizations_doc_string);
-
-    persistent_idType.ob_type = &PyType_Type;
-    Py_INCREF((PyObject *)&persistent_idType);
-    PyModule_AddObject(m, "persistent_idType", (PyObject *)&persistent_idType);
-}

src/ZODB/interfaces.py

+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Interfaces for ZODB."""
+
+try:
+    from zope.interface import Interface, Attribute
+except ImportError:
+    class Interface:
+        pass
+
+    class Attribute:
+        def __init__(self, __name__, __doc__):
+            self.__name__ = __name__
+            self.__doc__ = __doc__
+
+
+class IDataManager(Interface):
+    """Objects that manage transactional storage.
+
+    These object's may manage data for other objects, or they may manage
+    non-object storages, such as relational databases.
+    """
+
+    def abort_sub(transaction):
+        """Discard all subtransaction data.
+
+        See subtransaction.txt
+
+        This is called when top-level transactions are aborted.
+
+        No further subtransactions can be started once abort_sub()
+        has been called; this is only used when the transaction is
+        being aborted.
+
+        abort_sub also implies the abort of a 2-phase commit.
+
+        This should never fail.
+        """
+
+    def commit_sub(transaction):
+        """Commit all changes made in subtransactions and begin 2-phase commit
+
+        Data are saved *as if* they are part of the current transaction.
+        That is, they will not be persistent unless the current transaction
+        is committed.
+
+        This is called when the current top-level transaction is committed.
+
+        No further subtransactions can be started once commit_sub()
+        has been called; this is only used when the transaction is
+        being committed.
+
+        This call also implied the beginning of 2-phase commit.
+        """
+
+    # Two-phase commit protocol.  These methods are called by the
+    # ITransaction object associated with the transaction being
+    # committed.
+
+    def tpc_begin(transaction, subtransaction=False):
+        """Begin commit of a transaction, starting the two-phase commit.
+
+        transaction is the ITransaction instance associated with the
+        transaction being committed.
+
+        subtransaction is a Boolean flag indicating whether the
+        two-phase commit is being invoked for a subtransaction.
+
+        Important note: Subtransactions are modelled in the sense that
+        when you commit a subtransaction, subsequent commits should be
+        for subtransactions as well.  That is, there must be a
+        commit_sub() call between a tpc_begin() call with the
+        subtransaction flag set to true and a tpc_begin() with the
+        flag set to false.
+
+        """
+        
+
+    def tpc_abort(transaction):
+        """Abort a transaction.
+
+        This is always called after a tpc_begin call.
+
+        transaction is the ITransaction instance associated with the
+        transaction being committed.
+
+        This should never fail.
+        """
+
+    def tpc_finish(transaction):
+        """Indicate confirmation that the transaction is done.
+
+        transaction is the ITransaction instance associated with the
+        transaction being committed.
+
+        This should never fail. If this raises an exception, the
+        database is not expected to maintain consistency; it's a
+        serious error.
+
+        """
+
+    def tpc_vote(transaction):
+        """Verify that a data manager can commit the transaction
+
+        This is the last chance for a data manager to vote 'no'.  A
+        data manager votes 'no' by raising an exception.
+
+        transaction is the ITransaction instance associated with the
+        transaction being committed.
+        """
+
+    def commit(object, transaction):
+        """CCCommit changes to an object
+
+        Save the object as part of the data to be made persistent if
+        the transaction commits.
+        """
+
+    def abort(object, transaction):
+        """Abort changes to an object
+
+        Only changes made since the last transaction or
+        sub-transaction boundary are discarded.
+
+        This method may be called either:
+
+        o Outside of two-phase commit, or
+
+        o In the first phase of two-phase commit
+
+        """
+
+    
+
+
+class ITransaction(Interface):
+    """Object representing a running transaction.
+
+    Objects with this interface may represent different transactions
+    during their lifetime (.begin() can be called to start a new
+    transaction using the same instance).
+    """
+
+    user = Attribute(
+        "user",
+        "The name of the user on whose behalf the transaction is being\n"
+        "performed.  The format of the user name is defined by the\n"
+        "application.")
+    # XXX required to be a string?
+
+    description = Attribute(
+        "description",
+        "Textual description of the transaction.")
+
+    def begin(info=None, subtransaction=None):
+        """Begin a new transaction.
+
+        If the transaction is in progress, it is aborted and a new
+        transaction is started using the same transaction object.
+        """
+
+    def commit(subtransaction=None):
+        """Finalize the transaction.
+
+        This executes the two-phase commit algorithm for all
+        IDataManager objects associated with the transaction.
+        """
+
+    def abort(subtransaction=0, freeme=1):
+        """Abort the transaction.
+
+        This is called from the application.  This can only be called
+        before the two-phase commit protocol has been started.
+        """
+
+    def join(datamanager):
+        """Add a datamanager to the transaction.
+
+        The datamanager must implement the
+        transactions.interfaces.IDataManager interface, and be
+        adaptable to ZODB.interfaces.IDataManager.
+        """
+
+    def register(object):
+        """Register the given object for transaction control."""
+
+    def note(text):
+        """Add text to the transaction description.
+
+        If a description has already been set, text is added to the
+        end of the description following two newline characters.
+        Surrounding whitespace is stripped from text.
+        """
+        # XXX does impl do the right thing with ''?  Not clear what
+        # the "right thing" is.
+
+    def setUser(user_name, path="/"):
+        """Set the user name.
+
+        path should be provided if needed to further qualify the
+        identified user.
+        """
+
+    def setExtendedInfo(name, value):
+        """Add extension data to the transaction.
+
+        name is the name of the extension property to set; value must
+        be a picklable value.
+
+        Storage implementations may limit the amount of extension data
+        which can be stored.
+        """
+        # XXX is this this allowed to cause an exception here, during
+        # the two-phase commit, or can it toss data silently?

src/ZODB/referencesf.py

@@ -15,33 +15,54 @@
 """
 import cPickle, cStringIO
 
-def referencesf(p, rootl=None,
-                Unpickler=cPickle.Unpickler,
-                StringIO=cStringIO.StringIO,
-                tt=type(()),
-                type=type):
-
-    if rootl is None: rootl=[]
-    u=Unpickler(StringIO(p))
-    l=len(rootl)
-    u.persistent_load=rootl
+def referencesf(p, rootl=None,):
+
+    if rootl is None:
+        rootl = []
+
+    u = cPickle.Unpickler(cStringIO.StringIO(p))
+    l = len(rootl)
+    u.persistent_load = rootl
+    u.noload()
+    try:
         u.noload()
-    try: u.noload()
     except:
         # Hm.  We failed to do second load.  Maybe there wasn't a
         # second pickle.  Let's check:
-        f=StringIO(p)
-        u=Unpickler(f)
-        u.persistent_load=[]
+        f = cStringIO.StringIO(p)
+        u = cPickle.Unpickler(f)
+        u.persistent_load = []
         u.noload()
-        if len(p) > f.tell(): raise ValueError, 'Error unpickling, %s' % p
-
-    # References may have class info, so we need to
-    # check for wrapped references.
-    for i in range(l, len(rootl)):
-        v=rootl[i]
-        if v:
-            if type(v) is tt: v=v[0]
-            rootl[i]=v
+        if len(p) > f.tell():
+            raise ValueError, 'Error unpickling, %s' % p
+
+
+    # References may be:
+    #
+    # - A tuple, in which case they are an oid and class.
+    #   In this case, just extract the first element, which is
+    #   the oid
+    #
+    # - A list, which is a weak reference. We skip those.
+    #
+    # - Anything else must be an oid. This means that an oid
+    #   may not be a list or a tuple. This is a bit lame.
+    #   We could avoid this lamosity by allowing single-element
+    #   tuples, so that we wrap oids that are lists or tuples in
+    #   tuples.
+    #
+    # - oids may *not* be false.  I'm not sure why. 
+
+    out = []
+    for v in rootl:
+        assert v # Let's see if we ever get empty ones
+        if type(v) is list:
+            # skip wekrefs
+            continue
+        if type(v) is tuple:
+            v = v[0]
+        out.append(v)
+
+    rootl[:] = out
 
     return rootl

src/ZODB/serialize.py

@@ -58,34 +58,35 @@ old class.
 import cPickle
 import cStringIO
 
-from ZODB.coptimizations import new_persistent_id
+from persistent import Persistent
+from persistent.wref import WeakRefMarker, WeakRef
+from ZODB.POSException import InvalidObjectReference
 
-_marker = object()
+# Might to update or redo to reflect weakrefs
+# from ZODB.coptimizations import new_persistent_id
 
-def myhasattr(obj, attr):
-    """Returns True or False or raises an exception."""
-    val = getattr(obj, attr, _marker)
-    return val is not _marker
+
+def myhasattr(obj, name, _marker=object()):
+    """Make sure we don't mask exceptions like hasattr().
+
+    We don't want exceptions other than AttributeError to be masked,
+    since that too often masks other programming errors.
+    Three-argument getattr() doesn't mask those, so we use that to
+    implement our own hasattr() replacement.
+    """
+    return getattr(obj, name, _marker) is not _marker
 
 def getClassMetadata(obj):
-    klass = obj.__class__
-    if issubclass(klass, type):
-        # Handle ZClasses.
-        d = obj.__dict__.copy()
-        del d["_p_jar"]
-        args = obj.__name__, obj.__bases__, d
-        return klass, args
-    else:
-        getinitargs = getattr(klass, "__getinitargs__", None)
-        if getinitargs is None:
-            args = None
-        else:
-            args = getinitargs()
-        mod = getattr(klass, "__module__", None)
-        if mod is None:
-            return klass, args
+
+    # We don't use __class__ here, because obj could be a persistent proxy.
+    # We don't want to be folled by proxies.
+    klass = type(obj)
+
+    newargs = getattr(klass, "__getnewargs__", None)
+    if newargs is None:
+        return klass
     else:
-            return (mod, klass.__name__), args
+        return klass, newargs(obj)
 
 class BaseObjectWriter:
     """Serializes objects for storage in the database.
@@ -93,23 +94,90 @@ class BaseObjectWriter:
     The ObjectWriter creates object pickles in the ZODB format.  It
     also detects new persistent objects reachable from the current
     object.
-
-    The client is responsible for calling the close() method to avoid
-    leaking memory.  The ObjectWriter uses a Pickler internally, and
-    Pickler objects do not participate in garbage collection.  (Note
-    that in Python 2.3 and higher, the close() method would be
-    unnecessary because Picklers participate in garbage collection.)
     """
 
     def __init__(self, jar=None):
         self._file = cStringIO.StringIO()
         self._p = cPickle.Pickler(self._file, 1)
         self._stack = []
-        self._p.persistent_id = new_persistent_id(jar, self._stack)
+        self._p.persistent_id = self.persistent_id
         if jar is not None:
             assert myhasattr(jar, "new_oid")
         self._jar = jar
 
+
+    def persistent_id(self, obj):
+
+        # Most objects are not persistent. The following cheap test
+        # identifies most of them.  For these, we return None,
+        # signalling that the object should be pickled normally.
+        if not isinstance(obj, (Persistent, type, WeakRef)):
+            # Not persistent, pickle normally
+            return None
+
+        # Any persistent object mosy have an oid:
+        try:
+            oid = obj._p_oid
+        except AttributeError:
+            # Not persistent, pickle normally
+            return None
+
+        if not (oid is None or isinstance(oid, str)):
+            # Deserves a closer look:
+
+            # Make sure it's not a descr
+            if hasattr(oid, '__get__'):
+                # The oid is a decriptor.  That means obj is a non-persistent
+                # class whose instances are persistent, so ...
+                # Not persistent, pickle normally
+                return None
+
+            if oid is WeakRefMarker:
+                # we have a weakref, see weakref.py
+
+                oid = obj.oid
+                if oid is None:
+                    obj = obj() # get the referenced object
+                    oid = obj._p_oid
+                    if oid is None:
+                        # Here we are causing the object to be saved in
+                        # the database. One could argue that we shouldn't
+                        # do this, because a wekref should not cause an object
+                        # to be added.  We'll be optimistic, though, and
+                        # assume that the object will be added eventually.
+                        
+                        oid = self._jar.new_oid()
+                        obj._p_jar = self._jar
+                        obj._p_oid = oid
+                        self._stack.append(obj)
+                return [oid]
+
+
+        # Since we have an oid, we have either a persistent instance
+        # (an instance of Persistent), or a persistent class.
+
+        # NOTE! Persistent classes don't (and can't) subclass persistent.
+
+        if oid is None:
+            oid = obj._p_oid = self._jar.new_oid()
+            obj._p_jar = self._jar
+            self._stack.append(obj)
+        elif obj._p_jar is not self._jar:
+            raise InvalidObjectReference(
+                "Attempt to store an object from a foreign "
+                "database connection"
+                )
+
+        klass = type(obj)
+        if hasattr(klass, '__getnewargs__'):
+            # We don't want to save newargs in object refs.
+            # It's possible that __getnewargs__ is degenerate and
+            # returns (), but we don't want to have to deghostify
+            # the object to find out.
+            return oid 
+
+        return oid, klass
+
     def serialize(self, obj):
         return self._dump(getClassMetadata(obj), obj.__getstate__())
 
@@ -168,7 +236,7 @@ class BaseObjectReader:
         return unpickler
 
     def _new_object(self, klass, args):
-        if not args and not myhasattr(klass, "__getinitargs__"):
+        if not args and not myhasattr(klass, "__getnewargs__"):
             obj = klass.__new__(klass)
         else:
             obj = klass(*args)
@@ -179,19 +247,32 @@ class BaseObjectReader:
 
     def getClassName(self, pickle):
         unpickler = self._get_unpickler(pickle)
-        klass, newargs = unpickler.load()
+        klass = unpickler.load()
+        if isinstance(klass, tuple):
+            klass, args = klass
             if isinstance(klass, tuple):
+                # old style reference
                 return "%s.%s" % klass
-        else:
-            return klass.__name__
+        return "%s.%s" % (klass.__module__, klass.__name__)
 
     def getGhost(self, pickle):
         unpickler = self._get_unpickler(pickle)
-        klass, args = unpickler.load()
+        klass = unpickler.load()
+        if isinstance(klass, tuple):
+            # Here we have a separate class and args.
+            # This could be an old record, so the class module ne a named
+            # refernce
+            klass, args = klass
             if isinstance(klass, tuple):
+                # Old module_name, class_name tuple
                 klass = self._get_class(*klass)
-
-        return self._new_object(klass, args)
+            if args is None:
+                return klass.__new__(klass)
+            else:
+                return klass.__new__(klass, *args)
+        else:
+            # Definately new style direct class reference
+            return klass.__new__(klass)
 
     def getState(self, pickle):
         unpickler = self._get_unpickler(pickle)
@@ -202,13 +283,6 @@ class BaseObjectReader:
         state = self.getState(pickle)
         obj.__setstate__(state)
 
-    def getObject(self, pickle):
-        unpickler = self._get_unpickler(pickle)
-        klass, args = unpickler.load()
-        obj = self._new_object(klass, args)
-        state = unpickler.load()
-        obj.__setstate__(state)
-        return obj
 
 class ExternalReference(object):
     pass
@@ -242,19 +316,18 @@ class ConnectionObjectReader(BaseObjectReader):
         if isinstance(oid, tuple):
             # Quick instance reference.  We know all we need to know
             # to create the instance w/o hitting the db, so go for it!
-            oid, klass_info = oid
+            oid, klass = oid
             obj = self._cache.get(oid, None) # XXX it's not a dict
             if obj is not None:
                 return obj
-
-            klass = self._get_class(*klass_info)
-            # XXX Why doesn't this have args?
-            obj = self._new_object(klass, None)
-            # XXX This doesn't address the last fallback that used to
-            # exist:
-##                    # Eek, we couldn't get the class. Hm.  Maybe there's
-##                    # more current data in the object's actual record!
-##                    return self._conn[oid]
+            if isinstance(klass, tuple):
+                klass = self._get_class(*klass)
+            try:
+                obj = klass.__new__(klass)
+            except TypeError:
+                # Couldn't create the instance.  Maybe there's more
+                # current data in the object's actual record!
+                return self._conn[oid]
 
             # XXX should be done by connection
             obj._p_oid = oid
@@ -267,7 +340,15 @@ class ConnectionObjectReader(BaseObjectReader):
             self._cache[oid] = obj
             return obj
 
-        obj = self._cache.get(oid)
+        elif isinstance(oid, list):
+            # see weakref.py
+            [oid] = oid
+            obj = WeakRef.__new__(WeakRef)
+            obj.oid = oid
+            obj.dm = self._conn
+            return obj
+
+        obj = self._cache.get(oid, None)
         if obj is not None:
             return obj
         return self._conn[oid]

src/ZODB/subtransactions.txt

+=========================
+Subtransactions in ZODB 3
+=========================
+
+ZODB 3 provides limited support for subtransactions. Subtransactions
+are nested to *one* level. There are top-level transactions and
+subtransactions.  When a transaction is committed, a flag is passed
+indicating whether it is a subtransaction or a top-level transaction.
+Consider the following exampler commit calls:
+
+- commit()
+
+  A regular top-level transaction is committed.
+
+- commit(1)
+
+  A subtransaction is committed. There is now one subtransaction of
+  the current top-level transaction.
+
+- commit(1)
+
+  A subtransaction is committed. There are now two subtransactions of
+  the current top-level transaction.
+
+- abort(1)
+
+  A subtransaction is aborted. There are still two subtransactions of
+  the current top-level transaction; work done since the last
+  commit(1) call is discarded.
+
+- commit()
+
+  We now commit a top-level transaction. The work done in the previous
+  two subtransactions *plus* work done since the last abort(1) call
+  is saved.
+
+- commit(1)
+
+  A subtransaction is committed. There is now one subtransaction of
+  the current top-level transaction.
+
+- commit(1)
+
+  A subtransaction is committed. There are now two subtransactions of
+  the current top-level transaction.
+
+- abort()
+
+  We now abort a top-level transaction. We discard the work done in
+  the previous two subtransactions *plus* work done since the last
+  commit(1) call.

src/ZODB/tests/ConflictResolution.py

@@ -102,9 +102,9 @@ class ConflictResolvingStorage:
             self.fail("Expected ConflictError")
 
     def checkZClassesArentResolved(self):
-        from ZODB.ConflictResolution import bad_class
+        from ZODB.ConflictResolution import find_global, BadClassName
         dummy_class_tuple = ('*foobar', ())
-        assert bad_class(dummy_class_tuple) == 1
+        self.assertRaises(BadClassName, find_global, '*foobar', ())
 
     def checkBuggyResolve1(self):
         obj = PCounter3()

src/ZODB/tests/StorageTestBase.py

@@ -79,7 +79,12 @@ def zodb_unpickle(data):
     u.persistent_load = persistent_load
     klass_info = u.load()
     if isinstance(klass_info, types.TupleType):
-        if isinstance(klass_info[0], types.TupleType):
+        if isinstance(klass_info[0], type):
+            # XXX what is the second part of klass_info?
+            klass, xxx = klass_info
+            assert not xxx
+        else:
+            if isinstance(klass_info[0], tuple):
                 modname, klassname = klass_info[0]
             else:
                 modname, klassname = klass_info

src/ZODB/tests/testCache.py

@@ -21,6 +21,7 @@ objects in memory under the assumption that they may be used again.
 import time
 import types
 import unittest
+import gc
 
 import ZODB
 import ZODB.MappingStorage
@@ -102,14 +103,14 @@ class DBMethods(CacheTestBase):
     def checkFullSweep(self):
         old_size = self.db.cacheSize()
         time.sleep(3)
-        self.db.cacheFullSweep(0)
+        self.db.cacheFullSweep()
         new_size = self.db.cacheSize()
         self.assert_(new_size < old_size, "%s < %s" % (old_size, new_size))
 
     def checkMinimize(self):
         old_size = self.db.cacheSize()
         time.sleep(3)
-        self.db.cacheMinimize(0)
+        self.db.cacheMinimize()
         new_size = self.db.cacheSize()
         self.assert_(new_size < old_size, "%s < %s" % (old_size, new_size))
 
@@ -184,7 +185,13 @@ class LRUCacheTests(CacheTestBase):
         self.assertEquals(len(details), CONNS)
         for d in details:
             self.assertEquals(d['ngsize'], CACHE_SIZE)
-            self.assertEquals(d['size'], CACHE_SIZE)
+
+            # The assertion below is non-sensical
+            # The (poorly named) cache size is a target for non-ghosts.
+            # The cache *usually* contains non-ghosts, so that the
+            # size normally exceeds the target size.
+            
+            #self.assertEquals(d['size'], CACHE_SIZE)
 
     def checkDetail(self):
         CACHE_SIZE = 10
@@ -194,6 +201,28 @@ class LRUCacheTests(CacheTestBase):
         for i in range(CONNS):
             self.noodle_new_connection()
 
+        gc.collect()
+
+        # XXX The above gc.collect call is necessary to make this test
+        # pass.
+        #
+        # This test then only works because the other of computations
+        # and object accesses in the "noodle" calls is such that the
+        # persistent mapping containing the MinPO objects is
+        # deactivated before the MinPO objects.
+        #
+        # - Without the gc call, the cache will contain ghost MinPOs
+        #   and the check of the MinPO count below will fail. That's 
+        #   because the counts returned by cacheDetail include ghosts.
+        #
+        # - If the mapping object containing the MinPOs isn't
+        #   deactivated, there will be one fewer non-ghost MinPO and
+        #   the test will fail anyway.
+        #
+        # This test really needs to be thought through and documented
+        # better. 
+
+
         for klass, count in self.db.cacheDetail():
             if klass.endswith('MinPO'):
                 self.assertEqual(count, CONNS * CACHE_SIZE)

src/ZODB/tests/testConnection.py

+##############################################################################
+#
+# Copyright (c) 2001, 2002 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Unit tests for the Connection class.
+"""
+
+import unittest
+from persistent import Persistent
+
+__metaclass__ = type
+
+class ConnectionDotAdd(unittest.TestCase):
+
+    def setUp(self):
+        from ZODB.Connection import Connection
+        self.datamgr = Connection()
+        self.db = StubDatabase()
+        self.datamgr._setDB(self.db)
+        self.transaction = StubTransaction()
+
+    def check_add(self):
+        from ZODB.POSException import InvalidObjectReference
+        obj = StubObject()
+        self.assert_(obj._p_oid is None)
+        self.assert_(obj._p_jar is None)
+        self.datamgr.add(obj)
+        self.assert_(obj._p_oid is not None)
+        self.assert_(obj._p_jar is self.datamgr)
+        self.assert_(self.datamgr[obj._p_oid] is obj)
+
+        # Only first-class persistent objects may be added.
+        self.assertRaises(TypeError, self.datamgr.add, object())
+
+        # Adding to the same connection does not fail. Object keeps the
+        # same oid.
+        oid = obj._p_oid
+        self.datamgr.add(obj)
+        self.assertEqual(obj._p_oid, oid)
+
+        # Cannot add an object from a diffrerent connection.
+        obj2 = StubObject()
+        obj2._p_jar = object()
+        self.assertRaises(InvalidObjectReference, self.datamgr.add, obj2)
+
+    def checkResetOnAbort(self):
+        # Check that _p_oid and _p_jar are reset when a transaction is
+        # aborted.
+        obj = StubObject()
+        self.datamgr.add(obj)
+        oid = obj._p_oid
+        self.datamgr.abort(obj, self.transaction)
+        self.assert_(obj._p_oid is None)
+        self.assert_(obj._p_jar is None)
+        self.assertRaises(KeyError, self.datamgr.__getitem__, oid)
+
+    def checkResetOnTpcAbort(self):
+        obj = StubObject()
+        self.datamgr.add(obj)
+        oid = obj._p_oid
+
+        # This case simulates when an error occurred committing some other
+        # object, so tpc_abort is called, clearing everything.
+        self.datamgr.tpc_begin(self.transaction)
+        # Let's pretend something bad happens here.
+        self.datamgr.tpc_abort(self.transaction)
+        self.assert_(obj._p_oid is None)
+        self.assert_(obj._p_jar is None)
+        self.assertRaises(KeyError, self.datamgr.__getitem__, oid)
+
+    def checkTcpAbortAfterCommit(self):
+        obj = StubObject()
+        self.datamgr.add(obj)
+        oid = obj._p_oid
+
+        self.datamgr.tpc_begin(self.transaction)
+        self.datamgr.commit(obj, self.transaction)
+        # Let's pretend something bad happened here.
+        self.datamgr.tpc_abort(self.transaction)
+        self.assert_(obj._p_oid is None)
+        self.assert_(obj._p_jar is None)
+        self.assertRaises(KeyError, self.datamgr.__getitem__, oid)
+        self.assertEquals(self.db._storage._stored, [oid])
+
+    def checkCommit(self):
+        obj = StubObject()
+        self.datamgr.add(obj)
+        oid = obj._p_oid
+
+        self.datamgr.tpc_begin(self.transaction)
+        self.datamgr.commit(obj, self.transaction)
+        self.datamgr.tpc_finish(self.transaction)
+        self.assert_(obj._p_oid is oid)
+        self.assert_(obj._p_jar is self.datamgr)
+
+        # This next assert_ is covered by an assert in tpc_finish.
+        ##self.assert_(not self.datamgr._added)
+
+        self.assertEquals(self.db._storage._stored, [oid])
+        self.assertEquals(self.db._storage._finished, [oid])
+
+    def checkModifyOnGetstate(self):
+        subobj = StubObject()
+        obj = ModifyOnGetStateObject(subobj)
+        
+        self.datamgr.tpc_begin(self.transaction)
+        self.datamgr.commit(obj, self.transaction)
+        self.datamgr.tpc_finish(self.transaction)
+        storage = self.db._storage
+        self.assert_(obj._p_oid in storage._stored, "object was not stored")
+        self.assert_(subobj._p_oid in storage._stored,
+                "subobject was not stored")
+        self.assert_(self.datamgr._added_during_commit is None)
+
+    def checkErrorDuringCommit(self):
+        # We need to check that _added_during_commit still gets set to None
+        # when there is an error during commit()/
+        obj = ErrorOnGetstateObject()
+        
+        self.datamgr.tpc_begin(self.transaction)
+        self.assertRaises(ErrorOnGetstateException,
+                self.datamgr.commit, obj, self.transaction)
+        self.assert_(self.datamgr._added_during_commit is None)
+
+    def checkUnusedAddWorks(self):
+        # When an object is added, but not committed, it shouldn't be stored,
+        # but also it should be an error.
+        obj = StubObject()
+        self.datamgr.add(obj)
+        self.datamgr.tpc_begin(self.transaction)
+        self.datamgr.tpc_finish(self.transaction)
+        self.assert_(obj._p_oid not in self.datamgr._storage._stored)
+
+# ---- stubs
+
+class StubObject(Persistent):
+    pass
+
+
+class StubTransaction:
+    pass
+
+class ErrorOnGetstateException(Exception):
+    pass
+    
+class ErrorOnGetstateObject(Persistent):
+
+    def __getstate__(self):
+        raise ErrorOnGetstateException
+
+class ModifyOnGetStateObject(Persistent):
+
+    def __init__(self, p):
+        self._v_p = p
+
+    def __getstate__(self):
+        self._p_jar.add(self._v_p)
+        self.p = self._v_p
+        return Persistent.__getstate__(self)
+
+
+class StubStorage:
+    """Very simple in-memory storage that does *just* enough to support tests.
+
+    Only one concurrent transaction is supported.
+    Voting is not supported.
+    Versions are not supported.
+
+    Inspect self._stored and self._finished to see how the storage has been
+    used during a unit test. Whenever an object is stored in the store()
+    method, its oid is appended to self._stored. When a transaction is
+    finished, the oids that have been stored during the transaction are
+    appended to self._finished.
+    """
+
+    sortKey = 'StubStorage sortKey'
+
+    # internal
+    _oid = 1
+    _transaction = None
+
+    def __init__(self):
+        # internal
+        self._stored = []
+        self._finished = []
+        self._data = {}
+        self._transdata = {}
+        self._transstored = []
+
+    def new_oid(self):
+        oid = str(self._oid)
+        self._oid += 1
+        return oid
+
+    def tpc_begin(self, transaction):
+        if transaction is None:
+            raise TypeError('transaction may not be None')
+        elif self._transaction is None:
+            self._transaction = transaction
+        elif self._transaction != transaction:
+            raise RuntimeError(
+                'StubStorage uses only one transaction at a time')
+
+    def tpc_abort(self, transaction):
+        if transaction is None:
+            raise TypeError('transaction may not be None')
+        elif self._transaction != transaction:
+            raise RuntimeError(
+                'StubStorage uses only one transaction at a time')
+        del self._transaction
+        self._transdata.clear()
+
+    def tpc_finish(self, transaction, callback):
+        if transaction is None:
+            raise TypeError('transaction may not be None')
+        elif self._transaction != transaction:
+            raise RuntimeError(
+                'StubStorage uses only one transaction at a time')
+        self._finished.extend(self._transstored)
+        self._data.update(self._transdata)
+        callback(transaction)
+        del self._transaction
+        self._transdata.clear()
+        self._transstored = []
+
+    def load(self, oid, version):
+        if version != '':
+            raise TypeError('StubStorage does not support versions.')
+        return self._data[oid]
+
+    def store(self, oid, serial, p, version, transaction):
+        if version != '':
+            raise TypeError('StubStorage does not support versions.')
+        if transaction is None:
+            raise TypeError('transaction may not be None')
+        elif self._transaction != transaction:
+            raise RuntimeError(
+                'StubStorage uses only one transaction at a time')
+        self._stored.append(oid)
+        self._transstored.append(oid)
+        self._transdata[oid] = (p, serial)
+        # Explicitly returing None, as we're not pretending to be a ZEO
+        # storage
+        return None
+
+
+class StubDatabase:
+
+    def __init__(self):
+        self._storage = StubStorage()
+
+    _classFactory = None
+
+    def invalidate(self, transaction, dict_with_oid_keys, connection):
+        pass
+
+
+def test_suite():
+    s = unittest.makeSuite(ConnectionDotAdd, 'check')
+    return s

src/ZODB/tests/testSerialize.py

+##############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Tests of the serializer."""
+
+import cPickle
+import cStringIO as StringIO
+import sys
+import unittest
+
+from ZODB import serialize
+
+
+class ClassWithNewargs(int):
+    def __new__(cls, value):
+        return int.__new__(cls, value)
+
+    def __getnewargs__(self):
+        return int(self),
+
+class ClassWithoutNewargs(object):
+    def __init__(self, value):
+        self.value = value
+
+def make_pickle(ob):
+    sio = StringIO.StringIO()
+    p = cPickle.Pickler(sio, 1)
+    p.dump(ob)
+    return sio.getvalue()
+
+
+class SerializerTestCase(unittest.TestCase):
+
+    # old format:  (module, name), None
+    old_style_without_newargs = make_pickle(
+        ((__name__, "ClassWithoutNewargs"), None))
+
+    # old format:  (module, name), argtuple
+    old_style_with_newargs = make_pickle(
+        ((__name__, "ClassWithNewargs"), (1,)))
+
+    # new format:  klass
+    new_style_without_newargs = make_pickle(
+        ClassWithoutNewargs)
+
+    # new format:  klass, argtuple
+    new_style_with_newargs = make_pickle(
+        (ClassWithNewargs, (1,)))
+
+    def test_getClassName(self):
+        r = serialize.BaseObjectReader()
+        eq = self.assertEqual
+        eq(r.getClassName(self.old_style_with_newargs),
+           __name__ + ".ClassWithNewargs")
+        eq(r.getClassName(self.new_style_with_newargs),
+           __name__ + ".ClassWithNewargs")
+        eq(r.getClassName(self.old_style_without_newargs),
+           __name__ + ".ClassWithoutNewargs")
+        eq(r.getClassName(self.new_style_without_newargs),
+           __name__ + ".ClassWithoutNewargs")
+
+    def test_getGhost(self):
+        # Use a TestObjectReader since we need _get_class() to be
+        # implemented; otherwise this is just a BaseObjectReader.
+
+        class TestObjectReader(serialize.BaseObjectReader):
+            # A production object reader would optimize this, but we
+            # don't need to in a test
+            def _get_class(self, module, name):
+                __import__(module)
+                return getattr(sys.modules[module], name)
+
+        r = TestObjectReader()
+        g = r.getGhost(self.old_style_with_newargs)
+        self.assert_(isinstance(g, ClassWithNewargs))
+        self.assertEqual(g, 1)
+        g = r.getGhost(self.old_style_without_newargs)
+        self.assert_(isinstance(g, ClassWithoutNewargs))
+        g = r.getGhost(self.new_style_with_newargs)
+        self.assert_(isinstance(g, ClassWithNewargs))
+        g = r.getGhost(self.new_style_without_newargs)
+        self.assert_(isinstance(g, ClassWithoutNewargs))
+
+    def test_myhasattr(self):
+
+        class OldStyle:
+            bar = "bar"
+            def __getattr__(self, name):
+                if name == "error":
+                    raise ValueError("whee!")
+                else:
+                    raise AttributeError, name
+
+        class NewStyle(object):
+            bar = "bar"
+            def _raise(self):
+                raise ValueError("whee!")
+            error = property(_raise)
+
+        self.assertRaises(ValueError,
+                          serialize.myhasattr, OldStyle(), "error")
+        self.assertRaises(ValueError,
+                          serialize.myhasattr, NewStyle(), "error")
+        self.assert_(serialize.myhasattr(OldStyle(), "bar"))
+        self.assert_(serialize.myhasattr(NewStyle(), "bar"))
+        self.assert_(not serialize.myhasattr(OldStyle(), "rat"))
+        self.assert_(not serialize.myhasattr(NewStyle(), "rat"))
+
+
+def test_suite():
+    return unittest.makeSuite(SerializerTestCase)

src/ZODB/tests/testTransaction.py

@@ -14,7 +14,7 @@
 
 """
 Revision information:
-$Id: testTransaction.py,v 1.17 2003/12/24 16:01:58 jeremy Exp $
+$Id: testTransaction.py,v 1.18 2004/02/19 02:59:10 jeremy Exp $
 """
 
 """
@@ -628,9 +628,39 @@ class HoserJar(BasicJar):
         HoserJar.committed += 1
 
 
+def test_join():
+    """White-box test of the join method
+
+    The join method is provided for "backward-compatability" with ZODB 4
+    data managers.
+
+    The argument to join must be a zodb4 data manager,
+    transaction.interfaces.IDataManager.
+
+    >>> from ZODB.tests.sampledm import DataManager
+    >>> from ZODB.Transaction import DataManagerAdapter
+    >>> t = Transaction.Transaction()
+    >>> dm = DataManager()
+    >>> t.join(dm)
+
+    The end result is that a data manager adapter is one of the
+    transaction's objects:
+
+    >>> isinstance(t._objects[0], DataManagerAdapter)
+    True
+    >>> t._objects[0]._datamanager is dm
+    True
+    
+    """
+
 def test_suite():
+    from doctest import DocTestSuite
+    return unittest.TestSuite((
+        DocTestSuite(),
+        unittest.makeSuite(TransactionTests),
+        ))
 
-    return unittest.makeSuite(TransactionTests)
 
 if __name__ == '__main__':
     unittest.TextTestRunner().run(test_suite())
+

src/ZODB/tests/testUtils.py

@@ -16,7 +16,6 @@
 import random
 import unittest
 from persistent import Persistent
-import ZODB.coptimizations
 
 NUM = 100
 
@@ -47,11 +46,12 @@ class TestUtils(unittest.TestCase):
         self.assertEquals(U64("\000\000\000\001\000\000\000\000"), 1L<<32)
 
     def checkPersistentIdHandlesDescriptor(self):
+        from ZODB.serialize import BaseObjectWriter
         class P(Persistent):
             pass
-        L = []
-        pid = ZODB.coptimizations.new_persistent_id(None, L)
-        pid(P)
+
+        writer = BaseObjectWriter(None)
+        self.assertEqual(writer.persistent_id(P), None)
 
 def test_suite():
     return unittest.makeSuite(TestUtils, 'check')

src/ZODB/tests/util.py

+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""Conventience function for creating test databases
+
+$Id: util.py,v 1.2 2004/02/19 02:59:10 jeremy Exp $
+"""
+
+import time
+import persistent
+from ZODB.DemoStorage import DemoStorage
+from ZODB.DB import DB as _DB
+try:
+    from transaction import get_transaction
+except ImportError:
+    pass # else assume ZODB will install it as a builtin
+
+def DB(name='Test'):
+    return _DB(DemoStorage(name))
+
+def commit():
+    get_transaction().commit()
+
+def pack(db):
+    db.pack(time.time()+1)
+
+class P(persistent.Persistent):
+    
+    def __init__(self, name):
+        self.name = name
+
+    def __repr__(self):
+        return 'P(%s)' % self.name