Added locking support for use by storages.

Added a working newTid that replaces the non-working newTimeStamp.

Removed WeakSet, which is moved to the transaction package.

src/ZODB/tests/testUtils.py

@@ -16,6 +16,7 @@
 import random
 import unittest
 from persistent import Persistent
+from zope.testing import doctest
 
 NUM = 100
 
@@ -89,9 +90,7 @@ class TestUtils(unittest.TestCase):
 
 
 def test_suite():
-    return unittest.makeSuite(TestUtils, 'check')
-
-if __name__ == "__main__":
-    loader = unittest.TestLoader()
-    loader.testMethodPrefix = "check"
-    unittest.main(testLoader=loader)
+    suite = unittest.TestSuite()
+    suite.addTest(unittest.makeSuite(TestUtils, 'check'))
+    suite.addTest(doctest.DocFileSuite('../utils.txt'))
+    return suite

src/ZODB/utils.py

@@ -31,17 +31,17 @@ __all__ = ['z64',
            'u64',
            'U64',
            'cp',
-           'newTimeStamp',
+           'newTid',
            'oid_repr',
            'serial_repr',
            'tid_repr',
            'positive_id',
            'readable_tid_repr',
-           'WeakSet',
            'DEPRECATED_ARGUMENT',
            'deprecated37',
            'deprecated38',
            'get_pickle_metadata',
+           'locked',
           ]
 
 # A unique marker to give as the default value for a deprecated argument.
@@ -113,14 +113,12 @@ def cp(f1, f2, length=None):
         write(data)
         length -= len(data)
 
-def newTimeStamp(old=None,
-                 TimeStamp=TimeStamp,
-                 time=time.time, gmtime=time.gmtime):
-    t = time()
-    ts = TimeStamp(gmtime(t)[:5]+(t%60,))
+def newTid(old):
+    t = time.time()
+    ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
     if old is not None:
-        return ts.laterThan(old)
-    return ts
+        ts = ts.laterThan(TimeStamp(old))
+    return `ts`
 
 
 def oid_repr(oid):
@@ -223,75 +221,52 @@ def get_pickle_metadata(data):
         classname = ''
     return modname, classname
 
-# A simple implementation of weak sets, supplying just enough of Python's
-# sets.Set interface for our needs.
-
-class WeakSet(object):
-    """A set of objects that doesn't keep its elements alive.
-
-    The objects in the set must be weakly referencable.
-    The objects need not be hashable, and need not support comparison.
-    Two objects are considered to be the same iff their id()s are equal.
-
-    When the only references to an object are weak references (including
-    those from WeakSets), the object can be garbage-collected, and
-    will vanish from any WeakSets it may be a member of at that time.
-    """
-
-    def __init__(self):
-        # Map id(obj) to obj.  By using ids as keys, we avoid requiring
-        # that the elements be hashable or comparable.
-        self.data = weakref.WeakValueDictionary()
-
-    def __len__(self):
-        return len(self.data)
-
-    def __contains__(self, obj):
-        return id(obj) in self.data
-
-    # Same as a Set, add obj to the collection.
-    def add(self, obj):
-        self.data[id(obj)] = obj
-
-    # Same as a Set, remove obj from the collection, and raise
-    # KeyError if obj not in the collection.
-    def remove(self, obj):
-        del self.data[id(obj)]
-
-    # f is a one-argument function.  Execute f(elt) for each elt in the
-    # set.  f's return value is ignored.
-    def map(self, f):
-        for wr in self.as_weakref_list():
-            elt = wr()
-            if elt is not None:
-                f(elt)
-
-    # Return a list of weakrefs to all the objects in the collection.
-    # Because a weak dict is used internally, iteration is dicey (the
-    # underlying dict may change size during iteration, due to gc or
-    # activity from other threads).  as_weakef_list() is safe.
-    #
-    # Something like this should really be a method of Python's weak dicts.
-    # If we invoke self.data.values() instead, we get back a list of live
-    # objects instead of weakrefs.  If gc occurs while this list is alive,
-    # all the objects move to an older generation (because they're strongly
-    # referenced by the list!).  They can't get collected then, until a
-    # less frequent collection of the older generation.  Before then, if we
-    # invoke self.data.values() again, they're still alive, and if gc occurs
-    # while that list is alive they're all moved to yet an older generation.
-    # And so on.  Stress tests showed that it was easy to get into a state
-    # where a WeakSet grows without bounds, despite that almost all its
-    # elements are actually trash.  By returning a list of weakrefs instead,
-    # we avoid that, although the decision to use weakrefs is now# very
-    # visible to our clients.
-    def as_weakref_list(self):
-        # We're cheating by breaking into the internals of Python's
-        # WeakValueDictionary here (accessing its .data attribute).
-        return self.data.data.values()
-
-
 def mktemp(dir=None):
     """Create a temp file, known by name, in a semi-secure manner."""
     handle, filename = mkstemp(dir=dir)
     os.close(handle)
     return filename
+
+class Locked(object):
+
+    def __init__(self, func, inst=None, class_=None, preconditions=()):
+        self.im_func = func
+        self.im_self = inst
+        self.im_class = class_
+        self.preconditions = preconditions
+
+    def __get__(self, inst, class_):
+        return self.__class__(self.im_func, inst, class_, self.preconditions)
+
+    def __call__(self, *args, **kw):
+        inst = self.im_self
+        if inst is None:
+            inst = args[0]
+        func = self.im_func.__get__(self.im_self, self.im_class)
+
+        inst._lock_acquire()
+        try:
+            for precondition in self.preconditions:
+                if not precondition(inst):
+                    raise AssertionError(
+                        "Failed precondition: ",
+                        precondition.__doc__.strip())
+            
+            return func(*args, **kw)
+        finally:
+            inst._lock_release()
+
+class locked(object):
+
+    def __init__(self, *preconditions):
+        self.preconditions = preconditions
+
+    def __get__(self, inst, class_):
+        # We didn't get any preconditions, so we have a single "precondition",
+        # which is actually the function to call.
+        func, = self.preconditions
+        return Locked(func, inst, class_)
+
+    def __call__(self, func):
+        return Locked(func, preconditions=self.preconditions)
+        

src/ZODB/utils.txt

+ZODB Utilits Module
+===================
+
+The ZODB.utils module provides a number of helpful, somewhat random
+:), utility functions.
+
+    >>> import ZODB.utils
+
+This document documents a few of them. Over time, it may document
+more. 
+
+64-bit integers and strings
+---------------------------------
+
+ZODB uses 64-bit transaction ids that are typically represented as
+strings, but are sometimes manipulated as integers.  Object ids are
+strings too and it is common to ise 64-bit strings that are just
+packed integers.
+
+Functions p64 and u64 pack and unpack integers as strings:
+
+    >>> ZODB.utils.p64(250347764455111456)
+    '\x03yi\xf7"\xa8\xfb '
+
+    >>> print ZODB.utils.u64('\x03yi\xf7"\xa8\xfb ')
+    250347764455111456
+
+The contant z64 has zero packed as a 64-bit string:
+
+    >>> ZODB.utils.z64
+    '\x00\x00\x00\x00\x00\x00\x00\x00'
+
+Transaction id generation
+-------------------------
+
+Storages assign transaction ids as transactions are committed.  These
+are based on UTC time, but must be strictly increasing.  The
+newTid function akes this pretty easy.
+
+To see this work (in a predictable way), we'll first hack time.time:
+
+    >>> import time
+    >>> old_time = time.time
+    >>> time.time = lambda : 1224825068.12
+
+Now, if we ask for a new time stamp, we'll get one based on our faux
+time:
+
+    >>> tid = ZODB.utils.newTid(None)
+    >>> tid
+    '\x03yi\xf7"\xa54\x88'
+
+newTid requires an old tid as an argument. The old tid may be None, if
+we don't have a previous transaction id.
+
+This time is based on the current time, which we can see by converting
+it to a time stamp.
+
+    >>> import ZODB.TimeStamp
+    >>> print ZODB.TimeStamp.TimeStamp(tid)
+    2008-10-24 05:11:08.120000
+
+To assure that we get a new tid that is later than the old, we can
+pass an existing tid.  Let's pass the tid we just got.
+
+    >>> tid2 = ZODB.utils.newTid(tid)
+    >>> ZODB.utils.u64(tid), ZODB.utils.u64(tid2)
+    (250347764454864008L, 250347764454864009L)
+
+Here, since we called it at the same time, we got a time stamp that
+was only slightly larger than the previos one.  Of course, at a later
+time, the time stamp we get will be based on the time:
+
+    >>> time.time = lambda : 1224825069.12
+    >>> tid = ZODB.utils.newTid(tid2)
+    >>> print ZODB.TimeStamp.TimeStamp(tid)
+    2008-10-24 05:11:09.120000
+
+
+    >>> time.time = old_time
+
+
+Locking support
+---------------
+
+Storages are required to be thread safe.  The locking descriptor helps
+automate that. It arranges for a lock to be acquired when a function
+is called and released when a function exits.  To demonstrate this,
+we'll create a "lock" type that simply prints when it is called:
+
+    >>> class Lock:
+    ...     def acquire(self):
+    ...         print 'acquire'
+    ...     def release(self):
+    ...         print 'release'
+
+Now we'll demonstrate the descriptor:
+
+    >>> class C:
+    ...     _lock = Lock()
+    ...     _lock_acquire = _lock.acquire
+    ...     _lock_release = _lock.release
+    ...
+    ...     @ZODB.utils.locked
+    ...     def meth(self, *args, **kw):
+    ...         print 'meth', args, kw
+
+The descriptor expects the instance it wraps to have a '_lock
+attribute.
+
+    >>> C().meth(1, 2, a=3)
+    acquire
+    meth (1, 2) {'a': 3}
+    release
+
+.. Edge cases
+
+   We can get the method from the class:
+
+    >>> C.meth # doctest: +ELLIPSIS
+    <ZODB.utils.Locked object at ...>
+
+    >>> C.meth(C())
+    acquire
+    meth () {}
+    release
+
+    >>> class C2:
+    ...     _lock = Lock()
+    ...     _lock_acquire = _lock.acquire
+    ...     _lock_release = _lock.release
+
+    >>> C.meth(C2()) # doctest: +NORMALIZE_WHITESPACE
+    Traceback (most recent call last):
+    ...
+    TypeError: unbound method meth() must be called with C instance
+    as first argument (got C2 instance instead)
+
+Preconditions
+-------------
+
+Often, we want to supply method preconditions. The locking descriptor
+supports optional method preconditions [1]_.
+
+    >>> class C:
+    ...     def __init__(self):
+    ...         _lock = Lock()
+    ...         self._lock_acquire = _lock.acquire
+    ...         self._lock_release = _lock.release
+    ...         self._opened = True
+    ...         self._transaction = None
+    ...
+    ...     def opened(self):
+    ...         """The object is open
+    ...         """
+    ...         print 'checking if open'
+    ...         return self._opened
+    ...
+    ...     def not_in_transaction(self):
+    ...         """The object is not in a transaction
+    ...         """
+    ...         print 'checking if in a transaction'
+    ...         return self._transaction is None
+    ...
+    ...     @ZODB.utils.locked(opened, not_in_transaction)
+    ...     def meth(self, *args, **kw):
+    ...         print 'meth', args, kw
+
+    >>> c = C()
+    >>> c.meth(1, 2, a=3)
+    acquire
+    checking if open
+    checking if in a transaction
+    meth (1, 2) {'a': 3}
+    release
+
+    >>> c._transaction = 1
+    >>> c.meth(1, 2, a=3) # doctest: +NORMALIZE_WHITESPACE
+    Traceback (most recent call last):
+    ...
+    AssertionError:
+    ('Failed precondition: ', 'The object is not in a transaction')
+
+    >>> c._opened = False
+    >>> c.meth(1, 2, a=3) # doctest: +NORMALIZE_WHITESPACE
+    Traceback (most recent call last):
+    ...
+    AssertionError: ('Failed precondition: ', 'The object is open')
+
+
+.. [1] Arguably, preconditions should be handled via separate
+   descriptors, but for ZODB storages, almost all methods need to be
+   locked.  Combining preconditions with locking provides both
+   efficiency and concise expressions.  A more general-purpose
+   facility would almost certainly provide separate descriptors for
+   preconditions.