Merge pull request #20 from NextThought/zodb-on-pypy-support

Support for ZODB on PyPy

.gitignore

@@ -13,3 +13,5 @@ nosetests.xml
 coverage.xml
 *.egg-info
 .installed.cfg
+.dir-locals.el
+dist

CHANGES.rst

@@ -4,6 +4,11 @@
 4.0.10 (unreleased)
 -------------------
 
+
+- The Python implementation of ``Persistent`` and ``PickleCache`` now
+  behave more similarly to the C implementation. In particular, the
+  Python version can now run the complete ZODB and ZEO test suites.
+
 - Fix the hashcode of the Python ``TimeStamp`` on 32-bit platforms.
 
 4.0.9 (2015-04-08)

persistent/ring.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2015 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+
+#pylint: disable=W0212,E0211,W0622,E0213,W0221,E0239
+
+from zope.interface import Interface
+from zope.interface import implementer
+
+class IRing(Interface):
+    """Conceptually, a doubly-linked list for efficiently keeping track of least-
+    and most-recently used :class:`persistent.interfaces.IPersistent` objects.
+
+    This is meant to be used by the :class:`persistent.picklecache.PickleCache`
+    and should not be considered a public API. This interface documentation exists
+    to assist development of the picklecache and alternate implementations by
+    explaining assumptions and performance requirements.
+    """
+
+    def __len__():
+        """Return the number of persistent objects stored in the ring.
+
+        Should be constant time.
+        """
+
+    def __contains__(object):
+        """Answer whether the given persistent object is found in the ring.
+
+        Must not rely on object equality or object hashing, but only
+        identity or the `_p_oid`. Should be constant time.
+        """
+
+    def add(object):
+        """Add the persistent object to the ring as most-recently used.
+
+        When an object is in the ring, the ring holds a strong
+        reference to it so it can be deactivated later by the pickle
+        cache. Should be constant time.
+
+        The object should not already be in the ring, but this is not necessarily
+        enforced.
+		"""
+
+    def delete(object):
+        """Remove the object from the ring if it is present.
+
+        Returns a true value if it was present and a false value
+        otherwise. An ideal implementation should be constant time,
+        but linear time is allowed.
+        """
+
+    def move_to_head(object):
+        """Place the object as the most recently used object in the ring.
+
+        The object should already be in the ring, but this is not
+        necessarily enforced, and attempting to move an object that is
+        not in the ring has undefined consequences. An ideal
+        implementation should be constant time, but linear time is
+        allowed.
+        """
+
+    def delete_all(indexes_and_values):
+        """Given a sequence of pairs (index, object), remove all of them from
+        the ring.
+
+        This should be equivalent to calling :meth:`delete` for each
+        value, but allows for a more efficient bulk deletion process.
+
+        If the index and object pairs do not match with the actual state of the
+        ring, this operation is undefined.
+
+        Should be at least linear time (not quadratic).
+        """
+
+    def __iter__():
+        """Iterate over each persistent object in the ring, in the order of least
+        recently used to most recently used.
+
+        Mutating the ring while an iteration is in progress has
+        undefined consequences.
+        """
+
+from collections import deque
+
+@implementer(IRing)
+class _DequeRing(object):
+    """A ring backed by the :class:`collections.deque` class.
+
+    Operations are a mix of constant and linear time.
+
+    It is available on all platforms.
+    """
+
+    __slots__ = ('ring', 'ring_oids')
+
+    def __init__(self):
+
+        self.ring = deque()
+        self.ring_oids = set()
+
+    def __len__(self):
+        return len(self.ring)
+
+    def __contains__(self, pobj):
+        return pobj._p_oid in self.ring_oids
+
+    def add(self, pobj):
+        self.ring.append(pobj)
+        self.ring_oids.add(pobj._p_oid)
+
+    def delete(self, pobj):
+        # Note that we do not use self.ring.remove() because that
+        # uses equality semantics and we don't want to call the persistent
+        # object's __eq__ method (which might wake it up just after we
+        # tried to ghost it)
+        for i, o in enumerate(self.ring):
+            if o is pobj:
+                del self.ring[i]
+                self.ring_oids.discard(pobj._p_oid)
+                return 1
+
+    def move_to_head(self, pobj):
+        self.delete(pobj)
+        self.add(pobj)
+
+    def delete_all(self, indexes_and_values):
+        for ix, value in reversed(indexes_and_values):
+            del self.ring[ix]
+            self.ring_oids.discard(value._p_oid)
+
+    def __iter__(self):
+        return iter(self.ring)
+
+
+try:
+    from cffi import FFI
+except ImportError: # pragma: no cover
+    _CFFIRing = None
+else:
+
+    import os
+    this_dir = os.path.dirname(os.path.abspath(__file__))
+
+    ffi = FFI()
+    with open(os.path.join(this_dir, 'ring.h')) as f:
+        ffi.cdef(f.read())
+
+    _FFI_RING = ffi.verify("""
+    #include "ring.c"
+    """, include_dirs=[this_dir])
+
+    _OGA = object.__getattribute__
+    _OSA = object.__setattr__
+
+    #pylint: disable=E1101
+    @implementer(IRing)
+    class _CFFIRing(object):
+        """A ring backed by a C implementation. All operations are constant time.
+
+        It is only available on platforms with ``cffi`` installed.
+        """
+
+        __slots__ = ('ring_home', 'ring_to_obj')
+
+        def __init__(self):
+            node = self.ring_home = ffi.new("CPersistentRing*")
+            node.r_next = node
+            node.r_prev = node
+
+            # In order for the CFFI objects to stay alive, we must keep
+            # a strong reference to them, otherwise they get freed. We must
+            # also keep strong references to the objects so they can be deactivated
+            self.ring_to_obj = dict()
+
+        def __len__(self):
+            return len(self.ring_to_obj)
+
+        def __contains__(self, pobj):
+            return getattr(pobj, '_Persistent__ring', self) in self.ring_to_obj
+
+        def add(self, pobj):
+            node = ffi.new("CPersistentRing*")
+            _FFI_RING.ring_add(self.ring_home, node)
+            self.ring_to_obj[node] = pobj
+            _OSA(pobj, '_Persistent__ring', node)
+
+        def delete(self, pobj):
+            its_node = getattr(pobj, '_Persistent__ring', None)
+            our_obj = self.ring_to_obj.pop(its_node, None)
+            if its_node is not None and our_obj is not None and its_node.r_next:
+                _FFI_RING.ring_del(its_node)
+                return 1
+
+        def move_to_head(self, pobj):
+            node = _OGA(pobj, '_Persistent__ring')
+            _FFI_RING.ring_move_to_head(self.ring_home, node)
+
+        def delete_all(self, indexes_and_values):
+            for _, value in indexes_and_values:
+                self.delete(value)
+
+        def iteritems(self):
+            head = self.ring_home
+            here = head.r_next
+            while here != head:
+                yield here
+                here = here.r_next
+
+        def __iter__(self):
+            ring_to_obj = self.ring_to_obj
+            for node in self.iteritems():
+                yield ring_to_obj[node]
+
+# Export the best available implementation
+Ring = _CFFIRing if _CFFIRing else _DequeRing

persistent/tests/test_ring.py

+##############################################################################
+#
+# Copyright (c) 2015 Zope Foundation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+import unittest
+
+from .. import ring
+
+#pylint: disable=R0904,W0212,E1101
+
+class DummyPersistent(object):
+    _p_oid = None
+
+    __next_oid = 0
+
+    @classmethod
+    def _next_oid(cls):
+        cls.__next_oid += 1
+        return cls.__next_oid
+
+    def __init__(self, oid=None):
+        if oid is None:
+            self._p_oid = self._next_oid()
+
+    def __repr__(self):
+        return "<Dummy %r>" % self._p_oid
+
+class _Ring_Base(object):
+
+    def _getTargetClass(self):
+        """Return the type of the ring to test"""
+        raise NotImplementedError()
+
+    def _makeOne(self):
+        return self._getTargetClass()()
+
+    def test_empty_len(self):
+        self.assertEqual(0, len(self._makeOne()))
+
+    def test_empty_contains(self):
+        r = self._makeOne()
+        self.assertFalse(DummyPersistent() in r)
+
+    def test_empty_iter(self):
+        self.assertEqual([], list(self._makeOne()))
+
+    def test_add_one_len1(self):
+        r = self._makeOne()
+        p = DummyPersistent()
+        r.add(p)
+        self.assertEqual(1, len(r))
+
+    def test_add_one_contains(self):
+        r = self._makeOne()
+        p = DummyPersistent()
+        r.add(p)
+        self.assertTrue(p in r)
+
+    def test_delete_one_len0(self):
+        r = self._makeOne()
+        p = DummyPersistent()
+        r.add(p)
+        r.delete(p)
+        self.assertEqual(0, len(r))
+
+    def test_delete_one_multiple(self):
+        r = self._makeOne()
+        p = DummyPersistent()
+        r.add(p)
+        r.delete(p)
+        self.assertEqual(0, len(r))
+        self.assertFalse(p in r)
+
+        r.delete(p)
+        self.assertEqual(0, len(r))
+        self.assertFalse(p in r)
+
+    def test_delete_from_wrong_ring(self):
+        r1 = self._makeOne()
+        r2 = self._makeOne()
+        p1 = DummyPersistent()
+        p2 = DummyPersistent()
+
+        r1.add(p1)
+        r2.add(p2)
+
+        r2.delete(p1)
+
+        self.assertEqual(1, len(r1))
+        self.assertEqual(1, len(r2))
+
+        self.assertEqual([p1], list(r1))
+        self.assertEqual([p2], list(r2))
+
+    def test_move_to_head(self):
+        r = self._makeOne()
+        p1 = DummyPersistent()
+        p2 = DummyPersistent()
+        p3 = DummyPersistent()
+
+        r.add(p1)
+        r.add(p2)
+        r.add(p3)
+
+        self.assertEqual([p1, p2, p3], list(r))
+        self.assertEqual(3, len(r))
+
+        r.move_to_head(p1)
+        self.assertEqual([p2, p3, p1], list(r))
+
+        r.move_to_head(p3)
+        self.assertEqual([p2, p1, p3], list(r))
+
+        r.move_to_head(p3)
+        self.assertEqual([p2, p1, p3], list(r))
+
+    def test_delete_all(self):
+        r = self._makeOne()
+        p1 = DummyPersistent()
+        p2 = DummyPersistent()
+        p3 = DummyPersistent()
+
+        r.add(p1)
+        r.add(p2)
+        r.add(p3)
+        self.assertEqual([p1, p2, p3], list(r))
+
+        r.delete_all([(0, p1), (2, p3)])
+        self.assertEqual([p2], list(r))
+        self.assertEqual(1, len(r))
+
+class DequeRingTests(unittest.TestCase, _Ring_Base):
+
+    def _getTargetClass(self):
+        return ring._DequeRing
+
+_add_to_suite = [DequeRingTests]
+
+if ring._CFFIRing:
+    class CFFIRingTests(unittest.TestCase, _Ring_Base):
+
+        def _getTargetClass(self):
+            return ring._CFFIRing
+
+    _add_to_suite.append(CFFIRingTests)
+
+def test_suite():
+    return unittest.TestSuite([unittest.makeSuite(x) for x in _add_to_suite])

persistent/tests/test_timestamp.py

@@ -14,6 +14,11 @@
 import operator
 import unittest
 
+import platform
+py_impl = getattr(platform, 'python_implementation', lambda: None)
+_is_jython = py_impl() == 'Jython'
+
+
 class Test__UTC(unittest.TestCase):
 
     def _getTargetClass(self):
@@ -271,26 +276,37 @@ class PyAndCComparisonTests(unittest.TestCase):
             py = self._makePy(*self.now_ts_args)
             self.assertEqual(hash(py), bit_32_hash)
 
+
             persistent.timestamp.c_long = ctypes.c_int64
             # call __hash__ directly to avoid interpreter truncation
             # in hash() on 32-bit platforms
-            self.assertEqual(py.__hash__(), bit_64_hash)
+            if not _is_jython:
+                self.assertEqual(py.__hash__(), bit_64_hash)
+            else:
+                # Jython 2.7's ctypes module doesn't properly
+                # implement the 'value' attribute by truncating.
+                # (It does for native calls, but not visibly to Python).
+                # Therefore we get back the full python long. The actual
+                # hash() calls are correct, though, because the JVM uses
+                # 32-bit ints for its hashCode methods.
+                self.assertEqual(py.__hash__(), 384009219096809580920179179233996861765753210540033)
         finally:
             persistent.timestamp.c_long = orig_c_long
 
+        # These are *usually* aliases, but aren't required
+        # to be (and aren't under Jython 2.7).
         if orig_c_long is ctypes.c_int32:
             self.assertEqual(py.__hash__(), bit_32_hash)
         elif orig_c_long is ctypes.c_int64:
             self.assertEqual(py.__hash__(), bit_64_hash)
-        else:
-            self.fail("Unknown bitness")
 
     def test_hash_equal_constants(self):
         # The simple constants make it easier to diagnose
         # a difference in algorithms
         import persistent.timestamp
         import ctypes
-        is_32_bit = persistent.timestamp.c_long == ctypes.c_int32
+        # We get 32-bit hash values of 32-bit platforms, or on the JVM
+        is_32_bit = persistent.timestamp.c_long == ctypes.c_int32 or _is_jython
 
         c, py = self._make_C_and_Py(b'\x00\x00\x00\x00\x00\x00\x00\x00')
         self.assertEqual(hash(c), 8)

tox.ini

 [tox]
-envlist = 
-# Jython support pending 2.7 support, due 2012-07-15 or so.  See:
-# http://fwierzbicki.blogspot.com/2012/03/adconion-to-fund-jython-27.html
-#   py26,py27,py32,jython,pypy,coverage,docs
-    py26,py27,py27-pure,pypy,py32,py33,py34,pypy3,coverage,docs
+envlist =
+# Jython 2.7rc2 does work, but unfortunately has an issue running
+# with Tox 1.9.2 (http://bugs.jython.org/issue2325)
+#   py26,py27,py27-pure,pypy,py32,py33,py34,pypy3,jython,coverage,docs
+    py26,py27,py27-pure,py27-pure-cffi,pypy,py32,py33,py34,pypy3,coverage,docs
 
 [testenv]
 deps =
     zope.interface
-commands = 
+commands =
     python setup.py test -q
 
 [testenv:jython]
-commands = 
+commands =
    jython setup.py test -q
 
 [testenv:py27-pure]
@@ -22,24 +22,40 @@ setenv =
     PURE_PYTHON = 1
 deps =
     {[testenv]deps}
-commands = 
+commands =
     python setup.py test -q
 
+[testenv:py27-pure-cffi]
+basepython =
+    python2.7
+setenv =
+    PURE_PYTHON = 1
+    USING_CFFI = 1
+deps =
+    {[testenv]deps}
+    cffi
+commands =
+    python setup.py test -q
+
+
 [testenv:coverage]
 basepython =
     python2.6
-commands = 
+setenv =
+    USING_CFFI = 1
+commands =
     nosetests --with-xunit --with-xcoverage
 deps =
     zope.interface
     nose
     coverage
     nosexcover
+    cffi
 
 [testenv:docs]
 basepython =
     python2.6
-commands = 
+commands =
     sphinx-build -b html -d docs/_build/doctrees docs docs/_build/html
     sphinx-build -b doctest -d docs/_build/doctrees docs docs/_build/doctest
 deps =