Commit 487664ec authored by Jason Madden's avatar Jason Madden

Require CFFI on CPython for pure-Python operation.

Fixes #77
parent ed81c6f7
...@@ -16,10 +16,18 @@ ...@@ -16,10 +16,18 @@
- The Python implementation raises ``AttributeError`` if a - The Python implementation raises ``AttributeError`` if a
persistent class doesn't have a ``p_jar`` attribute. persistent class doesn't have a ``p_jar`` attribute.
See `issue 102
<https://github.com/zopefoundation/persistent/issues/102>`_.
- Allow sweeping cache without ``cache_size``. ``cache_size_bytes`` - Allow sweeping cache without ``cache_size``. ``cache_size_bytes``
works with ``cache_size=0``, no need to set ``cache_size`` to a works with ``cache_size=0``, no need to set ``cache_size`` to a
large value. large value.
- Require ``CFFI`` on CPython for pure-Python operation. This drops
support for Jython (which was untested). See `issue 77
<https://github.com/zopefoundation/persistent/issues/77>`_.
4.4.3 (2018-10-22) 4.4.3 (2018-10-22)
------------------ ------------------
......
...@@ -13,11 +13,13 @@ ...@@ -13,11 +13,13 @@
# #
############################################################################## ##############################################################################
#pylint: disable=W0212,E0211,W0622,E0213,W0221,E0239 # pylint:disable=inherit-non-class,no-self-argument,redefined-builtin,c-extension-no-member
from zope.interface import Interface from zope.interface import Interface
from zope.interface import implementer from zope.interface import implementer
from persistent import _ring
class IRing(Interface): class IRing(Interface):
"""Conceptually, a doubly-linked list for efficiently keeping track of least- """Conceptually, a doubly-linked list for efficiently keeping track of least-
and most-recently used :class:`persistent.interfaces.IPersistent` objects. and most-recently used :class:`persistent.interfaces.IPersistent` objects.
...@@ -28,13 +30,13 @@ class IRing(Interface): ...@@ -28,13 +30,13 @@ class IRing(Interface):
explaining assumptions and performance requirements. explaining assumptions and performance requirements.
""" """
def __len__(): def __len__(): # pylint:disable=no-method-argument
"""Return the number of persistent objects stored in the ring. """Return the number of persistent objects stored in the ring.
Should be constant time. Should be constant time.
""" """
def __contains__(object): def __contains__(object): # pylint:disable=unexpected-special-method-signature
"""Answer whether the given persistent object is found in the ring. """Answer whether the given persistent object is found in the ring.
Must not rely on object equality or object hashing, but only Must not rely on object equality or object hashing, but only
...@@ -83,7 +85,7 @@ class IRing(Interface): ...@@ -83,7 +85,7 @@ class IRing(Interface):
Should be at least linear time (not quadratic). Should be at least linear time (not quadratic).
""" """
def __iter__(): def __iter__(): # pylint:disable=no-method-argument
"""Iterate over each persistent object in the ring, in the order of least """Iterate over each persistent object in the ring, in the order of least
recently used to most recently used. recently used to most recently used.
...@@ -91,127 +93,72 @@ class IRing(Interface): ...@@ -91,127 +93,72 @@ class IRing(Interface):
undefined consequences. undefined consequences.
""" """
from collections import deque
@implementer(IRing) ffi = _ring.ffi
class _DequeRing(object): _FFI_RING = _ring.lib
"""A ring backed by the :class:`collections.deque` class.
Operations are a mix of constant and linear time. _OGA = object.__getattribute__
_OSA = object.__setattr__
It is available on all platforms.
@implementer(IRing)
class _CFFIRing(object):
"""A ring backed by a C implementation. All operations are constant time.
It is only available on platforms with ``cffi`` installed.
""" """
__slots__ = ('ring', 'ring_oids') __slots__ = ('ring_home', 'ring_to_obj')
def __init__(self): def __init__(self):
node = self.ring_home = ffi.new("CPersistentRing*")
node.r_next = node
node.r_prev = node
self.ring = deque() # In order for the CFFI objects to stay alive, we must keep
self.ring_oids = set() # a strong reference to them, otherwise they get freed. We must
# also keep strong references to the objects so they can be deactivated
self.ring_to_obj = dict()
def __len__(self): def __len__(self):
return len(self.ring) return len(self.ring_to_obj)
def __contains__(self, pobj): def __contains__(self, pobj):
return pobj._p_oid in self.ring_oids return getattr(pobj, '_Persistent__ring', self) in self.ring_to_obj
def add(self, pobj): def add(self, pobj):
self.ring.append(pobj) node = ffi.new("CPersistentRing*")
self.ring_oids.add(pobj._p_oid) _FFI_RING.ring_add(self.ring_home, node)
self.ring_to_obj[node] = pobj
_OSA(pobj, '_Persistent__ring', node)
def delete(self, pobj): def delete(self, pobj):
# Note that we do not use self.ring.remove() because that its_node = getattr(pobj, '_Persistent__ring', None)
# uses equality semantics and we don't want to call the persistent our_obj = self.ring_to_obj.pop(its_node, None)
# object's __eq__ method (which might wake it up just after we if its_node is not None and our_obj is not None and its_node.r_next:
# tried to ghost it) _FFI_RING.ring_del(its_node)
for i, o in enumerate(self.ring): return 1
if o is pobj: return None
del self.ring[i]
self.ring_oids.discard(pobj._p_oid)
return 1
def move_to_head(self, pobj): def move_to_head(self, pobj):
self.delete(pobj) node = _OGA(pobj, '_Persistent__ring')
self.add(pobj) _FFI_RING.ring_move_to_head(self.ring_home, node)
def delete_all(self, indexes_and_values): def delete_all(self, indexes_and_values):
for ix, value in reversed(indexes_and_values): for _, value in indexes_and_values:
del self.ring[ix] self.delete(value)
self.ring_oids.discard(value._p_oid)
def __iter__(self):
return iter(self.ring)
def iteritems(self):
head = self.ring_home
here = head.r_next
while here != head:
yield here
here = here.r_next
try: def __iter__(self):
from persistent import _ring ring_to_obj = self.ring_to_obj
except ImportError: # pragma: no cover for node in self.iteritems():
_CFFIRing = None yield ring_to_obj[node]
else:
ffi = _ring.ffi
_FFI_RING = _ring.lib
_OGA = object.__getattribute__
_OSA = object.__setattr__
#pylint: disable=E1101
@implementer(IRing)
class _CFFIRing(object):
"""A ring backed by a C implementation. All operations are constant time.
It is only available on platforms with ``cffi`` installed.
"""
__slots__ = ('ring_home', 'ring_to_obj')
def __init__(self):
node = self.ring_home = ffi.new("CPersistentRing*")
node.r_next = node
node.r_prev = node
# In order for the CFFI objects to stay alive, we must keep
# a strong reference to them, otherwise they get freed. We must
# also keep strong references to the objects so they can be deactivated
self.ring_to_obj = dict()
def __len__(self):
return len(self.ring_to_obj)
def __contains__(self, pobj):
return getattr(pobj, '_Persistent__ring', self) in self.ring_to_obj
def add(self, pobj):
node = ffi.new("CPersistentRing*")
_FFI_RING.ring_add(self.ring_home, node)
self.ring_to_obj[node] = pobj
_OSA(pobj, '_Persistent__ring', node)
def delete(self, pobj):
its_node = getattr(pobj, '_Persistent__ring', None)
our_obj = self.ring_to_obj.pop(its_node, None)
if its_node is not None and our_obj is not None and its_node.r_next:
_FFI_RING.ring_del(its_node)
return 1
def move_to_head(self, pobj):
node = _OGA(pobj, '_Persistent__ring')
_FFI_RING.ring_move_to_head(self.ring_home, node)
def delete_all(self, indexes_and_values):
for _, value in indexes_and_values:
self.delete(value)
def iteritems(self):
head = self.ring_home
here = head.r_next
while here != head:
yield here
here = here.r_next
def __iter__(self):
ring_to_obj = self.ring_to_obj
for node in self.iteritems():
yield ring_to_obj[node]
# Export the best available implementation # Export the best available implementation
Ring = _CFFIRing if _CFFIRing else _DequeRing Ring = _CFFIRing
...@@ -22,7 +22,6 @@ from persistent._compat import copy_reg ...@@ -22,7 +22,6 @@ from persistent._compat import copy_reg
_is_pypy3 = platform.python_implementation() == 'PyPy' and sys.version_info[0] > 2 _is_pypy3 = platform.python_implementation() == 'PyPy' and sys.version_info[0] > 2
_is_jython = platform.python_implementation() == 'Jython'
# pylint:disable=R0904,W0212,E1101 # pylint:disable=R0904,W0212,E1101
# pylint:disable=attribute-defined-outside-init,too-many-lines # pylint:disable=attribute-defined-outside-init,too-many-lines
...@@ -985,7 +984,7 @@ class _Persistent_Base(object): ...@@ -985,7 +984,7 @@ class _Persistent_Base(object):
self.assertEqual(inst.baz, 'bam') self.assertEqual(inst.baz, 'bam')
self.assertEqual(inst.qux, 'spam') self.assertEqual(inst.qux, 'spam')
if not _is_pypy3 and not _is_jython: if not _is_pypy3:
def test___setstate___interns_dict_keys(self): def test___setstate___interns_dict_keys(self):
class Derived(self._getTargetClass()): class Derived(self._getTargetClass()):
pass pass
......
...@@ -12,9 +12,7 @@ ...@@ -12,9 +12,7 @@
# #
############################################################################## ##############################################################################
import gc import gc
import os
import platform import platform
import sys
import unittest import unittest
from persistent.interfaces import UPTODATE from persistent.interfaces import UPTODATE
...@@ -22,7 +20,6 @@ from persistent.interfaces import UPTODATE ...@@ -22,7 +20,6 @@ from persistent.interfaces import UPTODATE
# pylint:disable=protected-access,too-many-lines,too-many-public-methods # pylint:disable=protected-access,too-many-lines,too-many-public-methods
_is_pypy = platform.python_implementation() == 'PyPy' _is_pypy = platform.python_implementation() == 'PyPy'
_is_jython = 'java' in sys.platform
_marker = object() _marker = object()
...@@ -32,22 +29,12 @@ def skipIfNoCExtension(o): ...@@ -32,22 +29,12 @@ def skipIfNoCExtension(o):
persistent._cPickleCache is None, persistent._cPickleCache is None,
"The C extension is not available")(o) "The C extension is not available")(o)
def skipIfPurePython(o):
if _is_jython: # pragma: no cover import persistent._compat
def with_deterministic_gc(f): return unittest.skipIf(
def test(self): persistent._compat.PURE_PYTHON,
# pylint:disable=no-member "Cannot mix and match implementations"
old_flags = gc.getMonitorGlobal() )(o)
gc.setMonitorGlobal(True)
try:
f(self, force_collect=True)
finally:
gc.setMonitorGlobal(old_flags)
return test
else:
def with_deterministic_gc(f):
return f
class PickleCacheTests(unittest.TestCase): class PickleCacheTests(unittest.TestCase):
...@@ -657,14 +644,6 @@ class PickleCacheTests(unittest.TestCase): ...@@ -657,14 +644,6 @@ class PickleCacheTests(unittest.TestCase):
self.assertTrue(pclass.invalidated) self.assertTrue(pclass.invalidated)
def test_ring_impl(self):
from .. import ring
expected = (ring._CFFIRing
if _is_pypy or ring._CFFIRing is not None or os.environ.get('USING_CFFI')
else ring._DequeRing)
self.assertIs(ring.Ring, expected)
class PythonPickleCacheTests(PickleCacheTests): class PythonPickleCacheTests(PickleCacheTests):
# Tests that depend on the implementation details of the # Tests that depend on the implementation details of the
...@@ -1007,9 +986,8 @@ class PythonPickleCacheTests(PickleCacheTests): ...@@ -1007,9 +986,8 @@ class PythonPickleCacheTests(PickleCacheTests):
self.assertTrue(items[0][1] is candidate) self.assertTrue(items[0][1] is candidate)
self.assertEqual(candidate._p_state, UPTODATE) self.assertEqual(candidate._p_state, UPTODATE)
@with_deterministic_gc
def test_cache_garbage_collection_bytes_also_deactivates_object(self, def test_cache_garbage_collection_bytes_also_deactivates_object(self,
force_collect=_is_pypy or _is_jython): force_collect=_is_pypy):
class MyPersistent(self._getDummyPersistentClass()): class MyPersistent(self._getDummyPersistentClass()):
def _p_deactivate(self): def _p_deactivate(self):
...@@ -1070,9 +1048,7 @@ class PythonPickleCacheTests(PickleCacheTests): ...@@ -1070,9 +1048,7 @@ class PythonPickleCacheTests(PickleCacheTests):
candidate._p_jar = None candidate._p_jar = None
self.assertRaises(KeyError, cache.new_ghost, key, candidate) self.assertRaises(KeyError, cache.new_ghost, key, candidate)
@with_deterministic_gc def test_cache_garbage_collection_bytes_with_cache_size_0(self):
def test_cache_garbage_collection_bytes_with_cache_size_0(
self, force_collect=_is_pypy or _is_jython):
class MyPersistent(self._getDummyPersistentClass()): class MyPersistent(self._getDummyPersistentClass()):
def _p_deactivate(self): def _p_deactivate(self):
...@@ -1116,6 +1092,7 @@ class PythonPickleCacheTests(PickleCacheTests): ...@@ -1116,6 +1092,7 @@ class PythonPickleCacheTests(PickleCacheTests):
@skipIfNoCExtension @skipIfNoCExtension
@skipIfPurePython
class CPickleCacheTests(PickleCacheTests): class CPickleCacheTests(PickleCacheTests):
def _getTargetClass(self): def _getTargetClass(self):
......
...@@ -15,7 +15,7 @@ import unittest ...@@ -15,7 +15,7 @@ import unittest
from .. import ring from .. import ring
#pylint: disable=R0904,W0212,E1101 # pylint:disable=protected-access
class DummyPersistent(object): class DummyPersistent(object):
_p_oid = None _p_oid = None
...@@ -34,11 +34,11 @@ class DummyPersistent(object): ...@@ -34,11 +34,11 @@ class DummyPersistent(object):
def __repr__(self): # pragma: no cover def __repr__(self): # pragma: no cover
return "<Dummy %r>" % self._p_oid return "<Dummy %r>" % self._p_oid
class _Ring_Base(object):
class CFFIRingTests(unittest.TestCase):
def _getTargetClass(self): def _getTargetClass(self):
"""Return the type of the ring to test""" return ring._CFFIRing
raise NotImplementedError()
def _makeOne(self): def _makeOne(self):
return self._getTargetClass()() return self._getTargetClass()()
...@@ -137,21 +137,3 @@ class _Ring_Base(object): ...@@ -137,21 +137,3 @@ class _Ring_Base(object):
r.delete_all([(0, p1), (2, p3)]) r.delete_all([(0, p1), (2, p3)])
self.assertEqual([p2], list(r)) self.assertEqual([p2], list(r))
self.assertEqual(1, len(r)) self.assertEqual(1, len(r))
class DequeRingTests(unittest.TestCase, _Ring_Base):
def _getTargetClass(self):
return ring._DequeRing
_add_to_suite = [DequeRingTests]
if ring._CFFIRing:
class CFFIRingTests(unittest.TestCase, _Ring_Base):
def _getTargetClass(self):
return ring._CFFIRing
_add_to_suite.append(CFFIRingTests)
def test_suite():
return unittest.TestSuite([unittest.makeSuite(x) for x in _add_to_suite])
...@@ -232,12 +232,6 @@ class PyAndCComparisonTests(unittest.TestCase): ...@@ -232,12 +232,6 @@ class PyAndCComparisonTests(unittest.TestCase):
from persistent.timestamp import pyTimeStamp from persistent.timestamp import pyTimeStamp
return pyTimeStamp(*args, **kwargs) return pyTimeStamp(*args, **kwargs)
@property
def _is_jython(self):
import platform
py_impl = getattr(platform, 'python_implementation', lambda: None)
return py_impl() == 'Jython'
def _make_C_and_Py(self, *args, **kwargs): def _make_C_and_Py(self, *args, **kwargs):
return self._makeC(*args, **kwargs), self._makePy(*args, **kwargs) return self._makeC(*args, **kwargs), self._makePy(*args, **kwargs)
...@@ -300,18 +294,7 @@ class PyAndCComparisonTests(unittest.TestCase): ...@@ -300,18 +294,7 @@ class PyAndCComparisonTests(unittest.TestCase):
MUT.c_long = c_int64 MUT.c_long = c_int64
# call __hash__ directly to avoid interpreter truncation # call __hash__ directly to avoid interpreter truncation
# in hash() on 32-bit platforms # in hash() on 32-bit platforms
if not self._is_jython: self.assertEqual(py.__hash__(), bit_64_hash)
self.assertEqual(py.__hash__(), bit_64_hash)
else: # pragma: no cover
# Jython 2.7's ctypes module doesn't properly
# implement the 'value' attribute by truncating.
# (It does for native calls, but not visibly to Python).
# Therefore we get back the full python long. The actual
# hash() calls are correct, though, because the JVM uses
# 32-bit ints for its hashCode methods.
self.assertEqual(
py.__hash__(),
384009219096809580920179179233996861765753210540033)
finally: finally:
MUT._MAXINT = orig_maxint MUT._MAXINT = orig_maxint
if orig_c_long is not None: if orig_c_long is not None:
...@@ -320,7 +303,7 @@ class PyAndCComparisonTests(unittest.TestCase): ...@@ -320,7 +303,7 @@ class PyAndCComparisonTests(unittest.TestCase):
del MUT.c_long del MUT.c_long
# These are *usually* aliases, but aren't required # These are *usually* aliases, but aren't required
# to be (and aren't under Jython 2.7). # to be
expected_hash = bit_32_hash if is_32_bit_hash else bit_64_hash expected_hash = bit_32_hash if is_32_bit_hash else bit_64_hash
self.assertEqual(py.__hash__(), expected_hash) self.assertEqual(py.__hash__(), expected_hash)
...@@ -328,9 +311,9 @@ class PyAndCComparisonTests(unittest.TestCase): ...@@ -328,9 +311,9 @@ class PyAndCComparisonTests(unittest.TestCase):
# The simple constants make it easier to diagnose # The simple constants make it easier to diagnose
# a difference in algorithms # a difference in algorithms
import persistent.timestamp as MUT import persistent.timestamp as MUT
# We get 32-bit hash values on 32-bit platforms, or on the JVM # We get 32-bit hash values on 32-bit platforms,
# OR on Windows (whether compiled in 64 or 32-bit mode) # OR on Windows (whether compiled in 64 or 32-bit mode)
is_32_bit = MUT._MAXINT == (2**31 - 1) or self._is_jython or sys.platform == 'win32' is_32_bit = MUT._MAXINT == (2**31 - 1) or sys.platform == 'win32'
c, py = self._make_C_and_Py(b'\x00\x00\x00\x00\x00\x00\x00\x00') c, py = self._make_C_and_Py(b'\x00\x00\x00\x00\x00\x00\x00\x00')
self.assertEqual(hash(c), 8) self.assertEqual(hash(c), 8)
......
...@@ -20,7 +20,7 @@ from setuptools import Extension ...@@ -20,7 +20,7 @@ from setuptools import Extension
from setuptools import find_packages from setuptools import find_packages
from setuptools import setup from setuptools import setup
version = '4.4.4.dev0' version = '4.5.0.dev0'
here = os.path.abspath(os.path.dirname(__file__)) here = os.path.abspath(os.path.dirname(__file__))
...@@ -33,12 +33,12 @@ def _read_file(filename): ...@@ -33,12 +33,12 @@ def _read_file(filename):
README = (_read_file('README.rst') + '\n\n' + _read_file('CHANGES.rst')) README = (_read_file('README.rst') + '\n\n' + _read_file('CHANGES.rst'))
is_pypy = platform.python_implementation() == 'PyPy' is_pypy = platform.python_implementation() == 'PyPy'
is_jython = 'java' in sys.platform
# Jython cannot build the C optimizations, while on PyPy they are # On PyPy the C optimizations are
# anti-optimizations (the C extension compatibility layer is known-slow, # anti-optimizations (the C extension compatibility layer is known-slow,
# and defeats JIT opportunities). # and defeats JIT opportunities); PyPy 6.0 can compile them, but the
if is_pypy or is_jython: # tests fail and they actually crash the VM.
if is_pypy:
# Note that all the lists we pass to setuptools must be distinct # Note that all the lists we pass to setuptools must be distinct
# objects, or bad things happen. See https://github.com/zopefoundation/persistent/issues/88 # objects, or bad things happen. See https://github.com/zopefoundation/persistent/issues/88
ext_modules = [] ext_modules = []
...@@ -120,7 +120,6 @@ setup(name='persistent', ...@@ -120,7 +120,6 @@ setup(name='persistent',
extras_require={ extras_require={
'test': [ 'test': [
'zope.testrunner', 'zope.testrunner',
"cffi ; platform_python_implementation == 'CPython'",
'manuel', 'manuel',
], ],
'testing': (), 'testing': (),
...@@ -131,6 +130,10 @@ setup(name='persistent', ...@@ -131,6 +130,10 @@ setup(name='persistent',
}, },
install_requires=[ install_requires=[
'zope.interface', 'zope.interface',
"cffi ; platform_python_implementation == 'CPython'",
],
setup_requires=[
"cffi ; platform_python_implementation == 'CPython'",
], ],
entry_points={}, entry_points={},
) )
[tox] [tox]
envlist = envlist =
# Jython 2.7rc2 does work, but unfortunately has an issue running
# with Tox 1.9.2 (http://bugs.jython.org/issue2325)
# py27,py27-pure,pypy,py33,py34,pypy3,jython,coverage,docs
py27,py27-pure,py27-pure-cffi,pypy,py34,py35,py36,py37,pypy3,coverage,docs py27,py27-pure,py27-pure-cffi,pypy,py34,py35,py36,py37,pypy3,coverage,docs
[testenv] [testenv]
deps = deps =
cffi
.[test] .[test]
commands = commands =
zope-testrunner --test-path=. zope-testrunner --test-path=.
...@@ -23,7 +19,6 @@ basepython = ...@@ -23,7 +19,6 @@ basepython =
python2.7 python2.7
setenv = setenv =
PURE_PYTHON = 1 PURE_PYTHON = 1
USING_CFFI = 1
[testenv:coverage] [testenv:coverage]
usedevelop = true usedevelop = true
...@@ -35,8 +30,6 @@ commands = ...@@ -35,8 +30,6 @@ commands =
deps = deps =
{[testenv]deps} {[testenv]deps}
coverage coverage
setenv =
USING_CFFI = 1
[testenv:docs] [testenv:docs]
basepython = basepython =
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment