bigfile/zodb: Factor-out LivePersistent into -> lib/zodb

It was from long-ago marked as "XXX move to common place".

bigarray/array_zodb.py

@@ -29,7 +29,8 @@ inherits most of the properties and specifics from its parents.
 """
 
 from wendelin.bigarray import BigArray
-from wendelin.bigfile.file_zodb import ZBigFile, LivePersistent
+from wendelin.bigfile.file_zodb import ZBigFile
+from wendelin.lib.zodb import LivePersistent
 import inspect
 
 

bigarray/tests/test_arrayzodb.py

@@ -18,9 +18,10 @@
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
 from wendelin.bigarray.array_zodb import ZBigArray
-from wendelin.bigfile.tests.test_filezodb import kkey, cacheInfo, ram_reclaim_all
+from wendelin.bigfile.tests.test_filezodb import ram_reclaim_all
 from wendelin.bigfile.tests.test_thread import NotifyChannel
 from wendelin.lib.zodb import dbclose
+from wendelin.lib.tests.test_zodb import cacheInfo, kkey
 from wendelin.lib.testing import getTestDB
 from persistent import UPTODATE
 import transaction

bigfile/file_zodb.py

@@ -133,7 +133,7 @@ will be our future approach after we teach NEO about object deduplication.
 
 from wendelin.bigfile import BigFile, WRITEOUT_STORE, WRITEOUT_MARKSTORED
 from wendelin.lib.mem import bzero, memcpy
-from wendelin.lib.zodb import deactivate_btree
+from wendelin.lib.zodb import LivePersistent, deactivate_btree
 
 from transaction.interfaces import IDataManager, ISynchronizer
 from persistent import Persistent, PickleCache, GHOST
@@ -482,33 +482,6 @@ class _ZBigFile(BigFile):
 
 
 
-# Persistent that never goes to ghost state, if it was ever uptodate.
-#
-# NOTE
-#
-# On invalidation LivePersistent still goes to ghost state, because
-# invalidation cannot be ignored, i.e. they indicate the object has been
-# changed externally.
-#
-# Invalidation can happen only at transaction boundary, so during the course of
-# transaction LivePersistent is guaranteed to stay uptodate.
-#
-# XXX move to common place?
-class LivePersistent(Persistent):
-    # don't allow us to go to ghost
-    #
-    # NOTE we can't use STICKY as that state is assumed as
-    # short-lived-temporary by ZODB and is changed back to UPTODATE by
-    # persistent code. In fact ZODB says: STICKY is UPTODATE+keep in memory.
-    def _p_deactivate(self):
-        # just returning here won't allow Persistent._p_deactivate() run and
-        # thus we'll stay in non-ghost state.
-        return
-
-
-    # NOTE _p_invalidate() is triggered on invalidations. We do not override it.
-
-
 # ZBigFile implements BigFile backend with data stored in ZODB.
 #
 # NOTE Can't inherit from Persistent and BigFile at the same time - both are C

bigfile/tests/test_filezodb.py

@@ -17,12 +17,13 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-from wendelin.bigfile.file_zodb import LivePersistent, ZBigFile, ZBlk_fmt_registry
+from wendelin.bigfile.file_zodb import ZBigFile, ZBlk_fmt_registry
 from wendelin.bigfile import file_zodb, ram_reclaim
 from wendelin.bigfile.tests.test_thread import NotifyChannel
-from wendelin.lib.zodb import dbclose
+from wendelin.lib.zodb import LivePersistent, dbclose
+from wendelin.lib.tests.test_zodb import cacheInfo, kkey
 from wendelin.lib.testing import getTestDB
-from persistent import UPTODATE, GHOST, CHANGED
+from persistent import UPTODATE
 import transaction
 from transaction import TransactionManager
 from ZODB.POSException import ConflictError
@@ -68,149 +69,6 @@ def ram_reclaim_all():
     return reclaimed
 
 
-# like db.cacheDetail(), but {} instead of []
-def cacheInfo(db):
-    return dict(db.cacheDetail())
-
-# key for cacheInfo() result
-def kkey(klass):
-    return '%s.%s' % (klass.__module__, klass.__name__)
-
-@func
-def test_livepersistent():
-    root = dbopen()
-    transaction.commit()    # set root._p_jar
-    db = root._p_jar.db()
-
-    # ~~~ test `obj initially created` case
-    root['live'] = lp = LivePersistent()
-    assert lp._p_jar   is None          # connection does not know about it yet
-    assert lp._p_state == UPTODATE      # object initially created in uptodate
-
-    # should not be in cache yet & thus should stay after gc
-    db.cacheMinimize()
-    assert lp._p_jar   is None
-    assert lp._p_state == UPTODATE
-    ci = cacheInfo(db)
-    assert kkey(LivePersistent) not in ci
-
-    # should be registered to connection & cache after commit
-    transaction.commit()
-    assert lp._p_jar   is not None
-    assert lp._p_state == UPTODATE
-    ci = cacheInfo(db)
-    assert ci[kkey(LivePersistent)] == 1
-
-    # should stay that way after cache gc
-    db.cacheMinimize()
-    assert lp._p_jar   is not None
-    assert lp._p_state == UPTODATE
-    ci = cacheInfo(db)
-    assert ci[kkey(LivePersistent)] == 1
-
-
-    # ~~~ reopen & test `obj loaded from db` case
-    dbclose(root)
-    del root, db, lp
-
-    root = dbopen()
-    db = root._p_jar.db()
-
-    # known to connection & cache & GHOST
-    # right after first loading from DB
-    lp = root['live']
-    assert lp._p_jar   is not None
-    assert lp._p_state is GHOST
-    ci = cacheInfo(db)
-    assert ci[kkey(LivePersistent)] == 1
-
-    # should be UPTODATE for sure after read access
-    getattr(lp, 'attr', None)
-    assert lp._p_jar   is not None
-    assert lp._p_state is UPTODATE
-    ci = cacheInfo(db)
-    assert ci[kkey(LivePersistent)] == 1
-
-    # does not go back to ghost on cache gc
-    db.cacheMinimize()
-    assert lp._p_jar   is not None
-    assert lp._p_state == UPTODATE
-    ci = cacheInfo(db)
-    assert ci[kkey(LivePersistent)] == 1
-
-    # ok
-    dbclose(root)
-    del root, db, lp
-
-
-    # demo that upon cache invalidation LivePersistent can go back to ghost
-    root = dbopen()
-    conn = root._p_jar
-    db   = conn.db()
-    conn.close()
-    del root, conn
-
-    tm1 = TransactionManager()
-    tm2 = TransactionManager()
-
-    conn1 = db.open(transaction_manager=tm1)
-    root1 = conn1.root()
-    defer(lambda: dbclose(root1))
-    lp1 = root1['live']
-
-    conn2 = db.open(transaction_manager=tm2)
-    root2 = conn2.root()
-    defer(conn2.close)
-    lp2 = root2['live']
-
-    # 2 connections are setup running in parallel with initial obj state as ghost
-    assert lp1._p_jar   is conn1
-    assert lp2._p_jar   is conn2
-
-    assert lp1._p_state is GHOST
-    assert lp2._p_state is GHOST
-
-    # conn1: modify  ghost -> changed
-    lp1.attr = 1
-
-    assert lp1._p_state is CHANGED
-    assert lp2._p_state is GHOST
-
-    # conn2: read    ghost -> uptodate
-    assert getattr(lp1, 'attr', None) == 1
-    assert getattr(lp2, 'attr', None) is None
-
-    assert lp1._p_state is CHANGED
-    assert lp2._p_state is UPTODATE
-
-    # conn1: commit  changed -> uptodate; conn2 untouched
-    tm1.commit()
-
-    assert lp1._p_state is UPTODATE
-    assert lp2._p_state is UPTODATE
-
-    assert getattr(lp1, 'attr', None) == 1
-    assert getattr(lp2, 'attr', None) is None
-
-    # conn2: commit  (nothing changed - just transaction boundary)
-    #                 uptodate -> ghost (invalidation)
-    tm2.commit()
-
-    assert lp1._p_state is UPTODATE
-    assert lp2._p_state is GHOST
-
-    assert getattr(lp1, 'attr', None) == 1
-
-    # conn2: after reading, the state is again uptodate + changes from conn1 are here
-    a = getattr(lp2, 'attr', None)
-    assert lp2._p_state is UPTODATE
-    assert a == 1
-
-
-    del conn2, root2
-
-
-
 # i'th memory block as u32 ndarray
 blksize32 = blksize // 4
 def Blk(vma, i):

lib/tests/test_zodb.py

@@ -17,11 +17,12 @@
 #
 # See COPYING file for full licensing terms.
 # See https://www.nexedi.com/licensing for rationale and options.
-from wendelin.lib.zodb import deactivate_btree, dbclose
+from wendelin.lib.zodb import LivePersistent, deactivate_btree, dbclose
 from wendelin.lib.testing import getTestDB
-from persistent import Persistent, UPTODATE, GHOST
+from persistent import Persistent, UPTODATE, GHOST, CHANGED
 from BTrees.IOBTree import IOBTree
 import transaction
+from transaction import TransactionManager
 from golang import defer, func
 import gc
 
@@ -38,6 +39,146 @@ def setup_module():
 def teardown_module():
     testdb.teardown()
 
+# like db.cacheDetail(), but {} instead of []
+def cacheInfo(db):
+    return dict(db.cacheDetail())
+
+# key for cacheInfo() result
+def kkey(klass):
+    return '%s.%s' % (klass.__module__, klass.__name__)
+
+@func
+def test_livepersistent():
+    root = dbopen()
+    transaction.commit()    # set root._p_jar
+    db = root._p_jar.db()
+
+    # ~~~ test `obj initially created` case
+    root['live'] = lp = LivePersistent()
+    assert lp._p_jar   is None          # connection does not know about it yet
+    assert lp._p_state == UPTODATE      # object initially created in uptodate
+
+    # should not be in cache yet & thus should stay after gc
+    db.cacheMinimize()
+    assert lp._p_jar   is None
+    assert lp._p_state == UPTODATE
+    ci = cacheInfo(db)
+    assert kkey(LivePersistent) not in ci
+
+    # should be registered to connection & cache after commit
+    transaction.commit()
+    assert lp._p_jar   is not None
+    assert lp._p_state == UPTODATE
+    ci = cacheInfo(db)
+    assert ci[kkey(LivePersistent)] == 1
+
+    # should stay that way after cache gc
+    db.cacheMinimize()
+    assert lp._p_jar   is not None
+    assert lp._p_state == UPTODATE
+    ci = cacheInfo(db)
+    assert ci[kkey(LivePersistent)] == 1
+
+
+    # ~~~ reopen & test `obj loaded from db` case
+    dbclose(root)
+    del root, db, lp
+
+    root = dbopen()
+    db = root._p_jar.db()
+
+    # known to connection & cache & GHOST
+    # right after first loading from DB
+    lp = root['live']
+    assert lp._p_jar   is not None
+    assert lp._p_state is GHOST
+    ci = cacheInfo(db)
+    assert ci[kkey(LivePersistent)] == 1
+
+    # should be UPTODATE for sure after read access
+    getattr(lp, 'attr', None)
+    assert lp._p_jar   is not None
+    assert lp._p_state is UPTODATE
+    ci = cacheInfo(db)
+    assert ci[kkey(LivePersistent)] == 1
+
+    # does not go back to ghost on cache gc
+    db.cacheMinimize()
+    assert lp._p_jar   is not None
+    assert lp._p_state == UPTODATE
+    ci = cacheInfo(db)
+    assert ci[kkey(LivePersistent)] == 1
+
+    # ok
+    dbclose(root)
+    del root, db, lp
+
+
+    # demo that upon cache invalidation LivePersistent can go back to ghost
+    root = dbopen()
+    conn = root._p_jar
+    db   = conn.db()
+    conn.close()
+    del root, conn
+
+    tm1 = TransactionManager()
+    tm2 = TransactionManager()
+
+    conn1 = db.open(transaction_manager=tm1)
+    root1 = conn1.root()
+    defer(lambda: dbclose(root1))
+    lp1 = root1['live']
+
+    conn2 = db.open(transaction_manager=tm2)
+    root2 = conn2.root()
+    defer(conn2.close)
+    lp2 = root2['live']
+
+    # 2 connections are setup running in parallel with initial obj state as ghost
+    assert lp1._p_jar   is conn1
+    assert lp2._p_jar   is conn2
+
+    assert lp1._p_state is GHOST
+    assert lp2._p_state is GHOST
+
+    # conn1: modify  ghost -> changed
+    lp1.attr = 1
+
+    assert lp1._p_state is CHANGED
+    assert lp2._p_state is GHOST
+
+    # conn2: read    ghost -> uptodate
+    assert getattr(lp1, 'attr', None) == 1
+    assert getattr(lp2, 'attr', None) is None
+
+    assert lp1._p_state is CHANGED
+    assert lp2._p_state is UPTODATE
+
+    # conn1: commit  changed -> uptodate; conn2 untouched
+    tm1.commit()
+
+    assert lp1._p_state is UPTODATE
+    assert lp2._p_state is UPTODATE
+
+    assert getattr(lp1, 'attr', None) == 1
+    assert getattr(lp2, 'attr', None) is None
+
+    # conn2: commit  (nothing changed - just transaction boundary)
+    #                 uptodate -> ghost (invalidation)
+    tm2.commit()
+
+    assert lp1._p_state is UPTODATE
+    assert lp2._p_state is GHOST
+
+    assert getattr(lp1, 'attr', None) == 1
+
+    # conn2: after reading, the state is again uptodate + changes from conn1 are here
+    a = getattr(lp2, 'attr', None)
+    assert lp2._p_state is UPTODATE
+    assert a == 1
+
+    del conn2, root2
+
 
 class XInt(Persistent):
     def __init__(self, i):

lib/zodb.py

@@ -79,6 +79,31 @@ def dbclose(root):
     stor.close()
 
 
+# LivePersistent is Persistent that never goes to ghost state, if it was ever uptodate.
+#
+# NOTE
+#
+# On invalidation LivePersistent still goes to ghost state, because
+# invalidation cannot be ignored, i.e. they indicate the object has been
+# changed externally.
+#
+# Invalidation can happen only at transaction boundary, so during the course of
+# transaction LivePersistent is guaranteed to stay uptodate.
+class LivePersistent(Persistent):
+    # don't allow us to go to ghost
+    #
+    # NOTE we can't use STICKY as that state is assumed as
+    # short-lived-temporary by ZODB and is changed back to UPTODATE by
+    # persistent code. In fact ZODB says: STICKY is UPTODATE+keep in memory.
+    def _p_deactivate(self):
+        # just returning here won't allow Persistent._p_deactivate() run and
+        # thus we'll stay in non-ghost state.
+        return
+
+    # NOTE _p_invalidate() is triggered on invalidations. We do not override it.
+
+
+
 # deactivate a btree, including all internal buckets and leaf nodes
 def deactivate_btree(btree):
     # first activate btree, to make sure its first bucket is loaded at all.