Test coverage for ZODB{3,4,5}

Fixes to get all tests passing with all ZODB versions we care about.  

Organize testing coverage for all cases via tox, similarly to wendelin.core .

/reviewed-on nexedi/zodbtools!11

.gitignore

 /dist
 /zodbtools.egg-info
+
+/.tox/

tox.ini

+# zodbtools | tox setup
+[tox]
+envlist = py27-{ZODB3,ZODB4,ZODB5}
+
+[testenv]
+deps	=
+    # XXX tox does not consult extras_require['test'] -> install pytest explicitly
+    pytest
+
+    # latest ZODB from ZODB3 series
+    ZODB3:  ZODB3 >=3.10, <3.11dev
+    ZODB3:  transaction <2.0dev
+    # ZConfig 3.2.0 passes filename to ZEO config as unicode which eventualy breaks in FileStorage:
+    # https://github.com/zopefoundation/ZODB/blob/3.10.7-4-gb8d7a8567/src/ZODB/FileStorage/FileStorage.py#L1640
+    ZODB3:  ZConfig <3.2.0
+
+    # latest current ZODB 4
+    ZODB4:  ZODB >=4.0, <5.0dev
+    ZODB4:  ZEO  >=4.0, <5.0dev
+    # ZEO4 depends on transaction <2
+    ZODB4:  transaction <2.0dev
+
+    # latest current ZODB 5
+    ZODB5:  ZODB >=5.0,	<6.0dev
+    ZODB5:  ZEO	 >=5.0, <6.0dev
+
+
+commands= {envpython} -m pytest

zodbtools/test/conftest.py

+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+import pytest
+from zodbtools.test.testutil import zext_supported
+
+# zext is a test fixture function object that allows to exercise 2 cases:
+#
+# - when ZODB does not have txn.extension_bytes support
+# - when ZODB might have    txn.extension_bytes support
+#
+# in a test, zext should be used as as follows:
+#
+#   def test_something(zext):
+#       # bytes for an extension dict
+#       raw_ext = dumps({...})
+#
+#       # will be either same as raw_ext, or b'' if ZODB lacks txn.extension_bytes support
+#       raw_ext = zext(raw_ext)
+#
+#       # zext.disabled indicates whether testing for non-empty extension was disabled.
+#       if zext.disabled:
+#           ...
+@pytest.fixture(params=['!zext', 'zext'])
+def zext(request):
+    if request.param == '!zext':
+        # txn.extension_bytes is not working - always test with empty extension
+        def _(ext):
+            return b''
+        _.disabled = True
+        return _
+    else:
+        # txn.extension_bytes might be working - test with given extension and
+        # xfail if ZODB does not have necessary support.
+        def _(ext):
+            return ext
+        _.disabled = False
+        if not zext_supported():
+            request.applymarker(pytest.mark.xfail(reason='ZODB does not have txn.extension_bytes support'))
+        return _

zodbtools/test/gen_testdata.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright (C) 2017  Nexedi SA and Contributors.
+# Copyright (C) 2017-2019  Nexedi SA and Contributors.
 #                          Kirill Smelkov <kirr@nexedi.com>
 #
 # This program is free software: you can Use, Study, Modify and Redistribute
@@ -60,7 +60,12 @@ def hex64(packed):
     return '0x%016x' % unpack64(packed)
 
 # make time.time() predictable
-_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
+_xtime0 = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
+def xtime_reset():
+    global _xtime
+    _xtime = _xtime0
+xtime_reset()
+
 def xtime():
     global _xtime
     _xtime += 1.1
@@ -94,7 +99,7 @@ class Object(Persistent):
 
 # prepare extension dictionary for subject
 alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-def ext(subj):
+def ext4subj(subj):
     d = {"x-generator": "zodb/py%s (%s)" % (sys.version_info.major, subj)}
 
     # also add some random 'x-cookie'
@@ -115,8 +120,16 @@ def ext(subj):
 
     return ext
 
-# gen_testdb generates test FileStorage database @ outfs_path
-def gen_testdb(outfs_path):
+# gen_testdb generates test FileStorage database @ outfs_path.
+#
+# zext indicates whether or not to include non-empty extension into transactions.
+def gen_testdb(outfs_path, zext=True):
+    xtime_reset()
+
+    ext = ext4subj
+    if not zext:
+        def ext(subj): return {}
+
     logging.basicConfig()
 
     # generate random changes to objects hooked to top-level root by a/b/c/... key
@@ -196,12 +209,21 @@ def gen_testdb(outfs_path):
 # ----------------------------------------
 
 from zodbtools.zodbdump import zodbdump
+from zodbtools.test.testutil import zext_supported
 
 def main():
+    # check that ZODB supports txn.extension_bytes; refuse to work if not.
+    if not zext_supported():
+        raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")
+
     out = "testdata/1"
-    gen_testdb("%s.fs" % out)
-    stor = FileStorage("%s.fs" % out, read_only=True)
-    with open("%s.zdump.ok" % out, "w") as f:
+    for zext in [True, False]:
+        dbname = out
+        if not zext:
+            dbname += "_!zext"
+        gen_testdb("%s.fs" % dbname, zext=zext)
+        stor = FileStorage("%s.fs" % dbname, read_only=True)
+        with open("%s.zdump.ok" % dbname, "w") as f:
             zodbdump(stor, None, None, out=f)
 
 if __name__ == '__main__':

zodbtools/test/test_commit.py

@@ -20,7 +20,7 @@
 from zodbtools.zodbcommit import zodbcommit
 from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy
 from zodbtools.util import storageFromURL, sha1
-from ZODB.utils import p64, u64, z64, maxtid
+from ZODB.utils import p64, u64, z64
 from ZODB._compat import BytesIO, dumps, _protocol   # XXX can't yet commit with arbitrary ext.bytes
 
 from tempfile import mkdtemp
@@ -29,7 +29,7 @@ from golang import func, defer
 
 # verify zodbcommit.
 @func
-def test_zodbcommit():
+def test_zodbcommit(zext):
     tmpd = mkdtemp('', 'zodbcommit.')
     defer(lambda: rmtree(tmpd))
 
@@ -40,7 +40,7 @@ def test_zodbcommit():
 
     # commit some transactions via zodbcommit and verify if storage dump gives
     # what is expected.
-    t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
+    t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
         ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
         ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
 
@@ -53,7 +53,7 @@ def test_zodbcommit():
 
 
     buf = BytesIO()
-    zodbdump(stor, p64(u64(head)+1), maxtid, out=buf)
+    zodbdump(stor, p64(u64(head)+1), None, out=buf)
     dumped = buf.getvalue()
 
     assert dumped == ''.join([_.zdump() for _ in (t1, t2)])

zodbtools/test/test_dump.py

@@ -27,14 +27,16 @@ from cStringIO import StringIO
 
 from os.path import dirname
 
-from pytest import raises
+from zodbtools.test.testutil import zext_supported
+from pytest import raises, xfail
 
 # verify zodbdump output against golden
-def test_zodbdump():
+def test_zodbdump(zext):
     tdir  = dirname(__file__)
-    stor = FileStorage('%s/testdata/1.fs' % tdir, read_only=True)
+    zkind = '_!zext' if zext.disabled else ''
+    stor  = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
 
-    with open('%s/testdata/1.zdump.ok' % tdir) as f:
+    with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
         dumpok = f.read()
 
     out = StringIO()

zodbtools/test/testutil.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (C) 2019  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""utilities for testing"""
+
+from ZODB.FileStorage import FileStorage
+from ZODB import DB
+import transaction
+
+from tempfile import mkdtemp
+from shutil import rmtree
+from golang import func, defer
+
+# zext_supported checks whether ZODB supports txn.extension_bytes .
+_zext_supported_memo = None
+def zext_supported():
+    global _zext_supported_memo
+    if _zext_supported_memo is not None:
+        return _zext_supported_memo
+
+    _ = _zext_supported_memo = _zext_supported()
+    return _
+
+@func
+def _zext_supported():
+    tmpd = mkdtemp('', 'zext_check.')
+    defer(lambda: rmtree(tmpd))
+    dbfs = tmpd + '/1.fs'
+
+    stor = FileStorage(dbfs, create=True)
+    db   = DB(stor)
+    conn = db.open()
+    root = conn.root()
+    root._p_changed = True
+
+    txn = transaction.get()
+    txn.setExtendedInfo('a', 'b')
+    txn.commit()
+
+    for last_txn in stor.iterator(start=stor.lastTransaction()):
+        break
+    else:
+        assert False, "cannot see details of last transaction"
+
+    assert last_txn.extension == {'a': 'b'}
+    return hasattr(last_txn, 'extension_bytes')

zodbtools/zodbdump.py

@@ -58,7 +58,7 @@ from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRan
 from ZODB._compat import loads, _protocol, BytesIO
 from zodbpickle.slowpickle import Pickler as pyPickler
 #import pickletools
-from ZODB.interfaces import IStorageTransactionMetaData
+from ZODB.interfaces import IStorageTransactionInformation
 from zope.interface import implementer
 
 import sys
@@ -413,7 +413,7 @@ class DumpReader(object):
 
 
 # Transaction represents one transaction record in zodbdump stream.
-@implementer(IStorageTransactionMetaData)
+@implementer(IStorageTransactionInformation)    # TODO -> IStorageTransactionMetaData after switch to ZODB >= 5
 class Transaction(object):
     # .tid              p64         transaction ID
     # .status           char        status of the transaction
@@ -444,6 +444,11 @@ class Transaction(object):
             return {}
         return loads(self.extension_bytes)
 
+    # ZODB < 5 wants ._extension
+    @property
+    def _extension(self):
+        return self.extension
+
     # zdump returns text representation of a record in zodbdump format.
     def zdump(self):
         z  = 'txn %s %s\n' % (ashex(self.tid), qq(self.status))