Commit 25aeda62 authored by Kirill Smelkov's avatar Kirill Smelkov

Test coverage for ZODB{3,4,5}

Fixes to get all tests passing with all ZODB versions we care about.  

Organize testing coverage for all cases via tox, similarly to wendelin.core .

/reviewed-on !11
parents 8c76eae2 8ff7020c
/dist /dist
/zodbtools.egg-info /zodbtools.egg-info
/.tox/
# zodbtools | tox setup
[tox]
envlist = py27-{ZODB3,ZODB4,ZODB5}
[testenv]
deps =
# XXX tox does not consult extras_require['test'] -> install pytest explicitly
pytest
# latest ZODB from ZODB3 series
ZODB3: ZODB3 >=3.10, <3.11dev
ZODB3: transaction <2.0dev
# ZConfig 3.2.0 passes filename to ZEO config as unicode which eventualy breaks in FileStorage:
# https://github.com/zopefoundation/ZODB/blob/3.10.7-4-gb8d7a8567/src/ZODB/FileStorage/FileStorage.py#L1640
ZODB3: ZConfig <3.2.0
# latest current ZODB 4
ZODB4: ZODB >=4.0, <5.0dev
ZODB4: ZEO >=4.0, <5.0dev
# ZEO4 depends on transaction <2
ZODB4: transaction <2.0dev
# latest current ZODB 5
ZODB5: ZODB >=5.0, <6.0dev
ZODB5: ZEO >=5.0, <6.0dev
commands= {envpython} -m pytest
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import pytest
from zodbtools.test.testutil import zext_supported
# zext is a test fixture function object that allows to exercise 2 cases:
#
# - when ZODB does not have txn.extension_bytes support
# - when ZODB might have txn.extension_bytes support
#
# in a test, zext should be used as as follows:
#
# def test_something(zext):
# # bytes for an extension dict
# raw_ext = dumps({...})
#
# # will be either same as raw_ext, or b'' if ZODB lacks txn.extension_bytes support
# raw_ext = zext(raw_ext)
#
# # zext.disabled indicates whether testing for non-empty extension was disabled.
# if zext.disabled:
# ...
@pytest.fixture(params=['!zext', 'zext'])
def zext(request):
if request.param == '!zext':
# txn.extension_bytes is not working - always test with empty extension
def _(ext):
return b''
_.disabled = True
return _
else:
# txn.extension_bytes might be working - test with given extension and
# xfail if ZODB does not have necessary support.
def _(ext):
return ext
_.disabled = False
if not zext_supported():
request.applymarker(pytest.mark.xfail(reason='ZODB does not have txn.extension_bytes support'))
return _
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2017 Nexedi SA and Contributors. # Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -60,7 +60,12 @@ def hex64(packed): ...@@ -60,7 +60,12 @@ def hex64(packed):
return '0x%016x' % unpack64(packed) return '0x%016x' % unpack64(packed)
# make time.time() predictable # make time.time() predictable
_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y")) _xtime0 = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
def xtime_reset():
global _xtime
_xtime = _xtime0
xtime_reset()
def xtime(): def xtime():
global _xtime global _xtime
_xtime += 1.1 _xtime += 1.1
...@@ -94,7 +99,7 @@ class Object(Persistent): ...@@ -94,7 +99,7 @@ class Object(Persistent):
# prepare extension dictionary for subject # prepare extension dictionary for subject
alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
def ext(subj): def ext4subj(subj):
d = {"x-generator": "zodb/py%s (%s)" % (sys.version_info.major, subj)} d = {"x-generator": "zodb/py%s (%s)" % (sys.version_info.major, subj)}
# also add some random 'x-cookie' # also add some random 'x-cookie'
...@@ -115,8 +120,16 @@ def ext(subj): ...@@ -115,8 +120,16 @@ def ext(subj):
return ext return ext
# gen_testdb generates test FileStorage database @ outfs_path # gen_testdb generates test FileStorage database @ outfs_path.
def gen_testdb(outfs_path): #
# zext indicates whether or not to include non-empty extension into transactions.
def gen_testdb(outfs_path, zext=True):
xtime_reset()
ext = ext4subj
if not zext:
def ext(subj): return {}
logging.basicConfig() logging.basicConfig()
# generate random changes to objects hooked to top-level root by a/b/c/... key # generate random changes to objects hooked to top-level root by a/b/c/... key
...@@ -196,13 +209,22 @@ def gen_testdb(outfs_path): ...@@ -196,13 +209,22 @@ def gen_testdb(outfs_path):
# ---------------------------------------- # ----------------------------------------
from zodbtools.zodbdump import zodbdump from zodbtools.zodbdump import zodbdump
from zodbtools.test.testutil import zext_supported
def main(): def main():
# check that ZODB supports txn.extension_bytes; refuse to work if not.
if not zext_supported():
raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")
out = "testdata/1" out = "testdata/1"
gen_testdb("%s.fs" % out) for zext in [True, False]:
stor = FileStorage("%s.fs" % out, read_only=True) dbname = out
with open("%s.zdump.ok" % out, "w") as f: if not zext:
zodbdump(stor, None, None, out=f) dbname += "_!zext"
gen_testdb("%s.fs" % dbname, zext=zext)
stor = FileStorage("%s.fs" % dbname, read_only=True)
with open("%s.zdump.ok" % dbname, "w") as f:
zodbdump(stor, None, None, out=f)
if __name__ == '__main__': if __name__ == '__main__':
main() main()
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
from zodbtools.zodbcommit import zodbcommit from zodbtools.zodbcommit import zodbcommit
from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy
from zodbtools.util import storageFromURL, sha1 from zodbtools.util import storageFromURL, sha1
from ZODB.utils import p64, u64, z64, maxtid from ZODB.utils import p64, u64, z64
from ZODB._compat import BytesIO, dumps, _protocol # XXX can't yet commit with arbitrary ext.bytes from ZODB._compat import BytesIO, dumps, _protocol # XXX can't yet commit with arbitrary ext.bytes
from tempfile import mkdtemp from tempfile import mkdtemp
...@@ -29,7 +29,7 @@ from golang import func, defer ...@@ -29,7 +29,7 @@ from golang import func, defer
# verify zodbcommit. # verify zodbcommit.
@func @func
def test_zodbcommit(): def test_zodbcommit(zext):
tmpd = mkdtemp('', 'zodbcommit.') tmpd = mkdtemp('', 'zodbcommit.')
defer(lambda: rmtree(tmpd)) defer(lambda: rmtree(tmpd))
...@@ -40,7 +40,7 @@ def test_zodbcommit(): ...@@ -40,7 +40,7 @@ def test_zodbcommit():
# commit some transactions via zodbcommit and verify if storage dump gives # commit some transactions via zodbcommit and verify if storage dump gives
# what is expected. # what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [ t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
ObjectData(p64(1), b'data1', 'sha1', sha1('data1')), ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))]) ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])
...@@ -53,7 +53,7 @@ def test_zodbcommit(): ...@@ -53,7 +53,7 @@ def test_zodbcommit():
buf = BytesIO() buf = BytesIO()
zodbdump(stor, p64(u64(head)+1), maxtid, out=buf) zodbdump(stor, p64(u64(head)+1), None, out=buf)
dumped = buf.getvalue() dumped = buf.getvalue()
assert dumped == ''.join([_.zdump() for _ in (t1, t2)]) assert dumped == ''.join([_.zdump() for _ in (t1, t2)])
......
...@@ -27,14 +27,16 @@ from cStringIO import StringIO ...@@ -27,14 +27,16 @@ from cStringIO import StringIO
from os.path import dirname from os.path import dirname
from pytest import raises from zodbtools.test.testutil import zext_supported
from pytest import raises, xfail
# verify zodbdump output against golden # verify zodbdump output against golden
def test_zodbdump(): def test_zodbdump(zext):
tdir = dirname(__file__) tdir = dirname(__file__)
stor = FileStorage('%s/testdata/1.fs' % tdir, read_only=True) zkind = '_!zext' if zext.disabled else ''
stor = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
with open('%s/testdata/1.zdump.ok' % tdir) as f: with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
dumpok = f.read() dumpok = f.read()
out = StringIO() out = StringIO()
......
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""utilities for testing"""
from ZODB.FileStorage import FileStorage
from ZODB import DB
import transaction
from tempfile import mkdtemp
from shutil import rmtree
from golang import func, defer
# zext_supported checks whether ZODB supports txn.extension_bytes .
_zext_supported_memo = None
def zext_supported():
global _zext_supported_memo
if _zext_supported_memo is not None:
return _zext_supported_memo
_ = _zext_supported_memo = _zext_supported()
return _
@func
def _zext_supported():
tmpd = mkdtemp('', 'zext_check.')
defer(lambda: rmtree(tmpd))
dbfs = tmpd + '/1.fs'
stor = FileStorage(dbfs, create=True)
db = DB(stor)
conn = db.open()
root = conn.root()
root._p_changed = True
txn = transaction.get()
txn.setExtendedInfo('a', 'b')
txn.commit()
for last_txn in stor.iterator(start=stor.lastTransaction()):
break
else:
assert False, "cannot see details of last transaction"
assert last_txn.extension == {'a': 'b'}
return hasattr(last_txn, 'extension_bytes')
...@@ -58,7 +58,7 @@ from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRan ...@@ -58,7 +58,7 @@ from zodbtools.util import ashex, fromhex, sha1, txnobjv, parse_tidrange, TidRan
from ZODB._compat import loads, _protocol, BytesIO from ZODB._compat import loads, _protocol, BytesIO
from zodbpickle.slowpickle import Pickler as pyPickler from zodbpickle.slowpickle import Pickler as pyPickler
#import pickletools #import pickletools
from ZODB.interfaces import IStorageTransactionMetaData from ZODB.interfaces import IStorageTransactionInformation
from zope.interface import implementer from zope.interface import implementer
import sys import sys
...@@ -413,7 +413,7 @@ class DumpReader(object): ...@@ -413,7 +413,7 @@ class DumpReader(object):
# Transaction represents one transaction record in zodbdump stream. # Transaction represents one transaction record in zodbdump stream.
@implementer(IStorageTransactionMetaData) @implementer(IStorageTransactionInformation) # TODO -> IStorageTransactionMetaData after switch to ZODB >= 5
class Transaction(object): class Transaction(object):
# .tid p64 transaction ID # .tid p64 transaction ID
# .status char status of the transaction # .status char status of the transaction
...@@ -444,6 +444,11 @@ class Transaction(object): ...@@ -444,6 +444,11 @@ class Transaction(object):
return {} return {}
return loads(self.extension_bytes) return loads(self.extension_bytes)
# ZODB < 5 wants ._extension
@property
def _extension(self):
return self.extension
# zdump returns text representation of a record in zodbdump format. # zdump returns text representation of a record in zodbdump format.
def zdump(self): def zdump(self):
z = 'txn %s %s\n' % (ashex(self.tid), qq(self.status)) z = 'txn %s %s\n' % (ashex(self.tid), qq(self.status))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment