Commit 0b6f99da authored by Kirill Smelkov's avatar Kirill Smelkov

test/gen_testdata: Fix for ZODB5 > 5.5.1 + preserve database compatibility with ZODB3/py2

Starting with upcoming ZODB 5.5.2 ZODB tries to preserve
`extension_bytes` transaction metadata property in the raw form as it
was stored on disk in the database:

    https://github.com/zopefoundation/ZODB/commit/2f8cc67a

However now when running test/gen_testdata.py with ZODB with that patch (and
gen_testdata.py refuses to work if it detects that ZODB does not properly
supports .extension_bytes property because we want it to be present in the
generated test database [1,2]) it now breaks:

    $ ./gen_testdata.py
    Traceback (most recent call last):
      File "./gen_testdata.py", line 230, in <module>
        main()
      File "./gen_testdata.py", line 224, in main
        gen_testdb("%s.fs" % dbname, zext=zext)
      File "./gen_testdata.py", line 194, in gen_testdb
        stor.tpc_begin(txn)
      File "/home/kirr/src/wendelin/z/ZODB/src/ZODB/BaseStorage.py", line 193, in tpc_begin
        ext = transaction.extension_bytes
    AttributeError: 'Transaction' object has no attribute 'extension_bytes'

The breakage is because, as specified in ZODB interfaces[3,4], storage requires
ZODB.IStorageTransactionMetaData, not transaction.ITransaction instance
gen_testdata.py was using. The script used to work before just by luck.

The fix is to convert transaction instance into storage transaction metadata
object for the place where we talk to storage at raw level.

HOWEVER, when checking regenerated database and its dump I noticed:

ZODB >= 5.4.0 uses pickle protocol 3 on both python2 and python3

    https://github.com/zopefoundation/ZODB/commit/12ee41c4

In other words it saves e.g. OID of an object as pickle binary, which decodes
as bytes on py3 and zodbpickle.binary on py2 when decoding via zodbpickle.
However it will result in *DecodeError* when decoding on py2 with standard
pickle module. The latter means that ZODB3 will _fail_ to load data from test
database, because ZODB3 - contrary to ZODB4 and ZODB5 - uses std pickle module,
not zodbpickle.

We still care about ZODB3 and in particular it is included into
zodbtools test matrix:

    https://lab.nexedi.com/nexedi/zodbtools/blob/7bc0385e/tox.ini#L9-14

so we cannot break it.

-> Temporarily patch ZODB at runtime to make sure it emits data with
older protocol and without using zodbpickle.binary for oid, so that
generated test database could be loaded on ZODB3 as well.

gen_testdata.py now works with latest ZODB, but produces exactly the
same bit-to-bit output as before.

[1] https://lab.nexedi.com/nexedi/zodbtools/blob/7bc0385e/zodbtools/test/gen_testdata.py#L215-217
[2] https://lab.nexedi.com/nexedi/zodbtools/blob/7bc0385e/zodbtools/test/testutil.py#L31-63
[3] https://github.com/zopefoundation/ZODB/blob/5.5.1-35-gb5895a5c2/src/ZODB/interfaces.py#L815-L818
[4] https://github.com/zopefoundation/ZODB/blob/5.5.1-35-gb5895a5c2/src/ZODB/interfaces.py#L538-L575

/reviewed-on nexedi/zodbtools!15
parent 7bc0385e
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors. # Copyright (C) 2017-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -39,6 +39,7 @@ ...@@ -39,6 +39,7 @@
from ZODB.FileStorage import FileStorage from ZODB.FileStorage import FileStorage
from ZODB import DB from ZODB import DB
from ZODB.Connection import TransactionMetaData
from ZODB.POSException import UndoError from ZODB.POSException import UndoError
from persistent import Persistent from persistent import Persistent
import transaction import transaction
...@@ -120,10 +121,60 @@ def ext4subj(subj): ...@@ -120,10 +121,60 @@ def ext4subj(subj):
return ext return ext
# run_with_zodb3py2_compat(f) runs f preserving database compatibility with
# ZODB3/py2, which cannot load pickles encoded with protocol 3.
#
# ZODB5 started to use protocol 3 and binary for oids starting from ZODB 5.4.0:
# https://github.com/zopefoundation/ZODB/commit/12ee41c4
# Undo it, while we generate test database.
def run_with_zodb3py2_compat(f):
import ZODB.ConflictResolution
import ZODB.Connection
import ZODB.ExportImport
import ZODB.FileStorage.FileStorage
import ZODB._compat
import ZODB.broken
import ZODB.fsIndex
import ZODB.serialize
binary = getattr(ZODB.serialize, 'binary', None)
_protocol = getattr(ZODB.serialize, '_protocol', None)
Pz3 = 2
try:
ZODB.serialize.binary = bytes
# XXX cannot change just ZODB._compat._protocol, because many modules
# do `from ZODB._compat import _protocol` and just `import ZODB`
# imports many ZODB.X modules. In other words we cannot change
# _protocol just in one place.
ZODB.ConflictResolution._protocol = Pz3
ZODB.Connection._protocol = Pz3
ZODB.ExportImport._protocol = Pz3
ZODB.FileStorage.FileStorage._protocol = Pz3
ZODB._compat._protocol = Pz3
ZODB.broken._protocol = Pz3
ZODB.fsIndex._protocol = Pz3
ZODB.serialize._protocol = Pz3
f()
finally:
ZODB.serialize.binary = binary
ZODB.ConflictResolution._protocol = _protocol
ZODB.Connection._protocol = _protocol
ZODB.ExportImport._protocol = _protocol
ZODB.FileStorage.FileStorage._protocol = _protocol
ZODB._compat._protocol = _protocol
ZODB.broken._protocol = _protocol
ZODB.fsIndex._protocol = _protocol
ZODB.serialize._protocol = _protocol
# gen_testdb generates test FileStorage database @ outfs_path. # gen_testdb generates test FileStorage database @ outfs_path.
# #
# zext indicates whether or not to include non-empty extension into transactions. # zext indicates whether or not to include non-empty extension into transactions.
def gen_testdb(outfs_path, zext=True): def gen_testdb(outfs_path, zext=True):
def _():
_gen_testdb(outfs_path, zext)
run_with_zodb3py2_compat(_)
def _gen_testdb(outfs_path, zext):
xtime_reset() xtime_reset()
ext = ext4subj ext = ext4subj
...@@ -191,14 +242,16 @@ def gen_testdb(outfs_path, zext=True): ...@@ -191,14 +242,16 @@ def gen_testdb(outfs_path, zext=True):
''.join(chr(_) for _ in range(32)), # <- NOTE all control characters ''.join(chr(_) for _ in range(32)), # <- NOTE all control characters
u"delete %i\nalpha beta gamma'delta\"lambda\n\nqqq ..." % i, u"delete %i\nalpha beta gamma'delta\"lambda\n\nqqq ..." % i,
ext("delete %s" % unpack64(obj._p_oid))) ext("delete %s" % unpack64(obj._p_oid)))
stor.tpc_begin(txn) # at low level stor requires ZODB.IStorageTransactionMetaData not txn (ITransaction)
stor.deleteObject(obj._p_oid, obj_tid_lastchange, txn) txn_stormeta = TransactionMetaData(txn.user, txn.description, txn.extension)
stor.tpc_vote(txn) stor.tpc_begin(txn_stormeta)
stor.deleteObject(obj._p_oid, obj_tid_lastchange, txn_stormeta)
stor.tpc_vote(txn_stormeta)
# TODO different txn status vvv # TODO different txn status vvv
# XXX vvv it does the thing, but py fs iterator treats this txn as EOF # XXX vvv it does the thing, but py fs iterator treats this txn as EOF
#if i != Niter-1: #if i != Niter-1:
# stor.tpc_finish(txn) # stor.tpc_finish(txn_stormeta)
stor.tpc_finish(txn) stor.tpc_finish(txn_stormeta)
# close db & rest not to get conflict errors after we touched stor # close db & rest not to get conflict errors after we touched stor
# directly a bit. everything will be reopened on next iteration. # directly a bit. everything will be reopened on next iteration.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment