commit: Fix it wrt ZEO

Since 7ae5ff82 (Port zodbtools to py3) zodbtools/py3 requires
nexedi/pygolang!21 with which
running `zodb commit` wrt ZEO, as demonstrates by added
test_zodbcommit_cmd, yields:

    test/test_commit.py::test_zodbcommit_cmd[ZEO-!zext] Traceback (most recent call last):
      File "/usr/lib/python2.7/runpy.py", line 174, in _run_module_as_main
        "__main__", fname, loader, pkg_name)
      File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
        exec code in run_globals
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodb.py", line 133, in <module>
        main()
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodb.py", line 129, in main
        return command_module.main(argv)
      File "<decorator-gen-4>", line 2, in main
      File "/home/kirr/src/tools/go/pygolang/golang/__init__.py", line 125, in _
        return f(*argv, **kw)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 232, in main
        tid = zodbcommit(stor, at, txn)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 131, in zodbcommit
        _()
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 128, in _
        stor.store(obj.oid, current_serial(obj.oid), data, '', txn)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 87, in current_serial
        return _serial_at(stor, oid, at)
      File "/home/kirr/src/wendelin/z/zodbtools/zodbtools/zodbcommit.py", line 152, in _serial_at
        xdata = stor.loadBefore(oid, before)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/ClientStorage.py", line 616, in loadBefore
        return self._server.load_before(oid, tid)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/client.py", line 935, in load_before
        return self.io_call(self.load_before_co(oid, tid, self.timeout))
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/client.py", line 862, in io_call
        return future.result() if wait else future
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/futures.py", line 230, in result
        return Future.result(self)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/futures.py", line 81, in result
        raise self._result
    ClientDisconnected: [Errno 104] Connection reset by peer
    FAILED()
    zeo.log:

    2024-07-16T21:13:45 ERROR ZEO.asyncio.server Can't deserialize message
    Traceback (most recent call last):
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/server.py", line 100, in message_received
        message_id, async_, name, args = self.decode(message)
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/marshal.py", line 119, in pickle_server_decode
        return unpickler.load()  # msgid, flags, name, args
      File "/home/kirr/src/wendelin/z/ZEO5/src/ZEO/asyncio/marshal.py", line 176, in server_find_global
        raise ImportError("Module not allowed: %s" % (module,))
    ImportError: Module not allowed: golang

This happens because zodbcommit.main and zodbdump.DumpReader use bstr to
handle binary data and further pass as IStorageTransactionInformation
instance to ZEO via its RPC, but ZEO server explicitly checks safe to
import modules and rejects golang.

One way to fix would be to patch ZEO to allow golang imports, but in
general it is better to follow the interface strictly and stay 100%
compatible with its users. That's why the fix converts fields covered by
IStorageTransactionInformation to exact types specified by that
interface to stay 100% compatible with users of the interface because on
py3 e.g. (b' ' == ' ') returns False and so we need to be careful to
provide .status as exactly str instead bytes, and do the similar for
other fields.

zodbtools/test/test_commit.py

@@ -21,11 +21,16 @@
 
 from zodbtools.zodbcommit import zodbcommit
 from zodbtools.zodbdump import zodbdump, Transaction, ObjectData, ObjectDelete, ObjectCopy
-from zodbtools.util import storageFromURL, sha1
+from zodbtools.util import storageFromURL, sha1, ashex, fromhex
 from ZODB.utils import p64, u64, z64
 from ZODB._compat import BytesIO, dumps, _protocol   # XXX can't yet commit with arbitrary ext.bytes
 
 from golang import func, defer, b
+from golang.gcompat import qq
+
+import sys
+from subprocess import Popen, PIPE
+
 
 # verify zodbcommit.
 @func
@@ -66,3 +71,76 @@ def test_zodbcommit(zsrv, zext):
     data1_3, _, _ = stor.loadBefore(p64(1), p64(u64(t3.tid)+1))
     assert data1_1 == data1_3
     assert data1_1 == b'data1'  # just in case
+
+
+# verify zodbcommit via commandline / stdin.
+def test_zodbcommit_cmd(zsrv, zext):
+    # for ZEO sync is noop unless server_sync=True is specified in options,
+    # but server_sync is available only on ZEO5, not ZEO4. Work it around with
+    # opening/closing the storage on every query.
+    @func
+    def zsrv_do(f):
+        stor = storageFromURL(zsrv.zurl)
+        defer(stor.close)
+        return f(stor)
+    at0 = zsrv_do(lambda stor: stor.lastTransaction())
+
+    # zodbcommit_cmd does `zodb commit` via command line and returns TID of
+    # committed transction.
+    def zodbcommit_cmd(at, stdin): # -> tid
+        p = Popen([sys.executable, '-m', 'zodbtools.zodb', 'commit',
+                   zsrv.zurl, ashex(at)], stdin=PIPE, stdout=PIPE)
+        stdout, _ = p.communicate(stdin)
+        assert p.returncode == 0,  stdout
+        return fromhex(stdout.rstrip())
+
+    t1 = b'''\
+user "user name"
+description "description ..."
+extension %s
+obj 0000000000000001 5 sha1:%s
+data1
+obj 0000000000000002 5 sha1:%s
+data2
+
+''' % (qq(zext(dumps({'a': 'b'}, _protocol))), ashex(sha1(b'data1')), ashex(sha1(b'data2')))
+
+    t2 = b'''\
+user "user2"
+description "desc2"
+extension ""
+obj 0000000000000002 delete
+
+'''
+
+    at1 = zodbcommit_cmd(at0, t1)
+    at2 = zodbcommit_cmd(at1, t2)
+
+    t1 = (b'txn %s " "\n' % ashex(at1)) + t1
+    t2 = (b'txn %s " "\n' % ashex(at2)) + t2
+
+    def _(stor):
+        buf = BytesIO()
+        zodbdump(stor, p64(u64(at0)+1), None, out=buf)
+        return buf.getvalue()
+    dumped = zsrv_do(_)
+
+    assert dumped == b''.join([t1, t2])
+
+    t3 = b'''\
+user "user3"
+description "desc3"
+extension ""
+obj 0000000000000001 from %s
+
+''' % ashex(at1)
+
+    # XXX see note about ObjectCopy in test_zodbcommit
+    at3 = zodbcommit_cmd(at1, t3)
+
+    def _(stor):
+        data1_1, _, _ = stor.loadBefore(p64(1), p64(u64(at1)+1))
+        data1_3, _, _ = stor.loadBefore(p64(1), p64(u64(at3)+1))
+        assert data1_1 == data1_3
+        assert data1_1 == b'data1'  # just in case
+    zsrv_do(_)

zodbtools/zodbdump.py

@@ -491,11 +491,19 @@ class Transaction(object):
     # .extension_bytes  bytes       transaction extension
     # .objv             []Object*   objects changed by transaction
     def __init__(self, tid, status, user, description, extension, objv):
-        self.tid                = tid
-        self.status             = status
-        self.user               = user
-        self.description        = description
-        self.extension_bytes    = extension
+        # NOTE we convert fields covered by IStorageTransactionInformation to
+        # exact types specified by that interface to stay 100% compatible with
+        # users of the interface because on py3 e.g. (b' ' == ' ') returns
+        # False and so we need to be careful to provide .status as exactly str
+        # instead bytes, and do the similar for other fields. It also would be
+        # generally incorrect to use bstr/ustr for the fields, because e.g. ZEO
+        # rejects messages with golang.bstr/ustr objects on the basis that they
+        # do not come from allowed list of modules.
+        self.tid                = _exactly_bytes(tid)
+        self.status             = _exactly_str(status)
+        self.user               = _exactly_bytes(user)
+        self.description        = _exactly_bytes(description)
+        self.extension_bytes    = _exactly_bytes(extension)
         self.objv               = objv
 
     # ZODB wants to work with extension as {} - try to convert it on the fly.
@@ -593,3 +601,28 @@ class HashOnly(object):
 
     def __eq__(a, b):
         return isinstance(b, HashOnly) and a.size == b.size
+
+
+# _exactly_bytes returns obj as an instance of exactly type bytes.
+#
+# obj must be initially an instance of bytes, e.g. bstr.
+def _exactly_bytes(obj): # -> bytes
+    assert isinstance(obj, bytes),  type(obj)
+    if type(obj) is not bytes:
+        obj = b(obj)        # bstr
+        obj = obj.encode()  # bytes
+    assert type(obj) is bytes,  type(obj)
+    return obj
+
+
+# _exactly_str returns obj as an instance of exactly type str.
+#
+# obj must be initially an instance of bytes or str/unicode, e.g. bstr or ustr.
+def _exactly_str(obj): # -> str
+    if type(obj) is not str:
+        obj = b(obj)        # bstr
+        obj = obj.encode()  # bytes
+        if str is not bytes:
+            obj = obj.decode('UTF-8')
+    assert type(obj) is str,  type(obj)
+    return obj