Commit 05de3cb4 authored by Kirill Smelkov's avatar Kirill Smelkov

More python3 support

Flushing changes from yet another attempt. Still not completely there yet, but closer.

Reviewed-by: @jerome
Reviewed-on: nexedi/zodbtools!16
parents a2e4dd23 2236aaaf
......@@ -113,7 +113,7 @@ def ext4subj(subj):
d[xcookie] = cookie
# shufle extension dict randomly - to likely trigger different ordering on save
keyv = d.keys()
keyv = list(d.keys())
random.shuffle(keyv)
ext = {}
for key in keyv:
......@@ -227,7 +227,7 @@ def _gen_testdb(outfs_path, zext):
break
# delete an object
name = random.choice(root.keys())
name = random.choice(list(root.keys()))
obj = root[name]
root[name] = Object("%s%i*" % (name, i))
# NOTE user/ext are kept empty on purpose - to also test this case
......@@ -276,7 +276,7 @@ def main():
dbname += "_!zext"
gen_testdb("%s.fs" % dbname, zext=zext)
stor = FileStorage("%s.fs" % dbname, read_only=True)
with open("%s.zdump.ok" % dbname, "w") as f:
with open("%s.zdump.ok" % dbname, "wb") as f:
zodbdump(stor, None, None, out=f)
if __name__ == '__main__':
......
# -*- coding: utf-8 -*-
# Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
......@@ -58,7 +58,7 @@ def test_zodbcommit(zext):
zodbdump(stor, p64(u64(head)+1), None, out=buf)
dumped = buf.getvalue()
assert dumped == ''.join([_.zdump() for _ in (t1, t2)])
assert dumped == b''.join([_.zdump() for _ in (t1, t2)])
# ObjectCopy. XXX zodbcommit handled ObjectCopy by actually copying data,
# not referencing previous transaction via backpointer.
......
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Copyright (C) 2017-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
......@@ -39,7 +39,7 @@ def test_zodbdump(zext):
zkind = '_!zext' if zext.disabled else ''
stor = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind), 'rb') as f:
dumpok = f.read()
out = BytesIO()
......@@ -116,11 +116,11 @@ extension "qqq"
assert r.readtxn() == None
z = ''.join([_.zdump() for _ in (t1, t2)])
z = b''.join([_.zdump() for _ in (t1, t2)])
assert z == in_
# unknown hash function
r = DumpReader(BytesIO("""\
r = DumpReader(BytesIO(b"""\
txn 0000000000000000 " "
user ""
description ""
......@@ -130,10 +130,10 @@ obj 0000000000000001 1 xyz:0123 -
"""))
with raises(RuntimeError) as exc:
r.readtxn()
assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",)
assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ("obj 0000000000000001 1 xyz:0123 -")""",)
# data integrity error
r = DumpReader(BytesIO("""\
r = DumpReader(BytesIO(b"""\
txn 0000000000000000 " "
user ""
description ""
......
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Copyright (C) 2019-2020 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
......@@ -27,6 +27,7 @@ from freezegun import freeze_time
import tzlocal
from zodbtools.util import TidRangeInvalid, TidInvalid, ashex, parse_tid, parse_tidrange
from golang import b
@pytest.fixture
......@@ -94,7 +95,7 @@ def test_parse_tid():
assert exc.value.args == ('', )
test_parameters = []
test_parameters = [] # of (reference_time, reference_tid, input_time)
with open(
os.path.join(
os.path.dirname(__file__), "testdata",
......@@ -109,7 +110,7 @@ with open(
test_parameters)
def test_parse_tid_time_format(fake_time, reference_time, reference_tid,
input_time):
assert reference_tid == ashex(parse_tid(input_time))
assert b(reference_tid) == ashex(parse_tid(input_time))
# check that the reference_tid matches the reference time, mainly
# to check that input is defined correctly.
assert reference_tid == ashex(parse_tid(reference_time))
assert b(reference_tid) == ashex(parse_tid(reference_time))
# Copyright (C) 2018-2019 Nexedi SA and Contributors.
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
......@@ -159,7 +159,7 @@ def main(argv):
stor = storageFromURL(storurl)
defer(stor.close)
zin = 'txn 0000000000000000 " "\n' # artificial transaction header
zin = b'txn 0000000000000000 " "\n' # artificial transaction header
zin += sys.stdin.read()
zin = BytesIO(zin)
zr = zodbdump.DumpReader(zin)
......
# -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 Nexedi SA and Contributors.
# Copyright (C) 2016-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com>
#
......@@ -67,7 +67,7 @@ import sys
import logging as log
import re
from golang.gcompat import qq
from golang import func, defer, strconv
from golang import func, defer, strconv, b
# txn_raw_extension returns raw extension from txn metadata
def txn_raw_extension(stor, txn):
......@@ -96,7 +96,7 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
for txn in stor.iterator(tidmin, tidmax):
# XXX .status not covered by IStorageTransactionInformation
# XXX but covered by BaseStorage.TransactionRecord
out.write("txn %s %s\nuser %s\ndescription %s\nextension %s\n" % (
out.write(b"txn %s %s\nuser %s\ndescription %s\nextension %s\n" % (
ashex(txn.tid), qq(txn.status),
qq(txn.user),
qq(txn.description),
......@@ -105,33 +105,33 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
objv = txnobjv(txn)
for obj in objv:
entry = "obj %s " % ashex(obj.oid)
entry = b"obj %s " % ashex(obj.oid)
write_data = False
if obj.data is None:
entry += "delete"
entry += b"delete"
# was undo and data taken from obj.data_txn
elif obj.data_txn is not None:
entry += "from %s" % ashex(obj.data_txn)
entry += b"from %s" % ashex(obj.data_txn)
else:
# XXX sha1 is hardcoded for now. Dump format allows other hashes.
entry += "%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data)))
entry += b"%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data)))
write_data = True
out.write(entry)
out.write(b(entry))
if write_data:
if hashonly:
out.write(" -")
out.write(b" -")
else:
out.write("\n")
out.write(b"\n")
out.write(obj.data)
out.write("\n")
out.write(b"\n")
out.write("\n")
out.write(b"\n")
# ----------------------------------------
# XPickler is Pickler that tries to save objects stably
......@@ -309,7 +309,7 @@ class DumpReader(object):
def _readline(self):
l = self._r.readline()
if l == '':
if l == b'':
self._line = None
return None # EOF
......@@ -320,7 +320,7 @@ class DumpReader(object):
# report a problem found around currently-read line
def _badline(self, msg):
raise RuntimeError("%s+%d: invalid line: %s (%r)" % (_ioname(self._r), self.lineno, msg, self._line))
raise RuntimeError("%s+%d: invalid line: %s (%s)" % (_ioname(self._r), self.lineno, msg, qq(self._line)))
# readtxn reads one transaction record from input stream and returns
# Transaction instance or None at EOF.
......@@ -350,7 +350,7 @@ class DumpReader(object):
objv = []
while 1:
l = self._readline()
if l == '':
if l == b'':
break # empty line - end of transaction
if l is None or not l.startswith(b'obj '):
......@@ -393,7 +393,7 @@ class DumpReader(object):
chunk = self._r.read(n)
data += chunk
n -= len(chunk)
self.lineno += data.count('\n')
self.lineno += data.count(b'\n')
self._line = None
if data[-1:] != b'\n':
raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno))
......@@ -452,15 +452,15 @@ class Transaction(object):
def _extension(self):
return self.extension
# zdump returns text representation of a record in zodbdump format.
def zdump(self):
z = 'txn %s %s\n' % (ashex(self.tid), qq(self.status))
z += 'user %s\n' % qq(self.user)
z += 'description %s\n' % qq(self.description)
z += 'extension %s\n' % qq(self.extension_bytes)
# zdump returns semi text-binary representation of a record in zodbdump format.
def zdump(self): # -> bytes
z = b'txn %s %s\n' % (ashex(self.tid), qq(self.status))
z += b'user %s\n' % qq(self.user)
z += b'description %s\n' % qq(self.description)
z += b'extension %s\n' % qq(self.extension_bytes)
for obj in self.objv:
z += obj.zdump()
z += '\n'
z += b'\n'
return z
......@@ -477,7 +477,7 @@ class ObjectDelete(Object):
super(ObjectDelete, self).__init__(oid)
def zdump(self):
return 'obj %s delete\n' % (ashex(self.oid))
return b'obj %s delete\n' % (ashex(self.oid))
# ObjectCopy represents object data copy.
class ObjectCopy(Object):
......@@ -487,7 +487,7 @@ class ObjectCopy(Object):
self.copy_from = copy_from
def zdump(self):
return 'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from))
return b'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from))
# ObjectData represents record with object data.
class ObjectData(Object):
......@@ -507,13 +507,13 @@ class ObjectData(Object):
size = data.size
else:
size = len(data)
z = 'obj %s %d %s:%s' % (ashex(self.oid), size, self.hashfunc, ashex(self.hash_))
z = b'obj %s %d %s:%s' % (ashex(self.oid), size, self.hashfunc, ashex(self.hash_))
if hashonly:
z += ' -'
z += b' -'
else:
z += '\n'
z += b'\n'
z += data
z += '\n'
z += b'\n'
return z
# HashOnly indicated that this ObjectData record contains only hash and does not contain object data.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment