Commit 05de3cb4 authored by Kirill Smelkov's avatar Kirill Smelkov

More python3 support

Flushing changes from yet another attempt. Still not completely there yet, but closer.

Reviewed-by: @jerome
Reviewed-on: !16
parents a2e4dd23 2236aaaf
...@@ -113,7 +113,7 @@ def ext4subj(subj): ...@@ -113,7 +113,7 @@ def ext4subj(subj):
d[xcookie] = cookie d[xcookie] = cookie
# shufle extension dict randomly - to likely trigger different ordering on save # shufle extension dict randomly - to likely trigger different ordering on save
keyv = d.keys() keyv = list(d.keys())
random.shuffle(keyv) random.shuffle(keyv)
ext = {} ext = {}
for key in keyv: for key in keyv:
...@@ -227,7 +227,7 @@ def _gen_testdb(outfs_path, zext): ...@@ -227,7 +227,7 @@ def _gen_testdb(outfs_path, zext):
break break
# delete an object # delete an object
name = random.choice(root.keys()) name = random.choice(list(root.keys()))
obj = root[name] obj = root[name]
root[name] = Object("%s%i*" % (name, i)) root[name] = Object("%s%i*" % (name, i))
# NOTE user/ext are kept empty on purpose - to also test this case # NOTE user/ext are kept empty on purpose - to also test this case
...@@ -276,7 +276,7 @@ def main(): ...@@ -276,7 +276,7 @@ def main():
dbname += "_!zext" dbname += "_!zext"
gen_testdb("%s.fs" % dbname, zext=zext) gen_testdb("%s.fs" % dbname, zext=zext)
stor = FileStorage("%s.fs" % dbname, read_only=True) stor = FileStorage("%s.fs" % dbname, read_only=True)
with open("%s.zdump.ok" % dbname, "w") as f: with open("%s.zdump.ok" % dbname, "wb") as f:
zodbdump(stor, None, None, out=f) zodbdump(stor, None, None, out=f)
if __name__ == '__main__': if __name__ == '__main__':
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2018-2019 Nexedi SA and Contributors. # Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com> # Jérome Perrin <jerome@nexedi.com>
# #
...@@ -58,7 +58,7 @@ def test_zodbcommit(zext): ...@@ -58,7 +58,7 @@ def test_zodbcommit(zext):
zodbdump(stor, p64(u64(head)+1), None, out=buf) zodbdump(stor, p64(u64(head)+1), None, out=buf)
dumped = buf.getvalue() dumped = buf.getvalue()
assert dumped == ''.join([_.zdump() for _ in (t1, t2)]) assert dumped == b''.join([_.zdump() for _ in (t1, t2)])
# ObjectCopy. XXX zodbcommit handled ObjectCopy by actually copying data, # ObjectCopy. XXX zodbcommit handled ObjectCopy by actually copying data,
# not referencing previous transaction via backpointer. # not referencing previous transaction via backpointer.
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2017-2019 Nexedi SA and Contributors. # Copyright (C) 2017-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com> # Jérome Perrin <jerome@nexedi.com>
# #
...@@ -39,7 +39,7 @@ def test_zodbdump(zext): ...@@ -39,7 +39,7 @@ def test_zodbdump(zext):
zkind = '_!zext' if zext.disabled else '' zkind = '_!zext' if zext.disabled else ''
stor = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True) stor = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)
with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f: with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind), 'rb') as f:
dumpok = f.read() dumpok = f.read()
out = BytesIO() out = BytesIO()
...@@ -116,11 +116,11 @@ extension "qqq" ...@@ -116,11 +116,11 @@ extension "qqq"
assert r.readtxn() == None assert r.readtxn() == None
z = ''.join([_.zdump() for _ in (t1, t2)]) z = b''.join([_.zdump() for _ in (t1, t2)])
assert z == in_ assert z == in_
# unknown hash function # unknown hash function
r = DumpReader(BytesIO("""\ r = DumpReader(BytesIO(b"""\
txn 0000000000000000 " " txn 0000000000000000 " "
user "" user ""
description "" description ""
...@@ -130,10 +130,10 @@ obj 0000000000000001 1 xyz:0123 - ...@@ -130,10 +130,10 @@ obj 0000000000000001 1 xyz:0123 -
""")) """))
with raises(RuntimeError) as exc: with raises(RuntimeError) as exc:
r.readtxn() r.readtxn()
assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ('obj 0000000000000001 1 xyz:0123 -')""",) assert exc.value.args == ("""+5: invalid line: unknown hash function "xyz" ("obj 0000000000000001 1 xyz:0123 -")""",)
# data integrity error # data integrity error
r = DumpReader(BytesIO("""\ r = DumpReader(BytesIO(b"""\
txn 0000000000000000 " " txn 0000000000000000 " "
user "" user ""
description "" description ""
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors. # Copyright (C) 2019-2020 Nexedi SA and Contributors.
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your # it under the terms of the GNU General Public License version 3, or (at your
...@@ -27,6 +27,7 @@ from freezegun import freeze_time ...@@ -27,6 +27,7 @@ from freezegun import freeze_time
import tzlocal import tzlocal
from zodbtools.util import TidRangeInvalid, TidInvalid, ashex, parse_tid, parse_tidrange from zodbtools.util import TidRangeInvalid, TidInvalid, ashex, parse_tid, parse_tidrange
from golang import b
@pytest.fixture @pytest.fixture
...@@ -94,7 +95,7 @@ def test_parse_tid(): ...@@ -94,7 +95,7 @@ def test_parse_tid():
assert exc.value.args == ('', ) assert exc.value.args == ('', )
test_parameters = [] test_parameters = [] # of (reference_time, reference_tid, input_time)
with open( with open(
os.path.join( os.path.join(
os.path.dirname(__file__), "testdata", os.path.dirname(__file__), "testdata",
...@@ -109,7 +110,7 @@ with open( ...@@ -109,7 +110,7 @@ with open(
test_parameters) test_parameters)
def test_parse_tid_time_format(fake_time, reference_time, reference_tid, def test_parse_tid_time_format(fake_time, reference_time, reference_tid,
input_time): input_time):
assert reference_tid == ashex(parse_tid(input_time)) assert b(reference_tid) == ashex(parse_tid(input_time))
# check that the reference_tid matches the reference time, mainly # check that the reference_tid matches the reference time, mainly
# to check that input is defined correctly. # to check that input is defined correctly.
assert reference_tid == ashex(parse_tid(reference_time)) assert b(reference_tid) == ashex(parse_tid(reference_time))
# Copyright (C) 2018-2019 Nexedi SA and Contributors. # Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# #
# This program is free software: you can Use, Study, Modify and Redistribute # This program is free software: you can Use, Study, Modify and Redistribute
...@@ -159,7 +159,7 @@ def main(argv): ...@@ -159,7 +159,7 @@ def main(argv):
stor = storageFromURL(storurl) stor = storageFromURL(storurl)
defer(stor.close) defer(stor.close)
zin = 'txn 0000000000000000 " "\n' # artificial transaction header zin = b'txn 0000000000000000 " "\n' # artificial transaction header
zin += sys.stdin.read() zin += sys.stdin.read()
zin = BytesIO(zin) zin = BytesIO(zin)
zr = zodbdump.DumpReader(zin) zr = zodbdump.DumpReader(zin)
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Copyright (C) 2016-2019 Nexedi SA and Contributors. # Copyright (C) 2016-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com> # Kirill Smelkov <kirr@nexedi.com>
# Jérome Perrin <jerome@nexedi.com> # Jérome Perrin <jerome@nexedi.com>
# #
...@@ -67,7 +67,7 @@ import sys ...@@ -67,7 +67,7 @@ import sys
import logging as log import logging as log
import re import re
from golang.gcompat import qq from golang.gcompat import qq
from golang import func, defer, strconv from golang import func, defer, strconv, b
# txn_raw_extension returns raw extension from txn metadata # txn_raw_extension returns raw extension from txn metadata
def txn_raw_extension(stor, txn): def txn_raw_extension(stor, txn):
...@@ -96,7 +96,7 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)): ...@@ -96,7 +96,7 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
for txn in stor.iterator(tidmin, tidmax): for txn in stor.iterator(tidmin, tidmax):
# XXX .status not covered by IStorageTransactionInformation # XXX .status not covered by IStorageTransactionInformation
# XXX but covered by BaseStorage.TransactionRecord # XXX but covered by BaseStorage.TransactionRecord
out.write("txn %s %s\nuser %s\ndescription %s\nextension %s\n" % ( out.write(b"txn %s %s\nuser %s\ndescription %s\nextension %s\n" % (
ashex(txn.tid), qq(txn.status), ashex(txn.tid), qq(txn.status),
qq(txn.user), qq(txn.user),
qq(txn.description), qq(txn.description),
...@@ -105,33 +105,33 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)): ...@@ -105,33 +105,33 @@ def zodbdump(stor, tidmin, tidmax, hashonly=False, out=asbinstream(sys.stdout)):
objv = txnobjv(txn) objv = txnobjv(txn)
for obj in objv: for obj in objv:
entry = "obj %s " % ashex(obj.oid) entry = b"obj %s " % ashex(obj.oid)
write_data = False write_data = False
if obj.data is None: if obj.data is None:
entry += "delete" entry += b"delete"
# was undo and data taken from obj.data_txn # was undo and data taken from obj.data_txn
elif obj.data_txn is not None: elif obj.data_txn is not None:
entry += "from %s" % ashex(obj.data_txn) entry += b"from %s" % ashex(obj.data_txn)
else: else:
# XXX sha1 is hardcoded for now. Dump format allows other hashes. # XXX sha1 is hardcoded for now. Dump format allows other hashes.
entry += "%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data))) entry += b"%i sha1:%s" % (len(obj.data), ashex(sha1(obj.data)))
write_data = True write_data = True
out.write(entry) out.write(b(entry))
if write_data: if write_data:
if hashonly: if hashonly:
out.write(" -") out.write(b" -")
else: else:
out.write("\n") out.write(b"\n")
out.write(obj.data) out.write(obj.data)
out.write("\n") out.write(b"\n")
out.write("\n") out.write(b"\n")
# ---------------------------------------- # ----------------------------------------
# XPickler is Pickler that tries to save objects stably # XPickler is Pickler that tries to save objects stably
...@@ -309,7 +309,7 @@ class DumpReader(object): ...@@ -309,7 +309,7 @@ class DumpReader(object):
def _readline(self): def _readline(self):
l = self._r.readline() l = self._r.readline()
if l == '': if l == b'':
self._line = None self._line = None
return None # EOF return None # EOF
...@@ -320,7 +320,7 @@ class DumpReader(object): ...@@ -320,7 +320,7 @@ class DumpReader(object):
# report a problem found around currently-read line # report a problem found around currently-read line
def _badline(self, msg): def _badline(self, msg):
raise RuntimeError("%s+%d: invalid line: %s (%r)" % (_ioname(self._r), self.lineno, msg, self._line)) raise RuntimeError("%s+%d: invalid line: %s (%s)" % (_ioname(self._r), self.lineno, msg, qq(self._line)))
# readtxn reads one transaction record from input stream and returns # readtxn reads one transaction record from input stream and returns
# Transaction instance or None at EOF. # Transaction instance or None at EOF.
...@@ -350,7 +350,7 @@ class DumpReader(object): ...@@ -350,7 +350,7 @@ class DumpReader(object):
objv = [] objv = []
while 1: while 1:
l = self._readline() l = self._readline()
if l == '': if l == b'':
break # empty line - end of transaction break # empty line - end of transaction
if l is None or not l.startswith(b'obj '): if l is None or not l.startswith(b'obj '):
...@@ -393,7 +393,7 @@ class DumpReader(object): ...@@ -393,7 +393,7 @@ class DumpReader(object):
chunk = self._r.read(n) chunk = self._r.read(n)
data += chunk data += chunk
n -= len(chunk) n -= len(chunk)
self.lineno += data.count('\n') self.lineno += data.count(b'\n')
self._line = None self._line = None
if data[-1:] != b'\n': if data[-1:] != b'\n':
raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno)) raise RuntimeError('%s+%d: no LF after obj data' % (_ioname(self._r), self.lineno))
...@@ -452,15 +452,15 @@ class Transaction(object): ...@@ -452,15 +452,15 @@ class Transaction(object):
def _extension(self): def _extension(self):
return self.extension return self.extension
# zdump returns text representation of a record in zodbdump format. # zdump returns semi text-binary representation of a record in zodbdump format.
def zdump(self): def zdump(self): # -> bytes
z = 'txn %s %s\n' % (ashex(self.tid), qq(self.status)) z = b'txn %s %s\n' % (ashex(self.tid), qq(self.status))
z += 'user %s\n' % qq(self.user) z += b'user %s\n' % qq(self.user)
z += 'description %s\n' % qq(self.description) z += b'description %s\n' % qq(self.description)
z += 'extension %s\n' % qq(self.extension_bytes) z += b'extension %s\n' % qq(self.extension_bytes)
for obj in self.objv: for obj in self.objv:
z += obj.zdump() z += obj.zdump()
z += '\n' z += b'\n'
return z return z
...@@ -477,7 +477,7 @@ class ObjectDelete(Object): ...@@ -477,7 +477,7 @@ class ObjectDelete(Object):
super(ObjectDelete, self).__init__(oid) super(ObjectDelete, self).__init__(oid)
def zdump(self): def zdump(self):
return 'obj %s delete\n' % (ashex(self.oid)) return b'obj %s delete\n' % (ashex(self.oid))
# ObjectCopy represents object data copy. # ObjectCopy represents object data copy.
class ObjectCopy(Object): class ObjectCopy(Object):
...@@ -487,7 +487,7 @@ class ObjectCopy(Object): ...@@ -487,7 +487,7 @@ class ObjectCopy(Object):
self.copy_from = copy_from self.copy_from = copy_from
def zdump(self): def zdump(self):
return 'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from)) return b'obj %s from %s\n' % (ashex(self.oid), ashex(self.copy_from))
# ObjectData represents record with object data. # ObjectData represents record with object data.
class ObjectData(Object): class ObjectData(Object):
...@@ -507,13 +507,13 @@ class ObjectData(Object): ...@@ -507,13 +507,13 @@ class ObjectData(Object):
size = data.size size = data.size
else: else:
size = len(data) size = len(data)
z = 'obj %s %d %s:%s' % (ashex(self.oid), size, self.hashfunc, ashex(self.hash_)) z = b'obj %s %d %s:%s' % (ashex(self.oid), size, self.hashfunc, ashex(self.hash_))
if hashonly: if hashonly:
z += ' -' z += b' -'
else: else:
z += '\n' z += b'\n'
z += data z += data
z += '\n' z += b'\n'
return z return z
# HashOnly indicated that this ObjectData record contains only hash and does not contain object data. # HashOnly indicated that this ObjectData record contains only hash and does not contain object data.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment