Commit 1b962f03 authored by Kirill Smelkov's avatar Kirill Smelkov

X Restructure: found bug that it was not marking objects as modified

As a result corrupt btrees were actually persisted to storage.
parent f0bf8db1
...@@ -456,8 +456,16 @@ def test_restructure(): ...@@ -456,8 +456,16 @@ def test_restructure():
zconn = db.open() zconn = db.open()
X = [] # X[i] -> XBlk corresponding to block #i X = [] # X[i] -> XBlk corresponding to block #i
for i in range(12): xv = 'abcdefghijkl'
X.append(XBlk('abcdefghijkl'[i])) for i in range(len(xv)):
X.append(XBlk(xv[i]))
def xdecode(v):
assert len(v) == 1
assert v in xv
return X[xv.index(v)]
def xencode(x):
assert isinstance(x, XBlk)
return x.data
# assertB wraps global assertB to automatically fill in X[k] values for specified keys. # assertB wraps global assertB to automatically fill in X[k] values for specified keys.
def assertB(znode, *keyv): def assertB(znode, *keyv):
...@@ -488,7 +496,7 @@ def test_restructure(): ...@@ -488,7 +496,7 @@ def test_restructure():
assert ztree[k] == v assert ztree[k] == v
if isinstance(newtopo, str): if isinstance(newtopo, str):
newStructure = xbtree.TopoDecode(newtopo) newStructure = xbtree.TopoDecode(newtopo, xdecode)
else: else:
assert isinstance(newtopo, xbtree.Tree) assert isinstance(newtopo, xbtree.Tree)
newStructure = newtopo newStructure = newtopo
...@@ -506,6 +514,9 @@ def test_restructure(): ...@@ -506,6 +514,9 @@ def test_restructure():
# S returns topo-encoded keys-only structure of ztree # S returns topo-encoded keys-only structure of ztree
def S(ztree): def S(ztree):
return xbtree.TopoEncode(xbtree.StructureOf(ztree, onlyKeys=True)) return xbtree.TopoEncode(xbtree.StructureOf(ztree, onlyKeys=True))
# Sv returns topo-encoded structure of ztree with values
def Sv(ztree):
return xbtree.TopoEncode(xbtree.StructureOf(ztree), xencode)
# Z0 creates new empty tree # Z0 creates new empty tree
def Z0(): def Z0():
...@@ -716,6 +727,23 @@ def test_restructure(): ...@@ -716,6 +727,23 @@ def test_restructure():
R(z, 'T/B1,3,5,7,11') R(z, 'T/B1,3,5,7,11')
# make sure changed objects are marked as such and so included into commit
z = Z(0,2)
transaction.commit()
R(z, 'T1/B0:a-B2:c')
transaction.commit()
assert Sv(z) == 'T1/B0:a-B2:c'
zconn.cacheMinimize() # force z state to be reloaded from storage
assert Sv(z) == 'T1/B0:a-B2:c' # will fail if T or B is not marked as changed
R(z, 'T/B0:a,2:c') # one more similar check
transaction.commit()
assert Sv(z) == 'T/B0:a,2:c'
zconn.cacheMinimize()
assert Sv(z) == 'T/B0:a,2:c'
# ---- tests on automatically generated topologies ---- # ---- tests on automatically generated topologies ----
# #
# ( we make sure that Restructure can make the restructurement and that # ( we make sure that Restructure can make the restructurement and that
......
...@@ -97,8 +97,9 @@ from persistent.mapping import PersistentMapping ...@@ -97,8 +97,9 @@ from persistent.mapping import PersistentMapping
XLOTree = xbtree_test.XLOTree XLOTree = xbtree_test.XLOTree
XLOTree.__module__ = 'BTrees.LOBTree' XLOTree.__module__ = 'BTrees.LOBTree'
XLOTree.__name__ = 'LOBTree' XLOTree.__name__ = 'LOBTree'
from BTrees import LOBTree import BTrees.LOBTree
LOBTree.LOBTree = XLOTree BTrees.LOBTree.LOBTree = XLOTree
from BTrees.LOBTree import LOBTree
# ZCtx represent treegen-level connection to ZODB. # ZCtx represent treegen-level connection to ZODB.
...@@ -152,7 +153,7 @@ def Trees(zstor, r): ...@@ -152,7 +153,7 @@ def Trees(zstor, r):
zctx = ZCtx(zstor) zctx = ZCtx(zstor)
defer(zctx.close) defer(zctx.close)
ztree = zctx.root['treegen/tree'] = XLOTree() ztree = zctx.root['treegen/tree'] = LOBTree()
head = commit('treegen/tree: init') head = commit('treegen/tree: init')
xprint("tree.srv start @%s root=%s" % (ashex(head), ashex(ztree._p_oid))) xprint("tree.srv start @%s root=%s" % (ashex(head), ashex(ztree._p_oid)))
treetxtPrev = zctx.ztreetxt(ztree) treetxtPrev = zctx.ztreetxt(ztree)
...@@ -240,7 +241,7 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None): ...@@ -240,7 +241,7 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
print("# allstructs %s %s" % (kv1txt, kv2txt)) print("# allstructs %s %s" % (kv1txt, kv2txt))
# create the tree # create the tree
ztree = zctx.root['ztree'] = XLOTree() ztree = zctx.root['ztree'] = LOBTree()
commit('init') commit('init')
# initial kv1 and kv2 states with topologies prepared as ZODB would do natively # initial kv1 and kv2 states with topologies prepared as ZODB would do natively
...@@ -380,7 +381,7 @@ def commit(description, skipIfEmpty=False): # -> tid | None ...@@ -380,7 +381,7 @@ def commit(description, skipIfEmpty=False): # -> tid | None
assert isinstance(zconn, Connection) assert isinstance(zconn, Connection)
assert len(zconn._registered_objects) > 0 assert len(zconn._registered_objects) > 0
obj = zconn._registered_objects[0] obj = zconn._registered_objects[0]
assert obj._p_state == CHANGED assert obj._p_state == CHANGED, obj._p_state
txn.commit() txn.commit()
return obj._p_serial return obj._p_serial
......
...@@ -199,6 +199,9 @@ func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) { ...@@ -199,6 +199,9 @@ func (tg *TreeSrv) Commit(tree string) (_ zodb.Tid, err error) {
reply, err := tg.pyout.ReadString('\n') reply, err := tg.pyout.ReadString('\n')
if err != nil { if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return zodb.InvalidTid, err return zodb.InvalidTid, err
} }
reply = strings.TrimSuffix(reply, "\n") reply = strings.TrimSuffix(reply, "\n")
...@@ -225,6 +228,9 @@ func (tg *AllStructsSrv) AllStructs(kv1, kv2 map[Key]string, maxdepth, maxsplit, ...@@ -225,6 +228,9 @@ func (tg *AllStructsSrv) AllStructs(kv1, kv2 map[Key]string, maxdepth, maxsplit,
for { for {
reply, err := tg.pyout.ReadString('\n') reply, err := tg.pyout.ReadString('\n')
if err != nil { if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return nil, err return nil, err
} }
reply = strings.TrimSuffix(reply, "\n") reply = strings.TrimSuffix(reply, "\n")
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment