Skip to content

W
wendelin.core
Commit f6c45548 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent d75d23bd
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 1126 deletions
wcfs/gen-set deleted 100755 → 0
View file @ d75d23bd
#!/bin/bash -e
# set.go.in -> specialized with concrete types
# gen-set package KIND VALUE out
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
PACKAGE=$1
KIND=$2
VALUE=$3
out=$4
input=$(dirname $0)/set.go.in
echo "// Code generated by gen-set $KIND $VALUE; DO NOT EDIT." >$out
echo >>$out
sed \
-e "s/PACKAGE/$PACKAGE/g" \
-e "s/VALUE/$VALUE/g" \
-e "s/Set/Set${KIND}/g" \
$input >>$out
wcfs/gen-δtail deleted 100755 → 0
View file @ d75d23bd
#!/bin/bash -e
# δtail.go.in -> specialized with concrete types
# gen-δtail KIND ID out
# Copyright (C) 2018-2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
KIND=$1
ID=$2
out=$3
zodb=lab.nexedi.com/kirr/neo/go/zodb
zdir=`go list -f '{{.Dir}}' $zodb`
zrev=`git -C $zdir describe --always`
echo "// Code generated by gen-δtail $KIND $ID; DO NOT EDIT." >$out
echo "// (from $zodb @ $zrev)" >>$out
echo >>$out
$zdir/δtail.go.cat-generic | sed \
-e "s/PACKAGE/main/g" \
-e "s/ID/$ID/g" \
-e "s/ΔTail/ΔTail${KIND}/g" \
-e "s/δRevEntry/δRevEntry${KIND}/g" \
>>$out
wcfs/internal/xbtree_test.py deleted 100644 → 0
View file @ d75d23bd
# -*- coding: utf-8 -*-
# Copyright (C) 2020 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
from __future__ import print_function, absolute_import
from wendelin.wcfs.internal import xbtree
from BTrees.LOBTree import LOBTree
from BTrees.IIBTree import IITreeSet, IISet
from BTrees.tests import testBTrees
from BTrees import check as zbcheck
from BTrees.check import BTREE_EMPTY, BTREE_ONE, BTREE_NORMAL
from ZODB.MappingStorage import MappingStorage
from ZODB import DB
from persistent import Persistent
import transaction
from pytest import raises
inf = float('inf')
# T/B are shorthands for Tree and Bucket without values.
# Bv is shorthand for Bucket with values.
T = xbtree.Tree
B = lambda *keyv: xbtree.Bucket(keyv, None)
Bv = lambda keyv, *valuev: xbtree.Bucket(keyv, valuev)
# buildDegenerateZTree builds ztree with known degenerate topology, see:
# https://github.com/zopefoundation/ZODB/commit/6cd24e99f89b
# https://github.com/zopefoundation/BTrees/blob/4.7.2-1-g078ba60/BTrees/tests/testBTrees.py#L20-L57
def buildDegenerateZTree():
ztree, keys = testBTrees.DegenerateBTree("testBasicOps")._build_degenerate_tree()
assert keys == [1, 3, 5, 7, 11]
assert xbtree.StructureOf(ztree) == T([4],
T([2],
T([], B(1)), T([], B(3))),
T([],
T([6, 10],
T([], T([], B(5))),
T([], B(7)),
T([], B(11))) ))
return ztree
def test_structureOf():
# empty tree
t = LOBTree()
assert xbtree.StructureOf(t) == T([], Bv([]))
# tree with 1 k->v
t = LOBTree()
t[10] = 'hello'
assert xbtree.StructureOf(t) == T([], Bv([10], 'hello'))
# known degenerate topology
t = buildDegenerateZTree()
assert xbtree.StructureOf(t) == T([4],
T([2],
T([], B(1)), T([], B(3))),
T([],
T([6, 10],
T([], T([], B(5))),
T([], B(7)),
T([], B(11))) ))
def test_topoEncoding():
def X(tree):
topo = xbtree.TopoEncode(tree)
t2 = xbtree.TopoDecode(topo)
assert t2 == tree
return topo
assert X(T([], B())) == 'T/B'
assert X(T([], B(1))) == 'T/B1'
assert X(T([], B(1,3))) == 'T/B1,3'
assert X(T([], T([], B()))) == 'T/T/B'
assert X(T([3],
T([], B(1)),
T([],
T([5], B(), B(7,8,9))))) == "T3/T-T/B1-T5/B-B7,8,9"
# degenerate btree from ZODB
assert X(T([4],
T([2],
T([], B(1)), T([], B(3))),
T([],
T([6, 10],
T([], T([], B(5))),
T([], B(7)),
T([], B(11))) ))) == "T4/T2-T/T-T-T6,10/B1-B3-T-T-T/T-B7-B11/B5"
# tree with key->value
assert X(T([], Bv([]))) == 'T/B:'
assert X(T([], Bv([1], 4))) == 'T/B1:4'
assert X(T([], Bv([1,2], 4,5))) == 'T/B1:4,2:5'
assert X(T([3],
Bv([1], 10),
Bv([4,5], 11,12))) == 'T3/B1:10-B4:11,5:12'
# TopoEncode/TopoDecode on autogenerated topologies.
for tree in xbtree.AllStructs([1,3,7,8], 1,2, allowEmptyBuckets=True):
t2 = xbtree.TopoDecode(xbtree.TopoEncode(tree))
assert t2 == tree
def test_allStructs():
# X = AllStructs(..., allowEmptyBuckets=True)
# Y = AllStructs(..., allowEmptyBuckets=False)
# XY = X = Y + assert X == Y
def X(keys, maxdepth, maxsplit, allowEmptyBuckets=True, kv=None):
return list(xbtree.AllStructs(keys, maxdepth, maxsplit, allowEmptyBuckets, kv))
def Y(keys, maxdepth, maxsplit, kv=None):
return X(keys, maxdepth, maxsplit, allowEmptyBuckets=False, kv=kv)
def XY(keys, maxdepth, maxsplit, kv=None):
x = X(keys, maxdepth, maxsplit, kv=kv)
y = Y(keys, maxdepth, maxsplit, kv=kv)
assert x == y
return x
assert XY([], 0, 0) == [ T([], B()) ]
assert XY([1], 0, 0) == [ T([], B(1)) ]
assert XY([1,3], 0, 0) == [ T([], B(1,3)) ]
assert XY([], 0, 1) == [ T([], B()) ] # nothing to split
assert X([], 1, 0) == [ T([], B()),
T([],
T([], B())) ]
assert Y([], 1, 0) == [ T([], B()) ]
assert X([], 2, 0) == [ T([], B()),
T([],
T([], B())),
T([],
T([],
T([], B()))) ]
assert Y([], 2, 0) == [ T([], B()) ]
assert XY([1,3], 0, 0) == [ T([], B(1,3)) ]
assert X([1,3], 0, 1) == [ T([], B(1,3)),
# nsplit=1
T([0], B(), B(1,3)),
T([1], B(), B(1,3)),
T([2], B(1), B(3)),
T([3], B(1), B(3)),
T([4], B(1,3), B()),
]
assert Y([1,3], 0, 1) == [ T([], B(1,3)),
# nsplit=1
T([2], B(1), B(3)),
T([3], B(1), B(3)),
]
assert XY([1,3], 1, 0) == [ T([], B(1,3)),
# depth=1
T([],
T([], B(1,3)))
]
assert X([1,3], 1, 1) == [
# T/
T([], B(1,3)), # nsplit=0
T([], # nsplit=0,0
T([], B(1,3))),
T([], # nsplit=0,1
T([0], B(), B(1,3))),
T([],
T([1], B(), B(1,3))),
T([],
T([2], B(1), B(3))),
T([],
T([3], B(1), B(3))),
T([],
T([4], B(1,3), B())),
# T0/
T([0], B(), B(1,3)), # nsplit=1
T([0], # nsplit=1,(0,0)
T([], B()),
T([], B(1,3))),
T([0], # nsplit=1,(0,1)
T([], B()),
T([1], B(), B(1,3))),
T([0],
T([], B()),
T([2], B(1), B(3))),
T([0],
T([], B()),
T([3], B(1), B(3))),
T([0],
T([], B()),
T([4], B(1,3), B())),
# nsplit=1,(1,*) -> ø
# T1/
T([1], B(), B(1,3)), # nsplit=1
T([1], # nsplit=1,(0,0)
T([], B()),
T([], B(1,3))),
T([1],
T([], B()), # nsplit=1,(0,1)
T([2], B(1), B(3))),
T([1],
T([], B()),
T([3], B(1), B(3))),
T([1],
T([], B()),
T([4], B(1,3), B())),
T([1], # nsplit=1,(1,0)
T([0], B(), B()),
T([], B(1,3))),
T([1], # nsplit=1,(1,1)
T([0], B(), B()),
T([2], B(1), B(3))),
T([1],
T([0], B(), B()),
T([3], B(1), B(3))),
T([1],
T([0], B(), B()),
T([4], B(1,3), B())),
# T2/
T([2], B(1), B(3)), # nsplit=1
T([2], # nsplit=1,(0,0)
T([], B(1)),
T([], B(3))),
T([2], # nsplit=1,(0,1)
T([], B(1)),
T([3], B(), B(3))),
T([2],
T([], B(1)),
T([4], B(3), B())),
T([2], # nsplit=1,(1,0)
T([0], B(), B(1)),
T([], B(3))),
T([2], # nsplit=1,(1,1)
T([0], B(), B(1)),
T([3], B(), B(3))),
T([2],
T([0], B(), B(1)),
T([4], B(3), B())),
T([2], # nsplit=1,(1,0)
T([1], B(), B(1)),
T([], B(3))),
T([2], # nsplit=1,(1,1)
T([1], B(), B(1)),
T([3], B(), B(3))),
T([2],
T([1], B(), B(1)),
T([4], B(3), B())),
# T3/
T([3], B(1), B(3)), # nsplit=1
T([3], # nsplit=1,(0,0)
T([], B(1)),
T([], B(3))),
T([3], # nsplit=1,(0,1)
T([], B(1)),
T([4], B(3), B())),
T([3], # nsplit=1,(1,0)
T([0], B(), B(1)),
T([], B(3))),
T([3], # nsplit=1,(1,1)
T([0], B(), B(1)),
T([4], B(3), B())),
T([3], # nsplit=1,(1,0)
T([1], B(), B(1)),
T([], B(3))),
T([3], # nsplit=1,(1,1)
T([1], B(), B(1)),
T([4], B(3), B())),
T([3], # nsplit=1,(1,0)
T([2], B(1), B()),
T([], B(3))),
T([3], # nsplit=1,(1,1)
T([2], B(1), B()),
T([4], B(3), B())),
# T4/
T([4], B(1,3), B()), # nsplit=1
T([4], # nsplit=1,(0,0)
T([], B(1,3)),
T([], B())),
# nsplit=1,(0,1) -> ø
T([4], # nsplit=1,(1,0)
T([0], B(), B(1,3)),
T([], B())),
T([4],
T([1], B(), B(1,3)),
T([], B())),
T([4],
T([2], B(1), B(3)),
T([], B())),
T([4],
T([3], B(1), B(3)),
T([], B())),
# nsplit=1,(1,1) -> ø
]
assert Y([1,3], 1, 1) == [
# T/
T([], B(1,3)), # nsplit=0
T([], # nsplit=0,0
T([], B(1,3))),
T([], # nsplit=0,1
T([2], B(1), B(3))),
T([],
T([3], B(1), B(3))),
# T0/
# nothing - leftmost bucket is always empty
# T1/
# nothing - leftmost bucket is always empty
# T2/
T([2], B(1), B(3)), # nsplit=1
T([2], # nsplit=1,(0,0)
T([], B(1)),
T([], B(3))),
# T3/
T([3], B(1), B(3)), # nsplit=1
T([3], # nsplit=1,(0,0)
T([], B(1)),
T([], B(3))),
# T4/
# nothing - rightmost bucket is always empty
]
# TODO test for maxsplit=2 / maxdepth=2 vvv
def TY(keys, maxdepth, maxsplit, kv=None):
yv = Y(keys, maxdepth, maxsplit, kv=kv)
return list([xbtree.TopoEncode(_, vencode=lambda v: v) for _ in yv])
assert TY([1,3], 1, 1) == [
'T/B1,3',
'T/T/B1,3',
'T/T2/B1-B3',
'T/T3/B1-B3',
'T2/B1-B3',
'T2/T-T/B1-B3',
'T3/B1-B3',
'T3/T-T/B1-B3',
]
# with values
assert TY([1,3], 1,1, kv={1:'a',3:'c'}) == [
'T/B1:a,3:c',
'T/T/B1:a,3:c',
'T/T2/B1:a-B3:c',
'T/T3/B1:a-B3:c',
'T2/B1:a-B3:c',
'T2/T-T/B1:a-B3:c',
'T3/B1:a-B3:c',
'T3/T-T/B1:a-B3:c',
]
# XBlk simulates ZBlk without xbtree_test.py depending on file_zodb.py
class XBlk(Persistent):
def __init__(self, data):
self.data = data
def __str__(self):
return 'X%s' % self.data
__repr__ = __str__
# XLOTree is like LOBTree but with small max tree and bucket node sizes.
# It's tree and bucket nodes are split often on regular tree updates.
class XLOTree(LOBTree):
#_bucket_type = XLOBucket
max_leaf_size = 2
max_internal_size = 2
zbcheck._type2kind[XLOTree] = (zbcheck.TYPE_BTREE, True)
zbcheck._btree2bucket[XLOTree] = XLOTree._bucket_type
def crack_btree(ztree):
assert xbtree._zclassify(ztree).is_ztree, ztree
return zbcheck.crack_btree(ztree, is_mapping=True)
def crack_bucket(zbucket):
assert xbtree._zclassify(zbucket).is_zbucket, zbucket
return zbcheck.crack_bucket(zbucket, is_mapping=True)
# assertT asserts that znode is normal tree node + has specified keys and children.
#
# by default children are checked exactly via "is"
# if a child is represented as 'T' or 'B' - it is only verified to be of tree
# or bucket type correspondingly.
def assertT(znode, keyv, *children): # -> [] of children marked with 'T'/'B'
_ = xbtree._zclassify(znode)
assert _.is_ztree
kind, keys, kids = zbcheck.crack_btree(znode, _.is_map)
assert kind == BTREE_NORMAL
assert keys == keyv
assert len(kids) == len(children)
retv = []
for (child, childOK) in zip(kids, children):
if childOK == 'T':
assert type(child) is type(znode)
retv.append(child)
elif childOK == 'B':
assert type(child) is znode._bucket_type
retv.append(child)
else:
assert child is childOK
return retv
# assertB asserts that znode is bucket node with specified keys and values
def assertB(znode, *kvv):
_ = xbtree._zclassify(znode)
assert _.is_zbucket
keys, values = zbcheck.crack_bucket(znode, _.is_map)
if not _.is_map:
assert values == []
assert keys == kvv
else:
assert len(keys) == len(values)
assert len(keys) == len(kvv)
for (i,(k,v)) in enumerate(zip(keys, values)):
kok, vok = kvv[i]
assert k == kok
assert v is vok
def test_restructure():
# do restructure tests under ZODB because without ZODB connection it is not
# always possible to __setstate__ for e.g. .../T/B. We also want to make
# sure Restructure correctly marks modified nodes as changed so that the
# changes are actually persisted to storage on commit.
zstor = MappingStorage()
db = DB(zstor)
zconn = db.open()
X = [] # X[i] -> XBlk corresponding to block #i
xv = 'abcdefghijkl'
for i in range(len(xv)):
X.append(XBlk(xv[i]))
def xdecode(v):
assert len(v) == 1
assert v in xv
return X[xv.index(v)]
def xencode(x):
assert isinstance(x, XBlk)
return x.data
# assertB wraps global assertB to automatically fill in X[k] values for specified keys.
def assertB(znode, *keyv):
globals()['assertB'](znode, *[(k,X[k]) for k in keyv])
# Z prepares XLOTree ztree with given keys via usual way.
# the tree is setup as {} k -> X[k].
def Z(*keys):
ztree = XLOTree()
zconn.add(ztree)
for k in keys:
ztree[k] = X[k]
# check all keys via iterating (this verifies firstbucket and B->next pointers)
keys2 = set(ztree.keys())
assert keys2 == set(keys)
# check all keys by [] access
for k in keys:
assert ztree[k] is X[k]
return ztree
# R restructures ztree to have specified new topology.
# The result is committed unless dontcommit=Y specified.
def R(ztree, newtopo, dontcommit=False):
# verify ztree consistency
items = list(ztree.items())
for (k,v) in items:
assert ztree[k] == v
if isinstance(newtopo, str):
newStructure = xbtree.TopoDecode(newtopo, xdecode)
else:
assert isinstance(newtopo, xbtree.Tree)
newStructure = newtopo
xbtree.Restructure(ztree, newStructure)
if not dontcommit:
transaction.commit()
# force objects state to be reloaded from storage.
# this leads further checks to also verify if Restructure modified a
# node, but did not marked it as changed. If this bug is indeed there -
# then the modifications will be lost after live cache clearance.
zconn.cacheMinimize()
assert xbtree.StructureOf(ztree, onlyKeys=True) == \
newStructure.copy(onlyKeys=True)
# verify iteration produces the same [] of (key, v)
assert list(ztree.items()) == items
# verify [k] gives the same v (for all k)
for (k,v) in items:
assert ztree[k] == v
# S returns topo-encoded keys-only structure of ztree.
# Sv returns topo-encoded structure of ztree with values.
def S(ztree):
return xbtree.TopoEncode(xbtree.StructureOf(ztree, onlyKeys=True))
def Sv(ztree):
return xbtree.TopoEncode(xbtree.StructureOf(ztree), xencode)
# Z0 creates new empty tree
def Z0():
z = Z()
assert crack_btree(z) == (BTREE_EMPTY, [], [])
return z
# ---- tests with manual verification of resulting topology and nodes ----
# ø -> T/B
z = Z0()
R(z, 'T/B')
assert crack_btree(z) == (BTREE_EMPTY, [], [])
with raises(ValueError, match="new keys != old keys"):
R(z, 'T/B1')
# ø -> T/T/B (don't - we don't emit topologies with empty buckets for
# tests since ZODB breaks on them)
"""
z = Z0()
R(z, 'T/T/B')
t, = assertT(z, [], 'T')
b, = assertT(t, [], 'B')
assertB(b)
"""
# ø -> T/T-T/B-B (don't - see ^^^)
"""
z = Z0()
R(z, 'T0/T-T/B-B')
Tl, Tr = assertT(z, [0], 'T','T')
bl, = assertT(Tl, [], 'B')
br, = assertT(Tr, [], 'B')
assertB(bl)
assertB(br)
"""
# tree with 1 k->v (not yet committed bucket)
z = Z(1)
assert crack_btree(z) == (BTREE_ONE, ((1, X[1]),), None)
R(z, 'T/B1', dontcommit=True)
assert crack_btree(z) == (BTREE_ONE, ((1, X[1]),), None)
R(z, 'T/T/B1', dontcommit=True)
t, = assertT(z, [], 'T')
b1, = assertT(t, [], 'B')
assertB(b1, 1)
assert b1._p_oid is not None
R(z, 'T/B1', dontcommit=True)
assertT(z, [], b1)
assertB(b1, 1)
# tree with 2 k->v (not-yet committed bucket)
z = Z(1,3)
assert crack_btree(z) == (BTREE_ONE, ((1, X[1], 3, X[3]),), None)
R(z, 'T2/B1-B3', dontcommit=True)
b1, b3 = assertT(z, [2], 'B','B')
assert b1._p_oid is None
assert b3._p_oid is None
assertB(b1, 1)
assertB(b3, 3)
R(z, 'T/B1,3')
# buckets were not yet assigned oid -> collapsed back into T
assert crack_btree(z) == (BTREE_ONE, ((1, X[1], 3, X[3]),), None)
R(z, 'T3/B1-B3', dontcommit=True)
b1, b3 = assertT(z, [3], 'B','B')
assert b1._p_oid is None
assert b3._p_oid is None
assertB(b1, 1)
assertB(b3, 3)
transaction.commit() # force buckets to be assigned oid
assert b1._p_oid is not None
assert b3._p_oid is not None
# restructure back - buckets not collapsed back into T
R(z, 'T/B1,3')
b13, = assertT(z, [], 'B')
assertB(b13, 1,3)
# add 1 key -> B splits -> B + B
assert S(z) == 'T/B1,3'
z[5] = X[5]
assert S(z) == 'T3/B1-B3,5'
b1, b35 = assertT(z, [3], 'B','B')
assertB(b1, 1)
assertB(b35, 3,5)
# -> T2/T-T/B1-B3,5 (add intermediate T-T level)
R(z, 'T2/T-T/B1-B3,5')
tl, tr = assertT(z, [2], 'T','T')
assertT(tl, [], b1)
assertT(tr, [], b35)
assertB(b1, 1)
assertB(b35, 3,5)
# -> T2/T-T/B1-T/B3,5 (add intermediate T level in right arm)
R(z, 'T2/T-T/B1-T/B3,5')
assertT(z, [2], tl, tr)
assertT(tl, [], b1)
trr, = assertT(tr, [], 'T')
assert isinstance(trr, XLOTree)
assertT(trr, [], b35)
assertB(b1, 1)
assertB(b35, 3,5)
# -> T2,4/B1-B3-B5 (kill intermediate trees, split B35->B3+B5)
R(z, 'T2,4/B1-B3-B5')
b3, = assertT(z, [2,4], b1,'B',b35)
b5 = b35; del b35
assertB(b1, 1)
assertB(b3, 3)
assertB(b5, 5)
# -> T2/T-T4/B1-B3-B5 (add intermediate T-T4 level)
R(z, 'T2/T-T4/B1-B3-B5')
tl, tr = assertT(z, [2], 'T','T')
assertT(tl, [], b1)
assertT(tr, [4], b3,b5)
assertB(b1, 1)
assertB(b3, 3)
assertB(b5, 5)
# -> T2/T-T/B1-T4/B3-B5 (add intermediate level in right arm)
R(z, 'T2/T-T/B1-T4/B3-B5')
tr, = assertT(z, [2], tl,'T')
assertT(tl, [], b1)
trr, = assertT(tr, [], 'T')
assertT(trr, [4], b3,b5)
assertB(b1, 1)
assertB(b3, 3)
assertB(b5, 5)
# -> T/B1,3,5 (collapse into T/B)
R(z, 'T/B1,3,5')
assertT(z, [], b1)
b135 = b1
assertB(b135, 1,3,5)
# grow the tree with four more keys (6,7,8,9) till top-level tree node splits
assert S(z) == 'T/B1,3,5'
z[6] = X[6]
assert S(z) == 'T5/B1,3-B5,6'
z[7] = X[7]
assert S(z) == 'T5,6/B1,3-B5-B6,7'
z[8] = X[8]
assert S(z) == 'T6/T5-T7/B1,3-B5-B6-B7,8'
# rotate keys in T and reflow B to the left
tl, tr = assertT(z, [6], 'T','T')
b13, b5 = assertT(tl, [5], 'B','B')
b6, b78 = assertT(tr, [7], 'B','B')
assertB(b13, 1,3)
assertB(b5, 5)
assertB(b6, 6)
assertB(b78, 7,8)
R(z, 'T7/T4,6-T/B1,3-B5-B6-B7,8')
assertT(z, [7], tl,tr)
assertT(tl, [4,6], b13,b5,b6)
assertT(tr, [], b78)
assertB(b13, 1,3)
assertB(b5, 5)
assertB(b6, 6)
assertB(b78, 7,8)
# migrate keys in between buckets
R(z, 'T6/T3-T8/B1-B3,5-B6,7-B8')
assertT(z, [6], tl,tr)
assertT(tl, [3], b13,b5)
assertT(tr, [8], b6,b78)
b1 = b13; del b13
b35 = b5; del b5
b67 = b6; del b6
b8 = b78; del b78
assertB(b1, 1)
assertB(b35, 3,5)
assertB(b67, 6,7)
assertB(b8, 8)
# ---- new structure given with values ----
z = Z(0,2)
R(z, T([1], Bv([0],X[0]), Bv([2],X[2])))
b0, b2 = assertT(z, [1], 'B','B')
assertB(b0, 0)
assertB(b2, 2)
assert b0[0] is X[0]
assert b2[2] is X[2]
# [2] changes value from X[2] to X[3]
with raises(ValueError, match=r"target bucket changes \[2\]"):
R(z, T([1], Bv([0],X[0]), Bv([2],X[3])))
# ---- tricky cases
z = Z(0,1,2,3)
R(z, 'T2/T1-T3/B0-B1-T-T/B2-B3')
R(z, 'T2/T1-T/T-T-B2,3/B0-B1')
# degenerate topology from ZODB example
z = Z(1,3,5,7,11)
R(z, 'T4/T2-T/T-T-T6,10/B1-B3-T-T-T/T-B7-B11/B5')
R(z, 'T/B1,3,5,7,11')
# verify that changed objects are marked as such and so included into commit
# (just R also partly verifies this on every call)
z = Z(0,2,3)
transaction.commit()
def Rz(newtopo):
R(z, newtopo, dontcommit=True)
transaction.commit()
assert Sv(z) == newtopo
zconn.cacheMinimize() # force z state to be reloaded from storage
assert Sv(z) == newtopo # will fail if T or B is not marked as changed
xbtree.zcheck(z)
Rz('T/B0:a,2:c,3:d')
Rz('T1/B0:a-B2:c,3:d')
Rz('T2/B0:a-B2:c,3:d')
Rz('T3/B0:a,2:c-B3:d')
Rz('T2/T-T3/B0:a-B2:c-B3:d')
# make sure that only modified nodes are marked as changed.
z = Z(0,1,2,3)
R(z, 'T1/T-T2/B0-B1-B2,3')
tl, tr = assertT(z, [1], 'T','T')
b0, = assertT(tl, [], 'B')
b1, b23 = assertT(tr, [2], 'B','B')
assertB(b0, 0)
assertB(b1, 1)
assertB(b23, 2,3)
assert z._p_changed == False
assert tl._p_changed == False
assert tr._p_changed == False
assert b0._p_changed == False
assert b1._p_changed == False
assert b23._p_changed == False
R(z, 'T1/T-T3/B0-B1,2-B3', dontcommit=True) # reflow right arm
assertT(z, [1], tl, tr)
assertT(tl, [], b0)
assertT(tr, [3], b1, b23) # changed
assertB(b0, 0)
assertB(b1, 1,2) # changed
assertB(b23, 3) # changed
assert z._p_changed == False
assert tl._p_changed == False
assert tr._p_changed == True
assert b0._p_changed == False
assert b1._p_changed == True
assert b23._p_changed == True
transaction.commit()
# ---- tests on automatically generated topologies ----
#
# ( we make sure that Restructure can make the restructurement and that
# after restructure a tree remains valid without any error introduced )
for nkeys in range(5): # XXX !slow -> ↑
for xkeyv in xbtree._iterSplitByN(-1, 5+1, nkeys):
keyv = xkeyv[1:-1] # -1, ..., N -> ...
#print(keyv)
z = Z(*keyv)
# d s Nvariants Ttest
# 3 2 35·10³ 40s
# 3 1 18·10³ 20s
# 2 2 8·10³ 8s
# 2 1 3·103 4s
# 1 1 1·10³ 1s
for tree in xbtree.AllStructs(keyv, 2, 1): # XXX !slow -> d=3, s=2
#print('\t%s' % xbtree.TopoEncode(tree))
R(z, tree)
def test_walkBFS():
R = xbtree._Range
# T/B
b = B()
t = T([], b)
walkv = list(xbtree.__walkBFS(t))
assert len(walkv) == 2 # (t) (b)
_ = walkv[0]
assert len(_) == 1
assert _[0].range == R(-inf, inf)
assert _[0].node is t
_ = walkv[1]
assert len(_) == 1
assert _[0].range == R(-inf, inf)
assert _[0].node is b
# T0/T-T/B-B
bl = B(); br = B()
tl = T([], bl)
tr = T([], br)
t = T([0], tl, tr)
walkv = list(xbtree.__walkBFS(t))
assert len(walkv) == 3 # (t) (tl, tr), (bl, br)
_ = walkv[0]
assert len(_) == 1
assert _[0].range == R(-inf, inf)
assert _[0].node is t
_ = walkv[1]
assert len(_) == 2
assert _[0].range == R(-inf, 0)
assert _[0].node is tl
assert _[1].range == R(0, inf)
assert _[1].node is tr
_ = walkv[2]
assert len(_) == 2
assert _[0].range == R(-inf, 0)
assert _[0].node is bl
assert _[1].range == R(0, inf)
assert _[1].node is br
# XXX more tests?
def test_zwalkBFS():
zt = buildDegenerateZTree()
# assign oid to created objects to force btrees not to embed bucket state
zstor = MappingStorage()
db = DB(zstor)
zconn = db.open()
zroot = zconn.root()
zroot['x'] = zt
transaction.commit()
def assertT(znode, keyv, *children):
assert isinstance(znode, IITreeSet)
return globals()['assertT'](znode, keyv, *children)
# assertB asserts that znode is bucket + has specified keys
def assertB(znode, *keyv):
assert isinstance(znode, IISet)
globals()['assertB'](znode, *keyv)
R = xbtree._Range
zwalkv = list(xbtree.__zwalkBFS(zt))
assert len(zwalkv) == 6 # [-∞,∞)T4,
# [-∞,4)T2, [4,∞)T
# [-∞,2)T, [2,4)T, [4,∞)T6,10
# [-∞,2)B1, [2,4)B3, [4,6)T, [6,10)T, [10,∞]T
# [4,6)T, [6,10)B7, [10,∞)B11
# [4,6)B5
_ = zwalkv[5] # [4,6)B5
assert len(_) == 1
assert _[0].range == R(4,6)
b5 = _[0].node; assertB(b5, 5)
_ = zwalkv[4] # [4,6)T, [6,10)B7, [10,∞)B11
assert len(_) == 3
assert _[0].range == R(4,6)
assert _[1].range == R(6,10)
assert _[2].range == R(10,inf)
t4_b5= _[0].node; assertT(t4_b5, [], b5)
b7 = _[1].node; assertB(b7, 7)
b11 = _[2].node; assertB(b11, 11)
_ = zwalkv[3] # [-∞,2)B1, [2,4)B3, [4,6)T, [6,10)T, [10,∞]T
assert len(_) == 5
assert _[0].range == R(-inf,2)
assert _[1].range == R(2,4)
assert _[2].range == R(4,6)
assert _[3].range == R(6,10)
assert _[4].range == R(10,inf)
b1 = _[0].node; assertB(b1, 1)
b3 = _[1].node; assertB(b3, 3)
t3_t4_b5 = _[2].node; assertT(t3_t4_b5, [], t4_b5)
t3_b7 = _[3].node; assertT(t3_b7, [], b7)
t3_b11 = _[4].node; assertT(t3_b11, [], b11)
_ = zwalkv[2] # [-∞,2)T, [2,4)T, [4,∞)T6,10
assert len(_) == 3
assert _[0].range == R(-inf,2)
assert _[1].range == R(2,4)
assert _[2].range == R(4,inf)
t2_b1 = _[0].node; assertT(t2_b1, [], b1)
t2_b3 = _[1].node; assertT(t2_b3, [], b3)
t2_610= _[2].node; assertT(t2_610, [6,10], t3_t4_b5, t3_b7, t3_b11)
_ = zwalkv[1] # [-∞,4)T2, [4,∞)T
assert len(_) == 2
assert _[0].range == R(-inf, 4)
assert _[1].range == R(4, inf)
t1_2 = _[0].node; assertT(t1_2, [2], t2_b1, t2_b3)
t1_t2_610 = _[1].node; assertT(t1_t2_610, [], t2_610)
_ = zwalkv[0] # [-∞,∞)T4,
assert len(_) == 1
assert _[0].range == R(-inf, inf)
assertT(_[0].node, [4], t1_2, t1_t2_610)
def test_keyvSliceBy():
X = xbtree._keyvSliceBy
assert X([], 0,0) == []
assert X([1], 0,0) == []
assert X([1], 0,1) == []
assert X([1], 1,1) == []
assert X([1], 1,2) == [1]
assert X([1,3,5,10,17], 3,10) == [3,5]
def test_iterSplitByN():
def X(lo, hi, nsplit):
return tuple(xbtree._iterSplitByN(lo, hi, nsplit))
assert X(0,0, 0) == ( [0,0], )
assert X(0,0, 1) == ()
assert X(0,1, 0) == ( [0,1], )
assert X(0,1, 1) == ()
assert X(0,2, 0) == ( [0,2], )
assert X(0,2, 1) == ( [0,1,2], )
assert X(0,2, 2) == ()
assert X(0,3, 0) == ( [0,3], )
assert X(0,3, 1) == ( [0,1,3], [0,2,3] )
assert X(0,3, 2) == ( [0,1,2,3], )
assert X(0,3, 3) == ()
assert X(0,4, 0) == ( [0,4], )
assert X(0,4, 1) == ( [0,1,4], [0,2,4], [0,3,4] )
assert X(0,4, 2) == ( [0,1,2,4], [0,1,3,4], [0,2,3,4] )
assert X(0,4, 3) == ( [0,1,2,3,4], )
assert X(0,4, 4) == ()
assert X(0,5, 0) == ( [0,5], )
assert X(0,5, 1) == ( [0,1,5], [0,2,5], [0,3,5], [0,4,5] )
assert X(0,5, 2) == ( [0,1,2,5], [0,1,3,5], [0,1,4,5], [0,2,3,5], [0,2,4,5], [0,3,4,5] )
assert X(0,5, 3) == ( [0,1,2,3,5], [0,1,2,4,5], [0,1,3,4,5], [0,2,3,4,5] )
assert X(0,5, 4) == ( [0,1,2,3,4,5], )
assert X(0,5, 5) == ()
def test_iterSplitKeyvByN():
keyv = [1,3,4]
def X(lo, hi, nsplit):
return tuple(xbtree._iterSplitKeyvByN(lo, hi, keyv, nsplit))
assert X(0,7, 0) == ( [0,7], )
assert X(0,7, 1) == ( [0,2,7], [0,3,7], [0,4,7] )
assert X(0,7, 2) == ( [0,2,4,7], [0,3,4,7] )
assert X(0,7, 3) == ()
wcfs/set.go.in deleted 100644 → 0
View file @ d75d23bd
// Copyright (C) 2015-2020 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package PACKAGE
// Set is a set of VALUE.
type Set map[VALUE]struct{}
// Add adds v to the set.
func (s Set) Add(v VALUE) {
s[v] = struct{}{}
}
// Del removes v from the set.
// it is noop if v was not in the set.
func (s Set) Del(v VALUE) {
delete(s, v)
}
// Has checks whether the set contains v.
func (s Set) Has(v VALUE) bool {
_, ok := s[v]
return ok
}
// Update adds t values to s.
func (s Set) Update(t Set) {
for v := range t {
s.Add(v)
}
}
// Elements returns all elements of set as slice.
func (s Set) Elements() []VALUE {
ev := make([]VALUE, len(s))
i := 0
for e := range s {
ev[i] = e
i++
}
return ev
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7