Skip to content

W
wendelin.core
Commit 00b6ac98 authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent 84986acb
Hide whitespace changes
Inline Side-by-side
Showing with 1189 additions and 1180 deletions
wcfs/internal/xbtree/δbtail_test.go
View file @ 00b6ac98
...@@ -61,47 +61,6 @@ import ( ...@@ -61,47 +61,6 @@ import (
type Δstring = xbtreetest.Δstring type Δstring = xbtreetest.Δstring
// trackSet returns what should be ΔBtail.trackSet coverage for specified tracked key set.
func trackSet(rbs xbtreetest.RBucketSet, tracked setKey) blib.PPTreeSubSet {
// nil = don't compute keyCover
// (trackSet is called from inside hot inner loop of rebuild test)
return _trackSetWithCov(rbs, tracked, nil)
}
// trackSetWithCov returns what should be ΔBtail.trackSet and its key coverage for specified tracked key set.
func trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey) (trackSet blib.PPTreeSubSet, keyCover *blib.RangedKeySet) {
keyCover = &blib.RangedKeySet{}
trackSet = _trackSetWithCov(rbs, tracked, keyCover)
return trackSet, keyCover
}
func _trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey, outKeyCover *blib.RangedKeySet) (trackSet blib.PPTreeSubSet) {
trackSet = blib.PPTreeSubSet{}
for k := range tracked {
kb := rbs.Get(k)
if outKeyCover != nil {
outKeyCover.AddRange(kb.Keycov)
}
trackSet.AddPath(kb.Path())
}
return trackSet
}
// XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to t1..t2 db snapshots.
func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstring {
δkv := make(map[Key]Δstring, len(δkvOid))
for k, δvOid := range δkvOid {
δkv[k] = Δstring{
Old: t1.XGetBlkData(δvOid.Old),
New: t2.XGetBlkData(δvOid.New),
}
}
return δkv
}
// KAdjMatrix is adjacency matrix that describes how set of tracked keys // KAdjMatrix is adjacency matrix that describes how set of tracked keys
// changes (always grow) when tree topology is updated from A to B. // changes (always grow) when tree topology is updated from A to B.
// //
...@@ -129,786 +88,986 @@ func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstri ...@@ -129,786 +88,986 @@ func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstri
// //
// XXX fix definition for "and changed, or coverage changed" // XXX fix definition for "and changed, or coverage changed"
// //
// Use:
//
// - KAdj(A,B) to build adjacency matrix for A -> B transition.
// - kadj.Map(keys) to compute kadj·keys.
// - kadj1.Mul(kadj2) to compute kadj1·kadj2.
//
// Note: adjacency matrix is symmetric (KAdj verifies this at runtime): // Note: adjacency matrix is symmetric (KAdj verifies this at runtime):
// //
// kadj(A,B) == kadj(B,A) // KAdj(A,B) == KAdj(B,A)
type KAdjMatrix map[Key]setKey type KAdjMatrix map[Key]setKey
// Map returns kadj·keys .
func (kadj KAdjMatrix) Map(keys setKey) setKey {
res := make(setKey, len(keys))
for k := range keys {
to, ok := kadj[k]
if !ok {
panicf("kadj.Map: %d ∉ kadj\n\nkadj: %v", k, kadj)
}
res.Update(to)
}
return res
}
// Mul returns kadjA·kadjB .
//
// (kadjA·kadjB).Map(keys) = kadjA.Map(kadjB.Map(keys))
func (kadjA KAdjMatrix) Mul(kadjB KAdjMatrix) KAdjMatrix {
// ~ assert kadjA.keys == kadjB.keys
// check only len here; the rest will be asserted by Map
if len(kadjA) != len(kadjB) {
panicf("kadj.Mul: different keys:\n\nkadjA: %v\nkadjB: %v", kadjA, kadjB)
}
kadj := make(KAdjMatrix, len(kadjB)) // ΔBTestEntry represents one entry in ΔBTail tests.
for k, tob := range kadjB { type ΔBTestEntry struct {
kadj[k] = kadjA.Map(tob) tree string // next tree topology
} kadjOK KAdjMatrix // adjacency matrix against previous case (optional)
return kadj flags ΔBTestFlags
} }
// KAdj builds adjacency matrix for t1 -> t2 transition. type ΔBTestFlags int
// const ΔBTest_SkipUpdate ΔBTestFlags = 1 // skip verifying Update for this test entry
// The set of keys for which kadj matrix is computed can be optionally provided. const ΔBTest_SkipRebuild ΔBTestFlags = 2 // skip verifying rebuild for this test entry
// This set of keys defaults to allTestKeys(t1,t2).
//
// KAdj itself is verified by testΔBTail on entries with .kadjOK set.
func KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
// assert KAdj(A,B) == KAdj(B,A)
kadj12 := _KAdj(t1,t2, keysv...)
kadj21 := _KAdj(t2,t1, keysv...)
if !reflect.DeepEqual(kadj12, kadj21) {
panicf("KAdj not symmetric:\nt1: %s\nt2: %s\nkadj12: %v\nkadj21: %v",
t1.Tree, t2.Tree, kadj12, kadj21)
}
return kadj12
}
const debugKAdj = false // ΔBTest converts xtest into ΔBTestEntry.
func debugfKAdj(format string, argv ...interface{}) { // xtest can be string|ΔBTestEntry.
if debugKAdj { func ΔBTest(xtest interface{}) ΔBTestEntry {
fmt.Printf(format, argv...) var test ΔBTestEntry
switch xtest := xtest.(type) {
case string:
test.tree = xtest
test.kadjOK = nil
test.flags = 0
case ΔBTestEntry:
test = xtest
default:
panicf("BUG: ΔBTest: bad type %T", xtest)
} }
return test
} }
func _KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) { // TestΔBTail verifies ΔBTail for explicitly provided tree topologies.
var keys setKey func TestΔBTail(t *testing.T) {
switch len(keysv) { // K is shorthand for setKey
case 0: K := func(keyv ...Key) setKey {
keys = allTestKeys(t1, t2) ks := setKey{}
case 1: for _, k := range keyv { ks.Add(k) }
keys = keysv[0] return ks
default: }
panic("multiple key sets on the call") // oo is shorthand for KeyMax
const oo = KeyMax
// A is shorthand for KAdjMatrix
type A = KAdjMatrix
// Δ is shorthand for ΔBTestEntry
Δ := func(tree string, kadjOK A) (test ΔBTestEntry) {
test.tree = tree
test.kadjOK = kadjOK
return test
} }
debugfKAdj("\n\n_KAdj\n") // test known cases going through tree1 -> tree2 -> ...
debugfKAdj("t1: %s\n", t1.Tree) testv := []interface{} {
debugfKAdj("t2: %s\n", t2.Tree) // start from non-empty tree to verify both ->empty and empty-> transitions
debugfKAdj("keys: %s\n", keys) "T/B1:a,2:b",
defer func() {
debugfKAdj("kadj -> %v\n", kadj)
}()
// kadj = {} k -> adjacent keys. // empty
// if k is tracked and covered by changed leaf -> changes to adjacents must be in Update(t1->t2). "T/B:",
kadj = KAdjMatrix{}
for k := range keys {
adj1 := setKey{}
adj2 := setKey{}
q1 := &blib.RangedKeySet{}; q1.Add(k) // +1
q2 := &blib.RangedKeySet{}; q2.Add(k) Δ("T/B1:a",
done1 := &blib.RangedKeySet{} A{1: K(1,oo),
done2 := &blib.RangedKeySet{} oo: K(1,oo)}),
debugfKAdj("\nk%s\n", kstr(k)) // +2
for !q1.Empty() || !q2.Empty() { Δ("T/B1:a,2:b",
debugfKAdj("q1: %s\tdone1: %s\n", q1, done1) A{1: K(1,2,oo),
debugfKAdj("q2: %s\tdone2: %s\n", q2, done2) 2: K(1,2,oo),
for _, r1 := range q1.AllRanges() { oo: K(1,2,oo)}),
lo1 := r1.Lo
for {
b1 := t1.Xkv.Get(lo1)
debugfKAdj(" b1: %s\n", b1)
for k_ := range keys {
if b1.Keycov.Has(k_) {
adj1.Add(k_)
debugfKAdj(" adj1 += %s\t-> %s\n", kstr(k_), adj1)
}
}
done1.AddRange(b1.Keycov)
// q2 |= (b1.keyrange \ done2)
δq2 := &blib.RangedKeySet{}
δq2.AddRange(b1.Keycov)
δq2.DifferenceInplace(done2)
q2.UnionInplace(δq2)
debugfKAdj("q2 += %s\t-> %s\n", δq2, q2)
// continue with next right bucket until r1 coverage is complete // -1
if r1.Hi_ <= b1.Keycov.Hi_ { Δ("T/B2:b",
break A{1: K(1,2,oo),
} 2: K(1,2,oo),
lo1 = b1.Keycov.Hi_ + 1 oo: K(1,2,oo)}),
}
}
q1.Clear()
for _, r2 := range q2.AllRanges() { // 2: b->c
lo2 := r2.Lo Δ("T/B2:c",
for { A{2: K(2,oo),
b2 := t2.Xkv.Get(lo2) oo: K(2,oo)}),
debugfKAdj(" b2: %s\n", b2)
for k_ := range keys {
if b2.Keycov.Has(k_) {
adj2.Add(k_)
debugfKAdj(" adj2 += %s\t-> %s\n", kstr(k_), adj2)
}
}
done2.AddRange(b2.Keycov)
// q1 |= (b2.keyrange \ done1)
δq1 := &blib.RangedKeySet{}
δq1.AddRange(b2.Keycov)
δq1.DifferenceInplace(done1)
q1.UnionInplace(δq1)
debugfKAdj("q1 += %s\t-> %s\n", δq1, q1)
// continue with next right bucket until r2 coverage is complete // +1 in new bucket (to the left)
if r2.Hi_ <= b2.Keycov.Hi_ { Δ("T2/B1:a-B2:c",
break A{1: K(1,2,oo),
} 2: K(1,2,oo),
lo2 = b2.Keycov.Hi_ + 1 oo: K(1,2,oo)}),
}
}
q2.Clear()
}
adj := setKey{}; adj.Update(adj1); adj.Update(adj2) // +3 in new bucket (to the right)
kadj[k] = adj Δ("T2,3/B1:a-B2:c-B3:c",
} A{1: K(1),
2: K(2,3,oo),
3: K(2,3,oo),
oo: K(2,3,oo)}),
return kadj // bucket split; +3 in new bucket
} "T/B1:a,2:b",
Δ("T2/B1:a-B2:b,3:c",
A{1: K(1,2,3,oo),
2: K(1,2,3,oo),
3: K(1,2,3,oo),
oo: K(1,2,3,oo)}),
// bucket split; +3 in new bucket; +4 +5 in another new bucket
// everything becomes tracked because original bucket had [-∞,∞) coverage
"T/B1:a,2:b",
Δ("T2,4/B1:a-B2:b,3:c-B4:d,5:e",
A{1: K(1,2,3,4,5,oo),
2: K(1,2,3,4,5,oo),
3: K(1,2,3,4,5,oo),
4: K(1,2,3,4,5,oo),
5: K(1,2,3,4,5,oo),
oo: K(1,2,3,4,5,oo)}),
// xverifyΔBTail_Update verifies how ΔBTail handles ZODB update for a tree with changes in between t1->t2. // reflow of keys: even if tracked={1}, changes to all B nodes need to be rescanned:
// // +B12 forces to look in -B23 which adds -3 into δ, which
// Note: this test verifies only single treediff step of ΔBtail.Update. // forces to look into +B34 and so on.
// the cycling phase of update, that is responsible to recompute older "T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
// entries when key coverage grows, is exercised by Δ("T3,5,7/B1:g,2:f-B3:e,4:d-B5:c,6:b-B7:a",
// xverifyΔBTail_rebuild. A{1: K(1,2,3,4,5,6,7,oo),
func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *xbtreetest.Commit) { 2: K(1,2,3,4,5,6,7,oo),
// verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞ 3: K(1,2,3,4,5,6,7,oo),
t.Run(fmt.Sprintf("Update/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) { 4: K(1,2,3,4,5,6,7,oo),
allKeys := allTestKeys(t1, t2) 5: K(1,2,3,4,5,6,7,oo),
allKeyv := allKeys.SortedElements() 6: K(1,2,3,4,5,6,7,oo),
7: K(1,2,3,4,5,6,7,oo),
oo: K(1,2,3,4,5,6,7,oo)}),
kadj12 := KAdj(t1, t2) // reflow of keys for rebuild: even if tracked1={}, tracked2={1}, changes to
// all A/B/C nodes need to be rescanned. Contrary to the above case the reflow
// is not detectable at separate diff(A,B) and diff(B,C) runs.
"T3,5,7/B1:a,2:b-B3:c,4:d-B5:e,6:f-B7:g,8:h",
"T/B1:b",
"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
// similar situation where rebuild has to detect reflow in between non-neighbour trees
"T3,6/B1:a,2:b-B3:c,4:d-B6:f,7:g",
"T4,7/B1:b-B4:d,5:e-B7:g,8:h",
"T2,5,8/B1:a-B2:b,3:c-B5:e,6:f-B8:h,9:i",
// verify at1->at2 for all combination of initial tracked keys. // depth=2; bucket split; +3 in new bucket; left T remain
for kidx := range IntSets(len(allKeyv)) { // _unchanged_ even though B under it is modified.
keys := setKey{} "T/T/B1:a,2:b",
for _, idx := range kidx { Δ("T2/T-T/B1:a-B2:b,3:c",
keys.Add(allKeyv[idx]) A{1: K(1,2,3,oo),
} 2: K(1,2,3,oo),
3: K(1,2,3,oo),
oo: K(1,2,3,oo)}),
// this t.Run allocates and keeps too much memory in -verylong // depth=2; like prev. case, but additional right arm with +4 +5 is added.
// also it is not so useful as above "Update/t1->t2" "T/T/B1:a,2:b",
//t.Run(fmt.Sprintf(" track=%s", keys), func(t *testing.T) { Δ("T2,4/T-T-T/B1:a-B2:b,3:c-B4:d,5:e",
xverifyΔBTail_Update1(t, subj, db, treeRoot, t1,t2, keys, kadj12) A{1: K(1,2,3,4,5,oo),
//}) 2: K(1,2,3,4,5,oo),
} 3: K(1,2,3,4,5,oo),
}) 4: K(1,2,3,4,5,oo),
} 5: K(1,2,3,4,5,oo),
oo: K(1,2,3,4,5,oo)}),
// xverifyΔBTail_Update1 verifies how ΔBTail handles ZODB update at1->at2 from initial // depth=2; bucket split; +3 in new bucket; t0 and t1 split;
// tracked state defined by initialTrackedKeys. // +right arm (T7/B45-B89).
func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *xbtreetest.Commit, initialTrackedKeys setKey, kadj KAdjMatrix) { "T/T/B1:a,2:b",
X := exc.Raiseif Δ("T4/T2-T7/B1:a-B2:b,3:c-B4:d,5:e-B8:h,9:i",
//t.Logf("\n>>> Track=%s\n", initialTrackedKeys) A{1: K(1,2,3,4,5,8,9,oo),
2: K(1,2,3,4,5,8,9,oo),
3: K(1,2,3,4,5,8,9,oo),
4: K(1,2,3,4,5,8,9,oo),
5: K(1,2,3,4,5,8,9,oo),
8: K(1,2,3,4,5,8,9,oo),
9: K(1,2,3,4,5,8,9,oo),
oo: K(1,2,3,4,5,8,9,oo)}),
δZ := t2.ΔZ
d12 := t2.Δxkv
var TrackedδZ setKey = nil // 2 reflow to right B neighbour; 8 splits into new B; δ=ø
var kadjTrackedδZ setKey = nil "T3/B1:a,2:b-B4:d,8:h",
var δT, δTok map[Key]Δstring = nil, nil "T2,5/B1:a-B2:b,4:d-B8:h",
δZset := setOid{}
for _, oid := range δZ.Changev {
δZset.Add(oid)
}
// badf queues error message to be reported on return. // case where kadj does not grow too much as leafs coverage remains stable
var badv []string "T4,8/B1:a,2:b-B5:d,6:e-B10:g,11:h",
badf := func(format string, argv ...interface{}) { Δ("T4,8/B2:b,3:c-B6:e,7:f-B11:h,12:i",
badv = append(badv, fmt.Sprintf(format, argv...)) A{1: K(1,2,3),
} 2: K(1,2,3),
defer func() { 3: K(1,2,3),
if badv != nil || t.Failed() { 5: K(5,6,7),
emsg := fmt.Sprintf("%s ; tracked=%v :\n\n", subj, initialTrackedKeys) 6: K(5,6,7),
emsg += fmt.Sprintf("d12: %v\nδTok: %v\nδT: %v\n\n", d12, δTok, δT) 7: K(5,6,7,),
emsg += fmt.Sprintf("δZ: %v\n", δZset) 10: K(10,11,12,oo),
emsg += fmt.Sprintf("Tracked^δZ: %v\n", TrackedδZ) 11: K(10,11,12,oo),
emsg += fmt.Sprintf("kadj[Tracked^δZ]: %v\n", kadjTrackedδZ) 12: K(10,11,12,oo),
emsg += fmt.Sprintf("kadj: %v\n\n", kadj) oo: K(10,11,12,oo)}),
emsg += strings.Join(badv, "\n")
emsg += "\n"
t.Fatal(emsg)
}
}()
// tree deletion
// having ø in the middle of the test cases exercises all:
// * `ø -> Tree ...` (tree is created anew),
// * `... Tree -> ø` (tree is deleted), and
// * `Tree -> ø -> Tree` (tree is deleted and then recreated)
xbtreetest.DEL,
// δbtail @at1 with initial tracked set // tree rotation
δbtail := NewΔBtail(t1.At, db) "T3/B2:b-B3:c,4:d",
xtrackKeys(δbtail, t1, initialTrackedKeys) "T5/T3-T7/B2:a-B3:a,4:a-B6:a-B8:a",
// TrackedδZ = Tracked ^ δZ (i.e. a tracked node has changed, or its coverage was changed) // found by AllStructs ([1] is not changed, but because B1 is
TrackedδZ = setKey{} // unlinked and 1 migrates to other bucket, changes in that
for k := range initialTrackedKeys { // other bucket must be included into δT)
leaf1 := t1.Xkv.Get(k) "T1,2/B0:e-B1:d-B2:g,3:a",
oid1 := leaf1.Oid "T1/B0:d-B1:d,2:d",
if oid1 == zodb.InvalidOid { // embedded bucket // ----//---- with depth=2
oid1 = leaf1.Parent.Oid "T1,2/T-T-T/B0:a-B1:b-B2:c,3:d",
} "T1/T-T/B0:e-B1:b,2:f",
leaf2 := t2.Xkv.Get(k)
oid2 := leaf2.Oid
if oid2 == zodb.InvalidOid { // embedded bucket
oid2 = leaf2.Parent.Oid
}
if δZset.Has(oid1) || δZset.Has(oid2) || (leaf1.Keycov != leaf2.Keycov) {
TrackedδZ.Add(k)
}
}
kadjTrackedδZ = setKey{} // kadj[Tracked^δZ] (all keys adjacent to tracked^δZ) // XXX depth=3 (to verify recursion and selecting which tree children to follow or not)
for k := range TrackedδZ {
kadjTrackedδZ.Update(kadj[k])
}
// assert TrackedδZ ∈ kadj[TrackedδZ] // degenerate topology from ZODB tests
trackNotInKadj := TrackedδZ.Difference(kadjTrackedδZ) // https://github.com/zopefoundation/ZODB/commit/6cd24e99f89b
if len(trackNotInKadj) > 0 { // https://github.com/zopefoundation/BTrees/blob/4.7.2-1-g078ba60/BTrees/tests/testBTrees.py#L20-L57
badf("BUG: Tracked^δZ ∉ kadj[Tracked^δZ] ; extra=%v", trackNotInKadj) "T4/T2-T/T-T-T6,10/B1:a-B3:b-T-T-T/T-B7:c-B11:d/B5:e",
return "T/B1:e,5:d,7:c,8:b,11:a", // -3 +8
}
// k ∈ d12 // was leading treegen to generate corrupt trees
// k ∈ δT <=> "T/T1/T-T/B0:g-B1:e,2:d,3:h",
// k ∈ U kadj[·] "T1/T-T3/B0:g-T-T/B1:e,2:d-B3:h",
// ·∈tracking^δZ
δTok = map[Key]Δstring{} // d12[all keys that should be present in δT]
for k,δv := range d12 {
if kadjTrackedδZ.Has(k) {
δTok[k] = δv
}
}
ø := blib.PPTreeSubSet{} // was leading to wrongly computed trackSet2 due to top not
// being tracked to tree root.
"T/T1/B0:a-B1:b",
"T/T1/T-T/B0:c-B1:d",
// trackSet1 = xkv1[tracked1] // was leading to wrongly computed trackSet2: leaf bucket not
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]] // reparented to root.
trackSet1, tkeyCov1 := trackSetWithCov(t1.Xkv, initialTrackedKeys) "T/T/B0:a",
trackSet2, tkeyCov2 := trackSetWithCov(t2.Xkv, initialTrackedKeys.Union(kadjTrackedδZ)) "T/B0:a",
// verify δbtail.trackSet against @at1 // δtkeycov grows due to change in parent tree only
δbtail.assertTrack(t, "1", ø, trackSet1) "T3/B1:a-B8:c",
"T7/B1:a-B8:c",
// ----//----
"T3/B1:a,2:b-B8:c,9:d",
"T7/B1:a,2:b-B8:c,9:d",
// ----//---- depth=2
"T3/T-T/B1:a,2:b-B8:c,9:d",
"T7/T-T/B1:a,2:b-B8:c,9:d",
// ----//---- found by AllStructs
"T1,3/B0:d-B1:a-B3:d,4:g",
"T1,4/B0:e-B1:a-B4:c",
// ----//---- found by AllStructs
"T2,4/T-T-T/T1-T-B4:f/T-T-B3:f/B0:h-B1:f",
"T4/T-T/B3:f-T/B4:a",
// δB <- δZ
//
// also call _Update1 directly to verify δtkeycov return from treediff
// the result of Update and _Update1 should be the same since δbtail is initially empty.
δbtail_ := δbtail.Clone()
δB1, err := δbtail_._Update1(δZ); X(err) // XXX don't compute treediff twice
// assert tkeyCov1 ⊂ tkeyCov2 // ---- found by AllStructs ----
dkeycov12 := tkeyCov1.Difference(tkeyCov2)
if !dkeycov12.Empty() {
t.Errorf("BUG: tkeyCov1 ⊄ tkeyCov2:\n\ttkeyCov1: %s\n\ttkeyCov2: %s\n\ttkeyCov1 \\ tkeyCov2: %s", tkeyCov1, tkeyCov2, dkeycov12)
}
// assert δtkeycov == δ(tkeyCov1, tkeyCov2) // trackSet2 wrongly computed due to top not being tracked to tree root
δtkeycovOK := tkeyCov2.Difference(tkeyCov1) "T2/T1-T/B0:g-B1:b-T/B2:b,3:a",
δtkeycov := &blib.RangedKeySet{} "T2/T1-T/T-T-B2:a/B0:c-B1:g",
if __, ok := δB1.ByRoot[treeRoot]; ok {
δtkeycov = __.δtkeycov1
}
if !δtkeycov.Equal(δtkeycovOK) {
badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
}
δB, err := δbtail.Update(δZ); X(err) // unchanged node is reparented
// XXX assert δB.roots == δTKeyCov roots "T1/B0:c-B1:f",
// XXX assert δBtail[root].vδT = δBtail_[root].vδT "T1/T-T/B0:c-T/B1:h",
if δB.Rev != δZ.Tid {
badf("δB: rev != δZ.Tid ; rev=%s δZ.Tid=%s", δB.Rev, δZ.Tid)
return
}
// verify δbtail.trackSet against @at2 // SIGSEGV in ApplyΔ
δbtail.assertTrack(t, "2", trackSet2, ø) "T1/T-T2/T-B1:c-B2:c/B0:g",
"T1/T-T/B0:g-T/B1:e",
// trackSet corruption: oid is pointed by some .parent but is not present
"T1/T-T/B0:g-T2/B1:h-B2:g",
"T/T1/T-T2/B0:e-B1:f-B2:g",
// assert δB.ByRoot == {treeRoot -> ...} if δTok != ø // ApplyΔ -> xunion: node is reachable from multiple parents
// == ø if δTok == ø // ( because xdifference did not remove common non-leaf node
rootsOK := setOid{} // under which there were also other changed, but not initially
if len(δTok) > 0 { // tracked, node )
rootsOK.Add(treeRoot) "T4/T1-T/T-T2-B4:c/T-T-T/B0:f-B1:h-B2:g,3:b",
} "T1/T-T/T-T2/T-T-T/B0:f-B1:h-B2:f",
roots := setOid{} // ----//----
for root := range δB.ΔByRoot { "T3/T1-T/T-T2-T/B0:b-T-T-B3:h/B1:e-B2:a",
roots.Add(root) "T1/T-T4/T-T2-T/T-T-T-T/B0:b-B1:e-B2:a,3:c-B4:e",
} // ----//----
if !reflect.DeepEqual(roots, rootsOK) { "T/T1,3/T-T2-T4/B0:b-T-T-B3:g-B4:c/B1:b-B2:e",
badf("δB: roots != rootsOK ; roots=%v rootsOK=%v", roots, rootsOK) "T1,4/T-T-T/T-T2-B4:f/T-T-T/B0:h-B1:b-B2:h,3:a",
}
_, inδB := δB.ΔByRoot[treeRoot]
if !inδB {
return
}
"T2/B1:a-B7:g",
"T2,8/B1:a-B7:g-B9:i",
// δT <- δB "T2/B1:a-B2:b", "T/B1:a,2:b",
δToid := δB.ΔByRoot[treeRoot] // {} k -> δoid "T2,3/B1:a-B2:b-B3:c", "T/B1:a,2:b",
δT = XGetδKV(t1,t2, δToid) // {} k -> δ(ZBlk(oid).data) "T2,3/B1:a-B2:c-B3:c", "T/B1:a,2:b",
// δT must be subset of d12. "T2/B1:a-B2:c", "T2,3/B1:a-B2:c-B3:c",
// changed keys, that are
// - in tracked set -> must be present in δT
// - outside tracked set -> may be present in δT (kadj gives exact answer)
// δT is subset of d12 "T2/B1:a-B3:c",
for _, k := range sortedKeys(δT) { Δ("T2/T-T4/B1:b-B3:d-B99:h",
_, ind12 := d12[k] A{1: K(1),
if !ind12 { 3: K(3,99,oo),
badf("δT[%v] ∉ d12", k) 99: K(3,99,oo),
} oo: K(3,99,oo)}),
}
// direct tree_i -> tree_{i+1} -> _{i+2} ... plus
// reverse ... tree_i <- _{i+1} <- _{i+2}
kadjOK := ΔBTest(testv[len(testv)-1]).kadjOK
for i := len(testv)-2; i >= 0; i-- {
test := ΔBTest(testv[i])
kadjOK, test.kadjOK = test.kadjOK, kadjOK
testv = append(testv, test)
} }
// k ∈ tracked set -> must be present in δT testq := make(chan ΔBTestEntry)
// k ∉ tracked set -> may be present in δT (kadj gives exact answer) go func() {
for _, k := range sortedKeys(d12) { defer close(testq)
_, inδT := δT[k] for _, test := range testv {
_, inδTok := δTok[k] testq <- ΔBTest(test)
if inδT && !inδTok {
badf("δT[%v] ∉ δTok", k)
}
if !inδT && inδTok {
badf("δT ∌ δTok[%v]", k)
} }
}()
testΔBTail(t, testq)
if inδT {
if δT[k] != d12[k] {
badf("δT[%v] ≠ δTok[%v]", k, k)
}
}
}
} }
// assertTrack verifies state of .trackSet and ΔTtail.trackNew. // TestΔBTailRandom verifies ΔBtail on random tree topologies generated by AllStructs.
// it assumes that only one tree root is being tracked. func TestΔBTailRandom(t *testing.T) {
// XXX place X := exc.Raiseif
func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK blib.PPTreeSubSet, trackNewOK blib.PPTreeSubSet) {
t.Helper()
if !δBtail.trackSet.Equal(trackSetOK) {
t.Errorf("%s: trackSet:\n\thave: %v\n\twant: %v", subj, δBtail.trackSet, trackSetOK)
}
roots := setOid{} // considerations:
for root := range δBtail.vδTbyRoot { // - maxdepth↑ better for testing (more tricky topologies)
roots.Add(root) // - maxsplit↑ not so better for testing (leave s=1, max s=2)
} // - |kmin - kmax| affects N(variants) significantly
// -> keep key range small (dumb increase does not help testing)
// - N(keys) affects N(variants) significantly
// -> keep Nkeys reasonably small/medium (dumb increase does not help testing)
//
// - spawning python subprocess is very slow (takes 300-500ms for
// imports; https://github.com/pypa/setuptools/issues/510)
// -> we spawn `treegen allstructs` once and use request/response approach.
nrootsOK := 1 maxdepth := xbtreetest.N(2, 3, 4)
if trackSetOK.Empty() && trackNewOK.Empty() { maxsplit := xbtreetest.N(1, 2, 2)
nrootsOK = 0 n := xbtreetest.N(10,10,100)
} nkeys := xbtreetest.N(3, 5, 10)
if len(roots) != nrootsOK {
t.Errorf("%s: len(vδTbyRoot) != %d ; roots=%v", subj, nrootsOK, roots)
return
}
if nrootsOK == 0 {
return
}
root := roots.Elements()[0] // server to generate AllStructs(kv, ...)
sg, err := xbtreetest.StartAllStructsSrv(); X(err)
defer func() {
err := sg.Close(); X(err)
}()
δTtail := δBtail.vδTbyRoot[root] // random-number generator
rng, seed := xbtreetest.NewRand()
t.Logf("# maxdepth=%d maxsplit=%d nkeys=%d n=%d seed=%d", maxdepth, maxsplit, nkeys, n, seed)
trackNewRootsOK := setOid{} // generate (kv1, kv2, kv3) randomly
if !trackNewOK.Empty() {
trackNewRootsOK.Add(root)
}
if !δBtail.trackNewRoots.Equal(trackNewRootsOK) { // keysv1, keysv2 and keysv3 are random shuffle of IntSets
t.Errorf("%s: trackNewRoots:\n\thave: %v\n\twant: %v", subj, δBtail.trackNewRoots, trackNewRootsOK) var keysv1 [][]int
var keysv2 [][]int
var keysv3 [][]int
for keys := range IntSets(nkeys) {
keysv1 = append(keysv1, keys)
keysv2 = append(keysv2, keys)
keysv3 = append(keysv3, keys)
} }
v := keysv1
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv2
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv3
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
if !δTtail.trackNew.Equal(trackNewOK) { // given random (kv1, kv2, kv3) generate corresponding set of random tree
t.Errorf("%s: vδT.trackNew:\n\thave: %v\n\twant: %v", subj, δTtail.trackNew, trackNewOK) // topology sets (T1, T2, T3). Then iterate through T1->T2->T3->T1...
// elements such that all right-directed triplets are visited and only once.
// Test Update and rebuild on the generated tree sequences.
vv := "abcdefghij"
randv := func() string {
i := rng.Intn(len(vv))
return vv[i:i+1]
} }
}
// xverifyΔBTail_rebuild verifies ΔBtail.rebuild during t0->t1->t2 transition.
//
// t0->t1 exercises from-scratch rebuild,
// t1->t2 further exercises incremental rebuild.
//
// It also exercises rebuild phase of ΔBtail.Update.
func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *xbtreetest.Commit) {
t.Run(fmt.Sprintf("rebuild/%s→%s", t0.Tree, t1.Tree), func(t *testing.T) {
tAllKeys := allTestKeys(t0, t1, t2)
tAllKeyv := tAllKeys.SortedElements()
// tid -> "at_i"
xat := map[zodb.Tid]string{
t0.At: "at0",
t1.At: "at1",
t2.At: "at2",
}
//fmt.Printf("@%s: %v\n", xat[t0.At], t0.Xkv.Flatten())
//fmt.Printf("@%s: %v\n", xat[t1.At], t1.Xkv.Flatten())
//fmt.Printf("@%s: %v\n", xat[t2.At], t2.Xkv.Flatten())
kadj10 := KAdj(t1,t0, allTestKeys(t0,t1,t2)) // the number of pairs is 3·n^2
kadj21 := KAdj(t2,t1, allTestKeys(t0,t1,t2)) // the number of triplets is n^3
kadj12 := KAdj(t1,t2, allTestKeys(t0,t1,t2)) //
// limit n for emitted triplets, so that the amount of work for Update
// and rebuild tests is approximately of the same order.
nrebuild := int(math.Ceil(math.Pow(3*float64(n*n), 1./3)))
// in non-short mode rebuild tests are exercising more keys variants, plus every test case
// takes more time. Compensate for that as well.
if !testing.Short() {
nrebuild -= 3
}
// kadj210 = kadj10·kadj21 testq := make(chan ΔBTestEntry)
kadj210 := kadj10.Mul(kadj21) go func() {
defer close(testq)
for i := range keysv1 {
keys1 := keysv1[i]
keys2 := keysv2[i]
keys3 := keysv3[i]
ø := blib.PPTreeSubSet{} kv1 := map[Key]string{}
kv2 := map[Key]string{}
kv3 := map[Key]string{}
for _, k := range keys1 { kv1[Key(k)] = randv() }
for _, k := range keys2 { kv2[Key(k)] = randv() }
for _, k := range keys3 { kv3[Key(k)] = randv() }
// verify t0 -> t1 Track(keys1) Rebuild -> t2 Track(keys2) Rebuild treev1, err1 := sg.AllStructs(kv1, maxdepth, maxsplit, n, rng.Int63())
// for all combinations of keys1 and keys2 treev2, err2 := sg.AllStructs(kv2, maxdepth, maxsplit, n, rng.Int63())
for k1idx := range IntSets(len(tAllKeyv)) { treev3, err3 := sg.AllStructs(kv3, maxdepth, maxsplit, n, rng.Int63())
keys1 := setKey{} err := xerr.Merge(err1, err2, err3)
for _, idx1 := range k1idx { if err != nil {
keys1.Add(tAllKeyv[idx1]) t.Fatal(err)
} }
// δkv1_1 = t1.δxkv / kadj10(keys1) emit := func(tree string, flags ΔBTestFlags) {
keys1_0 := kadj10.Map(keys1) // skip emitting this entry if both Update and
δkv1_1 := map[Key]Δstring{} // Rebuild are requested to be skipped.
for k := range keys1_0 { if flags == (ΔBTest_SkipUpdate | ΔBTest_SkipRebuild) {
δv, ok := t1.Δxkv[k] return
if ok {
δkv1_1[k] = δv
} }
testq <- ΔBTestEntry{tree, nil, flags}
} }
Tkeys1 := trackSet(t1.Xkv, keys1) URSkipIf := func(ucond, rcond bool) ΔBTestFlags {
Tkeys1_0 := trackSet(t1.Xkv, keys1_0) var flags ΔBTestFlags
if ucond {
t.Run(fmt.Sprintf(" T%s;R", keys1), func(t *testing.T) { flags |= ΔBTest_SkipUpdate
δbtail := NewΔBtail(t0.At, db) }
if rcond {
flags |= ΔBTest_SkipRebuild
}
return flags
}
// assert trackSet=ø, trackNew=ø, vδB=[] for j := range treev1 {
δbtail.assertTrack(t, "@at0", ø, ø) for k := range treev2 {
assertΔTtail(t, "@at0", δbtail, t0, treeRoot, xat, for l := range treev3 {
/*vδT=ø*/) // limit rebuild to subset of tree topologies,
// because #(triplets) grow as n^3. See nrebuild
// definition above for details.
norebuild := (j >= nrebuild ||
k >= nrebuild ||
l >= nrebuild)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t0, t1, xat, // C_{l-1} -> Aj (pair first seen on k=0)
/*trackSet=*/ø, emit(treev1[j], URSkipIf(k != 0, norebuild))
/*vδT=ø*/)
xverifyΔBTail_rebuild_TR(t, δbtail, t1, treeRoot, xat,
// after Track(keys1)
keys1,
/*trackSet=*/ ø,
/*trackNew=*/ Tkeys1,
// after rebuild // Aj -> Bk (pair first seen on l=0)
/*trackSet=*/ Tkeys1_0, emit(treev2[k], URSkipIf(l != 0, norebuild))
/*vδT=*/ δkv1_1)
t.Run((" →" + t2.Tree), func(t *testing.T) { // Bk -> Cl (pair first seen on j=0)
// keys1R2 is full set of keys that should become tracked after emit(treev3[l], URSkipIf(j != 0, norebuild))
// Update() (which includes rebuild)
keys1R2 := kadj12.Map(keys1)
for {
keys1R2_ := kadj210.Map(keys1R2)
if keys1R2.Equal(keys1R2_) {
break
}
keys1R2 = keys1R2_
} }
}
}
}
}()
// δkvX_k1R2 = tX.δxkv / keys1R2 testΔBTail(t, testq)
δkv1_k1R2 := map[Key]Δstring{} }
δkv2_k1R2 := map[Key]Δstring{}
for k := range keys1R2 {
δv1, ok := t1.Δxkv[k]
if ok {
δkv1_k1R2[k] = δv1
}
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k1R2[k] = δv2
}
}
Tkeys1R2 := trackSet(t2.Xkv, keys1R2) // testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq.
func testΔBTail(t_ *testing.T, testq chan ΔBTestEntry) {
t := xbtreetest.NewT(t_)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t1, t2, xat, var t0 *xbtreetest.Commit
/*trackSet=*/ Tkeys1R2, for test := range testq {
/*vδT=*/ δkv1_k1R2, δkv2_k1R2) t1 := t.Head()
t2 := t.CommitTree(test.tree)
// tRestKeys2 = tAllKeys - keys1 subj := fmt.Sprintf("%s -> %s", t1.Tree, t2.Tree)
// reduce that to = tAllKeys - keys1R2 in short mode //t.Logf("\n\n\n**** %s ****\n\n", subj)
// ( if key from keys2 already became tracked after Track(keys1) + Update,
// adding Track(that-key), is not adding much testing coverage to recompute paths )
var tRestKeys2 setKey
if testing.Short() {
tRestKeys2 = tAllKeys.Difference(keys1R2)
} else {
tRestKeys2 = tAllKeys.Difference(keys1)
}
tRestKeyv2 := tRestKeys2.SortedElements() // KAdj
for k2idx := range IntSets(len(tRestKeyv2)) { if kadjOK := test.kadjOK; kadjOK != nil {
keys2 := setKey{} t.Run(fmt.Sprintf("KAdj/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) {
for _, idx2 := range k2idx { kadj := KAdj(t1, t2)
keys2.Add(tRestKeyv2[idx2]) if !reflect.DeepEqual(kadj, kadjOK) {
} t.Fatalf("BUG: computed kadj is wrong:\nkadjOK: %v\nkadj : %v\n\n", kadjOK, kadj)
}
})
}
// keys12R2 is full set of keys that should become tracked after // ΔBTail.Update
// Track(keys2) + rebuild if test.flags & ΔBTest_SkipUpdate == 0 {
keys12R2 := keys1R2.Union(keys2) xverifyΔBTail_Update(t.T, subj, t.DB, t.Root(), t1,t2)
for { }
keys12R2_ := kadj210.Map(keys12R2)
if keys12R2.Equal(keys12R2_) {
break
}
keys12R2 = keys12R2_
}
Tkeys2 := trackSet(t2.Xkv, keys2) // ΔBTail.rebuild
Tkeys12R2 := trackSet(t2.Xkv, keys12R2) if t0 != nil && (test.flags & ΔBTest_SkipRebuild == 0) {
/* xverifyΔBTail_rebuild(t.T, t.DB, t.Root(), t0,t1,t2)
fmt.Printf("\n\n\nKKK\nkeys1=%s keys2=%s\n", keys1, keys2) }
fmt.Printf("keys1R2: %s\n", keys1R2)
fmt.Printf("keys12R2: %s\n", keys12R2)
fmt.Printf("t0.Xkv: %v\n", t0.Xkv) t0, t1 = t1, t2
fmt.Printf("t1.Xkv: %v\n", t1.Xkv) }
fmt.Printf("t2.Xkv: %v\n", t2.Xkv) }
fmt.Printf("kadj21: %v\n", kadj21)
fmt.Printf("kadj12: %v\n", kadj12)
fmt.Printf("Tkeys2 -> %s\n", Tkeys2)
fmt.Printf("Tkeys1R2 -> %s\n", Tkeys1R2)
fmt.Printf("Tkeys2 \\ Tkeys1R2 -> %s\n", Tkeys2.Difference(Tkeys1R2))
fmt.Printf("\n\n\n")
*/
// δkvX_k12R2 = tX.δxkv / keys12R2 // xverifyΔBTail_Update verifies how ΔBTail handles ZODB update for a tree with changes in between t1->t2.
δkv1_k12R2 := make(map[Key]Δstring, len(t1.Δxkv)) //
δkv2_k12R2 := make(map[Key]Δstring, len(t2.Δxkv)) // Note: this test verifies only single treediff step of ΔBtail.Update.
for k := range keys12R2 { // the cycling phase of update, that is responsible to recompute older
δv1, ok := t1.Δxkv[k] // entries when key coverage grows, is exercised by
if ok { // xverifyΔBTail_rebuild.
δkv1_k12R2[k] = δv1 func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *xbtreetest.Commit) {
} // verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞
δv2, ok := t2.Δxkv[k] t.Run(fmt.Sprintf("Update/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) {
if ok { allKeys := allTestKeys(t1, t2)
δkv2_k12R2[k] = δv2 allKeyv := allKeys.SortedElements()
}
}
// t.Run is expensive at this level of nest kadj12 := KAdj(t1, t2)
//t.Run(" T"+keys2.String()+";R", func(t *testing.T) {
δbtail_ := δbtail.Clone()
xverifyΔBTail_rebuild_TR(t, δbtail_, t2, treeRoot, xat,
// after Track(keys2)
keys2,
/*trackSet*/ Tkeys1R2,
/*trackNew*/ Tkeys2.Difference(
// trackNew should not cover ranges that are
// already in trackSet
Tkeys1R2),
// after rebuild // verify at1->at2 for all combination of initial tracked keys.
/* trackSet=*/ Tkeys12R2, for kidx := range IntSets(len(allKeyv)) {
/*vδT=*/ δkv1_k12R2, δkv2_k12R2) keys := setKey{}
//}) for _, idx := range kidx {
} keys.Add(allKeyv[idx])
}) }
})
// this t.Run allocates and keeps too much memory in -verylong
// also it is not so useful as above "Update/t1->t2"
//t.Run(fmt.Sprintf(" track=%s", keys), func(t *testing.T) {
xverifyΔBTail_Update1(t, subj, db, treeRoot, t1,t2, keys, kadj12)
//})
} }
}) })
} }
// xverifyΔBTail_rebuild_U verifies ΔBtail state after Update(ti->tj). // xverifyΔBTail_Update1 verifies how ΔBTail handles ZODB update at1->at2 from initial
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid, ti, tj *xbtreetest.Commit, xat map[zodb.Tid]string, trackSet blib.PPTreeSubSet, vδTok ...map[Key]Δstring) { // tracked state defined by initialTrackedKeys.
t.Helper() func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1,t2 *xbtreetest.Commit, initialTrackedKeys setKey, kadj KAdjMatrix) {
X := exc.Raiseif X := exc.Raiseif
ø := blib.PPTreeSubSet{} //t.Logf("\n>>> Track=%s\n", initialTrackedKeys)
subj := fmt.Sprintf("after Update(@%s→@%s)", xat[ti.At], xat[tj.At])
// Update ati -> atj δZ := t2.ΔZ
δB, err := δbtail.Update(tj.ΔZ); X(err) d12 := t2.Δxkv
δbtail.assertTrack(t, subj, trackSet, ø)
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...)
// assert δB = vδTok[-1] var TrackedδZ setKey = nil
var δT, δTok map[Key]Δstring var kadjTrackedδZ setKey = nil
if l := len(vδTok); l > 0 { var δT, δTok map[Key]Δstring = nil, nil
δTok = vδTok[l-1] δZset := setOid{}
} for _, oid := range δZ.Changev {
if len(δTok) == 0 { δZset.Add(oid)
δTok = nil
}
δrootsOK := 1
if δTok == nil {
δrootsOK = 0
} }
δroots := setOid{} // badf queues error message to be reported on return.
for root := range δbtail.vδTbyRoot { var badv []string
δroots.Add(root) badf := func(format string, argv ...interface{}) {
badv = append(badv, fmt.Sprintf(format, argv...))
} }
δToid, ok := δB.ΔByRoot[treeRoot] defer func() {
if ok { if badv != nil || t.Failed() {
δT = XGetδKV(ti, tj, δToid) emsg := fmt.Sprintf("%s ; tracked=%v :\n\n", subj, initialTrackedKeys)
emsg += fmt.Sprintf("d12: %v\nδTok: %v\nδT: %v\n\n", d12, δTok, δT)
emsg += fmt.Sprintf("δZ: %v\n", δZset)
emsg += fmt.Sprintf("Tracked^δZ: %v\n", TrackedδZ)
emsg += fmt.Sprintf("kadj[Tracked^δZ]: %v\n", kadjTrackedδZ)
emsg += fmt.Sprintf("kadj: %v\n\n", kadj)
emsg += strings.Join(badv, "\n")
emsg += "\n"
t.Fatal(emsg)
}
}()
// δbtail @at1 with initial tracked set
δbtail := NewΔBtail(t1.At, db)
xtrackKeys(δbtail, t1, initialTrackedKeys)
// TrackedδZ = Tracked ^ δZ (i.e. a tracked node has changed, or its coverage was changed)
TrackedδZ = setKey{}
for k := range initialTrackedKeys {
leaf1 := t1.Xkv.Get(k)
oid1 := leaf1.Oid
if oid1 == zodb.InvalidOid { // embedded bucket
oid1 = leaf1.Parent.Oid
}
leaf2 := t2.Xkv.Get(k)
oid2 := leaf2.Oid
if oid2 == zodb.InvalidOid { // embedded bucket
oid2 = leaf2.Parent.Oid
}
if δZset.Has(oid1) || δZset.Has(oid2) || (leaf1.Keycov != leaf2.Keycov) {
TrackedδZ.Add(k)
}
} }
if δB.Rev != tj.At {
t.Errorf("%s: δB.Rev: have %s ; want %s", subj, δB.Rev, tj.At) kadjTrackedδZ = setKey{} // kadj[Tracked^δZ] (all keys adjacent to tracked^δZ)
for k := range TrackedδZ {
kadjTrackedδZ.Update(kadj[k])
} }
if len(δB.ΔByRoot) != δrootsOK {
t.Errorf("%s: len(δB.ΔByRoot) != %d ; δroots=%v", subj, δrootsOK, δroots)
// assert TrackedδZ ∈ kadj[TrackedδZ]
trackNotInKadj := TrackedδZ.Difference(kadjTrackedδZ)
if len(trackNotInKadj) > 0 {
badf("BUG: Tracked^δZ ∉ kadj[Tracked^δZ] ; extra=%v", trackNotInKadj)
return
} }
if !δTEqual(δT, δTok) {
t.Errorf("%s: δB.ΔBByRoot[%s]:\nhave: %v\nwant: %v", subj, treeRoot, δT, δTok) // k ∈ d12
// k ∈ δT <=>
// k ∈ U kadj[·]
// ·∈tracking^δZ
δTok = map[Key]Δstring{} // d12[all keys that should be present in δT]
for k,δv := range d12 {
if kadjTrackedδZ.Has(k) {
δTok[k] = δv
}
} }
}
// xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys setKey, trackSet blib.PPTreeSubSet, trackNew, trackSetAfterRebuild blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
ø := blib.PPTreeSubSet{} ø := blib.PPTreeSubSet{}
// Track(keys) // trackSet1 = xkv1[tracked1]
xtrackKeys(δbtail, tj, keys) // trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet1, tkeyCov1 := trackSetWithCov(t1.Xkv, initialTrackedKeys)
trackSet2, tkeyCov2 := trackSetWithCov(t2.Xkv, initialTrackedKeys.Union(kadjTrackedδZ))
subj := fmt.Sprintf("@%s: after Track%v", xat[tj.At], keys) // verify δbtail.trackSet against @at1
δbtail.assertTrack(t, subj, trackSet, trackNew) δbtail.assertTrack(t, "1", ø, trackSet1)
δbtail.rebuildAll() // δB <- δZ
//
// also call _Update1 directly to verify δtkeycov return from treediff
// the result of Update and _Update1 should be the same since δbtail is initially empty.
δbtail_ := δbtail.Clone()
δB1, err := δbtail_._Update1(δZ); X(err) // XXX don't compute treediff twice
subj += " + rebuild" // assert tkeyCov1 ⊂ tkeyCov2
δbtail.assertTrack(t, subj, trackSetAfterRebuild, ø) dkeycov12 := tkeyCov1.Difference(tkeyCov2)
if !dkeycov12.Empty() {
t.Errorf("BUG: tkeyCov1 ⊄ tkeyCov2:\n\ttkeyCov1: %s\n\ttkeyCov2: %s\n\ttkeyCov1 \\ tkeyCov2: %s", tkeyCov1, tkeyCov2, dkeycov12)
}
// XXX verify Get -> XXX assertΔTtail ? // assert δtkeycov == δ(tkeyCov1, tkeyCov2)
δtkeycovOK := tkeyCov2.Difference(tkeyCov1)
δtkeycov := &blib.RangedKeySet{}
if __, ok := δB1.ByRoot[treeRoot]; ok {
δtkeycov = __.δtkeycov1
}
if !δtkeycov.Equal(δtkeycovOK) {
badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
}
// verify δbtail.vδTbyRoot[treeRoot] δB, err := δbtail.Update(δZ); X(err)
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...) // XXX assert δB.roots == δTKeyCov roots
} // XXX assert δBtail[root].vδT = δBtail_[root].vδT
// assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail. if δB.Rev != δZ.Tid {
// it also verifies that δbtail.vδBroots matches ΔTtail data. badf("δB: rev != δZ.Tid ; rev=%s δZ.Tid=%s", δB.Rev, δZ.Tid)
func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) { return
t.Helper() }
// XXX +KVAtTail, +lastRevOf
l := len(vδTok) // verify δbtail.trackSet against @at2
var vatOK []zodb.Tid δbtail.assertTrack(t, "2", trackSet2, ø)
var vδTok_ []map[Key]Δstring
at2t := map[zodb.Tid]*xbtreetest.Commit{tj.At: tj}
t0 := tj // assert δB.ByRoot == {treeRoot -> ...} if δTok != ø
for i := 0; i<l; i++ { // == ø if δTok == ø
// empty vδTok entries means they should be absent in vδT rootsOK := setOid{}
if δTok := vδTok[l-i-1]; len(δTok) != 0 { if len(δTok) > 0 {
vatOK = append([]zodb.Tid{t0.At}, vatOK...) rootsOK.Add(treeRoot)
vδTok_ = append([]map[Key]Δstring{δTok}, vδTok_...)
}
t0 = t0.Prev
at2t[t0.At] = t0
} }
vδTok = vδTok_ roots := setOid{}
δTtail, ok := δbtail.vδTbyRoot[treeRoot] for root := range δB.ΔByRoot {
var vδToid []ΔTree roots.Add(root)
if ok {
vδToid = δTtail.vδT
} }
if !reflect.DeepEqual(roots, rootsOK) {
l = len(vδToid) badf("δB: roots != rootsOK ; roots=%v rootsOK=%v", roots, rootsOK)
var vat []zodb.Tid }
var vδT []map[Key]Δstring _, inδB := δB.ΔByRoot[treeRoot]
atPrev := t0.At if !inδB {
for _, δToid := range vδToid { return
vat = append(vat, δToid.Rev)
δT := XGetδKV(at2t[atPrev], at2t[δToid.Rev], δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
vδT = append(vδT, δT)
atPrev = δToid.Rev
} }
var vatδB []zodb.Tid // δbtail.vδBroots/treeRoot
for _, δBroots := range δbtail.vδBroots { // δT <- δB
if δBroots.ΔRoots.Has(treeRoot) { δToid := δB.ΔByRoot[treeRoot] // {} k -> δoid
vatδB = append(vatδB, δBroots.Rev) δT = XGetδKV(t1,t2, δToid) // {} k -> δ(ZBlk(oid).data)
// δT must be subset of d12.
// changed keys, that are
// - in tracked set -> must be present in δT
// - outside tracked set -> may be present in δT (kadj gives exact answer)
// δT is subset of d12
for _, k := range sortedKeys(δT) {
_, ind12 := d12[k]
if !ind12 {
badf("δT[%v] ∉ d12", k)
} }
} }
tok := tidvEqual(vat, vatOK) && vδTEqual(vδT, vδTok) // k ∈ tracked set -> must be present in δT
bok := tidvEqual(vatδB, vatOK) // k ∉ tracked set -> may be present in δT (kadj gives exact answer)
if !(tok && bok) { for _, k := range sortedKeys(d12) {
emsg := fmt.Sprintf("%s: vδT:\n", subj) _, inδT := δT[k]
have := "" _, inδTok := δTok[k]
for i := 0; i<len(vδT); i++ { if inδT && !inδTok {
have += fmt.Sprintf("\n\t@%s: %v", xat[vat[i]], vδT[i]) badf("δT[%v] ∉ δTok", k)
} }
emsg += fmt.Sprintf("have: %s\n", have)
if !tok { if !inδT && inδTok {
want := "" badf("δT ∌ δTok[%v]", k)
for i := 0; i<len(vδTok); i++ {
want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
}
emsg += fmt.Sprintf("want: %s\n", want)
} }
if !bok { if inδT {
vδb_root := "" if δT[k] != d12[k] {
for i := 0; i<len(vatδB); i++ { badf("δT[%v] ≠ δTok[%v]", k, k)
vδb_root += fmt.Sprintf("\n\t@%s", xat[vatδB[i]])
} }
emsg += fmt.Sprintf("vδb/root: %s\n", vδb_root)
} }
t.Error(emsg)
} }
} }
// xtrackKeys issues δbtail.Track requests for tree[keys]. // xverifyΔBTail_rebuild verifies ΔBtail.rebuild during t0->t1->t2 transition.
// XXX place //
func xtrackKeys(δbtail *ΔBtail, t *xbtreetest.Commit, keys setKey) { // t0->t1 exercises from-scratch rebuild,
X := exc.Raiseif // t1->t2 further exercises incremental rebuild.
head := δbtail.Head() //
if head != t.At { // It also exercises rebuild phase of ΔBtail.Update.
panicf("BUG: δbtail.head: %s ; t.at: %s", head, t.At) func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *xbtreetest.Commit) {
} t.Run(fmt.Sprintf("rebuild/%s→%s", t0.Tree, t1.Tree), func(t *testing.T) {
tAllKeys := allTestKeys(t0, t1, t2)
tAllKeyv := tAllKeys.SortedElements()
for k := range keys { // tid -> "at_i"
// NOTE: if tree is deleted - the following adds it to tracked xat := map[zodb.Tid]string{
// set with every key being a hole. This aligns with the t0.At: "at0",
// following situation t1.At: "at1",
// t2.At: "at2",
// T1 -> ø -> T2 }
//
// where after T1->ø, even though the tree becomes deleted, its root
// continues to be tracked and all keys migrate to holes in the
// tracking set. By aligning initial state to the same as after
// T1->ø, we test what will happen on ø->T2.
b := t.Xkv.Get(k)
err := δbtail.track(k, b.Path()); X(err)
}
}
// xverifyΔBTail_GetAt verifies δBtail.Get on series of vt ZODB changes. //fmt.Printf("@%s: %v\n", xat[t0.At], t0.Xkv.Flatten())
// XXX //fmt.Printf("@%s: %v\n", xat[t1.At], t1.Xkv.Flatten())
// XXX kill //fmt.Printf("@%s: %v\n", xat[t2.At], t2.Xkv.Flatten())
kadj10 := KAdj(t1,t0, allTestKeys(t0,t1,t2))
kadj21 := KAdj(t2,t1, allTestKeys(t0,t1,t2))
kadj12 := KAdj(t1,t2, allTestKeys(t0,t1,t2))
// kadj210 = kadj10·kadj21
kadj210 := kadj10.Mul(kadj21)
ø := blib.PPTreeSubSet{}
// verify t0 -> t1 Track(keys1) Rebuild -> t2 Track(keys2) Rebuild
// for all combinations of keys1 and keys2
for k1idx := range IntSets(len(tAllKeyv)) {
keys1 := setKey{}
for _, idx1 := range k1idx {
keys1.Add(tAllKeyv[idx1])
}
// δkv1_1 = t1.δxkv / kadj10(keys1)
keys1_0 := kadj10.Map(keys1)
δkv1_1 := map[Key]Δstring{}
for k := range keys1_0 {
δv, ok := t1.Δxkv[k]
if ok {
δkv1_1[k] = δv
}
}
Tkeys1 := trackSet(t1.Xkv, keys1)
Tkeys1_0 := trackSet(t1.Xkv, keys1_0)
t.Run(fmt.Sprintf(" T%s;R", keys1), func(t *testing.T) {
δbtail := NewΔBtail(t0.At, db)
// assert trackSet=ø, trackNew=ø, vδB=[]
δbtail.assertTrack(t, "@at0", ø, ø)
assertΔTtail(t, "@at0", δbtail, t0, treeRoot, xat,
/*vδT=ø*/)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t0, t1, xat,
/*trackSet=*/ø,
/*vδT=ø*/)
xverifyΔBTail_rebuild_TR(t, δbtail, t1, treeRoot, xat,
// after Track(keys1)
keys1,
/*trackSet=*/ ø,
/*trackNew=*/ Tkeys1,
// after rebuild
/*trackSet=*/ Tkeys1_0,
/*vδT=*/ δkv1_1)
t.Run((" →" + t2.Tree), func(t *testing.T) {
// keys1R2 is full set of keys that should become tracked after
// Update() (which includes rebuild)
keys1R2 := kadj12.Map(keys1)
for {
keys1R2_ := kadj210.Map(keys1R2)
if keys1R2.Equal(keys1R2_) {
break
}
keys1R2 = keys1R2_
}
// δkvX_k1R2 = tX.δxkv / keys1R2
δkv1_k1R2 := map[Key]Δstring{}
δkv2_k1R2 := map[Key]Δstring{}
for k := range keys1R2 {
δv1, ok := t1.Δxkv[k]
if ok {
δkv1_k1R2[k] = δv1
}
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k1R2[k] = δv2
}
}
Tkeys1R2 := trackSet(t2.Xkv, keys1R2)
xverifyΔBTail_rebuild_U(t, δbtail, treeRoot, t1, t2, xat,
/*trackSet=*/ Tkeys1R2,
/*vδT=*/ δkv1_k1R2, δkv2_k1R2)
// tRestKeys2 = tAllKeys - keys1
// reduce that to = tAllKeys - keys1R2 in short mode
// ( if key from keys2 already became tracked after Track(keys1) + Update,
// adding Track(that-key), is not adding much testing coverage to recompute paths )
var tRestKeys2 setKey
if testing.Short() {
tRestKeys2 = tAllKeys.Difference(keys1R2)
} else {
tRestKeys2 = tAllKeys.Difference(keys1)
}
tRestKeyv2 := tRestKeys2.SortedElements()
for k2idx := range IntSets(len(tRestKeyv2)) {
keys2 := setKey{}
for _, idx2 := range k2idx {
keys2.Add(tRestKeyv2[idx2])
}
// keys12R2 is full set of keys that should become tracked after
// Track(keys2) + rebuild
keys12R2 := keys1R2.Union(keys2)
for {
keys12R2_ := kadj210.Map(keys12R2)
if keys12R2.Equal(keys12R2_) {
break
}
keys12R2 = keys12R2_
}
Tkeys2 := trackSet(t2.Xkv, keys2)
Tkeys12R2 := trackSet(t2.Xkv, keys12R2)
/*
fmt.Printf("\n\n\nKKK\nkeys1=%s keys2=%s\n", keys1, keys2)
fmt.Printf("keys1R2: %s\n", keys1R2)
fmt.Printf("keys12R2: %s\n", keys12R2)
fmt.Printf("t0.Xkv: %v\n", t0.Xkv)
fmt.Printf("t1.Xkv: %v\n", t1.Xkv)
fmt.Printf("t2.Xkv: %v\n", t2.Xkv)
fmt.Printf("kadj21: %v\n", kadj21)
fmt.Printf("kadj12: %v\n", kadj12)
fmt.Printf("Tkeys2 -> %s\n", Tkeys2)
fmt.Printf("Tkeys1R2 -> %s\n", Tkeys1R2)
fmt.Printf("Tkeys2 \\ Tkeys1R2 -> %s\n", Tkeys2.Difference(Tkeys1R2))
fmt.Printf("\n\n\n")
*/
// δkvX_k12R2 = tX.δxkv / keys12R2
δkv1_k12R2 := make(map[Key]Δstring, len(t1.Δxkv))
δkv2_k12R2 := make(map[Key]Δstring, len(t2.Δxkv))
for k := range keys12R2 {
δv1, ok := t1.Δxkv[k]
if ok {
δkv1_k12R2[k] = δv1
}
δv2, ok := t2.Δxkv[k]
if ok {
δkv2_k12R2[k] = δv2
}
}
// t.Run is expensive at this level of nest
//t.Run(" T"+keys2.String()+";R", func(t *testing.T) {
δbtail_ := δbtail.Clone()
xverifyΔBTail_rebuild_TR(t, δbtail_, t2, treeRoot, xat,
// after Track(keys2)
keys2,
/*trackSet*/ Tkeys1R2,
/*trackNew*/ Tkeys2.Difference(
// trackNew should not cover ranges that are
// already in trackSet
Tkeys1R2),
// after rebuild
/* trackSet=*/ Tkeys12R2,
/*vδT=*/ δkv1_k12R2, δkv2_k12R2)
//})
}
})
})
}
})
}
// xverifyΔBTail_rebuild_U verifies ΔBtail state after Update(ti->tj).
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, treeRoot zodb.Oid, ti, tj *xbtreetest.Commit, xat map[zodb.Tid]string, trackSet blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
X := exc.Raiseif
ø := blib.PPTreeSubSet{}
subj := fmt.Sprintf("after Update(@%s→@%s)", xat[ti.At], xat[tj.At])
// Update ati -> atj
δB, err := δbtail.Update(tj.ΔZ); X(err)
δbtail.assertTrack(t, subj, trackSet, ø)
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...)
// assert δB = vδTok[-1]
var δT, δTok map[Key]Δstring
if l := len(vδTok); l > 0 {
δTok = vδTok[l-1]
}
if len(δTok) == 0 {
δTok = nil
}
δrootsOK := 1
if δTok == nil {
δrootsOK = 0
}
δroots := setOid{}
for root := range δbtail.vδTbyRoot {
δroots.Add(root)
}
δToid, ok := δB.ΔByRoot[treeRoot]
if ok {
δT = XGetδKV(ti, tj, δToid)
}
if δB.Rev != tj.At {
t.Errorf("%s: δB.Rev: have %s ; want %s", subj, δB.Rev, tj.At)
}
if len(δB.ΔByRoot) != δrootsOK {
t.Errorf("%s: len(δB.ΔByRoot) != %d ; δroots=%v", subj, δrootsOK, δroots)
}
if !δTEqual(δT, δTok) {
t.Errorf("%s: δB.ΔBByRoot[%s]:\nhave: %v\nwant: %v", subj, treeRoot, δT, δTok)
}
}
// xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
func xverifyΔBTail_rebuild_TR(t *testing.T, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys setKey, trackSet blib.PPTreeSubSet, trackNew, trackSetAfterRebuild blib.PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
ø := blib.PPTreeSubSet{}
// Track(keys)
xtrackKeys(δbtail, tj, keys)
subj := fmt.Sprintf("@%s: after Track%v", xat[tj.At], keys)
δbtail.assertTrack(t, subj, trackSet, trackNew)
δbtail.rebuildAll()
subj += " + rebuild"
δbtail.assertTrack(t, subj, trackSetAfterRebuild, ø)
// XXX verify Get -> XXX assertΔTtail ?
// verify δbtail.vδTbyRoot[treeRoot]
assertΔTtail(t, subj, δbtail, tj, treeRoot, xat, vδTok...)
}
// xverifyΔBTail_GetAt verifies δBtail.Get on series of vt ZODB changes.
// XXX
// XXX kill
/* /*
func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*xbtreetest.Commit) { func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*xbtreetest.Commit) {
subj := vt[0].Tree subj := vt[0].Tree
...@@ -1000,565 +1159,415 @@ func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*x ...@@ -1000,565 +1159,415 @@ func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*x
// ---------------------------------------- // ----------------------------------------
// ΔBTestEntry represents one entry in ΔBTail tests. func TestΔBtailForget(t_ *testing.T) {
type ΔBTestEntry struct { t := xbtreetest.NewT(t_)
tree string // next tree topology X := exc.Raiseif
kadjOK KAdjMatrix // adjacency matrix against previous case (optional)
flags ΔBTestFlags
}
type ΔBTestFlags int
const ΔBTest_SkipUpdate ΔBTestFlags = 1 // skip verifying Update for this test entry
const ΔBTest_SkipRebuild ΔBTestFlags = 2 // skip verifying rebuild for this test entry
// ΔBTest converts xtest into ΔBTestEntry. t0 := t.CommitTree("T/B:")
// xtest can be string|ΔBTestEntry. t1 := t.CommitTree("T/B1:a")
func ΔBTest(xtest interface{}) ΔBTestEntry { t2 := t.CommitTree("T2/B1:a-B2:b")
var test ΔBTestEntry t3 := t.CommitTree("T/B2:b")
switch xtest := xtest.(type) {
case string: δbtail := NewΔBtail(t0.At, t.DB)
test.tree = xtest _, err := δbtail.Update(t1.ΔZ); X(err)
test.kadjOK = nil _, err = δbtail.Update(t2.ΔZ); X(err)
test.flags = 0
case ΔBTestEntry: // start tracking. everything becomes tracked because t1's T/B1:a has [-∞,∞) coverage
test = xtest // By starting tracking after t2 we verify vδBroots update in both Update and rebuild
default: _0 := setKey{}; _0.Add(0)
panicf("BUG: ΔBTest: bad type %T", xtest) xtrackKeys(δbtail, t2, _0)
_, err = δbtail.Update(t3.ΔZ); X(err)
xat := map[zodb.Tid]string{
t0.At: "at0",
t1.At: "at1",
t2.At: "at2",
t3.At: "at3",
} }
return test assertΔTtail(t.T, "init", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t0.At)
assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t1.At)
assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t3.At)
assertΔTtail(t.T, "forget ≤ at3", δbtail, t3, t.Root(), xat, )
} }
// testΔBTail verifies ΔBTail on sequence of tree topologies coming from testq. func TestΔBtailClone(t_ *testing.T) {
func testΔBTail(t_ *testing.T, testq chan ΔBTestEntry) { // ΔBtail.Clone had bug that aliased klon data to orig
t := xbtreetest.NewT(t_) t := xbtreetest.NewT(t_)
X := exc.Raiseif
var t0 *xbtreetest.Commit t0 := t.CommitTree("T2/B1:a-B2:b")
for test := range testq { t1 := t.CommitTree("T2/B1:c-B2:d")
t1 := t.Head() δbtail := NewΔBtail(t0.At, t.DB)
t2 := t.CommitTree(test.tree) _, err := δbtail.Update(t1.ΔZ); X(err)
_2 := setKey{}; _2.Add(2)
xtrackKeys(δbtail, t1, _2)
err = δbtail.rebuildAll(); X(err)
subj := fmt.Sprintf("%s -> %s", t1.Tree, t2.Tree) xat := map[zodb.Tid]string{
//t.Logf("\n\n\n**** %s ****\n\n", subj) t0.At: "at0",
t1.At: "at1",
}
// KAdj δkv1_1 := map[Key]Δstring{2:{"b","d"}}
if kadjOK := test.kadjOK; kadjOK != nil { assertΔTtail(t.T, "orig @at1", δbtail, t1, t.Root(), xat, δkv1_1)
t.Run(fmt.Sprintf("KAdj/%s→%s", t1.Tree, t2.Tree), func(t *testing.T) { δbklon := δbtail.Clone()
kadj := KAdj(t1, t2) assertΔTtail(t.T, "klon @at1", δbklon, t1, t.Root(), xat, δkv1_1)
if !reflect.DeepEqual(kadj, kadjOK) {
t.Fatalf("BUG: computed kadj is wrong:\nkadjOK: %v\nkadj : %v\n\n", kadjOK, kadj)
}
})
}
// ΔBTail.Update t2 := t.CommitTree("T/B1:b,2:a")
if test.flags & ΔBTest_SkipUpdate == 0 { _, err = δbtail.Update(t2.ΔZ); X(err)
xverifyΔBTail_Update(t.T, subj, t.DB, t.Root(), t1,t2) xat[t2.At] = "at2"
}
// ΔBTail.rebuild δkv1_2 := map[Key]Δstring{1:{"a","c"}, 2:{"b","d"}}
if t0 != nil && (test.flags & ΔBTest_SkipRebuild == 0) { δkv2_2 := map[Key]Δstring{1:{"c","b"}, 2:{"d","a"}}
xverifyΔBTail_rebuild(t.T, t.DB, t.Root(), t0,t1,t2) assertΔTtail(t.T, "orig @at2", δbtail, t2, t.Root(), xat, δkv1_2, δkv2_2)
} assertΔTtail(t.T, "klon @at1 after orig @at->@at2", δbklon, t1, t.Root(), xat, δkv1_1)
}
t0, t1 = t1, t2
// -------- KAdj --------
// Map returns kadj·keys.
func (kadj KAdjMatrix) Map(keys setKey) setKey {
res := make(setKey, len(keys))
for k := range keys {
to, ok := kadj[k]
if !ok {
panicf("kadj.Map: %d ∉ kadj\n\nkadj: %v", k, kadj)
}
res.Update(to)
} }
return res
} }
// TestΔBTail verifies ΔBTail for explicitly provided tree topologies. // Mul returns kadjA·kadjB.
func TestΔBTail(t *testing.T) { //
// K is shorthand for setKey // (kadjA·kadjB).Map(keys) = kadjA.Map(kadjB.Map(keys))
K := func(keyv ...Key) setKey { func (kadjA KAdjMatrix) Mul(kadjB KAdjMatrix) KAdjMatrix {
ks := setKey{} // ~ assert kadjA.keys == kadjB.keys
for _, k := range keyv { ks.Add(k) } // check only len here; the rest will be asserted by Map
return ks if len(kadjA) != len(kadjB) {
} panicf("kadj.Mul: different keys:\n\nkadjA: %v\nkadjB: %v", kadjA, kadjB)
// oo is shorthand for KeyMax
const oo = KeyMax
// A is shorthand for KAdjMatrix
type A = KAdjMatrix
// Δ is shorthand for ΔBTestEntry
Δ := func(tree string, kadjOK A) (test ΔBTestEntry) {
test.tree = tree
test.kadjOK = kadjOK
return test
} }
// test known cases going through tree1 -> tree2 -> ... kadj := make(KAdjMatrix, len(kadjB))
testv := []interface{} { for k, tob := range kadjB {
// start from non-empty tree to verify both ->empty and empty-> transitions kadj[k] = kadjA.Map(tob)
"T/B1:a,2:b", }
return kadj
}
// empty // KAdj computes adjacency matrix for t1 -> t2 transition.
"T/B:", //
// The set of keys for which kadj matrix is computed can be optionally provided.
// This set of keys defaults to allTestKeys(t1,t2).
//
// KAdj itself is verified by testΔBTail on entries with .kadjOK set.
func KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
// assert KAdj(A,B) == KAdj(B,A)
kadj12 := _KAdj(t1,t2, keysv...)
kadj21 := _KAdj(t2,t1, keysv...)
if !reflect.DeepEqual(kadj12, kadj21) {
panicf("KAdj not symmetric:\nt1: %s\nt2: %s\nkadj12: %v\nkadj21: %v",
t1.Tree, t2.Tree, kadj12, kadj21)
}
return kadj12
}
// +1 const debugKAdj = false
Δ("T/B1:a", func debugfKAdj(format string, argv ...interface{}) {
A{1: K(1,oo), if debugKAdj {
oo: K(1,oo)}), fmt.Printf(format, argv...)
}
}
// +2 func _KAdj(t1, t2 *xbtreetest.Commit, keysv ...setKey) (kadj KAdjMatrix) {
Δ("T/B1:a,2:b", var keys setKey
A{1: K(1,2,oo), switch len(keysv) {
2: K(1,2,oo), case 0:
oo: K(1,2,oo)}), keys = allTestKeys(t1, t2)
case 1:
keys = keysv[0]
default:
panic("multiple key sets on the call")
}
// -1 debugfKAdj("\n\n_KAdj\n")
Δ("T/B2:b", debugfKAdj("t1: %s\n", t1.Tree)
A{1: K(1,2,oo), debugfKAdj("t2: %s\n", t2.Tree)
2: K(1,2,oo), debugfKAdj("keys: %s\n", keys)
oo: K(1,2,oo)}), defer func() {
debugfKAdj("kadj -> %v\n", kadj)
}()
// 2: b->c // kadj = {} k -> adjacent keys.
Δ("T/B2:c", // if k is tracked and covered by changed leaf -> changes to adjacents must be in Update(t1->t2).
A{2: K(2,oo), kadj = KAdjMatrix{}
oo: K(2,oo)}), for k := range keys {
adj1 := setKey{}
adj2 := setKey{}
// +1 in new bucket (to the left) q1 := &blib.RangedKeySet{}; q1.Add(k)
Δ("T2/B1:a-B2:c", q2 := &blib.RangedKeySet{}; q2.Add(k)
A{1: K(1,2,oo), done1 := &blib.RangedKeySet{}
2: K(1,2,oo), done2 := &blib.RangedKeySet{}
oo: K(1,2,oo)}),
// +3 in new bucket (to the right) debugfKAdj("\nk%s\n", kstr(k))
Δ("T2,3/B1:a-B2:c-B3:c", for !q1.Empty() || !q2.Empty() {
A{1: K(1), debugfKAdj("q1: %s\tdone1: %s\n", q1, done1)
2: K(2,3,oo), debugfKAdj("q2: %s\tdone2: %s\n", q2, done2)
3: K(2,3,oo), for _, r1 := range q1.AllRanges() {
oo: K(2,3,oo)}), lo1 := r1.Lo
for {
b1 := t1.Xkv.Get(lo1)
debugfKAdj(" b1: %s\n", b1)
for k_ := range keys {
if b1.Keycov.Has(k_) {
adj1.Add(k_)
debugfKAdj(" adj1 += %s\t-> %s\n", kstr(k_), adj1)
}
}
done1.AddRange(b1.Keycov)
// q2 |= (b1.keyrange \ done2)
δq2 := &blib.RangedKeySet{}
δq2.AddRange(b1.Keycov)
δq2.DifferenceInplace(done2)
q2.UnionInplace(δq2)
debugfKAdj("q2 += %s\t-> %s\n", δq2, q2)
// bucket split; +3 in new bucket // continue with next right bucket until r1 coverage is complete
"T/B1:a,2:b", if r1.Hi_ <= b1.Keycov.Hi_ {
Δ("T2/B1:a-B2:b,3:c", break
A{1: K(1,2,3,oo), }
2: K(1,2,3,oo), lo1 = b1.Keycov.Hi_ + 1
3: K(1,2,3,oo), }
oo: K(1,2,3,oo)}), }
q1.Clear()
// bucket split; +3 in new bucket; +4 +5 in another new bucket for _, r2 := range q2.AllRanges() {
// everything becomes tracked because original bucket had [-∞,∞) coverage lo2 := r2.Lo
"T/B1:a,2:b", for {
Δ("T2,4/B1:a-B2:b,3:c-B4:d,5:e", b2 := t2.Xkv.Get(lo2)
A{1: K(1,2,3,4,5,oo), debugfKAdj(" b2: %s\n", b2)
2: K(1,2,3,4,5,oo), for k_ := range keys {
3: K(1,2,3,4,5,oo), if b2.Keycov.Has(k_) {
4: K(1,2,3,4,5,oo), adj2.Add(k_)
5: K(1,2,3,4,5,oo), debugfKAdj(" adj2 += %s\t-> %s\n", kstr(k_), adj2)
oo: K(1,2,3,4,5,oo)}), }
}
// reflow of keys: even if tracked={1}, changes to all B nodes need to be rescanned: done2.AddRange(b2.Keycov)
// +B12 forces to look in -B23 which adds -3 into δ, which // q1 |= (b2.keyrange \ done1)
// forces to look into +B34 and so on. δq1 := &blib.RangedKeySet{}
"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g", δq1.AddRange(b2.Keycov)
Δ("T3,5,7/B1:g,2:f-B3:e,4:d-B5:c,6:b-B7:a", δq1.DifferenceInplace(done1)
A{1: K(1,2,3,4,5,6,7,oo), q1.UnionInplace(δq1)
2: K(1,2,3,4,5,6,7,oo), debugfKAdj("q1 += %s\t-> %s\n", δq1, q1)
3: K(1,2,3,4,5,6,7,oo),
4: K(1,2,3,4,5,6,7,oo),
5: K(1,2,3,4,5,6,7,oo),
6: K(1,2,3,4,5,6,7,oo),
7: K(1,2,3,4,5,6,7,oo),
oo: K(1,2,3,4,5,6,7,oo)}),
// reflow of keys for rebuild: even if tracked1={}, tracked2={1}, changes to
// all A/B/C nodes need to be rescanned. Contrary to the above case the reflow
// is not detectable at separate diff(A,B) and diff(B,C) runs.
"T3,5,7/B1:a,2:b-B3:c,4:d-B5:e,6:f-B7:g,8:h",
"T/B1:b",
"T2,4,6/B1:a-B2:b,3:c-B4:d,5:e-B6:f,7:g",
// similar situation where rebuild has to detect reflow in between non-neighbour trees
"T3,6/B1:a,2:b-B3:c,4:d-B6:f,7:g",
"T4,7/B1:b-B4:d,5:e-B7:g,8:h",
"T2,5,8/B1:a-B2:b,3:c-B5:e,6:f-B8:h,9:i",
// depth=2; bucket split; +3 in new bucket; left T remain
// _unchanged_ even though B under it is modified.
"T/T/B1:a,2:b",
Δ("T2/T-T/B1:a-B2:b,3:c",
A{1: K(1,2,3,oo),
2: K(1,2,3,oo),
3: K(1,2,3,oo),
oo: K(1,2,3,oo)}),
// depth=2; like prev. case, but additional right arm with +4 +5 is added.
"T/T/B1:a,2:b",
Δ("T2,4/T-T-T/B1:a-B2:b,3:c-B4:d,5:e",
A{1: K(1,2,3,4,5,oo),
2: K(1,2,3,4,5,oo),
3: K(1,2,3,4,5,oo),
4: K(1,2,3,4,5,oo),
5: K(1,2,3,4,5,oo),
oo: K(1,2,3,4,5,oo)}),
// depth=2; bucket split; +3 in new bucket; t0 and t1 split;
// +right arm (T7/B45-B89).
"T/T/B1:a,2:b",
Δ("T4/T2-T7/B1:a-B2:b,3:c-B4:d,5:e-B8:h,9:i",
A{1: K(1,2,3,4,5,8,9,oo),
2: K(1,2,3,4,5,8,9,oo),
3: K(1,2,3,4,5,8,9,oo),
4: K(1,2,3,4,5,8,9,oo),
5: K(1,2,3,4,5,8,9,oo),
8: K(1,2,3,4,5,8,9,oo),
9: K(1,2,3,4,5,8,9,oo),
oo: K(1,2,3,4,5,8,9,oo)}),
// 2 reflow to right B neighbour; 8 splits into new B; δ=ø
"T3/B1:a,2:b-B4:d,8:h",
"T2,5/B1:a-B2:b,4:d-B8:h",
// case where kadj does not grow too much as leafs coverage remains stable
"T4,8/B1:a,2:b-B5:d,6:e-B10:g,11:h",
Δ("T4,8/B2:b,3:c-B6:e,7:f-B11:h,12:i",
A{1: K(1,2,3),
2: K(1,2,3),
3: K(1,2,3),
5: K(5,6,7),
6: K(5,6,7),
7: K(5,6,7,),
10: K(10,11,12,oo),
11: K(10,11,12,oo),
12: K(10,11,12,oo),
oo: K(10,11,12,oo)}),
// tree deletion
// having ø in the middle of the test cases exercises all:
// * `ø -> Tree ...` (tree is created anew),
// * `... Tree -> ø` (tree is deleted), and
// * `Tree -> ø -> Tree` (tree is deleted and then recreated)
xbtreetest.DEL,
// tree rotation
"T3/B2:b-B3:c,4:d",
"T5/T3-T7/B2:a-B3:a,4:a-B6:a-B8:a",
// found by AllStructs ([1] is not changed, but because B1 is
// unlinked and 1 migrates to other bucket, changes in that
// other bucket must be included into δT)
"T1,2/B0:e-B1:d-B2:g,3:a",
"T1/B0:d-B1:d,2:d",
// ----//---- with depth=2
"T1,2/T-T-T/B0:a-B1:b-B2:c,3:d",
"T1/T-T/B0:e-B1:b,2:f",
// XXX depth=3 (to verify recursion and selecting which tree children to follow or not)
// degenerate topology from ZODB tests
// https://github.com/zopefoundation/ZODB/commit/6cd24e99f89b
// https://github.com/zopefoundation/BTrees/blob/4.7.2-1-g078ba60/BTrees/tests/testBTrees.py#L20-L57
"T4/T2-T/T-T-T6,10/B1:a-B3:b-T-T-T/T-B7:c-B11:d/B5:e",
"T/B1:e,5:d,7:c,8:b,11:a", // -3 +8
// was leading treegen to generate corrupt trees
"T/T1/T-T/B0:g-B1:e,2:d,3:h",
"T1/T-T3/B0:g-T-T/B1:e,2:d-B3:h",
// was leading to wrongly computed trackSet2 due to top not
// being tracked to tree root.
"T/T1/B0:a-B1:b",
"T/T1/T-T/B0:c-B1:d",
// was leading to wrongly computed trackSet2: leaf bucket not
// reparented to root.
"T/T/B0:a",
"T/B0:a",
// δtkeycov grows due to change in parent tree only
"T3/B1:a-B8:c",
"T7/B1:a-B8:c",
// ----//----
"T3/B1:a,2:b-B8:c,9:d",
"T7/B1:a,2:b-B8:c,9:d",
// ----//---- depth=2
"T3/T-T/B1:a,2:b-B8:c,9:d",
"T7/T-T/B1:a,2:b-B8:c,9:d",
// ----//---- found by AllStructs
"T1,3/B0:d-B1:a-B3:d,4:g",
"T1,4/B0:e-B1:a-B4:c",
// ----//---- found by AllStructs
"T2,4/T-T-T/T1-T-B4:f/T-T-B3:f/B0:h-B1:f",
"T4/T-T/B3:f-T/B4:a",
// ---- found by AllStructs ----
// trackSet2 wrongly computed due to top not being tracked to tree root
"T2/T1-T/B0:g-B1:b-T/B2:b,3:a",
"T2/T1-T/T-T-B2:a/B0:c-B1:g",
// unchanged node is reparented
"T1/B0:c-B1:f",
"T1/T-T/B0:c-T/B1:h",
// SIGSEGV in ApplyΔ
"T1/T-T2/T-B1:c-B2:c/B0:g",
"T1/T-T/B0:g-T/B1:e",
// trackSet corruption: oid is pointed by some .parent but is not present
"T1/T-T/B0:g-T2/B1:h-B2:g",
"T/T1/T-T2/B0:e-B1:f-B2:g",
// ApplyΔ -> xunion: node is reachable from multiple parents
// ( because xdifference did not remove common non-leaf node
// under which there were also other changed, but not initially
// tracked, node )
"T4/T1-T/T-T2-B4:c/T-T-T/B0:f-B1:h-B2:g,3:b",
"T1/T-T/T-T2/T-T-T/B0:f-B1:h-B2:f",
// ----//----
"T3/T1-T/T-T2-T/B0:b-T-T-B3:h/B1:e-B2:a",
"T1/T-T4/T-T2-T/T-T-T-T/B0:b-B1:e-B2:a,3:c-B4:e",
// ----//----
"T/T1,3/T-T2-T4/B0:b-T-T-B3:g-B4:c/B1:b-B2:e",
"T1,4/T-T-T/T-T2-B4:f/T-T-T/B0:h-B1:b-B2:h,3:a",
"T2/B1:a-B7:g",
"T2,8/B1:a-B7:g-B9:i",
"T2/B1:a-B2:b", "T/B1:a,2:b",
"T2,3/B1:a-B2:b-B3:c", "T/B1:a,2:b",
"T2,3/B1:a-B2:c-B3:c", "T/B1:a,2:b",
"T2/B1:a-B2:c", "T2,3/B1:a-B2:c-B3:c", // continue with next right bucket until r2 coverage is complete
if r2.Hi_ <= b2.Keycov.Hi_ {
break
}
lo2 = b2.Keycov.Hi_ + 1
}
}
q2.Clear()
}
"T2/B1:a-B3:c", adj := setKey{}; adj.Update(adj1); adj.Update(adj2)
Δ("T2/T-T4/B1:b-B3:d-B99:h", kadj[k] = adj
A{1: K(1),
3: K(3,99,oo),
99: K(3,99,oo),
oo: K(3,99,oo)}),
}
// direct tree_i -> tree_{i+1} -> _{i+2} ... plus
// reverse ... tree_i <- _{i+1} <- _{i+2}
kadjOK := ΔBTest(testv[len(testv)-1]).kadjOK
for i := len(testv)-2; i >= 0; i-- {
test := ΔBTest(testv[i])
kadjOK, test.kadjOK = test.kadjOK, kadjOK
testv = append(testv, test)
} }
testq := make(chan ΔBTestEntry) return kadj
go func() {
defer close(testq)
for _, test := range testv {
testq <- ΔBTest(test)
}
}()
testΔBTail(t, testq)
} }
// TestΔBTailRandom verifies ΔBtail on random tree topologies generated by AllStructs.
func TestΔBTailRandom(t *testing.T) {
X := exc.Raiseif
// considerations:
// - maxdepth↑ better for testing (more tricky topologies)
// - maxsplit↑ not so better for testing (leave s=1, max s=2)
// - |kmin - kmax| affects N(variants) significantly
// -> keep key range small (dumb increase does not help testing)
// - N(keys) affects N(variants) significantly
// -> keep Nkeys reasonably small/medium (dumb increase does not help testing)
//
// - spawning python subprocess is very slow (takes 300-500ms for
// imports; https://github.com/pypa/setuptools/issues/510)
// -> we spawn `treegen allstructs` once and use request/response approach.
maxdepth := xbtreetest.N(2, 3, 4) // ----------------------------------------
maxsplit := xbtreetest.N(1, 2, 2)
n := xbtreetest.N(10,10,100)
nkeys := xbtreetest.N(3, 5, 10)
// server to generate AllStructs(kv, ...)
sg, err := xbtreetest.StartAllStructsSrv(); X(err)
defer func() {
err := sg.Close(); X(err)
}()
// random-number generator
rng, seed := xbtreetest.NewRand()
t.Logf("# maxdepth=%d maxsplit=%d nkeys=%d n=%d seed=%d", maxdepth, maxsplit, nkeys, n, seed)
// generate (kv1, kv2, kv3) randomly
// keysv1, keysv2 and keysv3 are random shuffle of IntSets // assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail.
var keysv1 [][]int // it also verifies that δbtail.vδBroots matches ΔTtail data.
var keysv2 [][]int func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, tj *xbtreetest.Commit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
var keysv3 [][]int t.Helper()
for keys := range IntSets(nkeys) { // XXX +KVAtTail, +lastRevOf
keysv1 = append(keysv1, keys)
keysv2 = append(keysv2, keys)
keysv3 = append(keysv3, keys)
}
v := keysv1
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv2
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
v = keysv3
rng.Shuffle(len(v), func(i,j int) { v[i], v[j] = v[j], v[i] })
// given random (kv1, kv2, kv3) generate corresponding set of random tree l := len(vδTok)
// topology sets (T1, T2, T3). Then iterate through T1->T2->T3->T1... var vatOK []zodb.Tid
// elements such that all right-directed triplets are visited and only once. var vδTok_ []map[Key]Δstring
// Test Update and rebuild on the generated tree sequences. at2t := map[zodb.Tid]*xbtreetest.Commit{tj.At: tj}
vv := "abcdefghij" t0 := tj
randv := func() string { for i := 0; i<l; i++ {
i := rng.Intn(len(vv)) // empty vδTok entries means they should be absent in vδT
return vv[i:i+1] if δTok := vδTok[l-i-1]; len(δTok) != 0 {
vatOK = append([]zodb.Tid{t0.At}, vatOK...)
vδTok_ = append([]map[Key]Δstring{δTok}, vδTok_...)
}
t0 = t0.Prev
at2t[t0.At] = t0
} }
vδTok = vδTok_
// the number of pairs is 3·n^2 δTtail, ok := δbtail.vδTbyRoot[treeRoot]
// the number of triplets is n^3 var vδToid []ΔTree
// if ok {
// limit n for emitted triplets, so that the amount of work for Update vδToid = δTtail.vδT
// and rebuild tests is approximately of the same order.
nrebuild := int(math.Ceil(math.Pow(3*float64(n*n), 1./3)))
// in non-short mode rebuild tests are exercising more keys variants, plus every test case
// takes more time. Compensate for that as well.
if !testing.Short() {
nrebuild -= 3
} }
testq := make(chan ΔBTestEntry) l = len(vδToid)
go func() { var vat []zodb.Tid
defer close(testq) var vδT []map[Key]Δstring
for i := range keysv1 { atPrev := t0.At
keys1 := keysv1[i] for _, δToid := range vδToid {
keys2 := keysv2[i] vat = append(vat, δToid.Rev)
keys3 := keysv3[i] δT := XGetδKV(at2t[atPrev], at2t[δToid.Rev], δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
vδT = append(vδT, δT)
atPrev = δToid.Rev
}
kv1 := map[Key]string{} var vatδB []zodb.Tid // δbtail.vδBroots/treeRoot
kv2 := map[Key]string{} for _, δBroots := range δbtail.vδBroots {
kv3 := map[Key]string{} if δBroots.ΔRoots.Has(treeRoot) {
for _, k := range keys1 { kv1[Key(k)] = randv() } vatδB = append(vatδB, δBroots.Rev)
for _, k := range keys2 { kv2[Key(k)] = randv() } }
for _, k := range keys3 { kv3[Key(k)] = randv() } }
treev1, err1 := sg.AllStructs(kv1, maxdepth, maxsplit, n, rng.Int63()) tok := tidvEqual(vat, vatOK) && vδTEqual(vδT, vδTok)
treev2, err2 := sg.AllStructs(kv2, maxdepth, maxsplit, n, rng.Int63()) bok := tidvEqual(vatδB, vatOK)
treev3, err3 := sg.AllStructs(kv3, maxdepth, maxsplit, n, rng.Int63()) if !(tok && bok) {
err := xerr.Merge(err1, err2, err3) emsg := fmt.Sprintf("%s: vδT:\n", subj)
if err != nil { have := ""
t.Fatal(err) for i := 0; i<len(vδT); i++ {
} have += fmt.Sprintf("\n\t@%s: %v", xat[vat[i]], vδT[i])
}
emsg += fmt.Sprintf("have: %s\n", have)
emit := func(tree string, flags ΔBTestFlags) { if !tok {
// skip emitting this entry if both Update and want := ""
// Rebuild are requested to be skipped. for i := 0; i<len(vδTok); i++ {
if flags == (ΔBTest_SkipUpdate | ΔBTest_SkipRebuild) { want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
return
}
testq <- ΔBTestEntry{tree, nil, flags}
} }
emsg += fmt.Sprintf("want: %s\n", want)
}
URSkipIf := func(ucond, rcond bool) ΔBTestFlags { if !bok {
var flags ΔBTestFlags vδb_root := ""
if ucond { for i := 0; i<len(vatδB); i++ {
flags |= ΔBTest_SkipUpdate vδb_root += fmt.Sprintf("\n\t@%s", xat[vatδB[i]])
}
if rcond {
flags |= ΔBTest_SkipRebuild
}
return flags
} }
emsg += fmt.Sprintf("vδb/root: %s\n", vδb_root)
}
for j := range treev1 { t.Error(emsg)
for k := range treev2 { }
for l := range treev3 { }
// limit rebuild to subset of tree topologies,
// because #(triplets) grow as n^3. See nrebuild
// definition above for details.
norebuild := (j >= nrebuild ||
k >= nrebuild ||
l >= nrebuild)
// C_{l-1} -> Aj (pair first seen on k=0) // xtrackKeys issues δbtail.Track requests for tree[keys].
emit(treev1[j], URSkipIf(k != 0, norebuild)) func xtrackKeys(δbtail *ΔBtail, t *xbtreetest.Commit, keys setKey) {
X := exc.Raiseif
head := δbtail.Head()
if head != t.At {
panicf("BUG: δbtail.head: %s ; t.at: %s", head, t.At)
}
// Aj -> Bk (pair first seen on l=0) for k := range keys {
emit(treev2[k], URSkipIf(l != 0, norebuild)) // NOTE: if tree is deleted - the following adds it to tracked
// set with every key being a hole. This aligns with the
// following situation
//
// T1 -> ø -> T2
//
// where after T1->ø, even though the tree becomes deleted, its root
// continues to be tracked and all keys migrate to holes in the
// tracking set. By aligning initial state to the same as after
// T1->ø, we test what will happen on ø->T2.
b := t.Xkv.Get(k)
err := δbtail.track(k, b.Path()); X(err)
}
}
// Bk -> Cl (pair first seen on j=0) // trackSet returns what should be ΔBtail.trackSet coverage for specified tracked key set.
emit(treev3[l], URSkipIf(j != 0, norebuild)) func trackSet(rbs xbtreetest.RBucketSet, tracked setKey) blib.PPTreeSubSet {
} // nil = don't compute keyCover
} // (trackSet is called from inside hot inner loop of rebuild test)
} return _trackSetWithCov(rbs, tracked, nil)
} }
}()
testΔBTail(t, testq) // trackSetWithCov returns what should be ΔBtail.trackSet and its key coverage for specified tracked key set.
func trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey) (trackSet blib.PPTreeSubSet, keyCover *blib.RangedKeySet) {
keyCover = &blib.RangedKeySet{}
trackSet = _trackSetWithCov(rbs, tracked, keyCover)
return trackSet, keyCover
} }
func _trackSetWithCov(rbs xbtreetest.RBucketSet, tracked setKey, outKeyCover *blib.RangedKeySet) (trackSet blib.PPTreeSubSet) {
trackSet = blib.PPTreeSubSet{}
for k := range tracked {
kb := rbs.Get(k)
if outKeyCover != nil {
outKeyCover.AddRange(kb.Keycov)
}
trackSet.AddPath(kb.Path())
}
return trackSet
}
func TestΔBtailForget(t_ *testing.T) { // assertTrack verifies state of .trackSet and ΔTtail.trackNew.
t := xbtreetest.NewT(t_) // it assumes that only one tree root is being tracked.
X := exc.Raiseif func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK blib.PPTreeSubSet, trackNewOK blib.PPTreeSubSet) {
t.Helper()
if !δBtail.trackSet.Equal(trackSetOK) {
t.Errorf("%s: trackSet:\n\thave: %v\n\twant: %v", subj, δBtail.trackSet, trackSetOK)
}
t0 := t.CommitTree("T/B:") roots := setOid{}
t1 := t.CommitTree("T/B1:a") for root := range δBtail.vδTbyRoot {
t2 := t.CommitTree("T2/B1:a-B2:b") roots.Add(root)
t3 := t.CommitTree("T/B2:b") }
δbtail := NewΔBtail(t0.At, t.DB) nrootsOK := 1
_, err := δbtail.Update(t1.ΔZ); X(err) if trackSetOK.Empty() && trackNewOK.Empty() {
_, err = δbtail.Update(t2.ΔZ); X(err) nrootsOK = 0
}
if len(roots) != nrootsOK {
t.Errorf("%s: len(vδTbyRoot) != %d ; roots=%v", subj, nrootsOK, roots)
return
}
if nrootsOK == 0 {
return
}
// start tracking. everything becomes tracked because t1's T/B1:a has [-∞,∞) coverage root := roots.Elements()[0]
// By starting tracking after t2 we verify vδBroots update in both Update and rebuild
_0 := setKey{}; _0.Add(0)
xtrackKeys(δbtail, t2, _0)
_, err = δbtail.Update(t3.ΔZ); X(err) δTtail := δBtail.vδTbyRoot[root]
xat := map[zodb.Tid]string{ trackNewRootsOK := setOid{}
t0.At: "at0", if !trackNewOK.Empty() {
t1.At: "at1", trackNewRootsOK.Add(root)
t2.At: "at2",
t3.At: "at3",
} }
assertΔTtail(t.T, "init", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t0.At)
assertΔTtail(t.T, "forget ≤ at0", δbtail, t3, t.Root(), xat, t1.Δxkv, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t1.At)
assertΔTtail(t.T, "forget ≤ at1", δbtail, t3, t.Root(), xat, t2.Δxkv, t3.Δxkv)
δbtail.ForgetPast(t3.At)
assertΔTtail(t.T, "forget ≤ at3", δbtail, t3, t.Root(), xat, )
}
func TestΔBtailClone(t_ *testing.T) {
// ΔBtail.Clone had bug that aliased klon data to orig
t := xbtreetest.NewT(t_)
X := exc.Raiseif
t0 := t.CommitTree("T2/B1:a-B2:b") if !δBtail.trackNewRoots.Equal(trackNewRootsOK) {
t1 := t.CommitTree("T2/B1:c-B2:d") t.Errorf("%s: trackNewRoots:\n\thave: %v\n\twant: %v", subj, δBtail.trackNewRoots, trackNewRootsOK)
δbtail := NewΔBtail(t0.At, t.DB) }
_, err := δbtail.Update(t1.ΔZ); X(err)
_2 := setKey{}; _2.Add(2)
xtrackKeys(δbtail, t1, _2)
err = δbtail.rebuildAll(); X(err)
xat := map[zodb.Tid]string{ if !δTtail.trackNew.Equal(trackNewOK) {
t0.At: "at0", t.Errorf("%s: vδT.trackNew:\n\thave: %v\n\twant: %v", subj, δTtail.trackNew, trackNewOK)
t1.At: "at1",
} }
}
δkv1_1 := map[Key]Δstring{2:{"b","d"}}
assertΔTtail(t.T, "orig @at1", δbtail, t1, t.Root(), xat, δkv1_1)
δbklon := δbtail.Clone()
assertΔTtail(t.T, "klon @at1", δbklon, t1, t.Root(), xat, δkv1_1)
t2 := t.CommitTree("T/B1:b,2:a")
_, err = δbtail.Update(t2.ΔZ); X(err)
xat[t2.At] = "at2"
δkv1_2 := map[Key]Δstring{1:{"a","c"}, 2:{"b","d"}} // XGetδKV translates {k -> δ<oid>} to {k -> δ(ZBlk(oid).data)} according to t1..t2 db snapshots.
δkv2_2 := map[Key]Δstring{1:{"c","b"}, 2:{"d","a"}} func XGetδKV(t1, t2 *xbtreetest.Commit, δkvOid map[Key]ΔValue) map[Key]Δstring {
assertΔTtail(t.T, "orig @at2", δbtail, t2, t.Root(), xat, δkv1_2, δkv2_2) δkv := make(map[Key]Δstring, len(δkvOid))
assertΔTtail(t.T, "klon @at1 after orig @at->@at2", δbklon, t1, t.Root(), xat, δkv1_1) for k, δvOid := range δkvOid {
δkv[k] = Δstring{
Old: t1.XGetBlkData(δvOid.Old),
New: t2.XGetBlkData(δvOid.New),
}
}
return δkv
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7