Skip to content

W
wendelin.core
Commit 52c72dbb authored by Kirill Smelkov's avatar Kirill Smelkov
2 Browse files
Options

X ΔBtail.rebuild started to work draftly 

* t2: (50 commits)
  .
  X rebuild: Serial tests now started to PASS draftly
  .
  X rebuild: Don't return nil for empty ΔPPTreeSubSet - that leads to SIGSEGV
  X treediff: Fix BUG while computing AB coverage
  X ΔBtail.Clone had bug that it was aliasing klon and orig data
  X Fix rebuild with ø @at2
  X Fix bug in PPTreeSubSet.Difference  - it was always leaving root node alive
  .
  .
  X found why TestΔBTailAllStructs was not effective to find δtkeycov bugs
  .
  .
  X wcfs: assert that keycov only grow
  .
  X another bug in δtkeyconv computation
  .
  .
  .
  .
  ...
parents c144b4a4 8402f4c3
Hide whitespace changes
Inline Side-by-side
Showing with 806 additions and 441 deletions
wcfs/internal/set/set.go.in
View file @ 52c72dbb
......@@ -136,6 +136,15 @@ func (a Set) Equal(b Set) bool {
return true
}
// Clone returns copy of the set.
func (orig Set) Clone() Set {
klon := make(Set, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s Set) SortedElements() []VALUE {
......
wcfs/internal/set/zset_i64.go
View file @ 52c72dbb
......@@ -138,6 +138,15 @@ func (a SetI64) Equal(b SetI64) bool {
return true
}
// Clone returns copy of the set.
func (orig SetI64) Clone() SetI64 {
klon := make(SetI64, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s SetI64) SortedElements() []int64 {
......
wcfs/internal/set/zset_oid.go
View file @ 52c72dbb
......@@ -138,6 +138,15 @@ func (a SetOid) Equal(b SetOid) bool {
return true
}
// Clone returns copy of the set.
func (orig SetOid) Clone() SetOid {
klon := make(SetOid, len(orig))
for v := range orig {
klon.Add(v)
}
return klon
}
// --------
func (s SetOid) SortedElements() []_Oid {
......
wcfs/internal/xbtree/pptreesubset.go
View file @ 52c72dbb
......@@ -27,6 +27,7 @@ import (
)
const tracePPSet = false
const debugPPSet = false
// PPTreeSubSet represents PP-connected subset of tree node objects.
//
......@@ -58,9 +59,13 @@ type PPTreeSubSet map[zodb.Oid]*nodeInTree
type nodeInTree struct {
parent zodb.Oid // parent node | InvalidOid for root
nchild int // number of direct children in PPTreeSubSet referring to this node
// XXX + [lo,hi) range this node is coming under in its parent XXX -> in its tree ?
}
// Has returns whether node is in the set.
func (S PPTreeSubSet) Has(oid zodb.Oid) bool {
_, ok := S[oid]
return ok
}
// Path returns path leading to the node specified by oid.
//
......@@ -84,6 +89,8 @@ func (S PPTreeSubSet) Path(oid zodb.Oid) (path []zodb.Oid) {
}
// AddPath adds path to a node to the set.
//
// Note: embedded buckets (leaf node with InvalidOid) are removed from the path.
func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
S.verify()
defer S.verify()
......@@ -93,18 +100,11 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
panic("empty path")
}
// don't keep track of artificial empty tree
if l == 1 && path[0] == zodb.InvalidOid {
return
}
// don't explicitly keep track of embedded buckets - they all have
// InvalidOid, and thus, if kept in S, e.g. T/B1:a and another
// T/B2:b would lead to InvalidOid having multiple parents.
if l == 2 && path[1] == zodb.InvalidOid {
path = path[:1]
}
// normalize path: remove embedded bucket and check whether it was an
// artificial empty tree.
path = normPath(path)
// go through path and add nodes to the set
parent := zodb.InvalidOid
var pt *nodeInTree = nil
for _, oid := range path {
......@@ -132,11 +132,33 @@ func (S PPTreeSubSet) AddPath(path []zodb.Oid) {
}
}
// ---- Union/Difference/Intersetctior ----
// normPath normalizes path.
//
// It removes embedded buckets and artificial empty trees.
// Returned slice is subslice of path and aliases its memory.
func normPath(path []zodb.Oid) []zodb.Oid {
l := len(path)
// don't keep track of artificial empty tree
if l == 1 && path[0] == zodb.InvalidOid {
return nil
}
// don't explicitly keep track of embedded buckets - they all have
// InvalidOid, and thus, if kept in S, e.g. T/B1:a and another
// T/B2:b would lead to InvalidOid having multiple parents.
if l == 2 && path[1] == zodb.InvalidOid {
return path[:1]
}
return path
}
// ---- Union/Difference/Intersection ----
// Union returns U = PP(A.leafs | B.leafs)
//
// In other words it returns sum of A and B.
// In other words it adds A and B nodes.
func (A PPTreeSubSet) Union(B PPTreeSubSet) PPTreeSubSet {
U := A.Clone()
U.UnionInplace(B)
......@@ -163,7 +185,7 @@ func (A PPTreeSubSet) UnionInplace(B PPTreeSubSet) {
// Difference returns D = PP(A.leafs \ B.leafs)
//
// In other words ... XXX
// In other words it removes B nodes from A while still maintaining A as PP-connected.
func (A PPTreeSubSet) Difference(B PPTreeSubSet) PPTreeSubSet {
D := A.Clone()
D.DifferenceInplace(B)
......@@ -189,7 +211,7 @@ func (A PPTreeSubSet) DifferenceInplace(B PPTreeSubSet) {
A.xDifferenceInplace(B)
}
// XXX Intersection
// TODO Intersection
func (A PPTreeSubSet) xUnionInplace(B PPTreeSubSet) {
if tracePPSet {
......@@ -269,10 +291,16 @@ func (A PPTreeSubSet) fixup(δnchild map[zodb.Oid]int) {
A.xfixup(+1, δnchild)
}
func (A PPTreeSubSet) xfixup(sign int, δnchild map[zodb.Oid]int) {
//fmt.Printf("\nfixup:\n")
//fmt.Printf(" ·: %s\n", A)
//fmt.Printf(" δ: %v\n", δnchild)
//defer fmt.Printf(" ->·: %s\n\n", A)
if debugPPSet {
ssign := "+"
if sign < 0 {
ssign = "-"
}
fmt.Printf("\n fixup:\n")
fmt.Printf(" ·: %s\n", A)
fmt.Printf(" %sδ: %v\n", ssign, δnchild)
defer fmt.Printf(" ->·: %s\n\n", A)
}
gcq := []zodb.Oid{}
for oid, δnc := range δnchild {
......@@ -304,7 +332,7 @@ func (S PPTreeSubSet) gc1(oid zodb.Oid) {
for oid != zodb.InvalidOid {
t := S[oid]
t.nchild--
if t.nchild > 0 || /* root node */t.parent == zodb.InvalidOid {
if t.nchild > 0 {
break
}
delete(S, oid)
......@@ -395,6 +423,11 @@ func (A PPTreeSubSet) Equal(B PPTreeSubSet) bool {
return true
}
// Empty returns whether set is empty.
func (S PPTreeSubSet) Empty() bool {
return len(S) == 0
}
func (t nodeInTree) String() string {
return fmt.Sprintf("{p%s c%d}", t.parent, t.nchild)
}
......@@ -440,13 +473,22 @@ func (t nodeInTree) String() string {
// - δ.Add, and
// - xfixup(+1, δnchildNonLeafs)
//
// produce correctly PP-connected set.
// produces correctly PP-connected set.
type ΔPPTreeSubSet struct {
Del PPTreeSubSet
Add PPTreeSubSet
δnchildNonLeafs map[zodb.Oid]int
}
// NewΔPPTreeSubSet creates new empty ΔPPTreeSubSet.
func NewΔPPTreeSubSet() *ΔPPTreeSubSet {
return &ΔPPTreeSubSet{
Del: PPTreeSubSet{},
Add: PPTreeSubSet{},
δnchildNonLeafs: map[zodb.Oid]int{},
}
}
// Update updates δ to be combination of δ+δ2.
func (δ *ΔPPTreeSubSet) Update(δ2 *ΔPPTreeSubSet) {
δ.Del.UnionInplace(δ2.Del)
......
wcfs/internal/xbtree/pptreesubset_test.go
View file @ 52c72dbb
......@@ -20,6 +20,7 @@
package xbtree
import (
"strings"
"testing"
"lab.nexedi.com/kirr/neo/go/zodb"
......@@ -79,18 +80,43 @@ func TestPPTreeSubSetOps(t *testing.T) {
S{a:{ø,1}, b:{a,0}}, // B
S{a:{ø,1}, b:{a,0}}, // U
S{}), // D
E(
S{a:{ø,1}, b:{a,1}, c:{b,0}}, // A
S{a:{ø,1}, b:{a,1}, c:{b,0}}, // B (=A)
S{a:{ø,1}, b:{a,1}, c:{b,0}}, // U (=A)
S{}), // D
}
// assert1 asserts that result of op(A,B) == resOK.
assert1 := func(op string, A, B, res, resOK S) {
t.Helper()
if res.Equal(resOK) {
return
}
op1 := op[0:1]
t.Errorf("%s:\n A: %s\n B: %s\n ->%s: %s\n ok%s: %s\n",
strings.Title(op), A, B, op1, res, strings.ToUpper(op1), resOK)
}
for _, tt := range testv {
U := tt.A.Union(tt.B)
D := tt.A.Difference(tt.B)
Uab := tt.A.Union(tt.B)
Uba := tt.B.Union(tt.A)
Dab := tt.A.Difference(tt.B)
if !U.Equal(tt.Union) {
t.Errorf("Union:\n A: %s\n B: %s\n ->u: %s\n okU: %s\n", tt.A, tt.B, U, tt.Union)
}
if !D.Equal(tt.Difference) {
t.Errorf("Difference:\n A: %s\n B: %s\n ->d: %s\n okD: %s\n", tt.A, tt.B, D, tt.Difference)
}
assert1("union", tt.A, tt.B, Uab, tt.Union)
assert1("union", tt.B, tt.A, Uba, tt.Union)
assert1("difference", tt.A, tt.B, Dab, tt.Difference)
Uaa := tt.A.Union(tt.A)
Ubb := tt.B.Union(tt.B)
Daa := tt.A.Difference(tt.A)
Dbb := tt.B.Difference(tt.B)
assert1("union", tt.A, tt.A, Uaa, tt.A)
assert1("union", tt.B, tt.B, Ubb, tt.B)
assert1("difference", tt.A, tt.A, Daa, S{})
assert1("difference", tt.B, tt.B, Dbb, S{})
// XXX also verify U/D properties like (A+B)\B + (A+B)\A + (A^B) == (A+B) ?
}
......
wcfs/internal/xbtree/testprog/treegen.py
View file @ 52c72dbb
......@@ -20,7 +20,7 @@
# See https://www.nexedi.com/licensing for rationale and options.
"""Program treegen provides infrastructure to generate ZODB BTree states.
It is used as helper for ΔBTree tests.
It is used as helper for ΔBtail tests.
The following subcommands are provided:
......@@ -319,6 +319,9 @@ def AllStructs(kv1txt, kv2txt, maxdepth, maxsplit, n, seed=None):
t1structv.insert(0, t1struct0)
t2structv.insert(0, t2struct0)
# XXX rework allstructs to accept only 1 kv and emit n structs for that one kv only
# -> iterate through the pairs/triplets in the caller (TestΔBTailAllStructs)
# emit topologies for tree1->tree2 and tree1<-tree2 transitions for all
# combinations of tree1 and tree2.
t12travel = list(bitravel2Way(t1structv, t2structv))
......
wcfs/internal/xbtree/treediff.go
View file @ 52c72dbb
......@@ -159,7 +159,7 @@ func δZConnectTracked(δZv []zodb.Oid, T PPTreeSubSet) (δZTC SetOid, δtopsByR
// nodeInRange represents a Node coming under [lo, hi_] key range in its tree.
type nodeInRange struct {
prefix []zodb.Oid // path to this node goes via this objects
lo, hi_ Key // [lo, hi_] NOTE _not_ hi) not to overflow at ∞ XXX -> Range
lo, hi_ Key // [lo, hi_] NOTE _not_ hi) not to overflow at ∞ XXX -> keycov KeyRange?
node Node
done bool // whether this node was already taken into account while computing diff
}
......@@ -330,7 +330,7 @@ func treediff(ctx context.Context, root zodb.Oid, δtops SetOid, δZTC SetOid, t
defer xerr.Contextf(&err, "treediff %s..%s %s", zconnOld.At(), zconnNew.At(), root)
δT = map[Key]ΔValue{}
δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
tracefDiff("\ntreediff %s δtops: %v δZTC: %v\n", root, δtops, δZTC)
......@@ -429,7 +429,7 @@ func diffX(ctx context.Context, a, b Node, δZTC SetOid, trackSet PPTreeSubSet)
var δtrack *ΔPPTreeSubSet
δ, err := diffB(ctx, aB, bB)
if δ != nil {
δtrack = &ΔPPTreeSubSet{}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
}
return δ, δtrack, δtkeycov, err
......@@ -445,7 +445,7 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
defer xerr.Contextf(&err, "diffT %s %s", xidOf(A), xidOf(B))
δ = map[Key]ΔValue{}
δtrack = &ΔPPTreeSubSet{Del: PPTreeSubSet{}, Add: PPTreeSubSet{}, δnchildNonLeafs: map[zodb.Oid]int{}}
δtrack = NewΔPPTreeSubSet()
δtkeycov = &RangedKeySet{}
defer func() {
tracefDiff(" -> δ: %v\n", δ)
......@@ -487,6 +487,9 @@ func diffT(ctx context.Context, A, B *Tree, δZTC SetOid, trackSet PPTreeSubSet)
ABcov:
for i := len(ABpath)-2; i >= 0; i-- {
xparent, err := node.PJar().Get(ctx, ABpath[i]); /*X*/if err != nil { return nil,nil,nil, err }
err = xparent.PActivate(ctx); /*X*/if err != nil { return nil,nil,nil, err}
defer xparent.PDeactivate()
parent := xparent.(*Tree) // must succeed
// find node in parent children and constrain ABlo/ABhi accordingly
entryv := parent.Entryv()
......@@ -512,7 +515,14 @@ ABcov:
}
}
panicf("BUG: T%s points to T%s as parent in trackSet, but not found in T%s children", node.POid(), parent.POid(), parent.POid())
emsg := fmt.Sprintf("BUG: T%s points to T%s as parent in trackSet, but not found in T%s children\n", node.POid(), parent.POid(), parent.POid())
children := []string{}
for _, entry := range entryv {
children = append(children, vnode(entry.Child()))
}
emsg += fmt.Sprintf("T%s children: %v\n", parent.POid(), children)
emsg += fmt.Sprintf("trackSet: %s\n", trackSet)
panic(emsg)
}
......@@ -565,10 +575,11 @@ ABcov:
}
}
// {} oid -> parent for all nodes in Bv: current and previously expanded - up till top B
BtrackSet := PPTreeSubSet{}
// {} oid -> nodeInRange for all nodes we've came through in Bv:
// current and previously expanded - up till top B.
BnodeIdx := map[zodb.Oid]*nodeInRange{}
if !Bempty {
BtrackSet.AddPath(ABpath)
BnodeIdx[ABoid] = btop
}
// δtkeycov will be = BAdd \ ADel
......@@ -579,7 +590,8 @@ ABcov:
// by default a node contributes to δ-
// a node ac does not contribute to δ- and can be skipped, if:
// - ac is not tracked, or
// - ac ∉ δZTC && ∃ bc from B: ac.oid == bc.oid (ac+ac.children were not changed, and ac stays in the tree)
// - ac ∉ δZTC && ∃ bc from B: ac.oid == bc.oid && ac.keycov == bc.keycov
// (ac+ac.children were not changed, ac stays in the tree with the same key range coverage)
Aq := []*nodeInRange{atop} // queue for A nodes that contribute to δ-
for len(Aq) > 0 {
debugfDiff("\n")
......@@ -606,7 +618,7 @@ ABcov:
ra.done = true
case *Tree:
// empty tree - only queue holes covered by it
// empty tree - queue holes covered by it
if len(a.Entryv()) == 0 {
ar := KeyRange{ra.lo, ra.hi_}
δtrack.Del.AddPath(ra.Path())
......@@ -632,7 +644,7 @@ ABcov:
// ( this does not give exact answer but should be a reasonable heuristic;
// the diff is the same if heuristic does not work and we
// look into and load more nodes to compute δ )
_, found := BtrackSet[acOid]
bc, found := BnodeIdx[acOid]
if !found {
for {
blo := Bv.Get(ac.lo)
......@@ -652,11 +664,12 @@ ABcov:
}
bchildren := Bv.Expand(blo)
for _, bc := range bchildren {
bcOid := bc.node.POid()
BtrackSet.AddPath(bc.Path())
if acOid == bcOid {
for _, bc_ := range bchildren {
bc_Oid := bc_.node.POid()
BnodeIdx[bc_Oid] = bc_
if acOid == bc_Oid {
found = true
bc = bc_
}
}
if found {
......@@ -665,29 +678,23 @@ ABcov:
}
}
if found {
// ac can be skipped
// adjust trackSet since path to the node could have changed
apath := trackSet.Path(acOid)
bpath := BtrackSet.Path(acOid)
if !pathEqual(apath, bpath) {
δtrack.Del.AddPath(apath)
δtrack.Add.AddPath(bpath)
if nc := at.nchild; nc != 0 {
δtrack.δnchildNonLeafs[acOid] = nc
// XXX debugDiff δtrack [) ...
} else {
// adjust δtkeycov only if it was leaf bucket
ar := KeyRange{ac.lo, ac.hi_}
bc := Bv.Get(ac.lo)
br := KeyRange{bc.lo, bc.hi_}
δtkeycovADel.AddRange(ar)
δtkeycovBAdd.AddRange(br)
debugfDiff(" δtrack - %s %v KKK\n", ar, apath)
debugfDiff(" δtrack + %s %v KKK\n", br, bpath)
// ac can be skipped if key coverage stays the same
ar := KeyRange{ac.lo, ac.hi_}
br := KeyRange{bc.lo, bc.hi_}
if ar == br {
// adjust trackSet since path to the node could have changed
apath := ac.Path()
bpath := bc.Path()
if !pathEqual(apath, bpath) {
δtrack.Del.AddPath(apath)
δtrack.Add.AddPath(bpath)
if nc := at.nchild; nc != 0 {
δtrack.δnchildNonLeafs[acOid] = nc
}
}
}
continue
continue
}
}
}
......@@ -751,11 +758,10 @@ ABcov:
b.done = true
}
// stop if r coverage is complete
// continue with next right bucket until r coverage is complete
if r.hi_ <= b.hi_ {
break
}
// continue with next right bucket
lo = b.hi_ + 1
}
}
......
wcfs/internal/xbtree/δbtail.go
View file @ 52c72dbb
......@@ -33,10 +33,6 @@ import (
"lab.nexedi.com/nexedi/wendelin.core/wcfs/internal/xtail"
)
// XXX malfancitioning rebuild currently breaks wcfs tests
// TODO kill this after rebuild is finished
const XXX_killWhenRebuildWorks = true
const traceΔBtail = false
const debugΔBtail = false
......@@ -89,30 +85,25 @@ const debugΔBtail = false
// See also zodb.ΔTail
// XXX naming -> ΔBTail ?
type ΔBtail struct {
// raw ZODB changes; Kept to rebuild .byRoot after new Track.
// raw ZODB changes; Kept to rebuild .vδTbyRoot after new Track.
// includes all changed objects, not only tracked ones.
δZtail *zodb.ΔTail
// XXX vvv keys ∈ tracked -> keys ∈ kadj[tracked] ?
// vδB []ΔB // data with δB changes; Noted only by keys ∈ tracked subset
byRoot map[zodb.Oid]*ΔTtail // {} root -> [] k/v change history; only for keys ∈ tracked subset
// handle to make connections to access database.
// TODO allow client to optionally provide zconnOld/zconnNew on e.g. Update()
db *zodb.DB // to open connections to load new/old tree|buckets
vδBroots []ΔBroots // [] (rev, roots changed in this rev)
vδTbyRoot map[zodb.Oid]*ΔTtail // {} root -> [] k/v change history; only for keys ∈ tracked subset XXX -> byRoot?
// set of tracked nodes as of @head state.
// For this set all vδT are fully computed.
// The set of nodes that were requested to be tracked, but were not yet
// taken into account, is kept in ΔTtail.trackNew & co.
trackSet PPTreeSubSet
// set of nodes that were requested to be tracked, but for which vδB was not yet rebuilt
trackNew PPTreeSubSet
}
// set of trees for which .trackNew is non-empty
trackNewRoots SetOid
// ΔB represents a change in BTrees space.
type ΔB struct {
Rev zodb.Tid
ByRoot map[zodb.Oid]map[Key]ΔValue // {} root -> {}(key, δvalue)
// handle to make connections to access database.
// TODO allow client to optionally provide zconnOld/zconnNew on e.g. Update()
db *zodb.DB // to open connections to load new/old tree|buckets
}
// ΔTtail represent tail of revisional changes to one BTree.
......@@ -121,6 +112,11 @@ type ΔB struct {
type ΔTtail struct {
vδT []ΔTree // changes to tree keys; rev↑. covers keys ∈ tracked subset
// set of nodes that were requested to be tracked in this tree, but for
// which vδT was not yet rebuilt
trackNew PPTreeSubSet
// XXX + trackNewKeys RangedKeySet
// {}k/v @tail for keys that are changed in (tail, head].
KVAtTail map[Key]Value // XXX not needed since vδT has ΔValue ?
......@@ -128,6 +124,20 @@ type ΔTtail struct {
lastRevOf map[Key]zodb.Tid // {} key -> last
}
// ΔBroots represents roots-only part of ΔB.
//
// It describes which trees were changed, but does not provide δkv details for changed trees.
type ΔBroots struct {
Rev zodb.Tid
ΔRoots SetOid // which roots changed in this revision
}
// ΔB represents a change in BTrees space.
type ΔB struct {
Rev zodb.Tid
ΔByRoot map[zodb.Oid]map[Key]ΔValue // {} root -> {}(key, δvalue) XXX -> ByRoot?
}
// ΔTree describes changes to one BTree in one revision.
// XXX -> ΔT ?
type ΔTree struct {
......@@ -146,15 +156,25 @@ type ΔTree struct {
func NewΔBtail(at0 zodb.Tid, db *zodb.DB) *ΔBtail {
return &ΔBtail{
δZtail: zodb.NewΔTail(at0),
byRoot: map[zodb.Oid]*ΔTtail{},
vδBroots: nil,
vδTbyRoot: map[zodb.Oid]*ΔTtail{},
trackSet: PPTreeSubSet{},
trackNew: PPTreeSubSet{},
db: db,
trackNewRoots: SetOid{},
db: db,
}
}
// newΔTtail creates new empty ΔTtail object.
func newΔTtail() *ΔTtail {
return &ΔTtail{
trackNew: PPTreeSubSet{},
KVAtTail: make(map[Key]Value),
lastRevOf: make(map[Key]zodb.Tid),
}
}
// xverifyΔBTail_rebuild needs ΔBTree.clone because otherwise it is too slow to run that test.
func (orig *ΔBtail) clone() *ΔBtail {
// Clone returns copy of ΔBtail.
func (orig *ΔBtail) Clone() *ΔBtail {
klon := &ΔBtail{db: orig.db}
// δZtail
......@@ -163,35 +183,51 @@ func (orig *ΔBtail) clone() *ΔBtail {
klon.δZtail.Append(δZ.Rev, δZ.Changev)
}
// trackSet, trackNew
klon.trackSet = orig.trackSet.Clone()
klon.trackNew = orig.trackNew.Clone()
// byRoot
klon.byRoot = make(map[zodb.Oid]*ΔTtail, len(orig.byRoot))
for root, origΔTtail := range orig.byRoot {
klonΔTtail := &ΔTtail{}
klonΔTtail.vδT = append(klonΔTtail.vδT, origΔTtail.vδT...)
klonΔTtail.KVAtTail = make(map[Key]Value, len(origΔTtail.KVAtTail))
for k, v := range origΔTtail.KVAtTail {
klonΔTtail.KVAtTail[k] = v
// vδBroots
for _, origδBroots := range orig.vδBroots {
klonδBroots := ΔBroots{
Rev: origδBroots.Rev,
ΔRoots: origδBroots.ΔRoots.Clone(),
}
klonΔTtail.lastRevOf = make(map[Key]zodb.Tid, len(origΔTtail.lastRevOf))
for k, rev := range origΔTtail.lastRevOf {
klonΔTtail.lastRevOf[k] = rev
}
klon.byRoot[root] = klonΔTtail
klon.vδBroots = append(klon.vδBroots, klonδBroots)
}
// vδTbyRoot
klon.vδTbyRoot = make(map[zodb.Oid]*ΔTtail, len(orig.vδTbyRoot))
for root, origΔTtail := range orig.vδTbyRoot {
klon.vδTbyRoot[root] = origΔTtail.Clone()
}
// trackSet, trackNewRoots
klon.trackSet = orig.trackSet.Clone()
klon.trackNewRoots = orig.trackNewRoots.Clone()
return klon
}
// newΔTtail creates new empty ΔTtail object.
func newΔTtail() *ΔTtail {
return &ΔTtail{
KVAtTail: make(map[Key]Value),
lastRevOf: make(map[Key]zodb.Tid),
// Clone returns copy of ΔTtail.
func (orig *ΔTtail) Clone() *ΔTtail {
klon := &ΔTtail{}
for _, origδT := range orig.vδT {
klonδT := ΔTree{
Rev: origδT.Rev,
ΔKV: make(map[Key]ΔValue, len(origδT.ΔKV)),
}
for k, δv := range origδT.ΔKV {
klonδT.ΔKV[k] = δv
}
klon.vδT = append(klon.vδT, klonδT)
}
klon.trackNew = orig.trackNew.Clone()
klon.KVAtTail = make(map[Key]Value, len(orig.KVAtTail))
for k, v := range orig.KVAtTail {
klon.KVAtTail[k] = v
}
klon.lastRevOf = make(map[Key]zodb.Tid, len(orig.lastRevOf))
for k, rev := range orig.lastRevOf {
klon.lastRevOf[k] = rev
}
return klon
}
// (tail, head] coverage
......@@ -199,7 +235,6 @@ func (δBtail *ΔBtail) Head() zodb.Tid { return δBtail.δZtail.Head() }
func (δBtail *ΔBtail) Tail() zodb.Tid { return δBtail.δZtail.Tail() }
// XXX SliceByRev?
// XXX ForgetPast
// Track adds tree path to tracked set.
//
......@@ -215,166 +250,282 @@ func (δBtail *ΔBtail) Tail() zodb.Tid { return δBtail.δZtail.Tail() }
// XXX path -> []oid ?
//
// XXX catch cycles on add?
// XXX no need to pass keyPresent since holeIdx was removed
func (δBtail *ΔBtail) Track(key Key, keyPresent bool, nodePath []Node) error { // XXX Tree|Bucket; path[0] = root
// XXX no need to pass in key? (-> all keys, covered by leaf keyrange, will be added to tracking set of keys)
func (δBtail *ΔBtail) Track(key Key, nodePath []Node) error { // XXX Tree|Bucket; path[0] = root
path := nodePathToPath(nodePath)
pathv := []string{}
for _, node := range nodePath { pathv = append(pathv, vnode(node)) }
tracefΔBtail("\nTrack [%v] %s\n", key, strings.Join(pathv, " -> "))
tracefΔBtail("trackSet: %s\n", δBtail.trackSet) // XXX locking
return δBtail.track(key, keyPresent, path)
return δBtail.track(key, path)
}
func (δBtail *ΔBtail) track(key Key, keyPresent bool, path []zodb.Oid) error {
func (δBtail *ΔBtail) track(key Key, path []zodb.Oid) error {
// XXX locking
// first normalize path: remove embedded bucket and check if it was an
// empty artificial tree. We need to do the normalization because we
// later check whether leaf path[-1] ∈ trackSet and without
// normalization path[-1] can be InvalidOid.
path = normPath(path)
if len(path) == 0 {
return nil // empty tree
}
root := path[0]
δBtail.trackNew.AddPath(path)
// track is track of path[-1] (i.e. leaf)
// nothing to do if key is already tracked
leaf := path[len(path)-1]
if δBtail.trackSet.Has(leaf) {
tracefΔBtail("->T: nop\n")
path_ := δBtail.trackSet.Path(leaf)
if !pathEqual(path, path_) {
panicf("BUG: key %s is already tracked via path=%v\ntrack requests path=%v", kstr(key), path_, path)
}
return nil
}
// XXX hack - until rebuild is implemented
if XXX_killWhenRebuildWorks {
_, ok := δBtail.byRoot[root]
// queue path into trackNew
δTtail, ok := δBtail.vδTbyRoot[root]
if !ok {
δBtail.byRoot[root] = newΔTtail()
δTtail = newΔTtail()
δBtail.vδTbyRoot[root] = δTtail
}
δBtail.trackNewRoots.Add(root)
δTtail.trackNew.AddPath(path)
tracefΔBtail("->T: [%s].trackNew -> %s\n", root, δTtail.trackNew)
return nil
}
// XXX update diff XXX here? or as separate step?
// XXX update lastRevOf
// rebuildAll rebuilds ΔBtail taking all trackNew requests into account.
func (δBtail *ΔBtail) rebuildAll() (err error) {
defer xerr.Context(&err, "ΔBtail rebuildAll")
// XXX locking
trackNewRoots := δBtail.trackNewRoots
tracefΔBtail("\nRebuildAll @%s..@%s trackNewRoots: %s\n", δBtail.Tail(), δBtail.Head(), trackNewRoots)
for root := range trackNewRoots {
δTtail := δBtail.vδTbyRoot[root] // must be there
δtrackSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
if err != nil {
return err
}
δBtail.trackSet.UnionInplace(δtrackSet)
}
δBtail.trackNewRoots = SetOid{}
return nil
}
// rebuild rebuilds ΔBtail taking trackNew requests into account.
// rebuild rebuilds ΔTtail taking trackNew requests into account.
//
// It returns set of nodes that must be added to ΔBtail.trackSet to account for
// keys that becomes tracked. Note: this set is potentially wider compared to what was in .trackNew.
// XXX place
func (δBtail *ΔBtail) rebuild() (err error) {
defer xerr.Context(&err, "ΔBtail rebuild")
func (δTtail *ΔTtail) rebuild(root zodb.Oid, δZtail *zodb.ΔTail, db *zodb.DB) (δtrackSet PPTreeSubSet, err error) {
defer xerr.Context(&err, "ΔTtail rebuild")
// XXX locking
tracefΔBtail("\nRebuild @%s .. @%s\n", δBtail.Tail(), δBtail.Head())
tracefΔBtail("trackSet: %v\n", δBtail.trackSet)
tracefΔBtail("trackNew: %v\n", δBtail.trackNew)
tracefΔBtail("\nRebuild %s @%s .. @%s\n", root, δZtail.Tail(), δZtail.Head())
tracefΔBtail("trackNew: %v\n", δTtail.trackNew)
trackNew := δBtail.trackNew
δBtail.trackNew = PPTreeSubSet{}
trackNew := δTtail.trackNew
δTtail.trackNew = PPTreeSubSet{}
if len(trackNew) == 0 {
return
return nil, nil
}
// go backwards and merge vδT <- treediff(lo..hi/trackNew)
vδZ := δBtail.δZtail.Data()
vδtrack := []*ΔPPTreeSubSet{}
for i := len(vδZ)-1; i>=0; i-- {
δZ := vδZ[i]
vδZ := δZtail.Data()
for {
δtkeycov := &RangedKeySet{} // all keys coming into tracking set during this lo<-hi scan
trackNewCur := trackNew.Clone() // trackNew adjusted as of when going to i<- entry
for i := len(vδZ)-1; i>=0; i-- {
δZ := vδZ[i]
// XXX dup wrt Update?
var atPrev zodb.Tid
if i > 0 {
atPrev = vδZ[i-1].Rev
} else {
atPrev = δZtail.Tail()
}
// XXX dup wrt Update?
δtrackNew, δtkeycov_, err := δTtail.rebuild1(atPrev, δZ, trackNewCur, db)
if err != nil {
return nil, err
}
var atPrev zodb.Tid
if i > 0 {
atPrev = vδZ[i-1].Rev
} else {
atPrev = δBtail.δZtail.Tail()
trackNewCur.ApplyΔ(δtrackNew)
δtkeycov.UnionInplace(δtkeycov_)
// XXX update .KVAtTail, .lastRevOf
}
δZTC, δtopsByRoot := δZConnectTracked(δZ.Changev, trackNew)
// an iteration closer to tail may turn out to add a key to the tracking set.
// We have to recheck all entries newer that revision for changes to that key,
// for example:
//
// 8 5*
// / \ <- / \
// 2 8 2* 7
//
// here initial tracked set is 5*-2*. Going to earlier revision
// 2'th keycov range is widen from [-∞,5) to [-∞,7), so 5*-7 in
// later revision have to be rechecked because 7 was added into
// tracking set.
//
// Implement this via restarting from head and cycling until
// set of tracked keys does not grow anymore.
if δtkeycov.Empty() {
break
}
debugfΔBtail("\n rebuild @%s <- @%s\n", atPrev, δZ.Rev)
debugfΔBtail(" δZ:\t%v\n", δZ.Changev)
debugfΔBtail(" trackNew: %v\n", trackNew)
debugfΔBtail(" trackSet: %v\n", δBtail.trackSet) // XXX needed?
defer debugfΔBtail("\n\n")
err := widenTrackNew(trackNew, δtkeycov, root, δZtail.Head(), db)
if err != nil {
return nil, err
}
}
// XXX len(δtopsByRoot) == 0 -> skip
return trackNew, nil
}
// widenTrackNew widens trackNew to cover δtkeycov.
// XXX -> widenTrackSet?
func widenTrackNew(trackNew PPTreeSubSet, δtkeycov *RangedKeySet, root zodb.Oid, at zodb.Tid, db *zodb.DB) (err error) {
// XXX errctx, debug
defer xerr.Contextf(&err, "widenTrackNew tree<%s> @%s +%s", root, at, δtkeycov)
// open ZODB connection corresponding to "current" and "prev" states
txn, ctx := transaction.New(context.TODO()) // XXX
defer txn.Abort() // XXX -> into func() or don't use defer
txn, ctx := transaction.New(context.TODO()) // XXX
defer txn.Abort()
zconnPrev, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: atPrev})
if err != nil {
return err
}
zconnCurr, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: δZ.Rev})
if err != nil {
return err
zhead, err := db.Open(ctx, &zodb.ConnOptions{At: at}); /*X*/ if err != nil { return err }
xtree, err := zgetNodeOrNil(ctx, zhead, root); /*X*/ if err != nil { return err }
if xtree == nil {
// root deleted -> root node covers [-∞,∞)
trackNew.AddPath([]zodb.Oid{root})
return nil
}
tree := xtree.(*Tree) // must succeed XXX better explicit panic?
top := &nodeInRange{prefix: nil, lo: KeyMin, hi_: KeyMax, node: tree}
V := rangeSplit{top}
for _, r := range δtkeycov.AllRanges() {
lo := r.lo
for {
b, err := V.GetToLeaf(ctx, lo); /*X*/ if err != nil { return err }
trackNew.AddPath(b.Path())
// continue with next right bucket until r coverage is complete
if r.hi_ <= b.hi_ {
break
}
lo = b.hi_ + 1
}
}
return nil
}
for root, δtops := range δtopsByRoot {
// diff backwards curr -> prev
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
if err != nil {
return err
}
// FIXME use δtkeycov to recompute track coverage
_ = δtkeycov
// rebuild1 rebuilds δT for single δZ.
//
// δtrackNew/δtkeycov represents how trackNew changes when going through `atPrev <- δZ.Rev` .
func (δTtail *ΔTtail) rebuild1(atPrev zodb.Tid, δZ zodb.ΔRevEntry, trackNew PPTreeSubSet, db *zodb.DB) (δtrackNew *ΔPPTreeSubSet, δtkeycov *RangedKeySet, err error) {
defer xerr.Contextf(&err, "rebuild1 %s<-%s", atPrev, δZ.Rev)
debugfΔBtail(" -> root<%s> δkv*: %v δtrack*: %v\n", root, δT, δtrack)
debugfΔBtail("\n rebuild1 @%s <- @%s\n", atPrev, δZ.Rev)
debugfΔBtail(" δZ:\t%v\n", δZ.Changev)
debugfΔBtail(" trackNew: %v\n", trackNew)
defer func() {
debugfΔBtail("-> δtrackNew: %v\n", δtrackNew)
debugfΔBtail("-> δtkeycov: %v\n", δtkeycov)
debugfΔBtail("\n\n")
}()
trackNew.ApplyΔ(δtrack)
vδtrack = append([]*ΔPPTreeSubSet{δtrack}, vδtrack...)
δZTC, δtopsByRoot := δZConnectTracked(δZ.Changev, trackNew)
if len(δT) == 0 { // an object might be resaved without change
continue
}
// skip opening DB connections if there is no change to this tree
if len(δtopsByRoot) == 0 {
return NewΔPPTreeSubSet(), &RangedKeySet{}, nil
}
δTtail, ok := δBtail.byRoot[root]
if !ok {
// this root was not tracked before -> create δTtail for it with empty changes
δTtail = newΔTtail()
δBtail.byRoot[root] = δTtail
}
if len(δtopsByRoot) != 1 {
panicf("BUG: δtopsByRoot has > 1 entries: %v\ntrackNew: %v\nδZ: %v", δtopsByRoot, trackNew, δZ)
}
var root zodb.Oid
var δtops SetOid
for root_, δtops_ := range δtopsByRoot {
root = root_
δtops = δtops_
}
// δTtail.vδT <- merge δT*
l := len(δTtail.vδT)
j := sort.Search(l, func(k int) bool {
return δZ.Rev <= δTtail.vδT[k].Rev
})
if j == l || δTtail.vδT[j].Rev != δZ.Rev {
δTcurr := ΔTree{Rev: δZ.Rev, ΔKV: map[Key]ΔValue{}}
// insert(@j, δTcurr)
δTtail.vδT = append(δTtail.vδT[:j],
append([]ΔTree{δTcurr},
δTtail.vδT[j:]...)...)
}
δTcurr := δTtail.vδT[j]
// open ZODB connection corresponding to "current" and "prev" states
txn, ctx := transaction.New(context.TODO()) // XXX
defer txn.Abort()
for k, δv := range δT {
// the diff was backward; δTtail entries are with diff forward
δv.New, δv.Old = δv.Old, δv.New
zconnPrev, err := db.Open(ctx, &zodb.ConnOptions{At: atPrev})
if err != nil {
return nil, nil, err
}
zconnCurr, err := db.Open(ctx, &zodb.ConnOptions{At: δZ.Rev})
if err != nil {
return nil, nil, err
}
δv_, already := δTcurr.ΔKV[k]
if already {
if δv != δv_ {
panicf("[%v] inconsistent δv:\nδTcurr: %v\nδT: %v", k, δTcurr, δT)
}
} else {
δTcurr.ΔKV[k] = δv
}
}
// diff backwards curr -> prev
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, trackNew, zconnCurr, zconnPrev)
if err != nil {
return nil, nil, err
}
// XXX update .KVAtTail, .lastRevOf
}
debugfΔBtail(" -> root<%s> δkv*: %v δtrack*: %v δtkeycov*: %v\n", root, δT, δtrack, δtkeycov)
if len(δT) == 0 { // an object might be resaved without change
return δtrack, δtkeycov, nil
}
// trackNew was adjusted to correspond to @tail potentially growing its key coverage.
// Remap it back to @head and merge to .trackSet
for _, δtrack := range vδtrack {
δtrack.Reverse() // we saved it as lo<-hi; now we go lo->hi
trackNew.ApplyΔ(δtrack)
// δTtail.vδT <- merge δT*
l := len(δTtail.vδT)
j := sort.Search(l, func(k int) bool {
return δZ.Rev <= δTtail.vδT[k].Rev
})
if j == l || δTtail.vδT[j].Rev != δZ.Rev {
δTcurr := ΔTree{Rev: δZ.Rev, ΔKV: map[Key]ΔValue{}}
// insert(@j, δTcurr)
δTtail.vδT = append(δTtail.vδT[:j],
append([]ΔTree{δTcurr},
δTtail.vδT[j:]...)...)
}
δBtail.trackSet.UnionInplace(trackNew)
δTcurr := δTtail.vδT[j]
return nil
for k, δv := range δT {
// the diff was backward; δTtail entries are with diff forward
δv.New, δv.Old = δv.Old, δv.New
δv_, already := δTcurr.ΔKV[k]
if already {
if δv != δv_ {
panicf("[%v] inconsistent δv:\nδTcurr: %v\nδT: %v", k, δTcurr, δT)
}
} else {
δTcurr.ΔKV[k] = δv
}
}
// XXX update .KVAtTail, .lastRevOf (here?)
return δtrack, δtkeycov, nil
}
// Update updates δB with per-object level ZODB changes.
// Update updates δB with per-object-level ZODB changes.
//
// Only those objects from δZ that belong to tracked set are guaranteed to be
// taken into account. In other words a tree history will assuredly include
......@@ -384,36 +535,81 @@ func (δBtail *ΔBtail) rebuild() (err error) {
//
// TODO optionally accept zconnOld/zconnNew from client
func (δBtail *ΔBtail) Update(δZ *zodb.EventCommit) (_ ΔB, err error) {
δB, _, err := δBtail._Update(δZ)
headOld := δBtail.Head()
defer xerr.Contextf(&err, "ΔBtail.Update %s -> %s", headOld, δZ.Tid)
δB1, err := δBtail._Update1(δZ)
δB := ΔB{Rev: δZ.Tid, ΔByRoot: make(map[zodb.Oid]map[Key]ΔValue)}
for root, δT1 := range δB1.ByRoot {
δTtail := δBtail.vδTbyRoot[root] // must succeed
// δtkeycov1 != ø -> rebuild δTtail with trackNew ~= δtkeycov1
if !δT1.δtkeycov1.Empty() && δBtail.δZtail.Len() > 1 {
trackNew := PPTreeSubSet{}
err := widenTrackNew(trackNew, δT1.δtkeycov1, root, δBtail.Head(), δBtail.db)
if err != nil {
return ΔB{}, err
}
// XXX assert δTtail.trackNew.Empty()
δTtail.trackNew = trackNew
// XXX vvv we can skip computing diff for HEAD~..HEAD (we just
// computed it in _Update1)? (or not - trackNew is as of @head ?)
δtrackSet, err := δTtail.rebuild(root, δBtail.δZtail, δBtail.db)
if err != nil {
return ΔB{}, err
}
δBtail.trackSet.UnionInplace(δtrackSet)
}
// build δB. Even if δT=ø after _Update1, but δtkeycov1 != ø, above
// rebuild could result in head δT becoming != ø. Detect that δTtail head
// is anew by comparing to δZ.Rev.
l := len(δTtail.vδT)
if l > 0 {
δT := δTtail.vδT[l-1] // δT head
if δT.Rev == δZ.Tid {
δB.ΔByRoot[root] = δT.ΔKV
}
}
// XXX rebuild KVAtTail
// XXX rebuild lastRevOf
}
return δB, err
}
// _Update serves Update and also returns δtkeycov from treediff for testing.
type _ΔTrackKeyCov struct {
ByRoot map[zodb.Oid]*RangedKeySet // {} root -> δtrackedKeys (always grow)
// _Update1 serves Update and performs direct update of δTtail head elements from δZ.
// On key coverage growth rebuilding tail of the history is done by Update itself.
//
// _Update1 is also used in tests to verify δtkeycov return from treediff.
type _ΔBUpdate1 struct {
ByRoot map[zodb.Oid]*_ΔTUpdate1
}
func (δBtail *ΔBtail) _Update(δZ *zodb.EventCommit) (_ ΔB, δTKeyCov _ΔTrackKeyCov, err error) {
type _ΔTUpdate1 struct {
δtkeycov1 *RangedKeySet // {} root -> δtrackedKeys after first treediff (always grow)
δtrack *ΔPPTreeSubSet
}
func (δBtail *ΔBtail) _Update1(δZ *zodb.EventCommit) (δB1 _ΔBUpdate1, err error) {
headOld := δBtail.Head()
defer xerr.Contextf(&err, "ΔBtail update %s -> %s", headOld, δZ.Tid)
defer xerr.Contextf(&err, "ΔBtail.update1 %s -> %s", headOld, δZ.Tid)
tracefΔBtail("\nUpdate @%s -> @%s δZ: %v\n", δBtail.Head(), δZ.Tid, δZ.Changev)
tracefΔBtail("trackSet: %v\n", δBtail.trackSet)
tracefΔBtail("trackNew: %v\n", δBtail.trackNew)
for _, root := range δBtail.trackNewRoots.SortedElements() {
δTtail := δBtail.vδTbyRoot[root]
tracefΔBtail("[%s].trackNew: %v\n", root, δTtail.trackNew)
}
δTKeyCov = _ΔTrackKeyCov{ByRoot: make(map[zodb.Oid]*RangedKeySet)}
δB1 = _ΔBUpdate1{ByRoot: make(map[zodb.Oid]*_ΔTUpdate1)}
if XXX_killWhenRebuildWorks {
// XXX hack - until vvv is reenabled
δBtail.trackSet.UnionInplace(δBtail.trackNew)
δBtail.trackNew = PPTreeSubSet{}
} else {
// XXX reenable (currently breaks wcfs tests)
// update .trackSet and vδB from .trackNew
err = δBtail.rebuild()
err = δBtail.rebuildAll()
if err != nil {
return ΔB{}, δTKeyCov, err
return δB1, err
}
}
// XXX dup wrt rebuild?
......@@ -421,11 +617,9 @@ if XXX_killWhenRebuildWorks {
δZTC, δtopsByRoot := δZConnectTracked(δZ.Changev, δBtail.trackSet)
δB := ΔB{Rev: δZ.Tid, ByRoot: make(map[zodb.Oid]map[Key]ΔValue)}
// skip opening DB connections if there is no change to any tree node
if len(δtopsByRoot) == 0 {
return δB, δTKeyCov, nil
return δB1, nil
}
// open ZODB connections corresponding to "old" and "new" states
......@@ -434,55 +628,41 @@ if XXX_killWhenRebuildWorks {
defer txn.Abort()
zconnOld, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: headOld})
if err != nil {
return ΔB{}, δTKeyCov, err
return δB1, err
}
zconnNew, err := δBtail.db.Open(ctx, &zodb.ConnOptions{At: δZ.Tid})
if err != nil {
return ΔB{}, δTKeyCov, err
return δB1, err
}
for root, δtops := range δtopsByRoot {
δT, δtrack, δtkeycov, err := treediff(ctx, root, δtops, δZTC, δBtail.trackSet, zconnOld, zconnNew)
if err != nil {
return ΔB{}, δTKeyCov, err
return δB1, err
}
tracefΔBtail("\n-> root<%s> δkv: %v δtrack: %v δtkeycov: %v\n", root, δT, δtrack, δtkeycov)
δTtail := δBtail.vδTbyRoot[root] // must be there
if len(δT) > 0 { // an object might be resaved without change
δB.ByRoot[root] = δT
δTtail, ok := δBtail.byRoot[root]
if !ok {
// this root was not tracked before -> create δTtail for it with empty changes
δTtail = newΔTtail()
δBtail.byRoot[root] = δTtail
}
δTtail.vδT = append(δTtail.vδT, ΔTree{Rev: δZ.Tid, ΔKV: δT})
// XXX rebuild KVAtTail
// XXX rebuild lastRevOf
}
δBtail.trackSet.ApplyΔ(δtrack)
δTKeyCov.ByRoot[root] = δtkeycov
δB1.ByRoot[root] = &_ΔTUpdate1{δtkeycov1: δtkeycov, δtrack: δtrack}
}
return δB, δTKeyCov, nil
return δB1, nil
}
func (δBtail *ΔBtail) ForgetPast(revCut zodb.Tid) {
δBtail.δZtail.ForgetPast(revCut) // XXX stub
// XXX clean vδT
// TODO go through vδBroots till revcut -> find which trees to trim -> trim ΔTtails.
panic("TODO")
}
// update brings .δBtail up to date by recomputing diff XXX and taking new
// entries in .δZtail into account.
// func (δBtail *ΔBtail) update()
// Get returns root[key] as of @at database state plus revision that changed it.
//
// if revExact=False - rev is upper estimate for the revision.
......@@ -504,7 +684,7 @@ func (δBtail *ΔBtail) GetAt(ctx context.Context, root *Tree, key Key, at zodb.
// XXX dirty -> rebuild
// XXX -> index lastXXXOf(key) | linear scan ↓ looking for change <= at
δTtail := δBtail.byRoot[root.POid()]
δTtail := δBtail.vδTbyRoot[root.POid()]
if δTtail == nil {
panicf("δBtail: root<%s> not tracked", root.POid())
}
......@@ -573,7 +753,8 @@ func (δBtail *ΔBtail) GetAt(ctx context.Context, root *Tree, key Key, at zodb.
func (δBtail *ΔBtail) SliceByRootRev(root zodb.Oid, lo, hi zodb.Tid) /*readonly*/[]ΔTree {
xtail.AssertSlice(δBtail, lo, hi)
// XXX locking
δTtail, ok := δBtail.byRoot[root]
// XXX rebuild
δTtail, ok := δBtail.vδTbyRoot[root]
if !ok {
return []ΔTree{}
}
......
wcfs/internal/xbtree/δbtail_test.go
View file @ 52c72dbb
......@@ -300,13 +300,13 @@ func (rbs RBucketSet) coverage() string {
return s
}
// trackSet returns what should be ΔBtree.trackSet coverage for specified tracked key set.
// trackSet returns what should be ΔBtail.trackSet coverage for specified tracked key set.
func (rbs RBucketSet) trackSet(tracked SetKey) PPTreeSubSet {
trackSet, _ := rbs.trackSetWithCov(tracked)
return trackSet
}
// trackSetWithCov returns what should be ΔBtree.trackSet and its key coverage for specified tracked key set.
// trackSetWithCov returns what should be ΔBtail.trackSet and its key coverage for specified tracked key set.
func (rbs RBucketSet) trackSetWithCov(tracked SetKey) (trackSet PPTreeSubSet, keyCover *RangedKeySet) {
trackSet = PPTreeSubSet{}
keyCover = &RangedKeySet{}
......@@ -645,6 +645,11 @@ func _KAdj(t1, t2 *tTreeCommit, keysv ...SetKey) (kadj KAdjMatrix) {
// xverifyΔBTail_Update verifies how ΔBTail handles ZODB update for a tree with changes in between t1->t2.
//
// Note: this test verifies only single treediff step of ΔBtail.Update.
// the cycling phase of update, that is responsible to recompute older
// entries when key coverage grows, is exercised by
// xverifyΔBTail_rebuild.
func xverifyΔBTail_Update(t *testing.T, subj string, db *zodb.DB, treeRoot zodb.Oid, t1, t2 *tTreeCommit) {
// verify transition at1->at2 for all initial states of tracked {keys} from kv1 + kv2 + ∞
......@@ -690,7 +695,7 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
badv = append(badv, fmt.Sprintf(format, argv...))
}
defer func() {
if badv != nil {
if badv != nil || t.Failed() {
emsg := fmt.Sprintf("%s ; tracked=%v :\n\n", subj, initialTrackedKeys)
emsg += fmt.Sprintf("d12: %v\nδTok: %v\nδT: %v\n\n", d12, δTok, δT)
emsg += fmt.Sprintf("δZ: %v\n", δZset)
......@@ -705,38 +710,12 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
}()
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
// zconn, δbtail @at1 with initial tracked set
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: at1}); X(err)
δbtail := NewΔBtail(zconn.At(), db)
xtree, err := zgetNodeOrNil(ctx, zconn, treeRoot); X(err)
var ztree *Tree // = nil if treeRoot was deleted
if xtree != nil {
ztree = xtree.(*Tree)
}
for k := range initialTrackedKeys {
if ztree != nil {
_, ok, path, err := ZTreeGetBlkData(ctx, ztree, k); X(err)
err = δbtail.Track(k, ok, path); X(err)
} else {
// if treeRoot is deleted - add it to tracked set with every key
// being a hole. This aligns with the following situation
//
// T1 -> ø -> T2
//
// where after T1->ø, even though the tree becomes deleted, its root
// continues to be tracked and all keys migrate to holes in the
// tracking set. By aligning initial state to the same as after
// T1->ø, we test what will happen on ø->T2.
err = δbtail.track(k, false, []zodb.Oid{treeRoot}); X(err)
}
}
// δbtail @at1 with initial tracked set
δbtail := NewΔBtail(at1, db)
xtrackKeys(δbtail, treeRoot, initialTrackedKeys)
TrackedδZ = SetKey{} // Tracked ^ δZ
// TrackedδZ = Tracked ^ δZ (i.e. a tracked node has changed, or its coverage was changed)
TrackedδZ = SetKey{}
for k := range initialTrackedKeys {
leaf1 := xkv1.Get(k)
oid1 := leaf1.oid
......@@ -748,7 +727,7 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
if oid2 == zodb.InvalidOid { // embedded bucket
oid2 = leaf2.parent.oid
}
if δZset.Has(oid1) || δZset.Has(oid2) {
if δZset.Has(oid1) || δZset.Has(oid2) || (KeyRange{leaf1.lo,leaf1.hi_} != KeyRange{leaf2.lo,leaf2.hi_}) {
TrackedδZ.Add(k)
}
}
......@@ -786,35 +765,51 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
ø := PPTreeSubSet{}
// verify δbtail.trackSet against @at1
// trackSet1 = xkv1[tracked1]
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet1, tkeyCov1 := xkv1.trackSetWithCov(initialTrackedKeys)
if !δbtail.trackSet.Equal(ø) {
badf("δbtail.trackSet1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, ø)
trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
// verify δbtail.trackSet against @at1
δbtail.assertTrack(t, "1", ø, trackSet1)
// δB <- δZ
//
// also call _Update1 directly to verify δtkeycov return from treediff
// the result of Update and _Update1 should be the same since δbtail is initially empty.
δbtail_ := δbtail.Clone()
δB1, err := δbtail_._Update1(δZ); X(err) // XXX don't compute treediff twice
// assert tkeyCov1 ⊂ tkeyCov2
dkeycov12 := tkeyCov1.Difference(tkeyCov2)
if !dkeycov12.Empty() {
t.Errorf("BUG: tkeyCov1 ⊄ tkeyCov2:\n\ttkeyCov1: %s\n\ttkeyCov2: %s\n\ttkeyCov1 \\ tkeyCov2: %s", tkeyCov1, tkeyCov2, dkeycov12)
}
// assert δtkeycov == δ(tkeyCov1, tkeyCov2)
δtkeycovOK := tkeyCov2.Difference(tkeyCov1)
//fmt.Printf("tkeyCov1: %s\n", tkeyCov1)
//fmt.Printf("tkeyCov2: %s\n", tkeyCov2)
//fmt.Printf("δtkeycov: %s\n", δtkeycovOK)
δtkeycov := &RangedKeySet{}
if __, ok := δB1.ByRoot[treeRoot]; ok {
δtkeycov = __.δtkeycov1
}
if !δbtail.trackNew.Equal(trackSet1) {
badf("δbtail.trackNew1 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, trackSet1)
if !δtkeycov.Equal(δtkeycovOK) {
badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
}
// δbtail.assertTrack(t, "1", ø, trackSet1)
δB, err := δbtail.Update(δZ); X(err)
// XXX assert δB.roots == δTKeyCov roots
// XXX assert δBtail[root].vδT = δBtail_[root].vδT
// δB <- δZ
δB, δTKeyCov, err := δbtail._Update(δZ); X(err)
if δB.Rev != δZ.Tid {
badf("δB: rev != δZ.Tid ; rev=%s δZ.Tid=%s", δB.Rev, δZ.Tid)
return
}
// verify δbtail.trackSet against @at2
// trackSet2 = xkv2[tracked2] ( = xkv2[kadj[tracked1]]
trackSet2, tkeyCov2 := xkv2.trackSetWithCov(initialTrackedKeys.Union(kadjTrackedδZ))
if !δbtail.trackSet.Equal(trackSet2) {
badf("δbtail.trackSet2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackSet, trackSet2)
}
if !δbtail.trackNew.Equal(ø) {
badf("δbtail.trackNew2 wrong:\n\thave: %v\n\twant: %v", δbtail.trackNew, ø)
}
// δbtail.assertTrack(t, "2", trackSet2, ø)
δbtail.assertTrack(t, "2", trackSet2, ø)
// assert δB.ByRoot == {treeRoot -> ...} if δTok != ø
......@@ -824,27 +819,20 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
rootsOK.Add(treeRoot)
}
roots := SetOid{}
for root := range δB.ByRoot {
for root := range δB.ΔByRoot {
roots.Add(root)
}
if !reflect.DeepEqual(roots, rootsOK) {
badf("δB: roots != rootsOK ; roots=%v rootsOK=%v", roots, rootsOK)
}
_, inδB := δB.ByRoot[treeRoot]
_, inδB := δB.ΔByRoot[treeRoot]
if !inδB {
return
}
// assert δtkeycov == δ(tkeyCov1, tkeyCov2)
δtkeycovOK := tkeyCov2.Difference(tkeyCov1)
δtkeycov := δTKeyCov.ByRoot[treeRoot]
if !δtkeycov.Equal(δtkeycovOK) {
badf("δtkeycov wrong:\nhave: %s\nwant: %s", δtkeycov, δtkeycovOK)
}
// δT <- δB
δToid := δB.ByRoot[treeRoot] // {} k -> δoid
δToid := δB.ΔByRoot[treeRoot] // {} k -> δoid
δT = XGetδKV(db, at1,at2, δToid) // {} k -> δ(ZBlk(oid).data)
// δT must be subset of d12.
......@@ -883,36 +871,64 @@ func xverifyΔBTail_Update1(t *testing.T, subj string, db *zodb.DB, treeRoot zod
// assertTrack verifies that trackSet == trackSetOK.
// XXX place
func assertTrack(t *testing.T, subj string, trackSet, trackSetOK PPTreeSubSet) {
// XXX inline into δbtail.assertTrack?
func _assertTrack(t *testing.T, subj string, trackSet, trackSetOK PPTreeSubSet) {
t.Helper()
eq := trackSet.Equal(trackSetOK)
if !eq {
if !trackSet.Equal(trackSetOK) {
t.Errorf("%s:\n\thave: %v\n\twant: %v", subj, trackSet, trackSetOK)
}
}
// assertTrack verifies state of .trackSet and .trackNew.
// assertTrack verifies state of .trackSet and ΔTtail.trackNew.
// it assumes that only one tree root is being tracked.
// XXX place
func (δbtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK, trackNewOK PPTreeSubSet) {
func (δBtail *ΔBtail) assertTrack(t *testing.T, subj string, trackSetOK PPTreeSubSet, trackNewOK PPTreeSubSet) {
t.Helper()
assertTrack(t, subj + ": trackSet", δbtail.trackSet, trackSetOK)
assertTrack(t, subj + ": trackNew", δbtail.trackNew, trackNewOK)
_assertTrack(t, subj + ": trackSet", δBtail.trackSet, trackSetOK)
roots := SetOid{}
for root := range δBtail.vδTbyRoot {
roots.Add(root)
}
nrootsOK := 1
if trackSetOK.Empty() && trackNewOK.Empty() {
nrootsOK = 0
}
if len(roots) != nrootsOK {
t.Errorf("%s: len(vδTbyRoot) != %d ; roots=%v", subj, nrootsOK, roots)
return
}
if nrootsOK == 0 {
return
}
root := roots.Elements()[0]
δTtail := δBtail.vδTbyRoot[root]
trackNewRootsOK := SetOid{}
if !trackNewOK.Empty() {
trackNewRootsOK.Add(root)
}
if !δBtail.trackNewRoots.Equal(trackNewRootsOK) {
t.Errorf("%s: trackNewRoots:\n\thave: %v\n\twant: %v", subj, δBtail.trackNewRoots, trackNewRootsOK)
}
_assertTrack(t, subj + ": vδT.trackNew", δTtail.trackNew, trackNewOK)
}
// xverifyΔBTail_rebuild verifies δBtail.rebuild during t0->t1->t2 transition.
// xverifyΔBTail_rebuild verifies ΔBtail.rebuild during t0->t1->t2 transition.
//
// t0->t1 exercises from-scratch rebuild,
// t1->t2 further exercises incremental rebuild.
//
// It also exercises rebuild phase of ΔBtail.Update.
func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1, t2 *tTreeCommit) {
// XXX handle DEL
// XXX Update -> Track -> rebuild ...
// XXX can start with non-existing tree
// t1 := t2.prev
// t0 := t1.prev
t.Run(fmt.Sprintf("rebuild/%s→%s", t0.tree, t1.tree), func(t *testing.T) {
t.Skip("TODO") // FIXME rebuild is currently broken
tAllKeys := allTestKeys(t0, t1, t2)
tAllKeyv := tAllKeys.SortedElements()
......@@ -927,7 +943,6 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
fmt.Printf("@%s: %v\n", xat[t1.at], t1.xkv.Flatten())
fmt.Printf("@%s: %v\n", xat[t2.at], t2.xkv.Flatten())
kadj01 := KAdj(t0,t1, allTestKeys(t0,t1,t2))
kadj10 := KAdj(t1,t0, allTestKeys(t0,t1,t2))
kadj21 := KAdj(t2,t1, allTestKeys(t0,t1,t2))
kadj12 := KAdj(t1,t2, allTestKeys(t0,t1,t2))
......@@ -958,7 +973,9 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
// assert trackSet=ø, trackNew=ø, vδB=[]
δbtail.assertTrack(t, "@at0", ø, ø) // XXX + vδB
xverifyΔBTail_rebuild_U(t, δbtail, t0, t1, xat, /*trackSet=*/ø)
xverifyΔBTail_rebuild_U(t, δbtail, db, treeRoot, t0, t1, xat,
/*trackSet=*/ø,
/*vδT=ø*/)
xverifyΔBTail_rebuild_TR(t, db, δbtail, t1, treeRoot, xat,
// after Track(keys1)
keys1,
......@@ -970,10 +987,34 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
/*vδT=*/ δkv1_1)
t.Run((" →" + t2.tree), func(t *testing.T) {
// tracked keys1 becomes tracked keys1_2 after Update(t1->t2)
// keys1_2 := kadj12.Map(keys1)
keys1_2 := kadj12.Map(kadj01.Map(keys1_0))
xverifyΔBTail_rebuild_U(t, δbtail, t1, t2, xat, /*trackSet=*/t2.xkv.trackSet(keys1_2))
// keys1R2 is full set of keys that should become tracked after
// Update() (which includes rebuild)
keys1R2 := kadj12.Map(keys1)
for {
keys1R2_ := kadj10.Map(kadj21.Map(keys1R2))
if keys1R2.Equal(keys1R2_) {
break
}
keys1R2 = keys1R2_
}
// δkvX_k1R2 = tX.δxkv / keys1R2
δkv1_k1R2 := map[Key]Δstring{}
δkv2_k1R2 := map[Key]Δstring{}
for k := range keys1R2 {
δv1, ok := t1.δxkv[k]
if ok {
δkv1_k1R2[k] = δv1
}
δv2, ok := t2.δxkv[k]
if ok {
δkv2_k1R2[k] = δv2
}
}
xverifyΔBTail_rebuild_U(t, δbtail, db, treeRoot, t1, t2, xat,
/*trackSet=*/t2.xkv.trackSet(keys1R2),
/*vδT=*/ δkv1_k1R2, δkv2_k1R2)
// tRestKeys2 = tAllKeys - keys1
tRestKeys2 := tAllKeys.Difference(keys1)
......@@ -984,60 +1025,69 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
keys2.Add(tRestKeyv2[idx2])
}
// find out full set of keys that should become tracked after
// keys12R2 is full set of keys that should become tracked after
// Track(keys2) + rebuild
keys12R := keys1_2.Union(keys2)
keys12R2 := keys1R2.Union(keys2)
for {
keys12R_ := kadj10.Map(kadj21.Map(keys12R))
if keys12R.Equal(keys12R_) {
keys12R2_ := kadj10.Map(kadj21.Map(keys12R2))
if keys12R2.Equal(keys12R2_) {
break
}
keys12R = keys12R_
keys12R2 = keys12R2_
}
// δkv1_2 = t1.δxkv / kadj10(kadj21(kadj12(keys1) | keys2))
// δkv2_2 = t2.δxkv / kadj10(kadj21(kadj12(keys1) | keys2))
// keys12_2 := keys1_2.Union(keys2)
// keys12_0 := kadj10.Map(kadj21.Map(keys12_2))
/*
fmt.Printf("\n\n\nKKK\nkeys1=%s keys2=%s\n", keys1, keys2)
fmt.Printf("keys12R=%s\n", keys12R)
fmt.Printf("keys1R2: %s\n", keys1R2)
fmt.Printf("keys12R2: %s\n", keys12R2)
fmt.Printf("t0.xkv: %v\n", t0.xkv)
fmt.Printf("t1.xkv: %v\n", t1.xkv)
fmt.Printf("t2.xkv: %v\n", t2.xkv)
fmt.Printf("kadj21: %v\n", kadj21)
fmt.Printf("kadj12: %v\n", kadj12)
fmt.Printf("t2.xkv.trackSet(%s) -> %s", keys12R, t2.xkv.trackSet(keys12R))
fmt.Printf("t2.xkv.trackSet(keys2) -> %s\n", t2.xkv.trackSet(keys2))
fmt.Printf("t2.xkv.trackSet(keys1R2) -> %s\n", t2.xkv.trackSet(keys1R2))
fmt.Printf("t2.xkv.trackSet(keys2) \\ t2.xkv.trackSet(keys1R2) -> %s\n",
t2.xkv.trackSet(keys2).Difference(t2.xkv.trackSet(keys1R2)))
fmt.Printf("\n\n\n")
*/
δkv1_2 := map[Key]Δstring{}
δkv2_2 := map[Key]Δstring{}
for k := range keys12R {
// δkvX_k12R2 = tX.δxkv / keys12R2
δkv1_k12R2 := map[Key]Δstring{}
δkv2_k12R2 := map[Key]Δstring{}
for k := range keys12R2 {
δv1, ok := t1.δxkv[k]
if ok {
δkv1_2[k] = δv1
δkv1_k12R2[k] = δv1
}
δv2, ok := t2.δxkv[k]
if ok {
δkv2_2[k] = δv2
δkv2_k12R2[k] = δv2
}
}
// t.Run is expensive at this level of nest
// t.Run(" T"+keys2.String()+";R", func(t *testing.T) {
δbtail_ := δbtail.clone()
δbtail_ := δbtail.Clone()
xverifyΔBTail_rebuild_TR(t, db, δbtail_, t2, treeRoot, xat,
// after Track(keys2)
keys2,
// /*trackSet*/ t2.xkv.trackSet(keys1_0.Union(keys1_2)),
/*trackSet*/ t2.xkv.trackSet(keys1_2),
// FIXME vvv trackNew should not cover ranges that are already in trackSet
/*trackNew*/ t2.xkv.trackSet(keys2),
/*trackSet*/ t2.xkv.trackSet(keys1R2),
/*trackNew*/ t2.xkv.trackSet(keys2).Difference(
// trackNew should not cover ranges that are
// already in trackSet
t2.xkv.trackSet(keys1R2)),
// after rebuild
/* trackSet=*/ t2.xkv.trackSet(keys12R),
/*vδT=*/ δkv1_2, δkv2_2)
/* trackSet=*/ t2.xkv.trackSet(keys12R2),
/*vδT=*/ δkv1_k12R2, δkv2_k12R2)
// XXX move vvv to separate test (out of this inner loop)
// ΔBtail.Clone had bug that aliased klon data to orig
assertΔTtail(t, "BUG: after clone check", δbtail, db, t2, treeRoot, xat,
/*vδT=*/ δkv1_k1R2, δkv2_k1R2)
// })
}
})
......@@ -1046,51 +1096,54 @@ func xverifyΔBTail_rebuild(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, t0, t1
})
}
// xverifyΔBTail_rebuild_U verifies ΔBTree state after Update(ti->tj).
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, ti, tj *tTreeCommit, xat map[zodb.Tid]string, trackSet PPTreeSubSet) {
// xverifyΔBTail_rebuild_U verifies ΔBtail state after Update(ti->tj).
func xverifyΔBTail_rebuild_U(t *testing.T, δbtail *ΔBtail, db *zodb.DB, treeRoot zodb.Oid, ti, tj *tTreeCommit, xat map[zodb.Tid]string, trackSet PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
X := exc.Raiseif
ø := PPTreeSubSet{}
subj := fmt.Sprintf("after Update(@%s→@%s)", xat[ti.at], xat[tj.at])
// Update ati -> atj
δB, err := δbtail.Update(tj.δZ); X(err)
// XXX assert δB.Rev = tj.at; δB = δ(ti,tj)/initially tracked
δbtail.assertTrack(t, fmt.Sprintf("after Update(@%s→@%s)", xat[ti.at], xat[tj.at]), trackSet, ø)
δbtail.assertTrack(t, subj, trackSet, ø)
_ = δB
assertΔTtail(t, subj, δbtail, db, tj, treeRoot, xat, vδTok...)
}
// xverifyΔBTail_rebuild_TR verifies ΔBTree state after Track(keys) + rebuild.
func xverifyΔBTail_rebuild_TR(t *testing.T, db *zodb.DB, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys SetKey, trackSet PPTreeSubSet, trackNew, trackSetAfterRebuild PPTreeSubSet,vδTok ...map[Key]Δstring) {
// xverifyΔBTail_rebuild_TR verifies ΔBtail state after Track(keys) + rebuild.
func xverifyΔBTail_rebuild_TR(t *testing.T, db *zodb.DB, δbtail *ΔBtail, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, keys SetKey, trackSet PPTreeSubSet, trackNew, trackSetAfterRebuild PPTreeSubSet, vδTok ...map[Key]Δstring) {
t.Helper()
X := exc.Raiseif
ø := PPTreeSubSet{}
// Track(keys)
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := db.Open(ctx, &zodb.ConnOptions{At: tj.at}); X(err)
xtree, err := zconn.Get(ctx, treeRoot); X(err)
ztree := xtree.(*Tree)
for k := range keys {
_, ok, path, err := ZTreeGetBlkData(ctx, ztree, k); X(err)
err = δbtail.Track(k, ok, path); X(err)
}
xtrackKeys(δbtail, treeRoot, keys)
δbtail.assertTrack(t, fmt.Sprintf("@%s: after Track%v", xat[tj.at], keys), trackSet, trackNew)
subj := fmt.Sprintf("@%s: after Track%v", xat[tj.at], keys)
δbtail.assertTrack(t, subj, trackSet, trackNew)
// XXX vδB=[ø]
δbtail.rebuild()
δbtail.rebuildAll()
δbtail.assertTrack(t, fmt.Sprintf("@%s: after Track%v + rebuild", xat[tj.at], keys), trackSetAfterRebuild, ø)
subj += " + rebuild"
δbtail.assertTrack(t, subj, trackSetAfterRebuild, ø)
// XXX assert vδB=[δ1/T(keys)]
// XXX verify Get
// verify SliceByRootRev XXX -> verify δbtail.byRoot[treeRoot] directly
// verify δbtail.vδTbyRoot[treeRoot]
assertΔTtail(t, subj, δbtail, db, tj, treeRoot, xat, vδTok...)
}
// assertΔTtail verifies state of ΔTtail that corresponds to treeRoot in δbtail.
func assertΔTtail(t *testing.T, subj string, δbtail *ΔBtail, db *zodb.DB, tj *tTreeCommit, treeRoot zodb.Oid, xat map[zodb.Tid]string, vδTok ...map[Key]Δstring) {
t.Helper()
// XXX +KVAtTail, +lastRevOf
l := len(vδTok)
vatOK := []zodb.Tid{}
vδTok_ := []map[Key]Δstring{}
var vatOK []zodb.Tid
var vδTok_ []map[Key]Δstring
t0 := tj
for i := 0; i<l; i++ {
// empty vδTok entries means they should be absent in vδT
......@@ -1101,16 +1154,18 @@ func xverifyΔBTail_rebuild_TR(t *testing.T, db *zodb.DB, δbtail *ΔBtail, tj *
t0 = t0.prev
}
vδTok = vδTok_
lo := t0.at
hi := tj.at
vδToid := δbtail.SliceByRootRev(treeRoot, lo, hi)
δTtail, ok := δbtail.vδTbyRoot[treeRoot]
var vδToid []ΔTree
if ok {
vδToid = δTtail.vδT
}
l = len(vδToid)
vat := make([]zodb.Tid, l)
vδT := []map[Key]Δstring{}
atPrev := lo
for i, δToid := range vδToid {
vat[i] = δToid.Rev
var vat []zodb.Tid
var vδT []map[Key]Δstring
atPrev := t0.at
for _, δToid := range vδToid {
vat = append(vat, δToid.Rev)
δT := XGetδKV(db, atPrev, δToid.Rev, δToid.ΔKV) // {} k -> δ(ZBlk(oid).data)
vδT = append(vδT, δT)
atPrev = δToid.Rev
......@@ -1125,13 +1180,49 @@ func xverifyΔBTail_rebuild_TR(t *testing.T, db *zodb.DB, δbtail *ΔBtail, tj *
for i := 0; i<len(vδTok); i++ {
want += fmt.Sprintf("\n\t@%s: %v", xat[vatOK[i]], vδTok[i])
}
t.Errorf("@%s: after Track%v + rebuild: SliceByRootRev:\nhave: %v\nwant: %v", xat[tj.at], keys, have, want)
t.Errorf("%s: vδT:\nhave: %v\nwant: %v", subj, have, want)
}
}
// xtrackKeys issues δbtail.Track requests for tree[keys].
// XXX place
func xtrackKeys(δbtail *ΔBtail, treeRoot zodb.Oid, keys SetKey) {
X := exc.Raiseif
txn, ctx := transaction.New(context.Background())
defer txn.Abort()
zconn, err := δbtail.db.Open(ctx, &zodb.ConnOptions{At: δbtail.Head()}); X(err)
xtree, err := zgetNodeOrNil(ctx, zconn, treeRoot); X(err)
var ztree *Tree // = nil if treeRoot was deleted
if xtree != nil {
ztree = xtree.(*Tree)
}
for k := range keys {
if ztree != nil {
_, _, path, err := ZTreeGetBlkData(ctx, ztree, k); X(err)
err = δbtail.Track(k, path); X(err)
} else {
// if treeRoot is deleted - add it to tracked set with every key
// being a hole. This aligns with the following situation
//
// T1 -> ø -> T2
//
// where after T1->ø, even though the tree becomes deleted, its root
// continues to be tracked and all keys migrate to holes in the
// tracking set. By aligning initial state to the same as after
// T1->ø, we test what will happen on ø->T2.
err = δbtail.track(k, []zodb.Oid{treeRoot}); X(err)
}
}
}
// xverifyΔBTail_GetAt verifies δBtail.Get on series of vt ZODB changes.
// XXX
// XXX kill
/*
func ___xverifyΔBTail_GetAt(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt ...*tTreeCommit) {
subj := vt[0].tree
for _, t := range vt[1:] {
......@@ -1182,8 +1273,8 @@ func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*t
ztree := xtree.(*Tree)
for k := range keys {
_, ok, path, err := ZTreeGetBlkData(ctx, ztree, k); X(err)
err = δbtail.Track(k, ok, path); X(err)
_, _, path, err := ZTreeGetBlkData(ctx, ztree, k); X(err)
err = δbtail.Track(k, path); X(err)
}
// verify GetAt(k, @at) for all keys and @at
......@@ -1217,6 +1308,7 @@ func xverifyΔBTail_GetAt1(t *testing.T, db *zodb.DB, treeRoot zodb.Oid, vt []*t
}
}
}
*/
// ΔBTestEntry represents one entry in ΔBTail tests.
......@@ -1241,7 +1333,7 @@ func ΔBTest(xtest interface{}) ΔBTestEntry {
return test
}
// ΔBCommit represent test commit changing a tree.
// tTreeCommit represent test commit changing a tree.
type tTreeCommit struct {
tree string // the tree in toplogy-encoding
prev *tTreeCommit // previous commit
......@@ -1585,6 +1677,23 @@ func TestΔBTail(t *testing.T) {
"T/T/B0:a",
"T/B0:a",
// δtkeycov grows due to change in parent tree only
"T3/B1:a-B8:c",
"T7/B1:a-B8:c",
// ----//----
"T3/B1:a,2:b-B8:c,9:d",
"T7/B1:a,2:b-B8:c,9:d",
// ----//---- depth=2
"T3/T-T/B1:a,2:b-B8:c,9:d",
"T7/T-T/B1:a,2:b-B8:c,9:d",
// ----//---- found by AllStructs
"T1,3/B0:d-B1:a-B3:d,4:g",
"T1,4/B0:e-B1:a-B4:c",
// ----//---- found by AllStructs
"T2,4/T-T-T/T1-T-B4:f/T-T-B3:f/B0:h-B1:f",
"T4/T-T/B3:f-T/B4:a",
// "T/T/T/B1:a,2:b",
// "T/T/B1:a",
......@@ -1671,8 +1780,8 @@ func TestΔBTailAllStructs(t *testing.T) {
X := exc.Raiseif
// considerations:
// - depth↑ better for testing (more tricky topologies)
// - nsplit↑ not so better for testing (leave s=1, max s=2)
// - maxdepth↑ better for testing (more tricky topologies)
// - maxsplit↑ not so better for testing (leave s=1, max s=2)
// - |kmin - kmax| affects N(variants) significantly
// -> keep key range small (dumb increase does not help testing)
// - N(keys) affects N(variants) significantly
......@@ -1735,6 +1844,10 @@ func TestΔBTailAllStructs(t *testing.T) {
return vv[i:i+1]
}
// XXX rework to be effective with testing rebuild:
// - AllStructs accept only 1 kv
// - we call it 3 times for kv1 kv2 kv3
// - iterate via triTravelXXXway ... here instead of in treegen.py
testq := make(chan ΔBTestEntry)
go func() {
defer close(testq)
......@@ -1744,8 +1857,8 @@ func TestΔBTailAllStructs(t *testing.T) {
kv1 := map[Key]string{}
kv2 := map[Key]string{}
for k := range keys1 { kv1[Key(k)] = randv() }
for k := range keys2 { kv2[Key(k)] = randv() }
for _, k := range keys1 { kv1[Key(k)] = randv() }
for _, k := range keys2 { kv2[Key(k)] = randv() }
// given (kv1, kv2) - test on automatically generated (tree1 -> tree2)
reqSeed := rng.Int63()
......@@ -1902,39 +2015,6 @@ func allTestKeys(vt ...*tTreeCommit) SetKey {
return allKeys
}
/*
// easies debugging / makes error output stable from run to run.
func (ks SetKey) SortedElements() []Key {
keyv := ks.Elements()
sort.Slice(keyv, func(i, j int) bool {
return keyv[i] < keyv[j]
})
return keyv
}
func (ks SetKey) String() string {
strv := []string{}
for _, k := range ks.SortedElements() {
strv = append(strv, fmt.Sprintf("%d", k))
}
return "{" + strings.Join(strv, " ") + "}"
}
func (os SetOid) SortedElements() []zodb.Oid {
oidv := os.Elements()
sort.Slice(oidv, func(i, j int) bool {
return oidv[i] < oidv[j]
})
return oidv
}
func (os SetOid) String() string {
strv := []string{}
for _, oid := range os.SortedElements() {
strv = append(strv, fmt.Sprintf("%s", oid))
}
return "{" + strings.Join(strv, " ") + "}"
}
*/
func sortedKeys(kv map[Key]Δstring) []Key {
keyv := []Key{}
for k := range kv {
......
wcfs/internal/zdata/δftail.go
View file @ 52c72dbb
......@@ -160,7 +160,7 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
// XXX blk = ∞ from beginning ?
blk = xbtree.KeyMax
}
err := δFtail.δBtail.Track(blk, zblk != nil, path)
err := δFtail.δBtail.Track(blk, path)
if err != nil {
panic(err) // XXX -> error? errctx
}
......@@ -233,7 +233,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
// take btree changes into account
for root, δt := range δB.ByRoot {
for root, δt := range δB.ΔByRoot {
files := δFtail.fileIdx[root]
if len(files) == 0 {
panicf("BUG: ΔFtail: root<%s> -> ø files", root)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7