Commit 588a512a authored by Kirill Smelkov's avatar Kirill Smelkov

X zdata: Switch SliceByFileRev not to clone Zinblk

Zinblk is potentially unbounded large.
Should also help further steps on concurrency.
parent 3ae45a4c
...@@ -137,31 +137,55 @@ func (t *T) Head() *Commit { ...@@ -137,31 +137,55 @@ func (t *T) Head() *Commit {
// XGetCommit finds and returns Commit created with revision at. // XGetCommit finds and returns Commit created with revision at.
func (t *T) XGetCommit(at zodb.Tid) *Commit { func (t *T) XGetCommit(at zodb.Tid) *Commit {
commit, _, _ := t.getCommit(at)
if commit == nil {
panicf("no commit corresponding to @%s", at)
}
return commit
}
func (t *T) getCommit(at zodb.Tid) (commit, cprev, cnext *Commit) {
l := len(t.commitv) l := len(t.commitv)
i := sort.Search(l, func(i int) bool { i := sort.Search(l, func(i int) bool {
return at <= t.commitv[i].At return at <= t.commitv[i].At
}) })
var commit *Commit
if i < l { if i < l {
commit = t.commitv[i] commit = t.commitv[i]
if commit.At != at { if commit.At != at {
cnext = commit
commit = nil commit = nil
} else if i+1 < l {
cnext = t.commitv[i+1]
} }
} }
if commit == nil { if i > 0 {
panicf("no commit corresponding to @%s", at) cprev = t.commitv[i-1]
} }
if commit.idx != i { if commit != nil && commit.idx != i {
panicf("BUG: commit.idx (%d) != i (%d)", commit.idx, i) panicf("BUG: commit.idx (%d) != i (%d)", commit.idx, i)
} }
return commit return commit, cprev, cnext
} }
// AtSymb returns symbolic representation of at, for example "at3". // AtSymb returns symbolic representation of at, for example "at3".
// //
// at must correspond to a Commit. // at should correspond to a Commit.
func (t *T) AtSymb(at zodb.Tid) string { func (t *T) AtSymb(at zodb.Tid) string {
return t.XGetCommit(at).AtSymb() commit, cprev, cnext := t.getCommit(at)
if commit != nil {
return commit.AtSymb()
}
// at does not correspond to commit - return something like ~at2<xxxx>at3
s := "~"
if cprev != nil {
s += cprev.AtSymb() + "<"
}
s += at.String()
if cnext != nil {
s += ">" + cnext.AtSymb()
}
return s
} }
// AtSymb returns symbolic representation of c.At, for example "at3". // AtSymb returns symbolic representation of c.At, for example "at3".
......
...@@ -180,7 +180,7 @@ type _ΔFileEpoch struct { ...@@ -180,7 +180,7 @@ type _ΔFileEpoch struct {
type _RootTrack struct { type _RootTrack struct {
// XXX -> ftrackSet ? // XXX -> ftrackSet ?
files setOid // {}foid which ZBigFiles refer to this tree files setOid // {}foid which ZBigFiles refer to this tree
// XXX -> ztrackInBlk ? // XXX -> ztrackInBlk ? trackZInblk ?
Zinblk map[zodb.Oid]setI64 // {} zblk -> {}blk which blocks map to zblk Zinblk map[zodb.Oid]setI64 // {} zblk -> {}blk which blocks map to zblk
} }
...@@ -618,6 +618,18 @@ func (δftail *_ΔFileTail) _forgetPast(revCut zodb.Tid) { ...@@ -618,6 +618,18 @@ func (δftail *_ΔFileTail) _forgetPast(revCut zodb.Tid) {
// TODO if needed // TODO if needed
// func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF // func (δFtail *ΔFtail) SliceByRev(lo, hi zodb.Tid) /*readonly*/ []ΔF
// _ZinblkOverlay is used by SliceByFileRev.
// It combiness read-only Zinblk base with read-write adjustment.
// It provides the following operations:
//
// - Get(zblk) -> {blk},
// - AddBlk(zblk, blk),
// - DelBlk(zblk, blk)
type _ZinblkOverlay struct {
Base map[zodb.Oid]setI64 // taken from _RootTrack.Zinblk or _ΔFileEpoch.oldZinblk
Adj map[zodb.Oid]setI64 // blk<0 reresents whitout
}
// SliceByFileRev returns history of file changes in (lo, hi] range. // SliceByFileRev returns history of file changes in (lo, hi] range.
// //
// it must be called with the following condition: // it must be called with the following condition:
...@@ -706,9 +718,8 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado ...@@ -706,9 +718,8 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
var head zodb.Tid // head] of current epoch coverage var head zodb.Tid // head] of current epoch coverage
// state of Zinblk as we are scanning ← current epoch // state of Zinblk as we are scanning ← current epoch
// initially corresponds to head of the epoch (= @head for latest epoch) // initially corresponds to head of the epoch (= @head for latest epoch)
// // Zinblk := map[zodb.Oid]setI64{} // zblk -> which #blk refers to it
// XXX -> ZinblkOverlay(.base = Zinblk@head w/o Clone; .adj = δZinblk) Zinblk := _ZinblkOverlay{} // zblk -> which #blk refers to it
Zinblk := map[zodb.Oid]setI64{} // zblk -> which #blk refers to it
var ZinblkAt zodb.Tid // Zinblk covers [ZinblkAt,<next δT>) var ZinblkAt zodb.Tid // Zinblk covers [ZinblkAt,<next δT>)
if ie+1 == le { if ie+1 == le {
// head // head
...@@ -716,19 +727,19 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado ...@@ -716,19 +727,19 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
head = δFtail.Head() head = δFtail.Head()
rt, ok := δFtail.byRoot[root] // XXX locking rt, ok := δFtail.byRoot[root] // XXX locking
if ok { if ok {
for zblk, inblk := range rt.Zinblk { // XXX -> no clone Zinblk.Base = rt.Zinblk
Zinblk[zblk] = inblk.Clone() // for zblk, inblk := range rt.Zinblk { // XXX -> no clone
} // Zinblk[zblk] = inblk.Clone()
// }
} }
} else { } else {
δE := vδE[ie+1] δE := vδE[ie+1]
root = δE.oldRoot root = δE.oldRoot
head = δE.Rev - 1 // TODO better set to exact revision coming before δE.Rev head = δE.Rev - 1 // TODO better set to exact revision coming before δE.Rev
for zblk, inblk := range δE.oldZinblk { Zinblk.Base = δE.oldZinblk
Zinblk[zblk] = inblk.Clone() // XXX Clone needed ? -> just use oldTrackSetZBlk directly // for zblk, inblk := range δE.oldZinblk {
// XXX -> just use Zinblk = δE.oldTrackSetZBlk directly (without full clone) // Zinblk[zblk] = inblk.Clone() // XXX -> no Clone needed
// XXX no -> vvv we adjust Zinblk -> thinking // }
}
} }
//fmt.Printf("Zinblk: %v\n", Zinblk) //fmt.Printf("Zinblk: %v\n", Zinblk)
...@@ -759,25 +770,28 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado ...@@ -759,25 +770,28 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
// also all present @at2, @at3 and @at4 - because @at2 both 0 // also all present @at2, @at3 and @at4 - because @at2 both 0
// and 1 are changed in the same tracked bucket. Note that // and 1 are changed in the same tracked bucket. Note that
// changes to 2 should not be present at all. // changes to 2 should not be present at all.
ZinblkAdj := map[zodb.Oid]setI64{} Zinblk.Adj = map[zodb.Oid]setI64{}
for _, δT := range vδT { for _, δT := range vδT {
for blk, δzblk := range δT.KV { for blk, δzblk := range δT.KV {
// XXX VDEL -> whiteout?
if δzblk.Old != xbtree.VDEL { if δzblk.Old != xbtree.VDEL {
inblk, ok := ZinblkAdj[δzblk.Old] inblk, ok := Zinblk.Adj[δzblk.Old]
if ok { if ok {
inblk.Del(blk) inblk.Del(blk)
} }
} }
// XXX VDEL -> whiteout?
if δzblk.New != xbtree.VDEL { if δzblk.New != xbtree.VDEL {
inblk, ok := ZinblkAdj[δzblk.New] inblk, ok := Zinblk.Adj[δzblk.New]
if !ok { if !ok {
inblk = setI64{} inblk = setI64{}
ZinblkAdj[δzblk.New] = inblk Zinblk.Adj[δzblk.New] = inblk
} }
inblk.Add(blk) inblk.Add(blk)
} }
} }
} }
/*
for zblk, inblkAdj := range ZinblkAdj { for zblk, inblkAdj := range ZinblkAdj {
inblk, ok := Zinblk[zblk] inblk, ok := Zinblk[zblk]
if !ok { if !ok {
...@@ -786,6 +800,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado ...@@ -786,6 +800,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
inblk.Update(inblkAdj) inblk.Update(inblkAdj)
} }
} }
*/
// merge vδZ and vδT of current epoch // merge vδZ and vδT of current epoch
for ((iz >= 0 && vδZ[iz].Rev > epoch) || it >= 0) { for ((iz >= 0 && vδZ[iz].Rev > epoch) || it >= 0) {
...@@ -796,7 +811,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado ...@@ -796,7 +811,7 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
if ZinblkAt <= δZ.Rev { if ZinblkAt <= δZ.Rev {
//fmt.Printf("δZ @%s\n", δZ.Rev) //fmt.Printf("δZ @%s\n", δZ.Rev)
for _, oid := range δZ.Changev { for _, oid := range δZ.Changev {
inblk, ok := Zinblk[oid] inblk, ok := Zinblk.Get_(oid)
if ok && len(inblk) != 0 { if ok && len(inblk) != 0 {
δf := vδfTail(δZ.Rev) δf := vδfTail(δZ.Rev)
δf.Blocks.Update(inblk) δf.Blocks.Update(inblk)
...@@ -814,18 +829,20 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado ...@@ -814,18 +829,20 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
for blk, δzblk := range δT.KV { for blk, δzblk := range δT.KV {
// apply in reverse as we go ← // apply in reverse as we go ←
if δzblk.New != xbtree.VDEL { if δzblk.New != xbtree.VDEL {
inblk, ok := Zinblk[δzblk.New] Zinblk.DelBlk(δzblk.New, blk)
if ok { // inblk, ok := Zinblk[δzblk.New] // XXX
inblk.Del(blk) // if ok {
} // inblk.Del(blk)
// }
} }
if δzblk.Old != xbtree.VDEL { if δzblk.Old != xbtree.VDEL {
inblk, ok := Zinblk[δzblk.Old] Zinblk.AddBlk(δzblk.Old, blk)
if !ok { // inblk, ok := Zinblk[δzblk.Old] // XXX
inblk = setI64{} // if !ok {
Zinblk[δzblk.Old] = inblk // inblk = setI64{}
} // Zinblk[δzblk.Old] = inblk
inblk.Add(blk) // }
// inblk.Add(blk)
} }
if δT.Rev <= hi { if δT.Rev <= hi {
...@@ -865,6 +882,64 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado ...@@ -865,6 +882,64 @@ func (δFtail *ΔFtail) SliceByFileRev(zfile *ZBigFile, lo, hi zodb.Tid) /*reado
return vδf return vδf
} }
// ZinblkOverlay XXX move closer to end?
func (o *_ZinblkOverlay) Get_(zoid zodb.Oid) (inblk /*readonly*/setI64, ok bool) {
base, bok := o.Base[zoid]
adj, aok := o.Adj[zoid]
if !aok {
return base, bok
}
// combine base + adj
if bok {
inblk = base.Clone()
} else {
inblk = make(setI64, len(adj))
}
for blk := range adj {
if blk < 0 { // whiteout
inblk.Del(flipsign(blk))
} else {
inblk.Add(blk)
}
}
if len(inblk) == 0 {
return nil, false
}
return inblk, true
}
func (o *_ZinblkOverlay) DelBlk(zoid zodb.Oid, blk int64) {
if blk < 0 {
panic("blk < 0")
}
o._AddBlk(zoid, flipsign(blk))
}
func (o *_ZinblkOverlay) AddBlk(zoid zodb.Oid, blk int64) {
if blk < 0 {
panic("blk < 0")
}
o._AddBlk(zoid, blk)
}
func (o *_ZinblkOverlay) _AddBlk(zoid zodb.Oid, blk int64) {
adj, ok := o.Adj[zoid]
if !ok {
adj = make(setI64, 1)
o.Adj[zoid] = adj
}
adj.Add(blk)
adj.Del(flipsign(blk))
}
// flipsign returns x with sign bit flipped.
func flipsign(x int64) int64 {
return int64(uint64(x) ^ (1<<63))
}
// BlkRevAt returns last revision that changed file[blk] as of @at database state. // BlkRevAt returns last revision that changed file[blk] as of @at database state.
// //
// if exact=False - what is returned is only an upper bound for last block revision. // if exact=False - what is returned is only an upper bound for last block revision.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment