X ΔFtail: Keep reference to ZBigFile via Oid, not via *ZBigFile

*ZBigFile is live object associated with particular ZODB Connection.

wcfs/internal/zdata/zset_bigfile.go

-// Code generated by gen-set ZBigFile *ZBigFile; DO NOT EDIT.
-
-// Copyright (C) 2015-2021  Nexedi SA and Contributors.
-//                          Kirill Smelkov <kirr@nexedi.com>
-//
-// This program is free software: you can Use, Study, Modify and Redistribute
-// it under the terms of the GNU General Public License version 3, or (at your
-// option) any later version, as published by the Free Software Foundation.
-//
-// You can also Link and Combine this program with other software covered by
-// the terms of any of the Free Software licenses or any of the Open Source
-// Initiative approved licenses and Convey the resulting work. Corresponding
-// source of such a combination shall include the source code for all other
-// software used.
-//
-// This program is distributed WITHOUT ANY WARRANTY; without even the implied
-// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-//
-// See COPYING file for full licensing terms.
-// See https://www.nexedi.com/licensing for rationale and options.
-
-package zdata
-
-// SetZBigFile is a set of *ZBigFile.
-type SetZBigFile map[*ZBigFile]struct{}
-
-// Add adds v to the set.
-func (s SetZBigFile) Add(v *ZBigFile) {
-	s[v] = struct{}{}
-}
-
-// Del removes v from the set.
-// it is noop if v was not in the set.
-func (s SetZBigFile) Del(v *ZBigFile) {
-	delete(s, v)
-}
-
-// Has checks whether the set contains v.
-func (s SetZBigFile) Has(v *ZBigFile) bool {
-	_, ok := s[v]
-	return ok
-}
-
-// Update adds t values to s.
-func (s SetZBigFile) Update(t SetZBigFile) {
-	for v := range t {
-		s.Add(v)
-	}
-}
-
-// Elements returns all elements of set as slice.
-func (s SetZBigFile) Elements() []*ZBigFile {
-	ev := make([]*ZBigFile, len(s))
-	i := 0
-	for e := range s {
-		ev[i] = e
-		i++
-	}
-	return ev
-}
-
-// Union returns s ∪ t
-func (s SetZBigFile) Union(t SetZBigFile) SetZBigFile {
-	// l = max(len(s), len(t))
-	l := len(s)
-	if lt := len(t); lt > l {
-		l = lt
-	}
-
-	u := make(SetZBigFile, l)
-
-	for v := range s {
-		u.Add(v)
-	}
-	for v := range t {
-		u.Add(v)
-	}
-	return u
-}
-
-// Intersection returns s ∩ t
-func (s SetZBigFile) Intersection(t SetZBigFile) SetZBigFile {
-	i := SetZBigFile{}
-	for v := range s {
-		if t.Has(v) {
-			i.Add(v)
-		}
-	}
-	return i
-}
-
-// Difference returns s\t.
-func (s SetZBigFile) Difference(t SetZBigFile) SetZBigFile {
-	d := SetZBigFile{}
-	for v := range s {
-		if !t.Has(v) {
-			d.Add(v)
-		}
-	}
-	return d
-}
-
-// SymmetricDifference returns s Δ t.
-func (s SetZBigFile) SymmetricDifference(t SetZBigFile) SetZBigFile {
-	d := SetZBigFile{}
-	for v := range s {
-		if !t.Has(v) {
-			d.Add(v)
-		}
-	}
-	for v := range t {
-		if !s.Has(v) {
-			d.Add(v)
-		}
-	}
-	return d
-}
-
-// Equal returns whether a == b.
-func (a SetZBigFile) Equal(b SetZBigFile) bool {
-	if len(a) != len(b) {
-		return false
-	}
-
-	for v := range a {
-		_, ok := b[v]
-		if !ok {
-			return false
-		}
-	}
-
-	return true
-}

wcfs/internal/zdata/δftail.go

@@ -19,8 +19,6 @@
 
 package zdata
 
-//go:generate ../set/gen-set zdata ZBigFile *ZBigFile zset_bigfile.go
-
 import (
 	"context"
 	"fmt"
@@ -38,6 +36,7 @@ import (
 )
 
 type SetI64 = set.SetI64
+type SetOid = set.SetOid
 
 // ΔFtail represents tail of revisional changes to files.
 //
@@ -78,7 +77,7 @@ type SetI64 = set.SetI64
 type ΔFtail struct {
 	// ΔFtail merges ΔBtail with history of ZBlk
 	δBtail  *xbtree.ΔBtail
-	fileIdx map[zodb.Oid]SetZBigFile // tree-root -> {} ZBigFile as of @head
+	fileIdx map[zodb.Oid]SetOid // tree-root -> {} ZBigFile<oid> as of @head
 
 // XXX kill
 ///*
@@ -90,14 +89,14 @@ type ΔFtail struct {
 
 	// tracked ZBlk that are not yet taken into account in current vδF.
 	// grows on new track requests; flushes on queries and update.
-	trackNew map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail // {} file -> {} oid -> zblk
+	trackNew map[zodb.Oid]map[zodb.Oid]*zblkInΔFtail // {} foid -> {} zoid -> zblk
 //*/
 }
 
 // ΔF represents a change in files space.
 type ΔF struct {
 	Rev    zodb.Tid
-	ByFile map[*ZBigFile]*ΔFile // file -> δfile
+	ByFile map[zodb.Oid]*ΔFile // foid -> δfile
 }
 
 // ΔFile represents a change to one file.
@@ -120,7 +119,7 @@ type zblkInΔFtail struct {
 	//  tree nodes and for tree_root->file)
 
 	// with which files/blocks this ZBlk is associated with as of @head state
-	infile map[*ZBigFile]SetI64 // {} file -> set(#blk)
+	infile map[zodb.Oid]SetI64 // {} foid -> set(#blk)
 }
 
 type _ZBlkInΔFtail interface { inΔFtail() *zblkInΔFtail }
@@ -136,8 +135,8 @@ func (z *zblkInΔFtail) inΔFtail() *zblkInΔFtail { return z }
 func NewΔFtail(at0 zodb.Tid, db *zodb.DB) *ΔFtail {
 	return &ΔFtail{
 		δBtail:   xbtree.NewΔBtail(at0, db),
-		fileIdx:  make(map[zodb.Oid]SetZBigFile),
-		trackNew: make(map[*ZBigFile]map[zodb.Oid]*zblkInΔFtail),
+		fileIdx:  make(map[zodb.Oid]SetOid),
+		trackNew: make(map[zodb.Oid]map[zodb.Oid]*zblkInΔFtail),
 	}
 }
 
@@ -159,6 +158,7 @@ func (δFtail *ΔFtail) Tail() zodb.Tid { return δFtail.δBtail.Tail() }
 //
 // A root can be associated with several files (each provided on different Track call).
 func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zblk ZBlk) {
+	foid := file.POid()
 	if blk == -1 {
 		// XXX blk = ∞ from beginning ?
 		blk = xbtree.KeyMax
@@ -170,22 +170,22 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
 	root := path[0].(*btree.LOBTree)
 	files, ok := δFtail.fileIdx[root.POid()]
 	if !ok {
-		files = SetZBigFile{}
+		files = SetOid{}
 		δFtail.fileIdx[root.POid()] = files
 	}
-	files.Add(file)
+	files.Add(foid)
 
 	// associate zblk with file, if it was not hole
 	if zblk != nil {
 		z := zblk.inΔFtail()
 		z.mu.Lock()
-		blocks, ok := z.infile[file]
+		blocks, ok := z.infile[foid]
 		if !ok {
 			blocks = make(SetI64, 1)
 			if z.infile == nil {
-				z.infile = make(map[*ZBigFile]SetI64)
+				z.infile = make(map[zodb.Oid]SetI64)
 			}
-			z.infile[file] = blocks
+			z.infile[foid] = blocks
 		}
 		blocks.Add(blk)
 		z.mu.Unlock()
@@ -193,10 +193,10 @@ func (δFtail *ΔFtail) Track(file *ZBigFile, blk int64, path []btree.LONode, zb
 		// XXX locking
 		if !ok {
 			// zblk was not associated with this file
-			zt := δFtail.trackNew[file]
+			zt := δFtail.trackNew[foid]
 			if zt == nil {
 				zt = make(map[zodb.Oid]*zblkInΔFtail, 1)
-				δFtail.trackNew[file] = zt
+				δFtail.trackNew[foid] = zt
 			}
 			zt[zblk.POid()] = z
 		}
@@ -233,7 +233,7 @@ func (δFtail *ΔFtail) Update(δZ *zodb.EventCommit, zhead *xzodb.ZConn) (_ ΔF
 		return ΔF{}, err
 	}
 
-	δF := ΔF{Rev: δB.Rev, ByFile: make(map[*ZBigFile]*ΔFile)}
+	δF := ΔF{Rev: δB.Rev, ByFile: make(map[zodb.Oid]*ΔFile)}
 
 	// take btree changes into account
 	for root, δt := range δB.ΔByRoot {
@@ -316,15 +316,16 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
 	}
 
 	// let's see if we need to rebuild .vδF due to not-yet processed track requests
+	foid := file.POid()
 
 	// XXX locking
 	// XXX dumb
-	zt, dirty := δFtail.trackNew[file]
+	zt, dirty := δFtail.trackNew[foid]
 	if !dirty {
 		return
 	}
 
-	delete(δFtail.trackNew, file)
+	delete(δFtail.trackNew, foid)
 	// XXX unlock here
 
 	for i, δZ := range δFtail.δBtail.ΔZtail().Data() {
@@ -338,14 +339,14 @@ func (δFtail *ΔFtail) update(file *ZBigFile) {
 			}
 
 			// XXX locking
-			// XXX -> func δF.δfile(file) ?
-			δfile, ok := δF.ByFile[file]
+			// XXX -> func δF.δfile(foid) ?
+			δfile, ok := δF.ByFile[foid]
 			if !ok {
 				δfile = &ΔFile{Rev: δF.Rev, Blocks: make(SetI64)}
-				δF.ByFile[file] = δfile
+				δF.ByFile[foid] = δfile
 			}
 
-			δfile.Blocks.Update(z.infile[file])
+			δfile.Blocks.Update(z.infile[foid])
 		}
 	}
 }
@@ -401,9 +402,10 @@ func (δFtail *ΔFtail) SliceByFileRev(file *ZBigFile, lo, hi zodb.Tid) /*readon
 	vδF = vδF[i:j+1]
 
 	// filter found changed to have only file-related bits
+	foid := file.POid()
 	var vδfile []*ΔFile
 	for _, δF := range vδF {
-		δfile, ok := δF.ByFile[file]
+		δfile, ok := δF.ByFile[foid]
 		if ok {
 			vδfile = append(vδfile, δfile)
 		}

wcfs/wcfs.go

@@ -882,7 +882,7 @@ retry:
 	if log.V(2) {
 		// debug dump δF
 		log.Infof("\n\nS: handleδZ: δF (#%d):\n", len(δF.ByFile))
-		for zfile, δfile := range δF.ByFile {
+		for foid, δfile := range δF.ByFile {
 			blkv := δfile.Blocks.Elements()
 			sort.Slice(blkv, func(i, j int) bool {
 				return blkv[i] < blkv[j]
@@ -891,19 +891,19 @@ retry:
 			if δfile.Size {
 				size = "S"
 			}
-			log.Infof("S: \t- %s\t%s %v\n", zfile.POid(), size, blkv)
+			log.Infof("S: \t- %s\t%s %v\n", foid, size, blkv)
 		}
 		log.Infof("\n\n")
 	}
 
 	wg := xsync.NewWorkGroup(ctx)
-	for zfile, δfile := range δF.ByFile {
+	for foid, δfile := range δF.ByFile {
 //		// XXX needed?
 //		// XXX even though δBtail is complete, not all ZBlk are present here
 //		file.δtail.Append(δF.Rev, δfile.Blocks.Elements())
 
-		// zfile was requested to be tracked -> it must be present in fileTab
-		file := bfdir.fileTab[zfile.POid()]
+		// file was requested to be tracked -> it must be present in fileTab
+		file := bfdir.fileTab[foid]
 		for blk := range δfile.Blocks {
 			blk := blk
 			wg.Go(func(ctx context.Context) error {
@@ -921,11 +921,11 @@ retry:
 	//
 	// do it after completing data invalidations.
 	wg = xsync.NewWorkGroup(ctx)
-	for zfile, δfile := range δF.ByFile {
+	for foid, δfile := range δF.ByFile {
 		if !δfile.Size {
 			continue
 		}
-		file := bfdir.fileTab[zfile.POid()] // must be present
+		file := bfdir.fileTab[foid] // must be present
 		wg.Go(func(ctx context.Context) error {
 			return file.invalidateAttr() // NOTE does not accept ctx
 		})
@@ -950,13 +950,15 @@ retry:
 	// 2. restat invalidated ZBigFile
 	// NOTE no lock needed since .blksize and .size are constant during lifetime of one txn.
 	// XXX -> parallel
-	for zfile := range δF.ByFile {
+	for foid := range δF.ByFile {
+		file := bfdir.fileTab[foid] // must be present
+		zfile := file.zfile
+
 		size, sizePath, err := zfile.Size(ctx)
 		if err != nil {
 			return err
 		}
 
-		file := bfdir.fileTab[zfile.POid()] // must be present
 		file.size = size
 		bfdir.δFtail.Track(zfile, -1, sizePath, nil)