From d3c8658475a96353ed67be48b301956bb9b06011 Mon Sep 17 00:00:00 2001
From: Kirill Smelkov <kirr@nexedi.com>
Date: Tue, 17 Jul 2018 14:46:52 +0300
Subject: [PATCH] .
---
wcfs/misc.go | 2 +-
wcfs/wcfs.go | 3 +-
wcfs/weak.go | 2 +-
wcfs/xxx.go | 153 +++++++++++++++++++++++++++++++++++++++++
wcfs/zodb.go | 187 ++++++++-------------------------------------------
5 files changed, 185 insertions(+), 162 deletions(-)
create mode 100644 wcfs/xxx.go
diff --git a/wcfs/misc.go b/wcfs/misc.go
index d2bfba3..aaea2d9 100644
--- a/wcfs/misc.go
+++ b/wcfs/misc.go
@@ -55,7 +55,7 @@ func (f *StaticFile) Read(_ nodefs.File, dest []byte, off int64, _ *fuse.Context
// mkdir adds child to parent as directory.
//
-// Note: parent must to be already in the filesystem tree - i.e. associated
+// Note: parent must be already in the filesystem tree - i.e. associated
// with inode. if not - nodefs will panic in Inode.NewChild on nil dereference.
func mkdir(parent nodefs.Node, name string, child nodefs.Node) {
parent.Inode().NewChild(name, true, child)
diff --git a/wcfs/wcfs.go b/wcfs/wcfs.go
index f14aa9e..0f1b525 100644
--- a/wcfs/wcfs.go
+++ b/wcfs/wcfs.go
@@ -446,6 +446,7 @@ func (br *BigFileRoot) Mkdir(name string, mode uint32, _ *fuse.Context) (*nodefs
// ZData
// str (chunk)
+
// Read implements reading from /bigfile/<bigfileX>/head/data.
// XXX and from /bigfile/<bigfileX>/@<tidX>/data.
/*
@@ -464,7 +465,7 @@ func (bf *BigFile) Read(_ nodefs.File, dest []byte, off int64, _ fuse.Context) (
// For the data itself - we put it to kernel cache and always deactivate from
// ZODB right after that.
//
-// XXX set it to Connection.CacheControl
+// TODO set it to Connection.CacheControl
type zodbCacheControl struct {}
func (cc *zodbCacheControl) WantEvict(obj Object) {
diff --git a/wcfs/weak.go b/wcfs/weak.go
index eb7be92..ce4c0e0 100644
--- a/wcfs/weak.go
+++ b/wcfs/weak.go
@@ -22,7 +22,7 @@
// See https://www.nexedi.com/licensing for rationale and options.
package main
-// weak reference
+// weak references
import (
"runtime"
diff --git a/wcfs/xxx.go b/wcfs/xxx.go
new file mode 100644
index 0000000..17e4610
--- /dev/null
+++ b/wcfs/xxx.go
@@ -0,0 +1,153 @@
+package main
+/*
+// loadInProgress entry in Conn.objtab tells users, that try to get the entry,
+// that another goroutine is already in progress of loading it.
+type loadInProgress struct {
+ ready chan struct{} // closed when loading finishes
+
+ // result of the load
+ pyobj interface{} // XXX -> pyObject iface
+ err error
+}
+*/
+
+
+/*
+ conn.objmu.Lock() // XXX -> rlock
+ objentry := conn.objtab[oid]
+
+ // someone else is already loading the object.
+ // we have to wait for that someone to finish the load.
+ if load, ok := objentry.(*loadInProgress); ok {
+ conn.objmu.Unlock()
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err() // XXX err ctx
+ case <-load.ready:
+ // XXX check pyclass match
+ return load.pyobj, load.err
+ }
+ }
+
+ // noone else is loading the object. Investigate whether it is simply not
+ // in objtab, or it is lost weakref, and if so we have to initiate the load.
+ var xobj interface{}
+ if objentry != nil {
+ wobj := objentry.(*WeakRef)
+ xobj = wobj.Get()
+ }
+ if xpyobj != nil {
+ // the object is already there
+ conn.objmu.Unlock()
+ return xpyobj, nil // XXX + activate?
+ }
+
+ // the object is not there, we have to (re)load it.
+ load := &loadInProgress{ready: make(chan struct)}
+ conn.objTab[oid] = load
+ conn.objmu.Unlock()
+
+ pyobj, err = conn.loadpy(ctx, oid)
+ load.pyobj = pyobj
+ load.err = err
+ close(load.ready)
+
+ // replace objtab entry with WeakRef
+ conn.objmu.Lock()
+ defer conn.objmu.Unlock()
+ if x := conn.objtab[oid]; x != load {
+ panic(fmt.Sprintf("zodb.Conn: objtab[%s]: load finished, but it is: %#v", oid, x)
+ }
+ if err != nil {
+ // it was loading error - remove the entry not to cause uncontrolled objtab growth.
+ // if this object is accessed next time - the load will be retried.
+ delete(conn.objtab[oid])
+ } else {
+ conn.objtab[oid] = NewWeakRef(pyobj)
+ }
+*/
+
+/*
+{
+ objentry := conn.objtab[oid]
+
+ // someone else is already loading the object.
+ // we have to wait for that someone to finish the load.
+ if load, ok := objentry.(*loadInProgress); ok {
+ conn.objmu.Unlock()
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err() // XXX err ctx
+ case <-load.ready:
+ // XXX check pyclass match
+ return load.pyobj, err
+ }
+ }
+
+ // noone else is loading the object. Investigate whether it is simply not
+ // in objtab, or it is lost weakref, and if so we have to initiate the load.
+ var xobj interface{}
+ if objentry != nil {
+ wobj := objentry.(*WeakRef)
+ xobj = wobj.Get()
+ }
+ if xpyobj != nil {
+ // the object is already there
+ conn.objmu.Unlock()
+ return xpyobj, nil // XXX + activate?
+ }
+
+ // the object is not there, we have to (re)load it.
+ load := &loadInProgress{ready: make(chan struct)}
+ conn.objTab[oid] = load
+ conn.objmu.Unlock()
+
+ pyobj, err = conn.loadpy(ctx, oid)
+ load.pyobj = pyobj
+ load.err = err
+ close(load.ready)
+
+ // replace objtab entry with WeakRef
+ conn.objmu.Lock()
+ defer conn.objmu.Unlock()
+ if x := conn.objtab[oid]; x != load {
+ panic(fmt.Sprintf("zodb.Conn: objtab[%s]: load finished, but it is: %#v", oid, x)
+ }
+ if err != nil {
+ // it was loading error - remove the entry not to cause uncontrolled objtab growth.
+ // if this object is accessed next time - the load will be retried.
+ delete(conn.objtab[oid])
+ } else {
+ conn.objtab[oid] = NewWeakRef(pyobj)
+ }
+}
+*/
+
+/*
+ // XXX -> loadpy
+ buf, serial, err := conn.stor.Load(ctx, zodb.Xid{Oid: oid, At: conn.at})
+ if err != nil {
+ return nil, err
+ }
+
+ pyclass, pystate, err := zodb.PyData(buf.Data).Decode()
+ if err != nil {
+ return nil, err // XXX err ctx
+ }
+
+ buf.Release()
+
+ return &pyObject{
+ object: object{jar: conn,oid: oid, serial: serial},
+ pyclass: pyclass,
+ pystate: pystate,
+ }, nil
+}
+*/
+
+/*
+func (conn *Connection) load(ctx context.Context, oid zodb.Oid) (*pyObject, error) {
+ // XXX
+}
+*/
+
diff --git a/wcfs/zodb.go b/wcfs/zodb.go
index e8a54a5..00e8a1d 100644
--- a/wcfs/zodb.go
+++ b/wcfs/zodb.go
@@ -25,17 +25,22 @@ import (
pickle "github.com/kisielk/og-rek"
)
+// XXX make methods private, e.g. _pJar ?
+
// Object is the interface that every in-RAM object representing any ZODB object implements.
type Object interface {
- // XXX make methods private, e.g. _pJar ?
- PJar() *Connection
- POid() zodb.Oid
+ PJar() *Connection // Connection this in-RAM object is part of.
+ POid() zodb.Oid // object ID in the database.
+
+ // object serial as of database state for particular Connection (PJar).
+ // 0 if not yet loaded (XXX ok?)
PSerial() zodb.Tid
+
// PActivate brings object to live state.
//
// It requests to persistency layer that in-RAM object data to be present.
- // If object state is not in RAM - it is loaded from the database.
+ // If object state was not in RAM - it is loaded from the database.
//
// On successful return the object data is either the same as in the
// database or, if this data was previously modified by user of
@@ -53,7 +58,7 @@ type Object interface {
//
// Note that it is valid to have several concurrent uses of object
// data, each protected with corresponding PActivate/PDeactivate pair:
- // As long as there is still any PActivate not yet compensated with
+ // as long as there is still any PActivate not yet compensated with
// corresponding PDeactivate, object data will assuredly stay alive in RAM.
//
// Besides exotic cases, the caller thus must not use object's data
@@ -66,7 +71,18 @@ type Object interface {
// database, or it was modified, that in-RAM data must be forgotten.
//
// PInvalidate must not be called while there is any in-progress
- // object's data use (PActivate / PDeactivate).
+ // object's data use (PActivate till PDeactivate).
+ //
+ // In practice this means that:
+ //
+ // - application must make sure to finish all objects accesses
+ // before transaction boundary: at transaction boundary - either
+ // at abort or commit, the persistency layer will sync to
+ // database and process invalidations.
+ //
+ // - if PInvalidate is explicitly called by application, the
+ // application must care to make sure it does not access the
+ // object data simultaneously.
PInvalidate()
}
@@ -79,7 +95,7 @@ type PyObject interface {
}
-// object is common base for in-RAM representation of ZODB object.
+// object is common base for in-RAM representation of ZODB objects.
type object struct {
jar *Connection
oid zodb.Oid
@@ -119,7 +135,7 @@ type LiveCacheControl interface {
//
// The view is representing state of ZODB objects as of `at` transaction.
//
-// XXX Connection changes are private and are isolated from changes in other Connections.
+// Connection changes are private and are isolated from changes in other Connections.
//
// XXX Connection, and {Py}Object methods that relate to it, are not safe for
// modifications from multiple goroutines simultaneously.
@@ -178,23 +194,12 @@ type Connection struct {
// Hopefully we don't have cycles with ZBtree/ZBucket XXX verify this
objmu sync.Mutex
objtab map[zodb.Oid]*WeakRef // oid -> WeakRef(PyObject)
- //objtab map[zodb.Oid]interface{} // oid -> WeakRef(pyObject) | loadInProgress
- cacheControl ConnCacheControl
+ // hooks for application to influence live caching decisions.
+ cacheControl LiveCacheControl
}
-/*
-// loadInProgress entry in Conn.objtab tells users, that try to get the entry,
-// that another goroutine is already in progress of loading it.
-type loadInProgress struct {
- ready chan struct{} // closed when loading finishes
-
- // result of the load
- pyobj interface{} // XXX -> pyObject iface
- err error
-}
-*/
// Get returns in-RAM object corresponding to specified ZODB object according to current database view.
//
@@ -238,60 +243,6 @@ func (conn *Connection) Get(ctx context.Context, oid zodb.Oid) (interface{} /*Py
return xobj, nil
}
-/*
- conn.objmu.Lock() // XXX -> rlock
- objentry := conn.objtab[oid]
-
- // someone else is already loading the object.
- // we have to wait for that someone to finish the load.
- if load, ok := objentry.(*loadInProgress); ok {
- conn.objmu.Unlock()
- select {
- case <-ctx.Done():
- return nil, ctx.Err() // XXX err ctx
- case <-load.ready:
- // XXX check pyclass match
- return load.pyobj, load.err
- }
- }
-
- // noone else is loading the object. Investigate whether it is simply not
- // in objtab, or it is lost weakref, and if so we have to initiate the load.
- var xobj interface{}
- if objentry != nil {
- wobj := objentry.(*WeakRef)
- xobj = wobj.Get()
- }
- if xpyobj != nil {
- // the object is already there
- conn.objmu.Unlock()
- return xpyobj, nil // XXX + activate?
- }
-
- // the object is not there, we have to (re)load it.
- load := &loadInProgress{ready: make(chan struct)}
- conn.objTab[oid] = load
- conn.objmu.Unlock()
-
- pyobj, err = conn.loadpy(ctx, oid)
- load.pyobj = pyobj
- load.err = err
- close(load.ready)
-
- // replace objtab entry with WeakRef
- conn.objmu.Lock()
- defer conn.objmu.Unlock()
- if x := conn.objtab[oid]; x != load {
- panic(fmt.Sprintf("zodb.Conn: objtab[%s]: load finished, but it is: %#v", oid, x)
- }
- if err != nil {
- // it was loading error - remove the entry not to cause uncontrolled objtab growth.
- // if this object is accessed next time - the load will be retried.
- delete(conn.objtab[oid])
- } else {
- conn.objtab[oid] = NewWeakRef(pyobj)
- }
-*/
// wrongClassError is the error cause returned when object's class is not what was expected.
type wrongClassError struct {
@@ -347,90 +298,6 @@ func (conn *Connection) get(pyclass pickle.Class, oid zodb.Oid) (PyObject, error
}
-/*
-{
- objentry := conn.objtab[oid]
-
- // someone else is already loading the object.
- // we have to wait for that someone to finish the load.
- if load, ok := objentry.(*loadInProgress); ok {
- conn.objmu.Unlock()
- select {
- case <-ctx.Done():
- return nil, ctx.Err() // XXX err ctx
- case <-load.ready:
- // XXX check pyclass match
- return load.pyobj, err
- }
- }
-
- // noone else is loading the object. Investigate whether it is simply not
- // in objtab, or it is lost weakref, and if so we have to initiate the load.
- var xobj interface{}
- if objentry != nil {
- wobj := objentry.(*WeakRef)
- xobj = wobj.Get()
- }
- if xpyobj != nil {
- // the object is already there
- conn.objmu.Unlock()
- return xpyobj, nil // XXX + activate?
- }
-
- // the object is not there, we have to (re)load it.
- load := &loadInProgress{ready: make(chan struct)}
- conn.objTab[oid] = load
- conn.objmu.Unlock()
-
- pyobj, err = conn.loadpy(ctx, oid)
- load.pyobj = pyobj
- load.err = err
- close(load.ready)
-
- // replace objtab entry with WeakRef
- conn.objmu.Lock()
- defer conn.objmu.Unlock()
- if x := conn.objtab[oid]; x != load {
- panic(fmt.Sprintf("zodb.Conn: objtab[%s]: load finished, but it is: %#v", oid, x)
- }
- if err != nil {
- // it was loading error - remove the entry not to cause uncontrolled objtab growth.
- // if this object is accessed next time - the load will be retried.
- delete(conn.objtab[oid])
- } else {
- conn.objtab[oid] = NewWeakRef(pyobj)
- }
-}
-*/
-
-/*
- // XXX -> loadpy
- buf, serial, err := conn.stor.Load(ctx, zodb.Xid{Oid: oid, At: conn.at})
- if err != nil {
- return nil, err
- }
-
- pyclass, pystate, err := zodb.PyData(buf.Data).Decode()
- if err != nil {
- return nil, err // XXX err ctx
- }
-
- buf.Release()
-
- return &pyObject{
- object: object{jar: conn,oid: oid, serial: serial},
- pyclass: pyclass,
- pystate: pystate,
- }, nil
-}
-*/
-
-/*
-func (conn *Connection) load(ctx context.Context, oid zodb.Oid) (*pyObject, error) {
- // XXX
-}
-*/
-
func (conn *Connection) loadpy(ctx context.Context, oid zodb.Oid) (pyclass pickle.Class, pystate interface{}, serial zodb.Tid, _ error) {
buf, serial, err := conn.stor.Load(ctx, zodb.Xid{Oid: oid, At: conn.at})
if err != nil {
@@ -447,6 +314,7 @@ func (conn *Connection) loadpy(ctx context.Context, oid zodb.Oid) (pyclass pickl
return pyclass, pystate, serial, nil
}
+
// path(class) -> new(pyobj)
var classTab = make(map[string]func(*pyObject)PyObject)
@@ -576,5 +444,6 @@ func (pyobj *pyObject) PDeactivate() {
}
// XXX pyobj.PInvalidate() = deactivate without checking if state != modified
+// XXX panic if refcnt != 0 (object being used)
--
2.30.9