Commit f59e1cb4 authored by Kirill Smelkov's avatar Kirill Smelkov

.

parent 9d44f29e
// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
// All Rights Reserved.
//
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This software is subject to the provisions of the Zope Public License,
// Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
// FOR A PARTICULAR PURPOSE
package main
// ZODB BTree handling XXX -> zodb
import (
"context"
"sort"
"lab.nexedi.com/kirr/go123/xerr"
//"lab.nexedi.com/kirr/neo/go/zodb"
pickle "github.com/kisielk/og-rek"
)
// XXX -> template
type KEY int64
// ZBucket mimics ?OBucket from btree/py, with ? being any integer.
//
// py description:
//
// A Bucket wraps contiguous vectors of keys and values. Keys are unique,
// and stored in sorted order. The 'values' pointer may be NULL if the
// Bucket is used to implement a set. Buckets serving as leafs of BTrees
// are chained together via 'next', so that the entire BTree contents
// can be traversed in sorted order quickly and easily.
type ZBucket struct {
*PyPersistent
next *ZBucket // the bucket with the next-larger keys
keys []KEY // 'len' keys, in increasing order
values []interface{} // 'len' corresponding values
}
// zBTreeItem mimics BTreeItem from btree/py.
type zBTreeItem struct {
key KEY
child interface{} // ZBTree or ZBucket
}
// ZBTree mimics ?OBTree from btree/py, with ? being any integer.
//
// See https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/Development.txt#L198
// for details.
type ZBTree struct {
*PyPersistent
// firstbucket points to the bucket containing the smallest key in
// the BTree. This is found by traversing leftmost child pointers
// (data[0].child) until reaching a Bucket.
firstbucket *ZBucket
// The BTree points to 'len' children, via the "child" fields of the data
// array. There are len-1 keys in the 'key' fields, stored in increasing
// order. data[0].key is unused. For i in 0 .. len-1, all keys reachable
// from data[i].child are >= data[i].key and < data[i+1].key, at the
// endpoints pretending that data[0].key is minus infinity and
// data[len].key is positive infinity.
data []zBTreeItem
}
// Get searches BTree by key.
//
// It loads intermediate BTree nodes from database on demand as needed.
func (t *ZBTree) Get(ctx context.Context, key KEY) (_ interface{}, _ bool, err error) {
defer xerr.Contextf(&err, "btree(%s): get %s", t.POid(), key) // XXX + url?
err = t.PActivate(ctx)
if err != nil {
return nil, false, err
}
if len(t.data) == 0 {
// empty btree
t.PDeactivate()
return nil, false, nil
}
for {
// search i: K(i) ≤ k < K(i+1) ; K(0) = -∞, K(len) = +∞
i := sort.Search(len(t.data), func(i int) bool {
j := i + 1
if j == len(t.data) {
return true // [len].key = +∞
}
return key < t.data[j].key
})
switch child := t.data[i].child.(type) {
case *ZBTree:
t.PDeactivate()
t = child
err = t.PActivate(ctx)
if err != nil {
return nil, false, err
}
case *ZBucket:
t.PDeactivate()
return child.Get(ctx, key)
}
}
}
// Get searches Bucket by key.
func (b *ZBucket) Get(ctx context.Context, key KEY) (_ interface{}, _ bool, err error) {
defer xerr.Contextf(&err, "bucket(%s): get %s", b.POid(), key) // XXX + url?
err = b.PActivate(ctx)
if err != nil {
return nil, false, err
}
v, ok := b.get(key)
b.PDeactivate()
return v, ok, nil
}
// get searches Bucket by key.
//
// no loading from database is done. The bucket must be already activated.
func (b *ZBucket) get(key KEY) (interface{}, bool) {
// search i: K(i-1) < k ≤ K(i) ; K(-1) = -∞, K(len) = +∞
i := sort.Search(len(b.keys), func(i int) bool {
return key <= b.keys[i]
})
if i == len(b.keys) || b.keys[i] != key {
return nil, false // not found
}
return b.values[i], true
}
// XXX ZBucket.MinKey ?
// XXX ZBucket.MaxKey ?
// ---- serialization ----
// from https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BTreeTemplate.c#L1087:
//
// For an empty BTree (self->len == 0), None.
//
// For a BTree with one child (self->len == 1), and that child is a bucket,
// and that bucket has a NULL oid, a one-tuple containing a one-tuple
// containing the bucket's state:
//
// (
// (
// child[0].__getstate__(),
// ),
// )
//
// Else a two-tuple. The first element is a tuple interleaving the BTree's
// keys and direct children, of size 2*self->len - 1 (key[0] is unused and
// is not saved). The second element is the firstbucket:
//
// (
// (child[0], key[1], child[1], key[2], child[2], ...,
// key[len-1], child[len-1]),
// self->firstbucket
// )
//
// In the above, key[i] means self->data[i].key, and similarly for child[i].
// DropState implements Stateful.
func (t *ZBTree) DropState() {
t.firstbucket = nil
t.data = nil
}
// PySetState implements PyStateful to set btree data from pystate.
func (bt *ZBTree) PySetState(pystate interface{}) error {
// empty btree
if _, ok := pystate.(pickle.None); ok {
bt.firstbucket = nil
bt.data = nil
return nil
}
t, ok := pystate.(pickle.Tuple)
if !ok || !(1 <= len(t) && len(t) <= 2) {
// XXX
}
// btree with 1 child bucket without oid
if len(t) == 1 {
bucket := &ZBucket{PyPersistent: nil /* FIXME */}
err := bucket.PySetState(t[0])
if err != nil {
// XXX
}
bt.firstbucket = bucket
bt.data = []zBTreeItem{{key: 0, child: bucket}}
return nil
}
// regular btree
t, ok = t[0].(pickle.Tuple)
if !(ok && len(t) % 2 == 0) {
// XXX
}
bt.firstbucket, ok = t[1].(*ZBucket)
if !ok {
// XXX
}
n := len(t) / 2
bt.data = make([]zBTreeItem, 0, n)
for i, idx := 0, 0; i < n; i++ {
key := int64(0)
if i > 0 {
// key[0] is unused and not saved
key, ok = t[idx].(int64) // XXX Xint
if ! ok {
// XXX
}
idx++
}
child := t[idx]
idx++
switch child.(type) {
default:
// XXX
case *ZBTree: // ok
case *ZBucket: // ok
}
bt.data = append(bt.data, zBTreeItem{key: KEY(key), child: child})
}
return nil
}
// from https://github.com/zopefoundation/BTrees/blob/4.5.0-1-gc8bf24e/BTrees/BucketTemplate.c#L1195:
//
// For a mapping bucket (self->values is not NULL), a one-tuple or two-tuple.
// The first element is a tuple interleaving keys and values, of length
// 2 * self->len. The second element is the next bucket, present iff next is
// non-NULL:
//
// (
// (keys[0], values[0], keys[1], values[1], ...,
// keys[len-1], values[len-1]),
// <self->next iff non-NULL>
// )
// DropState implements Stateful to discard bucket state.
func (b *ZBucket) DropState() {
b.next = nil
b.keys = nil
b.values = nil
}
// PySetState implements PyStateful to set bucket data from pystate.
func (b *ZBucket) PySetState(pystate interface{}) error {
t, ok := pystate.(pickle.Tuple)
if !ok || !(1 <= len(t) && len(t) <= 2) {
// XXX complain
}
// .next present
if len(t) == 2 {
next, ok := t[1].(*ZBucket)
if !ok {
// XXX
}
b.next = next
}
// main part
t, ok = t[0].(pickle.Tuple)
// XXX if !ok || (len(t) % 2 != 0)
// reset arrays just in case
n := len(t) / 2
b.keys = make([]KEY, 0, n)
b.values = make([]interface{}, 0, n)
for i := 0; i < n; i++ {
xk := t[2*i]
v := t[2*i+1]
k, ok := xk.(int64) // XXX use KEY XXX -> Xint64
if !ok {
// XXX
}
// XXX check keys are sorted?
b.keys = append(b.keys, KEY(k)) // XXX cast
b.values = append(b.values, v)
}
return nil
}
// ---- register classes to ZODB ----
func bucketNew(base *PyPersistent) IPyPersistent { return &ZBucket{PyPersistent: base} }
func btreeNew(base *PyPersistent) IPyPersistent { return &ZBTree{PyPersistent: base} }
func init() {
registerPyClass("zodb.BTree.LOBucket", bucketNew)
registerPyClass("zodb.BTree.LOBtree", btreeNew)
}
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package main
import (
//"testing"
)
// TODO
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// based on:
// https://groups.google.com/d/msg/golang-nuts/PYWxjT2v6ps/dL71oJk1mXEJ
// https://play.golang.org/p/f9HY6-z8Pp
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package main
// weak references
import (
"runtime"
"sync"
"unsafe"
)
// iface is how Go runtime represents an interface.
//
// NOTE layout must be synchronized to Go runtime representation.
type iface struct {
typ uintptr // type
data uintptr // data
}
// weakRefState represents current state of an object WeakRef points to.
type weakRefState int32
const (
objGot weakRefState = +1 // WeakRef.Get returned !nil
objLive weakRefState = 0 // object is alive, Get did not run yet in this GC cycle
objReleased weakRefState = -1 // the finalizer marked object as released
)
// WeakRef is a weak reference.
//
// Create one with NewWeakRef and retrieve referenced object with Get.
//
// There must be no more than 1 weak reference to any object.
// Weak references must not be attached to an object on which runtime.SetFinalizer is also used.
// Weak references must not be copied.
type WeakRef struct {
iface
// XXX try to do without mutex and only with atomics
mu sync.Mutex
state weakRefState
}
// NewWeakRef creates new weak reference pointing to obj.
//
// XXX + onrelease callback?
func NewWeakRef(obj interface{}) *WeakRef {
// since starting from ~ Go1.4 the GC is precise, we can save interface
// pointers to uintptr and that won't prevent GC from garbage
// collecting the object.
w := &WeakRef{
iface: *(*iface)(unsafe.Pointer(&obj)),
state: objLive,
}
var release func(interface{})
release = func(obj interface{}) {
// GC decided that the object is no longer reachable and
// scheduled us to run as finalizer. During the time till we
// actually run, WeakRef.Get might have been come to run and
// "rematerializing" the object for use. Check if we do not
// race with any Get in progress, and reschedule us to retry at
// next GC if we do.
w.mu.Lock()
if w.state == objGot {
w.state = objLive
runtime.SetFinalizer(obj, release)
} else {
w.state = objReleased
}
w.mu.Unlock()
}
runtime.SetFinalizer(obj, release)
return w
}
// Get returns object pointed to by this weak reference.
//
// If original object is still alive - it is returned.
// If not - nil is returned.
func (w *WeakRef) Get() (obj interface{}) {
w.mu.Lock()
if w.state != objReleased {
w.state = objGot
// recreate interface{} from saved words.
// XXX do writes as pointers so that compiler emits write barriers to notify GC?
i := (*iface)(unsafe.Pointer(&obj))
*i = w.iface
}
w.mu.Unlock()
return obj
}
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package main
import (
"runtime"
"testing"
"time"
"unsafe"
)
// verify that interface <-> iface works ok.
func TestIface(t *testing.T) {
var i interface{}
var fi *iface
isize := unsafe.Sizeof(i)
fsize := unsafe.Sizeof(*fi)
if isize != fsize {
t.Fatalf("sizeof(interface{}) (%d) != sizeof(iface) (%d)", isize, fsize)
}
i = 3
var j interface{}
if !(j == nil && i != j) {
t.Fatalf("i == j ? (i: %#v, j: %#v}", i, j)
}
fi = (*iface)(unsafe.Pointer(&i))
fj := (*iface)(unsafe.Pointer(&j))
*fj = *fi
if i != j {
t.Fatalf("i (%#v) != j (%#v)", i, j)
}
}
func TestWeakRef(t *testing.T) {
type T struct { _ [8]int64 } // large enough not to go into tinyalloc
p := new(T)
w := NewWeakRef(p)
pptr := uintptr(unsafe.Pointer(p))
assertEq := func(a, b interface{}) {
t.Helper()
if a != b {
t.Fatalf("not equal: %#v != %#v", a, b)
}
}
// perform GC + give finalizers a chancet to run.
GC := func() {
runtime.GC()
// GC only queues finalizers, not runs them directly. Give it
// some time so that finalizers could have been run.
time.Sleep(10*time.Millisecond) // XXX hack
}
assertEq(w.state, objLive)
assertEq(w.Get(), p)
assertEq(w.state, objGot)
GC()
assertEq(w.state, objGot) // fin has not been run at all (p is live)
assertEq(w.Get(), p)
assertEq(w.state, objGot)
p = nil
GC()
assertEq(w.state, objLive) // fin ran and downgraded got -> live
switch p_ := w.Get().(type) {
default:
t.Fatalf("Get after objGot -> objLive: %#v", p_)
case *T:
if uintptr(unsafe.Pointer(p_)) != pptr {
t.Fatal("Get after objGot -> objLive: T, but ptr is not the same")
}
}
assertEq(w.state, objGot)
GC()
assertEq(w.state, objLive) // fin ran again and again downgraded got -> live
GC()
assertEq(w.state, objReleased) // fin ran again and released the object
assertEq(w.Get(), nil)
}
This diff is collapsed.
// Copyright (c) 2001, 2002 Zope Foundation and Contributors.
// All Rights Reserved.
//
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This software is subject to the provisions of the Zope Public License,
// Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
// THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
// WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
// FOR A PARTICULAR PURPOSE
package main
// Bits that should be in ZODB XXX -> zodb
import (
"context"
"fmt"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/neo/go/zodb"
pickle "github.com/kisielk/og-rek"
)
// IPyPersistent is the interface that every in-RAM object representing Python ZODB object implements.
type IPyPersistent interface {
IPersistent
PyClass() pickle.Class // python class of this object
// PyState() interface{} // object state. python passes this to pyclass.__new__().__setstate__()
// IPyPersistent must be stateful for persistency to work
// XXX try to move out of IPyPersistent? Rationale: we do not want e.g. PySetState to
// be available to user who holds IPyPersistent interface: it is confusing to have
// both PActivate and PySetState at the same time.
PyStateful
}
// PyPersistent is common base implementation for in-RAM representation of ZODB Python objects.
type PyPersistent struct {
Persistent
pyclass pickle.Class
}
func (pyobj *PyPersistent) PyClass() pickle.Class { return pyobj.pyclass }
//func (pyobj *PyPersistent) PyState() interface{} { return pyobj.pystate }
// PyStateful is the interface describing in-RAM object whose data state can be
// exchanged as Python data.
type PyStateful interface {
// PySetState should set state of the in-RAM object from Python data.
// Analog of __setstate__() in Python.
PySetState(pystate interface{}) error
// PyGetState should return state of the in-RAM object as Python data.
// Analog of __getstate__() in Python.
//PyGetState() interface{} TODO
}
// ---- PyPersistent <-> Persistent state exchange ----
// pyinstance returns .instance upcasted to IPyPersistent.
//
// this should be always safe because we always create pyObjects via
// newGhost which passes IPyPersistent as instance to IPersistent.
func (pyobj *PyPersistent) pyinstance() IPyPersistent {
return pyobj.instance.(IPyPersistent)
}
func (pyobj *PyPersistent) SetState(state *mem.Buf) error {
pyclass, pystate, err := zodb.PyData(state.Data).Decode()
if err != nil {
return err // XXX err ctx
}
if pyclass != pyobj.pyclass {
// complain that pyclass changed
// (both ref and object data use pyclass so it indeed can be different)
return &wrongClassError{want: pyobj.pyclass, have: pyclass} // XXX + err ctx
}
return pyobj.pyinstance().PySetState(pystate) // XXX err ctx = ok?
}
// TODO PyPersistent.GetState
// ---- pyclass -> new ghost ----
// function representing new of a class.
type pyClassNewFunc func(base *PyPersistent) IPyPersistent
// path(pyclass) -> new(pyobj)
var pyClassTab = make(map[string]pyClassNewFunc)
// registerPyClass registers python class to be transformed to Go instance
// created via classNew.
//
// must be called from global init().
func registerPyClass(pyClassPath string, classNew pyClassNewFunc) {
pyClassTab[pyClassPath] = classNew
// XXX + register so that zodb.PyData decode handles pyClassPath
}
// newGhost creates new ghost object corresponding to pyclass and oid.
func (conn *Connection) newGhost(pyclass pickle.Class, oid zodb.Oid) IPyPersistent {
pyobj := &PyPersistent{
Persistent: Persistent{jar: conn, oid: oid, serial: 0, state: GHOST},
pyclass: pyclass,
}
// switch on pyclass and transform e.g. "zodb.BTree.Bucket" -> *ZBucket
classNew := pyClassTab[pyclass.Module + "." + pyclass.Name]
var instance IPyPersistent
if classNew != nil {
instance = classNew(pyobj)
} else {
instance = &dummyPyInstance{PyPersistent: pyobj}
}
pyobj.instance = instance
return instance
}
// dummyPyInstance is used for python classes that were not registered.
type dummyPyInstance struct {
*PyPersistent
pystate interface{}
}
func (d *dummyPyInstance) DropState() {
d.pystate = nil
}
func (d *dummyPyInstance) PySetState(pystate interface{}) error {
d.pystate = pystate
return nil
}
// ----------------------------------------
// Get returns in-RAM object corresponding to specified ZODB object according to current database view.
//
// If there is already in-RAM object that corresponds to oid, that in-RAM object is returned.
// Otherwise new in-RAM object is created and filled with object's class loaded from the database.
//
// The scope of the object returned is the Connection. XXX ok?
//
// The object's data is not neccessarily loaded after Get returns. Use
// PActivate to make sure the object ifs fully loaded.
func (conn *Connection) Get(ctx context.Context, oid zodb.Oid) (IPyPersistent, error) {
conn.objmu.Lock() // XXX -> rlock
wobj := conn.objtab[oid]
var xobj interface{}
if wobj != nil {
xobj = wobj.Get()
}
conn.objmu.Unlock()
// object was already there in objtab.
if xobj != nil {
return xobj.(IPyPersistent), nil
}
// object is not there in objtab - raw load it, get its class -> get(pyclass, oid)
pyclass, pystate, serial, err := conn.loadpy(ctx, oid)
if err != nil {
return nil, err // XXX errctx
}
obj, err := conn.get(pyclass, oid)
if err != nil {
return nil, err
}
// XXX we are dropping just loaded pystate. Usually Get should be used
// to only load root object, so maybe that is ok.
//
// TODO -> use (pystate, serial) to activate.
_, _ = pystate, serial
return obj, nil
}
// wrongClassError is the error cause returned when python object's class is not what was expected.
type wrongClassError struct {
want, have pickle.Class
}
func (e *wrongClassError) Error() string {
return fmt.Sprintf("wrong class: want %q; have %q", e.want, e.have)
}
// get returns in-RAM object corresponding to specified ZODB object according to current database view.
//
// If there is already in-RAM object that corresponds to oid, that in-RAM object is returned.
// Otherwise new in-RAM object is created according to specified class.
//
// The object's data is not neccessarily loaded after get returns. Use
// PActivate to make sure the object is fully loaded.
//
// XXX object scope.
//
// Use-case: in ZODB references are (pyclass, oid), so new ghost is created
// without further loading anything.
func (conn *Connection) get(pyclass pickle.Class, oid zodb.Oid) (IPyPersistent, error) {
conn.objmu.Lock() // XXX -> rlock
wobj := conn.objtab[oid]
var pyobj IPyPersistent
checkClass := false
if wobj != nil {
if xobj := wobj.Get(); xobj != nil {
pyobj = xobj.(IPyPersistent)
}
}
if pyobj == nil {
pyobj = conn.newGhost(pyclass, oid)
conn.objtab[oid] = NewWeakRef(pyobj)
} else {
checkClass = true
}
conn.objmu.Unlock()
if checkClass {
if cls := pyobj.PyClass(); pyclass != cls {
return nil, &zodb.OpError{
URL: conn.stor.URL(),
Op: fmt.Sprintf("@%s: get", conn.at), // XXX abuse
Args: oid,
Err: &wrongClassError{pyclass, cls},
}
}
}
return pyobj, nil
}
// loadpy loads object specified by oid and decodes it as a ZODB Python object.
//
// loadpy does not create any in-RAM object associated with Connection.
// It only returns decoded database data.
func (conn *Connection) loadpy(ctx context.Context, oid zodb.Oid) (pyclass pickle.Class, pystate interface{}, serial zodb.Tid, _ error) {
buf, serial, err := conn.stor.Load(ctx, zodb.Xid{Oid: oid, At: conn.at})
if err != nil {
return pickle.Class{}, nil, 0, err
}
defer buf.Release()
pyclass, pystate, err = zodb.PyData(buf.Data).Decode()
if err != nil {
return pickle.Class{}, nil, 0, err // XXX err ctx
}
return pyclass, pystate, serial, nil
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment