Commit 532d014f authored by Kirill Smelkov's avatar Kirill Smelkov

go/zodb: PyStateful persistency support

As promised in 354e0e51 (go/zodb: Persistent - the base type to
implement IPersistent objects) add support to persistency machinery to
set object state from python pickles serialized by ZODB/py.

Persistent references are not yet handled.

As promised add some very minimal persistent tests.
parent abc11031
...@@ -22,6 +22,7 @@ package zodb ...@@ -22,6 +22,7 @@ package zodb
import ( import (
"context" "context"
"fmt"
"lab.nexedi.com/kirr/go123/mem" "lab.nexedi.com/kirr/go123/mem"
) )
...@@ -32,6 +33,15 @@ type Connection struct { ...@@ -32,6 +33,15 @@ type Connection struct {
at Tid // current view of database; stable inside a transaction. at Tid // current view of database; stable inside a transaction.
} }
// wrongClassError is the error cause returned when ZODB object's class is not what was expected.
type wrongClassError struct {
want, have string
}
func (e *wrongClassError) Error() string {
return fmt.Sprintf("wrong class: want %q; have %q", e.want, e.have)
}
// load loads object specified by oid. // load loads object specified by oid.
func (conn *Connection) load(ctx context.Context, oid Oid) (_ *mem.Buf, serial Tid, _ error) { func (conn *Connection) load(ctx context.Context, oid Oid) (_ *mem.Buf, serial Tid, _ error) {
return conn.stor.Load(ctx, Xid{Oid: oid, At: conn.at}) return conn.stor.Load(ctx, Xid{Oid: oid, At: conn.at})
......
...@@ -34,7 +34,7 @@ import ( ...@@ -34,7 +34,7 @@ import (
// representation of database objects. // representation of database objects.
// //
// To use - a class needs to embed Persistent and register itself additionally // To use - a class needs to embed Persistent and register itself additionally
// providing Ghostable and Stateful methods. For example: // providing Ghostable and (Py)Stateful methods. For example:
// //
// type MyObject struct { // type MyObject struct {
// Persistent // Persistent
...@@ -69,7 +69,7 @@ type Persistent struct { ...@@ -69,7 +69,7 @@ type Persistent struct {
refcnt int32 refcnt int32
// Persistent is the base for the instance. // Persistent is the base for the instance.
// instance, via its state type, is additionally Ghostable and Stateful. // instance, via its state type, is additionally Ghostable and (Stateful | PyStateful).
instance IPersistent instance IPersistent
loading *loadState loading *loadState
} }
...@@ -178,6 +178,10 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) { ...@@ -178,6 +178,10 @@ func (obj *Persistent) PActivate(ctx context.Context) (err error) {
err = istate.SetState(state) err = istate.SetState(state)
xerr.Context(&err, "setstate") xerr.Context(&err, "setstate")
case PyStateful:
err = pySetState(istate, obj.zclass.class, state)
xerr.Context(&err, "pysetstate")
default: default:
panic(obj.badf("activate: !stateful instance")) panic(obj.badf("activate: !stateful instance"))
} }
...@@ -242,7 +246,7 @@ func (obj *Persistent) PInvalidate() { ...@@ -242,7 +246,7 @@ func (obj *Persistent) PInvalidate() {
// istate returns .instance casted to corresponding stateType. // istate returns .instance casted to corresponding stateType.
// //
// returns: Ghostable + Stateful. // returns: Ghostable + (Stateful | PyStateful).
func (obj *Persistent) istate() Ghostable { func (obj *Persistent) istate() Ghostable {
xstateful := reflect.ValueOf(obj.instance).Convert(reflect.PtrTo(obj.zclass.stateType)) xstateful := reflect.ValueOf(obj.instance).Convert(reflect.PtrTo(obj.zclass.stateType))
return xstateful.Interface().(Ghostable) return xstateful.Interface().(Ghostable)
...@@ -282,6 +286,7 @@ var rIPersistent = reflect.TypeOf((*IPersistent)(nil)).Elem() // typeof(IPersist ...@@ -282,6 +286,7 @@ var rIPersistent = reflect.TypeOf((*IPersistent)(nil)).Elem() // typeof(IPersist
var rPersistent = reflect.TypeOf(Persistent{}) // typeof(Persistent) var rPersistent = reflect.TypeOf(Persistent{}) // typeof(Persistent)
var rGhostable = reflect.TypeOf((*Ghostable)(nil)).Elem() // typeof(Ghostable) var rGhostable = reflect.TypeOf((*Ghostable)(nil)).Elem() // typeof(Ghostable)
var rStateful = reflect.TypeOf((*Stateful)(nil)).Elem() // typeof(Stateful) var rStateful = reflect.TypeOf((*Stateful)(nil)).Elem() // typeof(Stateful)
var rPyStateful = reflect.TypeOf((*PyStateful)(nil)).Elem() // typeof(PyStateful)
// RegisterClass registers ZODB class to correspond to Go type. // RegisterClass registers ZODB class to correspond to Go type.
// //
...@@ -295,7 +300,7 @@ var rStateful = reflect.TypeOf((*Stateful)(nil)).Elem() // typeof(Stateful ...@@ -295,7 +300,7 @@ var rStateful = reflect.TypeOf((*Stateful)(nil)).Elem() // typeof(Stateful
// typ must embed Persistent; *typ must implement IPersistent. // typ must embed Persistent; *typ must implement IPersistent.
// //
// typ must be convertible to stateType; stateType must implement Ghostable and // typ must be convertible to stateType; stateType must implement Ghostable and
// Stateful(*). // either Stateful or PyStateful(*).
// //
// RegisterClass must be called from global init(). // RegisterClass must be called from global init().
// //
...@@ -333,8 +338,9 @@ func RegisterClass(class string, typ, stateType reflect.Type) { ...@@ -333,8 +338,9 @@ func RegisterClass(class string, typ, stateType reflect.Type) {
} }
stateful := ptstate.Implements(rStateful) stateful := ptstate.Implements(rStateful)
if !stateful { pystateful := ptstate.Implements(rPyStateful)
badf("%q does not implement Stateful", ptstate) if !(stateful || pystateful) {
badf("%q does not implement any of Stateful or PyStateful", ptstate)
} }
zc := &zclass{class: class, typ: typ, stateType: stateType} zc := &zclass{class: class, typ: typ, stateType: stateType}
......
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package zodb
import (
"fmt"
"reflect"
"testing"
"lab.nexedi.com/kirr/go123/mem"
"github.com/stretchr/testify/require"
)
// test Persistent type.
type MyObject struct {
Persistent
value string
}
type myObjectState MyObject
func (o *myObjectState) DropState() {
o.value = ""
}
func (o *myObjectState) PySetState(pystate interface{}) error {
s, ok := pystate.(string)
if !ok {
return fmt.Errorf("myobject: setstate: want str; got %T", pystate)
}
o.value = s
return nil
}
func init() {
t := reflect.TypeOf
RegisterClass("t.zodb.MyObject", t(MyObject{}), t(myObjectState{}))
}
func TestPersistent(t *testing.T) {
assert := require.New(t)
// checkObj verifies current state of persistent object.
checkObj := func(obj IPersistent, jar *Connection, oid Oid, serial Tid, state ObjectState, refcnt int32, loading *loadState) {
t.Helper()
xbase := reflect.ValueOf(obj).Elem().FieldByName("Persistent")
pbase := xbase.Addr().Interface().(*Persistent)
badf := func(format string, argv ...interface{}) {
t.Helper()
msg := fmt.Sprintf(format, argv...)
t.Fatalf("%#v: %s", obj, msg)
}
zc := pbase.zclass
//zc.class
if typ := reflect.TypeOf(obj).Elem(); typ != zc.typ {
badf("invalid zclass: .typ = %s ; want %s", zc.typ, typ)
}
//zc.stateType
if pbase.jar != jar {
badf("invalid jar")
}
if pbase.oid != oid {
badf("invalid oid: %s ; want %s", pbase.oid, oid)
}
if pbase.serial != serial {
badf("invalid serial: %s ; want %s", pbase.serial, serial)
}
if pbase.state != state {
badf("invalid state: %s ; want %s", pbase.state, state)
}
if pbase.refcnt != refcnt {
badf("invalid refcnt: %s ; want %s", pbase.refcnt, refcnt)
}
if pbase.instance != obj {
badf("base.instance != obj")
}
// XXX loading too?
}
// unknown type -> Broken
xobj := newGhost("t.unknown", 10, nil)
b, ok := xobj.(*Broken)
if !ok {
t.Fatalf("unknown -> %T; want Broken", xobj)
}
checkObj(b, nil, 10, InvalidTid, GHOST, 0, nil)
assert.Equal(b.class, "t.unknown")
assert.Equal(b.state, (*mem.Buf)(nil))
// t.zodb.MyObject -> *MyObject
xobj = newGhost("t.zodb.MyObject", 11, nil)
obj, ok := xobj.(*MyObject)
if !ok {
t.Fatalf("unknown -> %T; want Broken", xobj)
}
checkObj(obj, nil, 11, InvalidTid, GHOST, 0, nil)
assert.Equal(zclassOf(obj), "t.zodb.MyObject")
// TODO activate - jar has to load, state changes
// TODO activate again - refcnt++
// TODO deactivate - refcnt--
// TODO deactivate - state dropped
}
...@@ -58,6 +58,33 @@ func (d PyData) ClassName() string { ...@@ -58,6 +58,33 @@ func (d PyData) ClassName() string {
return pyclassPath(klass) return pyclassPath(klass)
} }
// decode decodes raw ZODB python data into Python class and state.
func (d PyData) decode() (pyclass pickle.Class, pystate interface{}, err error) {
defer xerr.Context(&err, "pydata: decode")
p := pickle.NewDecoderWithConfig(
bytes.NewReader([]byte(d)),
&pickle.DecoderConfig{/* TODO: handle persistent references */},
)
xklass, err := p.Decode()
if err != nil {
return pickle.Class{}, nil, fmt.Errorf("class description: %s", err)
}
klass, err := xpyclass(xklass)
if err != nil {
return pickle.Class{}, nil, fmt.Errorf("class description: %s", err)
}
state, err := p.Decode()
if err != nil {
return pickle.Class{}, nil, fmt.Errorf("object state: %s", err)
}
return klass, state, nil
}
// xpyclass verifies and extracts py class from unpickled value. // xpyclass verifies and extracts py class from unpickled value.
// //
// it normalizes py class that has just been decoded from a serialized ZODB // it normalizes py class that has just been decoded from a serialized ZODB
......
...@@ -99,6 +99,19 @@ ...@@ -99,6 +99,19 @@
// multiple goroutines simultaneously. // multiple goroutines simultaneously.
// //
// //
// Python data
//
// To maintain database data compatibility with ZODB/py, ZODB/go provides
// first class support for Python data. At storage-level PyData provides way to
// treat raw data record content as serialized by ZODB/py, and at application
// level types that are registered with state type providing PyStateful (see
// RegisterClass) are automatically (de)serialized as Python pickles(*).
//
// --------
//
// (*) for pickle support package github.com/kisielk/og-rek is used.
//
//
// Storage drivers // Storage drivers
// //
// To implement a ZODB storage one need to provide IStorageDriver interface and // To implement a ZODB storage one need to provide IStorageDriver interface and
......
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package zodb
// Support for python objects/data in ZODB.
import (
"lab.nexedi.com/kirr/go123/mem"
)
// PyStateful is the interface describing in-RAM object whose data state can be
// exchanged as Python data.
type PyStateful interface {
// PySetState should set state of the in-RAM object from Python data.
//
// It is analog of __setstate__() in Python.
//
// The error returned does not need to have object/setstate prefix -
// persistent machinery is adding such prefix automatically.
PySetState(pystate interface{}) error
// PyGetState should return state of the in-RAM object as Python data.
// Analog of __getstate__() in Python.
//PyGetState() interface{} TODO
}
// pySetState decodes raw state as zodb/py serialized stream, and sets decoded
// state on PyStateful obj.
//
// It is an error if decoded state has python class not as specified.
func pySetState(obj PyStateful, objClass string, state *mem.Buf) error {
pyclass, pystate, err := PyData(state.Data).decode()
if err != nil {
return err
}
class := pyclassPath(pyclass)
if class != objClass {
// complain that pyclass changed
// (both ref and object data use pyclass so it indeed can be different)
return &wrongClassError{want: objClass, have: class}
}
return obj.PySetState(pystate)
}
// TODO pyGetState
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment