go/zodb: Teach Persistent to serialize itself

This will be used in the future to commit object changes back into
database. No test, since this functionality will be used at first to
implement tests themselves (see next patch). Hopefully after
bootstrapping we'll have full tests.

go/zodb/persistent.go

@@ -75,6 +75,8 @@ type Persistent struct {
 	loading  *loadState
 }
 
+func (obj *Persistent) persistent() *Persistent { return obj }
+
 func (obj *Persistent) PJar() *Connection { return obj.jar }
 func (obj *Persistent) POid() Oid         { return obj.oid }
 
@@ -129,8 +131,32 @@ type Stateful interface {
 	SetState(state *mem.Buf) error
 }
 
+// ---- RAM → DB: serialize ----
+
+// pSerialize returns object in serialized form to be saved in the database.
+//
+// pSerialize is non-public method that is exposed and used only by ZODB internally.
+// pSerialize is called only on non-ghost objects.
+func (obj *Persistent) pSerialize() *mem.Buf {
+	obj.mu.Lock()
+	defer obj.mu.Unlock()
+	if obj.state == GHOST {
+		panic(obj.badf("serialize: ghost object"))
+	}
+
+	switch istate := obj.istate().(type) {
+	case Stateful:
+		return istate.GetState()
+
+	case PyStateful:
+		return pyGetState(istate, ClassOf(obj.instance))
+
+	default:
+		panic(obj.badf("serialize: !stateful instance"))
+	}
+}
 
-// ---- activate/deactivate/invalidate ----
+// ---- RAM ← DB: activate/deactivate/invalidate ----
 
 // PActivate implements IPersistent.
 func (obj *Persistent) PActivate(ctx context.Context) (err error) {

go/zodb/persistent_api.go

 // Copyright (c) 2001, 2002 Zope Foundation and Contributors.
 // All Rights Reserved.
 //
-// Copyright (C) 2018  Nexedi SA and Contributors.
+// Copyright (C) 2018-2019  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This software is subject to the provisions of the Zope Public License,
@@ -97,6 +97,10 @@ type IPersistent interface {
 
 	// XXX probably don't need this.
 	//PState()  ObjectState	// in-RAM object state.
+
+
+	// IPersistent can be implemented only by objects that embed Persistent.
+	persistent() *Persistent
 }
 
 // ObjectState describes state of in-RAM object.

go/zodb/pydata.go

-// Copyright (C) 2016-2018  Nexedi SA and Contributors.
+// Copyright (C) 2016-2019  Nexedi SA and Contributors.
 //                          Kirill Smelkov <kirr@nexedi.com>
 //
 // This program is free software: you can Use, Study, Modify and Redistribute
@@ -24,6 +24,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"strings"
 
 	pickle "github.com/kisielk/og-rek"
 	"lab.nexedi.com/kirr/go123/xerr"
@@ -59,7 +60,37 @@ func (d PyData) ClassName() string {
 	return pyclassPath(klass)
 }
 
-// TODO PyData.referencesf
+// encodePyData encodes Python class and state into raw ZODB python data.
+func encodePyData(pyclass pickle.Class, pystate interface{}) PyData {
+	// XXX better return mem.Buf instead of PyData?
+	buf := &bytes.Buffer{}
+
+	p := pickle.NewEncoderWithConfig(buf, &pickle.EncoderConfig{
+		// allow pristine python2 to decode the pickle.
+		// TODO 2 -> 3 since ZODB5 switched to it and uses zodbpickle.
+		Protocol:      2,
+		PersistentRef: persistentRef,
+	})
+
+	// emit: object type
+	err := p.Encode(pyclass)
+	if err != nil {
+		// buf.Write never errors, as well as pickle encoder on supported types.
+		// -> the error here is a bug.
+		panic(fmt.Errorf("pydata: encode: class: %s", err))
+	}
+
+	// emit: object state
+	err = p.Encode(pystate)
+	if err != nil {
+		// see ^^^
+		panic(fmt.Errorf("pydata: encode: state: %s", err))
+	}
+
+	return PyData(buf.Bytes())
+}
+
+// TODO PyData.References()	(=referencesf in zodb/py)
 
 // decode decodes raw ZODB python data into Python class and state.
 //
@@ -90,6 +121,26 @@ func (d PyData) decode(jar *Connection) (pyclass pickle.Class, pystate interface
 	return klass, state, nil
 }
 
+// persistentRef decides whether to encode obj as persistent reference, and if yes - how.
+func persistentRef(obj interface{}) *pickle.Ref {
+	pobj, ok := obj.(IPersistent)
+	if !ok {
+		// regular object - include its state when encoding referee
+		return nil
+	}
+
+	// Persistent object - when encoding someone who references it - don't
+	// include obj state and use just reference to obj.
+
+	// XXX check obj.jar is the same as in referee
+	// XXX if not -> use multidb ref format?
+
+	return &pickle.Ref{
+		// (oid, class)
+		Pid: pickle.Tuple{pobj.POid(), zpyclass(ClassOf(pobj))},
+	}
+}
+
 // loadref loads persistent references resolving them through jar.
 //
 // https://github.com/zopefoundation/ZODB/blob/a89485c1/src/ZODB/serialize.py#L80
@@ -122,6 +173,22 @@ func (jar *Connection) loadref(ref pickle.Ref) (_ interface{}, err error) {
 	return jar.get(class, oid)
 }
 
+// zpyclass converts ZODB class into Python class.
+func zpyclass(zclass string) pickle.Class {
+	// BTrees.LOBTree.LOBucket -> BTrees.LOBTree, LOBucket
+	var zmod, zname string
+	dot := strings.LastIndexByte(zclass, '.')
+	if dot == -1 {
+		// zmod remains ""
+		zname = zclass
+	} else {
+		zmod  = zclass[:dot]
+		zname = zclass[dot+1:]
+	}
+
+	return pickle.Class{Module: zmod, Name: zname}
+}
+
 // xpyclass verifies and extracts py class from unpickled value.
 //
 // it normalizes py class that has just been decoded from a serialized ZODB

go/zodb/zodbpy.go

@@ -71,8 +71,13 @@ func pySetState(obj PyStateful, objClass string, state *mem.Buf, jar *Connection
 	return obj.PySetState(pystate)
 }
 
-// TODO pyGetState
-
+// pyGetState encodes obj as zodb/py serialized stream.
+func pyGetState(obj PyStateful, objClass string) *mem.Buf {
+	pyclass := zpyclass(objClass)
+	pystate := obj.PyGetState()
+	data := encodePyData(pyclass, pystate)
+	return &mem.Buf{Data: data} // XXX -> better bufalloc (and in encodePyData)
+}
 
 
 // loadpy loads object specified by oid and decodes it as a ZODB Python object.