Commit 4bbc7942 authored by Kirill Smelkov's avatar Kirill Smelkov

X demo storage stub

parent f278d611
......@@ -73,10 +73,10 @@ func RegisterDriver(scheme string, opener DriverOpener) {
driverRegistry[scheme] = opener
}
// XXX
func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
// XXX ...
func OpenDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorageDriver, at0 Tid, _ error) {
// no scheme -> file://
if !strings.Contains(zurl, "://") {
if !strings.Contains(zurl, ":") {
zurl = "file://" + zurl
}
......@@ -91,7 +91,7 @@ func openDriver(ctx context.Context, zurl string, opt *DriverOptions) (_ IStorag
opener, ok := driverRegistry[u.Scheme]
if !ok {
return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s://\" not supported", u.Scheme)
return nil, InvalidTid, fmt.Errorf("zodb: URL scheme \"%s:\" not supported", u.Scheme)
}
storDriver, at0, err := opener(ctx, u, opt)
......@@ -116,7 +116,7 @@ func Open(ctx context.Context, zurl string, opt *OpenOptions) (IStorage, error)
Watchq: drvWatchq,
}
storDriver, at0, err := openDriver(ctx, zurl, drvOpt)
storDriver, at0, err := OpenDriver(ctx, zurl, drvOpt)
if err != nil {
return nil, err
}
......
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
// Package demo provides overlayed storage, similar to DemoStorage in ZODB/py.
//
// XXX
// XXX link to DemoStorage
package demo
// TODO tests
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"regexp"
"lab.nexedi.com/kirr/go123/mem"
"lab.nexedi.com/kirr/go123/xerr"
"lab.nexedi.com/kirr/neo/go/internal/task"
"lab.nexedi.com/kirr/neo/go/zodb"
)
// XXX
type Storage struct {
base zodb.IStorageDriver
δ zodb.IStorageDriver
baseAt0 zodb.Tid
}
// Close implements zodb.IStorageDriver .
func (ovl *Storage) Close() error {
// XXX errctx?
err1 := ovl.δ.Close()
err2 := ovl.base.Close()
// XXX close base watcher
return xerr.Merge(err1, err2)
}
// Sync implements zodb.IStorageDriver .
func (ovl *Storage) Sync(ctx context.Context) (head zodb.Tid, _ error) {
// XXX errctx?
head, err := ovl.δ.Sync(ctx)
if err != nil {
return zodb.InvalidTid, err
}
// δ is just created database
if head == 0 {
head = ovl.baseAt0
}
return head, nil
}
// Load implements zodb.IStorageDriver .
func (ovl *Storage) Load(ctx context.Context, xid zodb.Xid) (*mem.Buf, zodb.Tid, error) {
// XXX errctx?
data, serial, err := ovl.δ.Load(ctx, xid)
if data != nil {
// object data is present in δ
return data, serial, err
}
useBase := false
var eNoData *zodb.NoDataError
var eNoObject *zodb.NoObjectError
switch {
case errors.As(err, &eNoData):
if eNoData.DeletedAt != 0 {
// object deleted in δ -> whiteout
return data, serial, err
} else {
// object not created in δ
useBase = true
}
case errors.As(err, &eNoObject):
// object not created in δ
useBase = true
}
if !useBase {
return data, serial, err
}
// XXX cap .at in xid to base.At0 ? (and convert back on error return)
data, serial, err = ovl.base.Load(ctx, xid)
return data, serial, err
}
// Iterator implements zodb.IStorageDriver .
func (ovl *Storage) Iterate(ctx context.Context, tidMin, tidMax zodb.Tid) zodb.ITxnIterator {
// XXX errctx?
panic("TODO") // XXX
}
// URL implements zodb.IStorageDriver .
func (ovl *Storage) URL() string {
return "demo:(" + ovl.base.URL() + ")/(" + ovl.δ.URL() + ")"
}
var demoRe = regexp.MustCompile(`^\((.*)\)/\((.*)\)$`)
func openByURL(ctx context.Context, u *url.URL, opt *zodb.DriverOptions) (_ zodb.IStorageDriver, _ zodb.Tid, err error) {
// demo:(base_zurl)/(δ_zurl)
defer task.Runningf(&ctx, "demo: open %s", u)(&err)
if !(u.Opaque != "" && u.RawQuery == "" && u.Fragment == "") {
// opaque != "" makes sure user,host,path are empty
return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
}
argv := demoRe.FindStringSubmatch(u.Opaque)
if argv == nil {
return nil, zodb.InvalidTid, fmt.Errorf("invalid url")
}
baseZURL := argv[1]
δZURL := argv[2]
// open base - always readonly
baseWatchq := make(chan zodb.Event)
base, baseAt0, err := zodb.OpenDriver(ctx, baseZURL, &zodb.DriverOptions{
ReadOnly: true,
Watchq: baseWatchq,
})
if err != nil {
return nil, zodb.InvalidTid, err
}
defer func() {
if err != nil {
__ := base.Close()
err = xerr.Merge(err, __)
}
}()
// open δ - as requested
δ, δAt0, err := zodb.OpenDriver(ctx, δZURL, opt)
if err != nil {
return nil, zodb.InvalidTid, err
}
defer func() {
if err != nil {
__ := δ.Close()
err = xerr.Merge(err, __)
}
}()
// verify that either
// - δ is all empty (just created), or
// - all δ transactions come strictly after base.
at0 := baseAt0
if δAt0 != 0 {
if δAt0 < baseAt0 {
return nil, zodb.InvalidTid, fmt.Errorf("base is ahead of δ: base.head=%s δ.head=%s", baseAt0, δAt0)
}
:= δ.Iterate(ctx, 0, baseAt0)
for {
δtxni, _, err := .NextTxn(ctx)
if err == io.EOF {
break // all ok - nothing in δ
}
if err != nil {
return nil, zodb.InvalidTid, err
}
// there is a δ transaction ∈ [δAt0, baseAt0)
// TODO iδ.Stop()
return nil, zodb.InvalidTid, fmt.Errorf("base overlaps with δ: base.head=%s δ.tail=%s", baseAt0, δtxni.Tid)
}
at0 = δAt0
}
// XXX listen on baseWatchq and shutdown storage if base changes.
ovl := &Storage{
base: base,
δ : δ,
baseAt0: baseAt0,
}
return ovl, at0, nil
}
func init() {
zodb.RegisterDriver("demo", openByURL)
}
// Copyright (C) 2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
// option) any later version, as published by the Free Software Foundation.
//
// You can also Link and Combine this program with other software covered by
// the terms of any of the Free Software licenses or any of the Open Source
// Initiative approved licenses and Convey the resulting work. Corresponding
// source of such a combination shall include the source code for all other
// software used.
//
// This program is distributed WITHOUT ANY WARRANTY; without even the implied
// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
//
// See COPYING file for full licensing terms.
// See https://www.nexedi.com/licensing for rationale and options.
package demo
// XXX stdtest Demo(data, ø)
// XXX stdtest Demo(ø, data)
// XXX more?
// Copyright (C) 2017 Nexedi SA and Contributors.
// Copyright (C) 2017-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......@@ -31,4 +31,5 @@ import (
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/fs1"
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/zeo"
_ "lab.nexedi.com/kirr/neo/go/neo"
_ "lab.nexedi.com/kirr/neo/go/zodb/storage/demo"
)
// Copyright (C) 2016-2020 Nexedi SA and Contributors.
// Copyright (C) 2016-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
......
......@@ -45,6 +45,8 @@ There are also following simpler ways:
- zeo://<host>:<port> for a ZEO database
- /path/to/file for a FileStorage database
- demo:(zurl_base)/(zurl_δ) XXX overlay, DemoStorage ...
Please see zodburi documentation for full details:
http://docs.pylonsproject.org/projects/zodburi/
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment