Skip to content

N
neoppod
Commit 6ed71cfd authored by Kirill Smelkov's avatar Kirill Smelkov
Browse files
Options

.

parent 1dd5d4c1
Hide whitespace changes
Inline Side-by-side
Showing with 34 additions and 47 deletions
go/neo/storage/sqlite/pool.go
View file @ 6ed71cfd
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -29,7 +29,7 @@ import (
sqlite3 "github.com/gwenn/gosqlite"
)
// connPool is a pool of sqlite3.Conn
// connPool is a pool of sqlite3.Conn .
type connPool struct {
factory func() (*sqlite3.Conn, error) // =nil if pool closed
......
go/neo/storage/sqlite/sqlite.go
View file @ 6ed71cfd
......@@ -71,7 +71,6 @@ const schemaVersion = 3
// table "config" stores configuration parameters which affect the persistent data.
//
// XXX
// (name, nid, ptid, replicas, version, zodb=pickle...)
const config = `
name TEXT NOT NULL PRIMARY KEY,
......@@ -115,7 +114,7 @@ const obj = `
// `(partition, tid, oid)`
// `(data_id)`
// XXX reenable for ^^^
// TODO create indices for obj
//index_dict['obj'] = (
// "CREATE INDEX %s ON %s(partition, tid, oid)",
// "CREATE INDEX %s ON %s(data_id)")
......@@ -127,7 +126,7 @@ const data = `
compression INTEGER NOT NULL,
value BLOB NOT NULL
`
// XXX reenable for ^^^
// TODO reenable for ^^^
//if dedup:
// index_dict['data'] = (
// "CREATE UNIQUE INDEX %s ON %s(hash, compression)",)
......@@ -156,9 +155,7 @@ const tobj = `
PRIMARY KEY (tid, oid)
`
// ----------------------------------------
type Backend struct {
pool *connPool
......@@ -172,7 +169,7 @@ type row1 struct {
pool *connPool
conn *sqlite3.Conn
stmt *sqlite3.Stmt
err error // != nil on an error obtaining the row
err error // != nil on an error obtaining the row
}
var errNoRows = errors.New("sqlite: no rows in result set")
......@@ -304,18 +301,11 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
}()
obj := &proto.AnswerObject{Oid: xid.Oid, DataSerial: 0}
// TODO reenable, but XXX we have to use Query, not QueryRow for RawBytes support
//var data sql.RawBytes
var data []byte
// XXX recheck vvv with sqlite3 direct
// hash is variable-length BLOB - Scan refuses to put it into [20]byte
//var hash sql.RawBytes
var hash []byte
// obj.value_tid can be null
//var valueTid sql.NullInt64 // XXX ok not to uint64 - max tid is max signed int64
var valueTid int64 // XXX ok not to uint64 - max tid is max signed int64
var valueTid int64 // ok to be not uint64 - max tid is max signed int64
// FIXME pid = getReadablePartition (= oid % Np; error if pid not readable)
pid := 0
......@@ -323,7 +313,8 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
// XXX somehow detect errors in sql misuse and log them as 500 without reporting to client?
// XXX such errors start with "unsupported Scan, "
// XXX use conn for several query1 (see below) without intermediate returns to pool?
// TODO use conn for several query1 (see below) without intermediate returns to pool?
// TODO try to use ScanRawBytes for data
err = b.query1(
"SELECT tid, compression, data.hash, value, value_tid" +
......@@ -345,7 +336,7 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
case err == nil:
err = &zodb.NoDataError{
Oid: xid.Oid,
DeletedAt: 0, // XXX hardcoded
DeletedAt: 0, // FIXME hardcoded
}
case err == errNoRows:
......@@ -375,13 +366,13 @@ func (b *Backend) Load(ctx context.Context, xid zodb.Xid) (_ *proto.AnswerObject
obj.DataSerial = zodb.Tid(valueTid)
}
// data -> obj.Data
obj.Data = mem.BufAlloc(len(data))
copy(obj.Data.Data, data)
// find out nextSerial
// XXX kill nextSerial support after neo/py cache does not need it
// TODO kill nextSerial support after neo/py cache does not need it
// see also: https://github.com/zopefoundation/ZODB/pull/323
err = b.query1(
"SELECT tid from obj" +
" WHERE partition=? AND oid=? AND tid>?" +
......@@ -405,9 +396,13 @@ func (b *Backend) config(key string, pvalue *string) error {
return b.query1("SELECT value FROM config WHERE name=?", key).Scan(pvalue)
}
func (b *Backend) Close() error {
err := b.pool.Close()
return err // XXX err ctx
func (b *Backend) Close() (err error) {
defer func() {
if err != nil {
err = &zodb.OpError{URL: b.url, Op: "close", Err: err}
}
}()
return b.pool.Close()
}
// ---- open ----
......@@ -450,12 +445,12 @@ func openConn(dburl string) (*sqlite3.Conn, error) {
// primary SQLite author also confirms it should be working normally:
// https://bugzilla.mozilla.org/show_bug.cgi?id=993556#c1
//
// XXX neo/py does not use locking_mode=EXCLUSIVE.
// NOTE neo/py does not use locking_mode=EXCLUSIVE.
mode := "EXCLUSIVE"
mode_, err := conn.SetLockingMode("", mode)
if err != nil {
conn.Close()
return nil, err // XXX or other error?
return nil, err
}
mode_ = strings.ToUpper(mode_) // sqlite returns "exclusive"
......@@ -515,8 +510,6 @@ func Open(dburl string) (_ *Backend, err error) {
value := ""
err := b.config(name, &value)
// XXX prefix "b.path: config: %s:"
switch err {
case errNoRows:
err = fmt.Errorf("not found")
......@@ -533,18 +526,15 @@ func Open(dburl string) (_ *Backend, err error) {
}
}
checkConfig("version", schemaVersion)
checkConfig("nid", int(proto.NID(proto.STORAGE, 1)))
checkConfig("replicas", 0) // XXX neo/py uses nreplicas as 1 + n(replica)
checkConfig("version", schemaVersion)
checkConfig("nid", int(proto.NID(proto.STORAGE, 1)))
checkConfig("replicas", 0) // neo/py uses nreplicas as 1 + n(replica)
err = errv.Err()
if err != nil {
return nil, fmt.Errorf("%s: NEO/go POC: not ready to handle: %s", dburl, err)
}
// config("version")
// config("nid")
// config("replicas")
// config("name")
// config("ptid")
// config("backup_tid")
......
go/neo/storage/sqlite/sqlite_test.go
View file @ 6ed71cfd
// Copyright (C) 2018 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
// Copyright (C) 2018-2021 Nexedi SA and Contributors.
// Kirill Smelkov <kirr@nexedi.com>
//
// This program is free software: you can Use, Study, Modify and Redistribute
// it under the terms of the GNU General Public License version 3, or (at your
......@@ -33,19 +33,17 @@ import (
// TODO verify data can be read as the same
var bg = context.Background()
func BenchmarkLoad(b *testing.B) {
back, err := Open("testdata/1.sqlite")
exc.Raiseif(err)
X := exc.Raiseif
back, err := Open("testdata/1.sqlite"); X(err)
defer func() {
err := back.Close()
exc.Raiseif(err)
err := back.Close(); X(err)
}()
lastTid, err := back.LastTid(bg)
exc.Raiseif(err)
ctx := context.Background()
lastTid, err := back.LastTid(ctx); X(err)
xid := zodb.Xid{Oid: 0, At: lastTid}
......@@ -53,7 +51,7 @@ func BenchmarkLoad(b *testing.B) {
loop:
for i := 0; i < b.N; i++ {
obj, err := back.Load(bg, xid)
obj, err := back.Load(ctx, xid)
if err != nil {
switch errors.Cause(err).(type) {
case *zodb.NoObjectError:
......@@ -63,7 +61,6 @@ loop:
case *zodb.NoDataError:
panic(err) // XXX object was deleted
default:
b.Fatal(err)
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7