Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neo
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
Kirill Smelkov
neo
Commits
7a98ef72
Commit
7a98ef72
authored
Feb 15, 2018
by
Kirill Smelkov
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
.
parent
9a4004fe
Changes
8
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
35 additions
and
23 deletions
+35
-23
go/neo/client.go
go/neo/client.go
+5
-4
go/neo/neo.go
go/neo/neo.go
+5
-4
go/neo/nodetab.go
go/neo/nodetab.go
+1
-1
go/neo/parttab.go
go/neo/parttab.go
+3
-3
go/neo/proto/proto.go
go/neo/proto/proto.go
+2
-0
go/neo/server.go
go/neo/server.go
+3
-1
go/neo/storage.go
go/neo/storage.go
+6
-6
go/neo/storage/sqlite/sqlite.go
go/neo/storage/sqlite/sqlite.go
+10
-4
No files found.
go/neo/client.go
View file @
7a98ef72
...
@@ -18,6 +18,7 @@
...
@@ -18,6 +18,7 @@
// See https://www.nexedi.com/licensing for rationale and options.
// See https://www.nexedi.com/licensing for rationale and options.
package
neo
package
neo
// client node
// XXX old: Package client provides ZODB storage interface for accessing NEO cluster.
// XXX old: Package client provides ZODB storage interface for accessing NEO cluster.
...
@@ -43,20 +44,20 @@ import (
...
@@ -43,20 +44,20 @@ import (
"lab.nexedi.com/kirr/neo/go/xcommon/xio"
"lab.nexedi.com/kirr/neo/go/xcommon/xio"
)
)
// Client talks to NEO cluster and exposes access to it via ZODB interfaces.
// Client
is NEO node that
talks to NEO cluster and exposes access to it via ZODB interfaces.
type
Client
struct
{
type
Client
struct
{
node
*
NodeApp
node
*
NodeApp
talkMasterCancel
func
()
talkMasterCancel
func
()
// link to master - established and maintained by talkMaster.
// link to master - established and maintained by talkMaster.
// users retrieve it via masterLink.
// users retrieve it via masterLink
()
.
mlinkMu
sync
.
Mutex
mlinkMu
sync
.
Mutex
mlink
*
neonet
.
NodeLink
mlink
*
neonet
.
NodeLink
mlinkReady
chan
struct
{}
// reinitialized at each new talk cycle
mlinkReady
chan
struct
{}
// reinitialized at each new talk cycle
// operational state in node is maintained by recvMaster.
// operational state in node is maintained by recvMaster.
// users retrieve it via withOperational.
// users retrieve it via withOperational
()
.
//
//
// NOTE being operational means:
// NOTE being operational means:
// - link to master established and is ok
// - link to master established and is ok
...
@@ -237,7 +238,7 @@ func (c *Client) talkMaster1(ctx context.Context) (err error) {
...
@@ -237,7 +238,7 @@ func (c *Client) talkMaster1(ctx context.Context) (err error) {
return
err
return
err
}
}
//
XXX vvv dup from Server.talkMaster1 ?
//
FIXME vvv dup from Storage.talkMaster1
// XXX -> node.Dial ?
// XXX -> node.Dial ?
if
accept
.
YourUUID
!=
c
.
node
.
MyInfo
.
UUID
{
if
accept
.
YourUUID
!=
c
.
node
.
MyInfo
.
UUID
{
...
...
go/neo/neo.go
View file @
7a98ef72
...
@@ -49,7 +49,7 @@ type NodeApp struct {
...
@@ -49,7 +49,7 @@ type NodeApp struct {
ClusterName
string
ClusterName
string
Net
xnet
.
Networker
// network AP we are sending/receiving on
Net
xnet
.
Networker
// network AP we are sending/receiving on
MasterAddr
string
// address of
master XXX -> Address
?
MasterAddr
string
// address of
current master XXX put under StateMu
?
StateMu
sync
.
RWMutex
// <- XXX just embed?
StateMu
sync
.
RWMutex
// <- XXX just embed?
NodeTab
*
NodeTable
// information about nodes in the cluster
NodeTab
*
NodeTable
// information about nodes in the cluster
...
@@ -69,7 +69,7 @@ func NewNodeApp(net xnet.Networker, typ proto.NodeType, clusterName, masterAddr,
...
@@ -69,7 +69,7 @@ func NewNodeApp(net xnet.Networker, typ proto.NodeType, clusterName, masterAddr,
}
}
app
:=
&
NodeApp
{
app
:=
&
NodeApp
{
MyInfo
:
proto
.
NodeInfo
{
Type
:
typ
,
Addr
:
addr
,
IdTime
:
proto
.
IdTimeNone
},
MyInfo
:
proto
.
NodeInfo
{
Type
:
typ
,
Addr
:
addr
,
UUID
:
0
,
IdTime
:
proto
.
IdTimeNone
},
ClusterName
:
clusterName
,
ClusterName
:
clusterName
,
Net
:
net
,
Net
:
net
,
MasterAddr
:
masterAddr
,
MasterAddr
:
masterAddr
,
...
@@ -86,8 +86,9 @@ func NewNodeApp(net xnet.Networker, typ proto.NodeType, clusterName, masterAddr,
...
@@ -86,8 +86,9 @@ func NewNodeApp(net xnet.Networker, typ proto.NodeType, clusterName, masterAddr,
// Dial connects to another node in the cluster.
// Dial connects to another node in the cluster.
//
//
// It handshakes, requests identification and checks peer type. If successful returned are:
// It handshakes, requests identification and checks peer type. If successful returned are:
// - established link
//
// - accept identification reply
// - established link
// - accept identification reply
//
//
// Dial does not update .NodeTab or its node entries in any way.
// Dial does not update .NodeTab or its node entries in any way.
// For establishing links to peers present in .NodeTab use Node.Dial.
// For establishing links to peers present in .NodeTab use Node.Dial.
...
...
go/neo/nodetab.go
View file @
7a98ef72
...
@@ -347,7 +347,7 @@ func (p *Node) dial(ctx context.Context) (_ *neonet.NodeLink, err error) {
...
@@ -347,7 +347,7 @@ func (p *Node) dial(ctx context.Context) (_ *neonet.NodeLink, err error) {
err
=
fmt
.
Errorf
(
"connected, but peer gives us uuid %v (our is %v)"
,
accept
.
YourUUID
,
app
.
MyInfo
.
UUID
)
err
=
fmt
.
Errorf
(
"connected, but peer gives us uuid %v (our is %v)"
,
accept
.
YourUUID
,
app
.
MyInfo
.
UUID
)
case
!
(
accept
.
NumPartitions
==
1
&&
accept
.
NumReplicas
==
1
)
:
case
!
(
accept
.
NumPartitions
==
1
&&
accept
.
NumReplicas
==
1
)
:
err
=
fmt
.
Errorf
(
"connected but TODO peer works with !
1x1 partition table."
)
err
=
fmt
.
Errorf
(
"connected but TODO peer works with !1x1 partition table."
)
}
}
if
err
!=
nil
{
if
err
!=
nil
{
...
...
go/neo/parttab.go
View file @
7a98ef72
...
@@ -32,7 +32,7 @@ import (
...
@@ -32,7 +32,7 @@ import (
//
//
// It is
// It is
//
//
//
Oid -> []Storage # XXX actually
oid -> []uuid
// oid -> []uuid
//
//
// mapping associating object id with list of storage nodes on where data for
// mapping associating object id with list of storage nodes on where data for
// this oid should be written-to/loaded-from. This mapping is organized as follows:
// this oid should be written-to/loaded-from. This mapping is organized as follows:
...
@@ -52,8 +52,8 @@ import (
...
@@ -52,8 +52,8 @@ import (
//
//
// mapping so that
// mapping so that
//
//
//
- redundancy level set by R is met
//
- redundancy level set by R is met
//
- storages associated with adjacent pids are different
//
- storages associated with adjacent pids are different
//
//
// when such organization is reached the partition table is called operational
// when such organization is reached the partition table is called operational
// and non-operational otherwise. XXX and if storages are ready
// and non-operational otherwise. XXX and if storages are ready
...
...
go/neo/proto/proto.go
View file @
7a98ef72
...
@@ -239,6 +239,8 @@ const (
...
@@ -239,6 +239,8 @@ const (
// order bit is really important and the 31 other bits could be random.
// order bit is really important and the 31 other bits could be random.
// Extra namespace information and non-randomness of 3 LOB help to read logs.
// Extra namespace information and non-randomness of 3 LOB help to read logs.
//
//
// 0 is invalid NodeUUID XXX correct?
//
// TODO -> back to 16-bytes randomly generated UUID
// TODO -> back to 16-bytes randomly generated UUID
type
NodeUUID
int32
type
NodeUUID
int32
...
...
go/neo/server.go
View file @
7a98ef72
...
@@ -67,6 +67,7 @@ func Serve(ctx context.Context, l *neo.Listener, srv Server) error {
...
@@ -67,6 +67,7 @@ func Serve(ctx context.Context, l *neo.Listener, srv Server) error {
}
}
*/
*/
/*
// FIXME kill vvv
// FIXME kill vvv
// ----------------------------------------
// ----------------------------------------
...
@@ -81,7 +82,7 @@ func IdentifyPeer(ctx context.Context, link *neonet.NodeLink, myNodeType proto.N
...
@@ -81,7 +82,7 @@ func IdentifyPeer(ctx context.Context, link *neonet.NodeLink, myNodeType proto.N
defer xerr.Contextf(&err, "%s: identify", link)
defer xerr.Contextf(&err, "%s: identify", link)
// the first conn must come with RequestIdentification packet
// the first conn must come with RequestIdentification packet
conn
,
err
:=
link
.
Accept
(
/*ctx*/
)
conn, err := link.Accept(
) //+ctx
if err != nil {
if err != nil {
return nodeInfo, err
return nodeInfo, err
}
}
...
@@ -117,6 +118,7 @@ func IdentifyPeer(ctx context.Context, link *neonet.NodeLink, myNodeType proto.N
...
@@ -117,6 +118,7 @@ func IdentifyPeer(ctx context.Context, link *neonet.NodeLink, myNodeType proto.N
return req, nil
return req, nil
}
}
*/
// ----------------------------------------
// ----------------------------------------
...
...
go/neo/storage.go
View file @
7a98ef72
...
@@ -183,7 +183,7 @@ func (stor *Storage) talkMaster(ctx context.Context) (err error) {
...
@@ -183,7 +183,7 @@ func (stor *Storage) talkMaster(ctx context.Context) (err error) {
// it returns error describing why such cycle had to finish.
// it returns error describing why such cycle had to finish.
// XXX distinguish between temporary problems and non-temporary ones?
// XXX distinguish between temporary problems and non-temporary ones?
func
(
stor
*
Storage
)
talkMaster1
(
ctx
context
.
Context
)
(
err
error
)
{
func
(
stor
*
Storage
)
talkMaster1
(
ctx
context
.
Context
)
(
err
error
)
{
//
XXX dup in Client.talkMaster1 ?
//
FIXME dup in Client.talkMaster1
mlink
,
accept
,
err
:=
stor
.
node
.
Dial
(
ctx
,
proto
.
MASTER
,
stor
.
node
.
MasterAddr
)
mlink
,
accept
,
err
:=
stor
.
node
.
Dial
(
ctx
,
proto
.
MASTER
,
stor
.
node
.
MasterAddr
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
...
@@ -316,7 +316,7 @@ func (stor *Storage) m1initialize1(ctx context.Context, req neonet.Request) erro
...
@@ -316,7 +316,7 @@ func (stor *Storage) m1initialize1(ctx context.Context, req neonet.Request) erro
// handling transaction commit (with master) and syncing data with other
// handling transaction commit (with master) and syncing data with other
// storage nodes (XXX correct?).
// storage nodes (XXX correct?).
//
//
// it always returns with an error describing why serve ha
s
to be stopped -
// it always returns with an error describing why serve ha
d
to be stopped -
// either due to master commanding us to stop, or context cancel or some other
// either due to master commanding us to stop, or context cancel or some other
// error.
// error.
func
(
stor
*
Storage
)
m1serve
(
ctx
context
.
Context
,
reqStart
*
neonet
.
Request
)
(
err
error
)
{
func
(
stor
*
Storage
)
m1serve
(
ctx
context
.
Context
,
reqStart
*
neonet
.
Request
)
(
err
error
)
{
...
@@ -420,7 +420,7 @@ func (stor *Storage) withWhileOperational(ctx context.Context) (context.Context,
...
@@ -420,7 +420,7 @@ func (stor *Storage) withWhileOperational(ctx context.Context) (context.Context,
}
}
// serveLink serves incoming node-node link connection
// serveLink serves incoming node-node link connection
.
func
(
stor
*
Storage
)
serveLink
(
ctx
context
.
Context
,
req
*
neonet
.
Request
,
idReq
*
proto
.
RequestIdentification
)
(
err
error
)
{
func
(
stor
*
Storage
)
serveLink
(
ctx
context
.
Context
,
req
*
neonet
.
Request
,
idReq
*
proto
.
RequestIdentification
)
(
err
error
)
{
link
:=
req
.
Link
()
link
:=
req
.
Link
()
defer
task
.
Runningf
(
&
ctx
,
"serve %s"
,
link
)(
&
err
)
defer
task
.
Runningf
(
&
ctx
,
"serve %s"
,
link
)(
&
err
)
...
@@ -526,7 +526,7 @@ func (stor *Storage) serveClient1(ctx context.Context, req proto.Msg) (resp prot
...
@@ -526,7 +526,7 @@ func (stor *Storage) serveClient1(ctx context.Context, req proto.Msg) (resp prot
xid
.
At
=
before2At
(
req
.
Tid
)
xid
.
At
=
before2At
(
req
.
Tid
)
}
}
resp
,
err
:=
stor
.
back
.
Load
(
ctx
,
xid
)
obj
,
err
:=
stor
.
back
.
Load
(
ctx
,
xid
)
if
err
!=
nil
{
if
err
!=
nil
{
// translate err to NEO protocol error codes
// translate err to NEO protocol error codes
e
:=
err
.
(
*
zodb
.
OpError
)
// XXX move this to ErrEncode?
e
:=
err
.
(
*
zodb
.
OpError
)
// XXX move this to ErrEncode?
...
@@ -536,7 +536,7 @@ func (stor *Storage) serveClient1(ctx context.Context, req proto.Msg) (resp prot
...
@@ -536,7 +536,7 @@ func (stor *Storage) serveClient1(ctx context.Context, req proto.Msg) (resp prot
// compatibility with py side:
// compatibility with py side:
// for loadSerial - check we have exact hit - else "nodata"
// for loadSerial - check we have exact hit - else "nodata"
if
req
.
Serial
!=
proto
.
INVALID_TID
{
if
req
.
Serial
!=
proto
.
INVALID_TID
{
if
resp
.
Serial
!=
req
.
Serial
{
if
obj
.
Serial
!=
req
.
Serial
{
return
&
proto
.
Error
{
return
&
proto
.
Error
{
Code
:
proto
.
OID_NOT_FOUND
,
Code
:
proto
.
OID_NOT_FOUND
,
Message
:
fmt
.
Sprintf
(
"%s: no data with serial %s"
,
xid
.
Oid
,
req
.
Serial
),
Message
:
fmt
.
Sprintf
(
"%s: no data with serial %s"
,
xid
.
Oid
,
req
.
Serial
),
...
@@ -544,7 +544,7 @@ func (stor *Storage) serveClient1(ctx context.Context, req proto.Msg) (resp prot
...
@@ -544,7 +544,7 @@ func (stor *Storage) serveClient1(ctx context.Context, req proto.Msg) (resp prot
}
}
}
}
return
resp
return
obj
case
*
proto
.
LastTransaction
:
case
*
proto
.
LastTransaction
:
lastTid
,
err
:=
stor
.
back
.
LastTid
(
ctx
)
lastTid
,
err
:=
stor
.
back
.
LastTid
(
ctx
)
...
...
go/neo/storage/sqlite/sqlite.go
View file @
7a98ef72
...
@@ -162,13 +162,19 @@ func (b *Backend) LastOid(ctx context.Context) (zodb.Oid, error) {
...
@@ -162,13 +162,19 @@ func (b *Backend) LastOid(ctx context.Context) (zodb.Oid, error) {
panic
(
"TODO"
)
panic
(
"TODO"
)
}
}
func
(
b
*
Backend
)
Load
(
ctx
context
.
Context
,
xid
zodb
.
Xid
)
(
*
proto
.
AnswerObject
,
error
)
{
func
(
b
*
Backend
)
Load
(
ctx
context
.
Context
,
xid
zodb
.
Xid
)
(
_
*
proto
.
AnswerObject
,
err
error
)
{
// XXX err ctx zodb.OpError{URL: b.url, Op: "load", Err: ...}
defer
func
()
{
if
err
!=
nil
{
err
=
&
zodb
.
OpError
{
URL
:
b
.
url
,
Op
:
"load"
,
Err
:
err
}
}
}()
obj
:=
&
proto
.
AnswerObject
{
Oid
:
xid
.
Oid
}
obj
:=
&
proto
.
AnswerObject
{
Oid
:
xid
.
Oid
}
var
data
sql
.
RawBytes
var
data
sql
.
RawBytes
// XXX pid = getReadablePartition (= oid % Np, raise if pid not readable)
// XXX pid = getReadablePartition (= oid % Np; error if pid not readable)
err
:=
b
.
query1
(
ctx
,
err
=
b
.
query1
(
ctx
,
"SELECT tid, compression, data.hash, value, value_tid"
+
"SELECT tid, compression, data.hash, value, value_tid"
+
" FROM obj LEFT JOIN data ON obj.data_id = data.id"
+
" FROM obj LEFT JOIN data ON obj.data_id = data.id"
+
" WHERE partition=? AND oid=? AND tid<=?"
+
" WHERE partition=? AND oid=? AND tid<=?"
+
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment