Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neo
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Stefane Fermigier
neo
Commits
d3780906
Commit
d3780906
authored
Jan 23, 2017
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
client: use a class instead of a simple dict to hold transaction information
parent
97e57031
Changes
7
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
158 additions
and
145 deletions
+158
-145
neo/client/app.py
neo/client/app.py
+49
-91
neo/client/handlers/master.py
neo/client/handlers/master.py
+1
-1
neo/client/handlers/storage.py
neo/client/handlers/storage.py
+10
-13
neo/client/pool.py
neo/client/pool.py
+0
-37
neo/client/transactions.py
neo/client/transactions.py
+95
-0
neo/tests/client/testClientApp.py
neo/tests/client/testClientApp.py
+1
-1
neo/tests/threaded/test.py
neo/tests/threaded/test.py
+2
-2
No files found.
neo/client/app.py
View file @
d3780906
This diff is collapsed.
Click to expand it.
neo/client/handlers/master.py
View file @
d3780906
...
...
@@ -147,7 +147,7 @@ class PrimaryNotificationsHandler(MTEventHandler):
logging
.
critical
(
msg
)
app
.
master_conn
=
None
for
txn_context
in
app
.
txn_contexts
():
txn_context
[
'error'
]
=
msg
txn_context
.
error
=
msg
try
:
del
app
.
pt
except
AttributeError
:
...
...
neo/client/handlers/storage.py
View file @
d3780906
...
...
@@ -22,7 +22,7 @@ from neo.lib.util import dump
from
neo.lib.exception
import
NodeNotReady
from
neo.lib.handler
import
MTEventHandler
from
.
import
AnswerBaseHandler
from
..
pool
import
InvolvedNodeDict
from
..
transactions
import
Transaction
from
..exception
import
NEOStorageError
,
NEOStorageNotFoundError
from
..exception
import
NEOStorageDoesNotExistError
...
...
@@ -82,26 +82,26 @@ class StorageAnswersHandler(AnswerBaseHandler):
if
conflict
!=
MAX_TID
:
# If this conflict is not already resolved, mark it for
# resolution.
if
conflict
<=
txn_context
[
'resolved_dict'
]
.
get
(
oid
,
''
):
if
conflict
<=
txn_context
.
resolved_dict
.
get
(
oid
,
''
):
return
txn_context
[
'conflict_dict'
]
[
oid
]
=
serial
,
conflict
txn_context
.
conflict_dict
[
oid
]
=
serial
,
conflict
else
:
try
:
data
=
txn_context
[
'data_dict'
]
.
pop
(
oid
)
data
=
txn_context
.
data_dict
.
pop
(
oid
)
except
KeyError
:
# replica, or multiple undo
return
if
type
(
data
)
is
str
:
size
=
len
(
data
)
txn_context
[
'data_size'
]
-=
size
size
+=
txn_context
[
'cache_size'
]
txn_context
.
data_size
-=
size
size
+=
txn_context
.
cache_size
if
size
<
self
.
app
.
_cache
.
_max_size
:
txn_context
[
'cache_size'
]
=
size
txn_context
.
cache_size
=
size
else
:
# Do not cache data past cache max size, as it
# would just flush it on tpc_finish. This also
# prevents memory errors for big transactions.
data
=
None
txn_context
[
'cache_dict'
]
[
oid
]
=
data
txn_context
.
cache_dict
[
oid
]
=
data
answerCheckCurrentSerial
=
answerStoreObject
...
...
@@ -112,11 +112,8 @@ class StorageAnswersHandler(AnswerBaseHandler):
def
connectionClosed
(
self
,
conn
):
txn_context
=
self
.
app
.
getHandlerData
()
# XXX: A 'Transaction' class would be cleaner.
if
type
(
txn_context
)
is
dict
:
involved_nodes
=
txn_context
.
get
(
'involved_nodes'
)
if
type
(
involved_nodes
)
is
InvolvedNodeDict
:
involved_nodes
[
conn
.
getUUID
()]
=
2
if
type
(
txn_context
)
is
Transaction
:
txn_context
.
involved_nodes
[
conn
.
getUUID
()]
=
2
super
(
StorageAnswersHandler
,
self
).
connectionClosed
(
conn
)
def
answerTIDsFrom
(
self
,
conn
,
tid_list
):
...
...
neo/client/pool.py
View file @
d3780906
...
...
@@ -37,23 +37,6 @@ CELL_GOOD = 0
CELL_FAILED
=
1
class
InvolvedNodeDict
(
dict
):
# Keys are node ids instead of Node objects because a node may disappear
# from the cluster. In any case, we always have to check if the id is
# still known by the NodeManager.
def
ask
(
self
,
conn
):
def
ask
(
*
args
,
**
kw
):
try
:
conn
.
ask
(
*
args
,
**
kw
)
except
ConnectionClosed
:
self
[
conn
.
getUUID
()]
=
2
else
:
self
.
fail
=
0
return
conn
.
getUUID
()
return
ask
class
ConnectionPool
(
object
):
"""This class manages a pool of connections to storage nodes."""
...
...
@@ -135,26 +118,6 @@ class ConnectionPool(object):
if
self
.
app
.
master_conn
is
None
:
raise
NEOPrimaryMasterLost
def
iterateForWrite
(
self
,
object_id
,
involved
,
store
=
1
):
pt
=
self
.
app
.
pt
involved
.
fail
=
1
for
cell
in
pt
.
getCellList
(
pt
.
getPartition
(
object_id
)):
node
=
cell
.
getNode
()
uuid
=
node
.
getUUID
()
status
=
involved
.
setdefault
(
uuid
,
store
)
if
status
<
store
:
involved
[
uuid
]
=
store
elif
status
>
1
:
continue
conn
=
self
.
getConnForNode
(
node
)
if
conn
is
None
:
involved
[
uuid
]
=
2
else
:
yield
involved
.
ask
(
conn
)
if
involved
.
fail
:
raise
NEOStorageError
(
'no storage available for write to partition %s'
%
object_id
)
def
getConnForNode
(
self
,
node
):
"""Return a locked connection object to a given node
If no connection exists, create a new one"""
...
...
neo/client/transactions.py
0 → 100644
View file @
d3780906
#
# Copyright (C) 2017 Nexedi SA
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from
ZODB.POSException
import
StorageTransactionError
from
neo.lib.connection
import
ConnectionClosed
from
neo.lib.locking
import
SimpleQueue
from
.exception
import
NEOStorageError
class
Transaction
(
object
):
cache_size
=
0
# size of data in cache_dict
data_size
=
0
# size of data in data_dict
error
=
None
voted
=
False
ttid
=
None
# XXX: useless, except for testBackupReadOnlyAccess
def
__init__
(
self
,
txn
):
self
.
queue
=
SimpleQueue
()
self
.
txn
=
txn
# data being stored
self
.
data_dict
=
{}
# data stored: this will go to the cache on tpc_finish
self
.
cache_dict
=
{}
# conflicts to resolve
self
.
conflict_dict
=
{}
# {oid: (base_serial, serial)}
# resolved conflicts
self
.
resolved_dict
=
{}
# {oid: serial}
# Keys are node ids instead of Node objects because a node may
# disappear from the cluster. In any case, we always have to check
# if the id is still known by the NodeManager.
# status: 0 -> check only, 1 -> store, 2 -> failed
self
.
involved_nodes
=
{}
# {node_id: status}
def
write
(
self
,
app
,
packet
,
object_id
,
store
=
1
,
**
kw
):
uuid_list
=
[]
pt
=
app
.
pt
involved
=
self
.
involved_nodes
object_id
=
pt
.
getPartition
(
object_id
)
for
cell
in
pt
.
getCellList
(
object_id
):
node
=
cell
.
getNode
()
uuid
=
node
.
getUUID
()
status
=
involved
.
setdefault
(
uuid
,
store
)
if
status
<
store
:
involved
[
uuid
]
=
store
elif
status
>
1
:
continue
conn
=
app
.
cp
.
getConnForNode
(
node
)
if
conn
is
not
None
:
try
:
conn
.
ask
(
packet
,
queue
=
self
.
queue
,
**
kw
)
uuid_list
.
append
(
uuid
)
continue
except
ConnectionClosed
:
pass
involved
[
uuid
]
=
2
if
uuid_list
:
return
uuid_list
raise
NEOStorageError
(
'no storage available for write to partition %s'
%
object_id
)
class
TransactionContainer
(
dict
):
# IDEA: Drop this container and use the new set_data/data API on
# transactions (requires transaction >= 1.6).
def
pop
(
self
,
txn
):
return
dict
.
pop
(
self
,
id
(
txn
),
None
)
def
get
(
self
,
txn
):
try
:
return
self
[
id
(
txn
)]
except
KeyError
:
raise
StorageTransactionError
(
"unknown transaction %r"
%
txn
)
def
new
(
self
,
txn
):
key
=
id
(
txn
)
if
key
in
self
:
raise
StorageTransactionError
(
"commit of transaction %r"
" already started"
%
txn
)
context
=
self
[
key
]
=
Transaction
(
txn
)
return
context
neo/tests/client/testClientApp.py
View file @
d3780906
...
...
@@ -70,7 +70,7 @@ class ClientApplicationTests(NeoUnitTestBase):
def
_begin
(
self
,
app
,
txn
,
tid
):
txn_context
=
app
.
_txn_container
.
new
(
txn
)
txn_context
[
'ttid'
]
=
tid
txn_context
.
ttid
=
tid
return
txn_context
def
getApp
(
self
,
master_nodes
=
None
,
name
=
'test'
,
**
kw
):
...
...
neo/tests/threaded/test.py
View file @
d3780906
...
...
@@ -370,7 +370,7 @@ class Test(NEOThreadedTest):
resolved
=
[]
last
=
lambda
txn
:
txn
.
_extension
[
'last'
]
# BBB
def
_handleConflicts
(
orig
,
txn_context
):
resolved
.
append
(
last
(
txn_context
[
'txn'
]
))
resolved
.
append
(
last
(
txn_context
.
txn
))
orig
(
txn_context
)
def
tpc_vote
(
orig
,
transaction
):
(
l3
if
last
(
transaction
)
else
l2
)()
...
...
@@ -969,7 +969,7 @@ class Test(NEOThreadedTest):
txn
=
transaction
.
Transaction
()
client
.
tpc_begin
(
None
,
txn
)
txn_context
=
client
.
_txn_container
.
get
(
txn
)
txn_context
[
'ttid'
]
=
add64
(
txn_context
[
'ttid'
]
,
1
)
txn_context
.
ttid
=
add64
(
txn_context
.
ttid
,
1
)
self
.
assertRaises
(
POSException
.
StorageError
,
client
.
tpc_finish
,
txn
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment