Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
N
neoppod
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
1
Issues
1
List
Boards
Labels
Milestones
Merge Requests
2
Merge Requests
2
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
nexedi
neoppod
Commits
574b64df
Commit
574b64df
authored
May 26, 2024
by
Julien Muchembled
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
master: simplify code
parent
eb3a3937
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
49 additions
and
104 deletions
+49
-104
neo/master/app.py
neo/master/app.py
+6
-7
neo/master/transactions.py
neo/master/transactions.py
+42
-96
neo/tests/threaded/testReplication.py
neo/tests/threaded/testReplication.py
+1
-1
No files found.
neo/master/app.py
View file @
574b64df
...
...
@@ -559,23 +559,22 @@ class Application(BaseApplication):
# I have received all the lock answers now:
# - send a Notify Transaction Finished to the initiated client node
# - Invalidate Objects to the other client nodes
ttid
=
txn
.
getTTID
()
tid
=
txn
.
getTID
()
transaction_node
=
txn
.
getNode
()
invalidate_objects
=
Packets
.
InvalidateObjects
(
tid
,
txn
.
getOIDList
()
)
ttid
=
txn
.
ttid
tid
=
txn
.
tid
transaction_node
=
txn
.
node
invalidate_objects
=
Packets
.
InvalidateObjects
(
tid
,
txn
.
oid_list
)
client_list
=
self
.
nm
.
getClientList
(
only_identified
=
True
)
for
client_node
in
client_list
:
if
client_node
is
transaction_node
:
client_node
.
send
(
Packets
.
AnswerTransactionFinished
(
ttid
,
tid
),
msg_id
=
txn
.
getMessageId
()
)
msg_id
=
txn
.
msg_id
)
else
:
client_node
.
send
(
invalidate_objects
)
# Unlock Information to relevant storage nodes.
notify_unlock
=
Packets
.
NotifyUnlockInformation
(
ttid
)
getByUUID
=
self
.
nm
.
getByUUID
txn_storage_list
=
txn
.
getUUIDList
()
for
storage_uuid
in
txn_storage_list
:
for
storage_uuid
in
txn
.
involved
:
getByUUID
(
storage_uuid
).
send
(
notify_unlock
)
# Notify storage nodes about new pack order if any.
...
...
neo/master/transactions.py
View file @
574b64df
...
...
@@ -27,22 +27,18 @@ class Transaction(object):
"""
A pending transaction
"""
_tid
=
None
_msg_id
=
None
_oid_list
=
None
_failed
=
frozenset
()
_prepared
=
False
# uuid dict hold flag to known who has locked the transaction
_uuid_set
=
None
_lock_wait_uuid_set
=
None
tid
=
None
oid_list
=
()
failed
=
\
involved
=
frozenset
()
def
__init__
(
self
,
node
,
storage_readiness
,
ttid
):
"""
Prepare the transaction, set OIDs and UUIDs related to it
"""
self
.
_
node
=
node
self
.
_
storage_readiness
=
storage_readiness
self
.
_
ttid
=
ttid
self
.
node
=
node
self
.
storage_readiness
=
storage_readiness
self
.
ttid
=
ttid
self
.
_birth
=
time
()
# store storage uuids that must be notified at commit
self
.
_notification_set
=
set
()
...
...
@@ -50,57 +46,14 @@ class Transaction(object):
def
__repr__
(
self
):
return
"<%s(client=%r, tid=%r, oids=%r, storages=%r, age=%.2fs) at %x>"
%
(
self
.
__class__
.
__name__
,
self
.
_
node
,
dump
(
self
.
_
tid
),
map
(
dump
,
self
.
_oid_list
or
()
),
map
(
uuid_str
,
self
.
_uuid_set
or
()
),
self
.
node
,
dump
(
self
.
tid
),
map
(
dump
,
self
.
oid_list
),
map
(
uuid_str
,
self
.
involved
),
time
()
-
self
.
_birth
,
id
(
self
),
)
def
getNode
(
self
):
"""
Return the node that had began the transaction
"""
return
self
.
_node
def
getTTID
(
self
):
"""
Return the temporary transaction ID.
"""
return
self
.
_ttid
def
getTID
(
self
):
"""
Return the transaction ID
"""
return
self
.
_tid
def
getMessageId
(
self
):
"""
Returns the packet ID to use in the answer
"""
return
self
.
_msg_id
def
getUUIDList
(
self
):
"""
Returns the list of node's UUID that lock the transaction
"""
return
list
(
self
.
_uuid_set
)
def
getOIDList
(
self
):
"""
Returns the list of OIDs used in the transaction
"""
return
list
(
self
.
_oid_list
)
def
isPrepared
(
self
):
"""
Returns True if the commit has been requested by the client
"""
return
self
.
_prepared
def
registerForNotification
(
self
,
uuid
):
"""
Register a node that requires a notification at commit
...
...
@@ -114,14 +67,12 @@ class Transaction(object):
"""
return
list
(
self
.
_notification_set
)
def
prepare
(
self
,
tid
,
oid_list
,
uuid_set
,
msg_id
):
self
.
_tid
=
tid
self
.
_oid_list
=
oid_list
self
.
_msg_id
=
msg_id
self
.
_uuid_set
=
uuid_set
self
.
_lock_wait_uuid_set
=
uuid_set
.
copy
()
self
.
_prepared
=
True
def
prepare
(
self
,
tid
,
oid_list
,
involved
,
msg_id
):
self
.
tid
=
tid
self
.
oid_list
=
oid_list
self
.
msg_id
=
msg_id
self
.
involved
=
involved
self
.
locking
=
involved
.
copy
()
def
storageLost
(
self
,
uuid
):
"""
...
...
@@ -133,16 +84,17 @@ class Transaction(object):
# XXX: We might lose information that a storage successfully locked
# data but was later found to be disconnected. This loss has no impact
# on current code, but it might be disturbing to reader or future code.
if
self
.
_prepared
:
self
.
_lock_wait_uuid_set
.
discard
(
uuid
)
self
.
_uuid_set
.
discard
(
uuid
)
return
self
.
locked
()
if
self
.
tid
:
locking
=
self
.
locking
locking
.
discard
(
uuid
)
self
.
involved
.
discard
(
uuid
)
return
not
locking
return
False
def
clientLost
(
self
,
node
):
if
self
.
_
node
is
node
:
if
self
.
_prepare
d
:
self
.
_
node
=
None
# orphan
if
self
.
node
is
node
:
if
self
.
ti
d
:
self
.
node
=
None
# orphan
else
:
return
True
# abort
else
:
...
...
@@ -154,14 +106,9 @@ class Transaction(object):
Define that a node has locked the transaction
Returns true if all nodes are locked
"""
self
.
_lock_wait_uuid_set
.
remove
(
uuid
)
return
self
.
locked
()
def
locked
(
self
):
"""
Returns true if all nodes are locked
"""
return
not
self
.
_lock_wait_uuid_set
locking
=
self
.
locking
locking
.
remove
(
uuid
)
return
not
locking
class
TransactionManager
(
EventQueue
):
...
...
@@ -302,7 +249,7 @@ class TransactionManager(EventQueue):
txn
=
self
[
ttid
]
# The client does not know which nodes are not expected to have
# transactions in full. Let's filter out them.
failed
=
app
.
getStorageReadySet
(
txn
.
_
storage_readiness
)
failed
=
app
.
getStorageReadySet
(
txn
.
storage_readiness
)
failed
.
intersection_update
(
uuid_list
)
if
failed
:
operational
=
app
.
pt
.
operational
...
...
@@ -312,16 +259,15 @@ class TransactionManager(EventQueue):
return
False
all_failed
=
failed
.
copy
()
for
t
in
self
.
_ttid_dict
.
itervalues
():
all_failed
|=
t
.
_
failed
all_failed
|=
t
.
failed
if
not
operational
(
all_failed
):
# Other transactions were voted and unless they're aborted,
# we won't be able to finish this one, because that would make
# the cluster non-operational. Let's tell the caller to retry
# later.
# the cluster non-operational. Let's retry later.
raise
DelayEvent
# Allow the client to finish the transaction,
# even if this will disconnect storage nodes.
txn
.
_
failed
=
failed
txn
.
failed
=
failed
return
True
def
prepare
(
self
,
app
,
ttid
,
oid_list
,
checked_list
,
msg_id
):
...
...
@@ -331,22 +277,22 @@ class TransactionManager(EventQueue):
txn
=
self
[
ttid
]
pt
=
app
.
pt
failed
=
txn
.
_
failed
failed
=
txn
.
failed
if
failed
and
not
pt
.
operational
(
failed
):
return
None
,
None
ready
=
app
.
getStorageReadySet
(
txn
.
_
storage_readiness
)
ready
=
app
.
getStorageReadySet
(
txn
.
storage_readiness
)
getPartition
=
pt
.
getPartition
partition_set
=
set
(
map
(
getPartition
,
oid_list
))
partition_set
.
update
(
map
(
getPartition
,
checked_list
))
partition_set
.
add
(
getPartition
(
ttid
))
node_list
=
[]
uuid_set
=
set
()
involved
=
set
()
for
partition
in
partition_set
:
for
cell
in
pt
.
getCellList
(
partition
):
node
=
cell
.
getNode
()
if
node
.
isIdentified
():
uuid
=
node
.
getUUID
()
if
uuid
in
uuid_set
:
if
uuid
in
involved
:
continue
if
uuid
in
failed
:
# This will commit a new PT with outdated cells before
...
...
@@ -354,7 +300,7 @@ class TransactionManager(EventQueue):
# the verification phase.
node
.
getConnection
().
close
()
elif
uuid
in
ready
:
uuid_set
.
add
(
uuid
)
involved
.
add
(
uuid
)
node_list
.
append
(
node
)
# A node that was not ready at the beginning of the transaction
# can't have readable cells. And if we're still operational without
...
...
@@ -369,8 +315,8 @@ class TransactionManager(EventQueue):
tid
=
self
.
_nextTID
(
ttid
,
pt
.
getPartitions
())
self
.
_queue
.
append
(
ttid
)
logging
.
debug
(
'Finish TXN %s for %s (was %s)'
,
dump
(
tid
),
txn
.
getNode
()
,
dump
(
ttid
))
txn
.
prepare
(
tid
,
oid_list
,
uuid_set
,
msg_id
)
dump
(
tid
),
txn
.
node
,
dump
(
ttid
))
txn
.
prepare
(
tid
,
oid_list
,
involved
,
msg_id
)
# check if greater and foreign OID was stored
if
oid_list
:
self
.
setLastOID
(
max
(
oid_list
))
...
...
@@ -382,7 +328,7 @@ class TransactionManager(EventQueue):
"""
logging
.
debug
(
'Abort TXN %s for %s'
,
dump
(
ttid
),
uuid_str
(
uuid
))
txn
=
self
[
ttid
]
if
txn
.
isPrepared
()
:
if
txn
.
tid
:
raise
ProtocolError
(
"commit already requested for ttid %s"
%
dump
(
ttid
))
del
self
[
ttid
]
...
...
@@ -424,7 +370,7 @@ class TransactionManager(EventQueue):
while
queue
:
ttid
=
queue
[
0
]
txn
=
self
.
_ttid_dict
[
ttid
]
if
not
txn
.
locked
()
:
if
txn
.
locking
:
break
del
queue
[
0
],
self
.
_ttid_dict
[
ttid
]
self
.
_on_commit
(
txn
)
...
...
@@ -433,7 +379,7 @@ class TransactionManager(EventQueue):
def
clientLost
(
self
,
node
):
for
txn
in
self
.
_ttid_dict
.
values
():
if
txn
.
clientLost
(
node
):
tid
=
txn
.
getTTID
()
tid
=
txn
.
ttid
del
self
[
tid
]
yield
tid
,
txn
.
getNotificationUUIDList
()
...
...
neo/tests/threaded/testReplication.py
View file @
574b64df
...
...
@@ -220,7 +220,7 @@ class ReplicationTests(NEOThreadedTest):
counts
[
0
]
+=
1
if
counts
[
0
]
>
1
:
node_list
=
orig
.
im_self
.
nm
.
getClientList
(
only_identified
=
True
)
node_list
.
remove
(
txn
.
getNode
()
)
node_list
.
remove
(
txn
.
node
)
node_list
[
0
].
getConnection
().
close
()
return
orig
(
txn
)
with
NEOCluster
(
partitions
=
np
,
replicas
=
0
,
storage_count
=
1
)
as
upstream
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment