Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Z
ZODB
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Kirill Smelkov
ZODB
Commits
b5895a5c
Commit
b5895a5c
authored
Jan 23, 2020
by
Julien Muchembled
Committed by
GitHub
Jan 23, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
mvccadapter: fix race with invalidations when starting a new transaction (#291)
Fixes #290
parent
12b52a71
Changes
5
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
75 additions
and
22 deletions
+75
-22
CHANGES.rst
CHANGES.rst
+5
-0
src/ZODB/FileStorage/tests.py
src/ZODB/FileStorage/tests.py
+4
-4
src/ZODB/mvccadapter.py
src/ZODB/mvccadapter.py
+23
-8
src/ZODB/tests/testConnection.py
src/ZODB/tests/testConnection.py
+36
-4
src/ZODB/tests/testmvcc.py
src/ZODB/tests/testmvcc.py
+7
-6
No files found.
CHANGES.rst
View file @
b5895a5c
...
@@ -5,6 +5,11 @@
...
@@ -5,6 +5,11 @@
5.6.0 (unreleased)
5.6.0 (unreleased)
==================
==================
- Fix race with invalidations when starting a new transaction. The bug
affected Storage implementations that rely on mvccadapter, and could result
in data corruption (oid loaded at wrong serial after a concurrent commit).
See `issue 290 <https://github.com/zopefoundation/ZODB/issues/290>`_.
- Improve volatile attribute ``_v_`` documentation.
- Improve volatile attribute ``_v_`` documentation.
- Make repozo's recover mode atomic by recovering the backup in a
- Make repozo's recover mode atomic by recovering the backup in a
...
...
src/ZODB/FileStorage/tests.py
View file @
b5895a5c
...
@@ -113,15 +113,15 @@ def pack_with_repeated_blob_records():
...
@@ -113,15 +113,15 @@ def pack_with_repeated_blob_records():
fixed by the time you read this, but there might still be
fixed by the time you read this, but there might still be
transactions in the wild that have duplicate records.
transactions in the wild that have duplicate records.
>>> fs = ZODB.FileStorage.FileStorage('t', blob_dir='bobs')
>>> db = ZODB.DB(ZODB.FileStorage.FileStorage('t', blob_dir='bobs'))
>>> db = ZODB.DB(fs)
>>> conn = db.open()
>>> conn = db.open()
>>> conn.root()[1] = ZODB.blob.Blob()
>>> conn.root()[1] = ZODB.blob.Blob()
>>> transaction.commit()
>>> transaction.commit()
>>> tm = transaction.TransactionManager()
>>> tm = transaction.TransactionManager()
>>> oid = conn.root()[1]._p_oid
>>> oid = conn.root()[1]._p_oid
>>> from ZODB.utils import load_current
>>> fs = db._mvcc_storage.new_instance()
>>> blob_record, oldserial = load_current(fs, oid)
>>> _ = fs.poll_invalidations()
>>> blob_record, oldserial = fs.load(oid)
Now, create a transaction with multiple saves:
Now, create a transaction with multiple saves:
...
...
src/ZODB/mvccadapter.py
View file @
b5895a5c
...
@@ -49,6 +49,7 @@ class MVCCAdapter(Base):
...
@@ -49,6 +49,7 @@ class MVCCAdapter(Base):
instance
=
MVCCAdapterInstance
(
self
)
instance
=
MVCCAdapterInstance
(
self
)
with
self
.
_lock
:
with
self
.
_lock
:
self
.
_instances
.
add
(
instance
)
self
.
_instances
.
add
(
instance
)
instance
.
_lastTransaction
()
return
instance
return
instance
def
before_instance
(
self
,
before
=
None
):
def
before_instance
(
self
,
before
=
None
):
...
@@ -77,13 +78,13 @@ class MVCCAdapter(Base):
...
@@ -77,13 +78,13 @@ class MVCCAdapter(Base):
def
invalidate
(
self
,
transaction_id
,
oids
):
def
invalidate
(
self
,
transaction_id
,
oids
):
with
self
.
_lock
:
with
self
.
_lock
:
for
instance
in
self
.
_instances
:
for
instance
in
self
.
_instances
:
instance
.
_invalidate
(
oids
)
instance
.
_invalidate
(
transaction_id
,
oids
)
def
_invalidate_finish
(
self
,
oids
,
committing_instance
):
def
_invalidate_finish
(
self
,
tid
,
oids
,
committing_instance
):
with
self
.
_lock
:
with
self
.
_lock
:
for
instance
in
self
.
_instances
:
for
instance
in
self
.
_instances
:
if
instance
is
not
committing_instance
:
if
instance
is
not
committing_instance
:
instance
.
_invalidate
(
oids
)
instance
.
_invalidate
(
tid
,
oids
)
references
=
serialize
.
referencesf
references
=
serialize
.
referencesf
transform_record_data
=
untransform_record_data
=
lambda
self
,
data
:
data
transform_record_data
=
untransform_record_data
=
lambda
self
,
data
:
data
...
@@ -98,14 +99,26 @@ class MVCCAdapterInstance(Base):
...
@@ -98,14 +99,26 @@ class MVCCAdapterInstance(Base):
'checkCurrentSerialInTransaction'
,
'tpc_abort'
,
'checkCurrentSerialInTransaction'
,
'tpc_abort'
,
)
)
_start
=
None
# Transaction start time
_ltid
=
None
# Last storage transaction id
def
__init__
(
self
,
base
):
def
__init__
(
self
,
base
):
self
.
_base
=
base
self
.
_base
=
base
Base
.
__init__
(
self
,
base
.
_storage
)
Base
.
__init__
(
self
,
base
.
_storage
)
self
.
_lock
=
Lock
()
self
.
_lock
=
Lock
()
self
.
_invalidations
=
set
()
self
.
_invalidations
=
set
()
self
.
_start
=
None
# Transaction start time
self
.
_sync
=
getattr
(
self
.
_storage
,
'sync'
,
lambda
:
None
)
self
.
_sync
=
getattr
(
self
.
_storage
,
'sync'
,
lambda
:
None
)
def
_lastTransaction
(
self
):
ltid
=
self
.
_storage
.
lastTransaction
()
# At this precise moment, a transaction may be
# committed and we have already received the new tid.
with
self
.
_lock
:
# So make sure we won't override with a smaller value.
if
self
.
_ltid
is
None
:
# Calling lastTransaction() here could result in a deadlock.
self
.
_ltid
=
ltid
def
release
(
self
):
def
release
(
self
):
self
.
_base
.
_release
(
self
)
self
.
_base
.
_release
(
self
)
...
@@ -115,8 +128,9 @@ class MVCCAdapterInstance(Base):
...
@@ -115,8 +128,9 @@ class MVCCAdapterInstance(Base):
with
self
.
_lock
:
with
self
.
_lock
:
self
.
_invalidations
=
None
self
.
_invalidations
=
None
def
_invalidate
(
self
,
oids
):
def
_invalidate
(
self
,
tid
,
oids
):
with
self
.
_lock
:
with
self
.
_lock
:
self
.
_ltid
=
tid
try
:
try
:
self
.
_invalidations
.
update
(
oids
)
self
.
_invalidations
.
update
(
oids
)
except
AttributeError
:
except
AttributeError
:
...
@@ -128,8 +142,8 @@ class MVCCAdapterInstance(Base):
...
@@ -128,8 +142,8 @@ class MVCCAdapterInstance(Base):
self
.
_sync
()
self
.
_sync
()
def
poll_invalidations
(
self
):
def
poll_invalidations
(
self
):
self
.
_start
=
p64
(
u64
(
self
.
_storage
.
lastTransaction
())
+
1
)
with
self
.
_lock
:
with
self
.
_lock
:
self
.
_start
=
p64
(
u64
(
self
.
_ltid
)
+
1
)
if
self
.
_invalidations
is
None
:
if
self
.
_invalidations
is
None
:
self
.
_invalidations
=
set
()
self
.
_invalidations
=
set
()
return
None
return
None
...
@@ -175,7 +189,8 @@ class MVCCAdapterInstance(Base):
...
@@ -175,7 +189,8 @@ class MVCCAdapterInstance(Base):
self
.
_modified
=
None
self
.
_modified
=
None
def
invalidate_finish
(
tid
):
def
invalidate_finish
(
tid
):
self
.
_base
.
_invalidate_finish
(
modified
,
self
)
self
.
_base
.
_invalidate_finish
(
tid
,
modified
,
self
)
self
.
_ltid
=
tid
func
(
tid
)
func
(
tid
)
return
self
.
_storage
.
tpc_finish
(
transaction
,
invalidate_finish
)
return
self
.
_storage
.
tpc_finish
(
transaction
,
invalidate_finish
)
...
@@ -260,7 +275,7 @@ class UndoAdapterInstance(Base):
...
@@ -260,7 +275,7 @@ class UndoAdapterInstance(Base):
def
tpc_finish
(
self
,
transaction
,
func
=
lambda
tid
:
None
):
def
tpc_finish
(
self
,
transaction
,
func
=
lambda
tid
:
None
):
def
invalidate_finish
(
tid
):
def
invalidate_finish
(
tid
):
self
.
_base
.
_invalidate_finish
(
self
.
_undone
,
None
)
self
.
_base
.
_invalidate_finish
(
tid
,
self
.
_undone
,
None
)
func
(
tid
)
func
(
tid
)
self
.
_storage
.
tpc_finish
(
transaction
,
invalidate_finish
)
self
.
_storage
.
tpc_finish
(
transaction
,
invalidate_finish
)
src/ZODB/tests/testConnection.py
View file @
b5895a5c
...
@@ -14,6 +14,7 @@
...
@@ -14,6 +14,7 @@
"""Unit tests for the Connection class."""
"""Unit tests for the Connection class."""
from
__future__
import
print_function
from
__future__
import
print_function
from
contextlib
import
contextmanager
import
doctest
import
doctest
import
re
import
re
import
six
import
six
...
@@ -535,13 +536,13 @@ class InvalidationTests(unittest.TestCase):
...
@@ -535,13 +536,13 @@ class InvalidationTests(unittest.TestCase):
>>> mvcc_storage.invalidate(p64(1), {p1._p_oid: 1})
>>> mvcc_storage.invalidate(p64(1), {p1._p_oid: 1})
Transaction start times are based on storage's last
Transaction start times are based on storage's last
transaction,
transaction. (Previousely, they were based on the first
which is known from invalidations. (Previousely, they were
invalidation seen in a transaction.)
based on the first
invalidation seen in a transaction.)
>>> mvcc_instance.poll_invalidations() == [p1._p_oid]
>>> mvcc_instance.poll_invalidations() == [p1._p_oid]
True
True
>>> mvcc_instance._start == p64(
u64(db.storage.lastTransaction()) + 1
)
>>> mvcc_instance._start == p64(
2
)
True
True
>>> mvcc_storage.invalidate(p64(10), {p2._p_oid: 1, p64(76): 1})
>>> mvcc_storage.invalidate(p64(10), {p2._p_oid: 1, p64(76): 1})
...
@@ -570,6 +571,36 @@ class InvalidationTests(unittest.TestCase):
...
@@ -570,6 +571,36 @@ class InvalidationTests(unittest.TestCase):
>>> db.close()
>>> db.close()
"""
"""
def
test_mvccadapterNewTransactionVsInvalidations
(
self
):
"""
Check that polled invalidations are consistent with the TID at which
the transaction operates. Otherwise, it's like we miss invalidations.
"""
db
=
databaseFromString
(
"<zodb>
\
n
<mappingstorage/>
\
n
</zodb>"
)
try
:
t1
=
transaction
.
TransactionManager
()
c1
=
db
.
open
(
t1
)
r1
=
c1
.
root
()
r1
[
'a'
]
=
1
t1
.
commit
()
t2
=
transaction
.
TransactionManager
()
c2
=
db
.
open
(
t2
)
c2
.
root
()[
'b'
]
=
1
s1
=
c1
.
_storage
l1
=
s1
.
_lock
@
contextmanager
def
beforeLock1
():
s1
.
_lock
=
l1
t2
.
commit
()
with
l1
:
yield
s1
.
_lock
=
beforeLock1
()
t1
.
begin
()
self
.
assertIs
(
s1
.
_lock
,
l1
)
self
.
assertIn
(
'b'
,
r1
)
finally
:
db
.
close
()
def
doctest_invalidateCache
():
def
doctest_invalidateCache
():
"""The invalidateCache method invalidates a connection's cache.
"""The invalidateCache method invalidates a connection's cache.
...
@@ -1395,4 +1426,5 @@ def test_suite():
...
@@ -1395,4 +1426,5 @@ def test_suite():
s
.
addTest
(
doctest
.
DocTestSuite
(
checker
=
checker
))
s
.
addTest
(
doctest
.
DocTestSuite
(
checker
=
checker
))
s
.
addTest
(
unittest
.
makeSuite
(
TestConnection
))
s
.
addTest
(
unittest
.
makeSuite
(
TestConnection
))
s
.
addTest
(
unittest
.
makeSuite
(
EstimatedSizeTests
))
s
.
addTest
(
unittest
.
makeSuite
(
EstimatedSizeTests
))
s
.
addTest
(
unittest
.
makeSuite
(
InvalidationTests
))
return
s
return
s
src/ZODB/tests/testmvcc.py
View file @
b5895a5c
...
@@ -85,13 +85,14 @@ storage has seen.
...
@@ -85,13 +85,14 @@ storage has seen.
>>> cn = db.open()
>>> cn = db.open()
>>> cn._storage._start == p64(u64(st.lastTransaction()) + 1)
>>> ltid = u64(st.lastTransaction())
>>> cn._storage._start == p64(ltid + 1)
True
True
>>> cn.db()._mvcc_storage.invalidate(
100
, dict.fromkeys([1, 2]))
>>> cn.db()._mvcc_storage.invalidate(
p64(ltid+100)
, dict.fromkeys([1, 2]))
>>> cn._storage._start == p64(
u64(st.lastTransaction())
+ 1)
>>> cn._storage._start == p64(
ltid
+ 1)
True
True
>>> cn.db()._mvcc_storage.invalidate(
200
, dict.fromkeys([1, 2]))
>>> cn.db()._mvcc_storage.invalidate(
p64(ltid+200)
, dict.fromkeys([1, 2]))
>>> cn._storage._start == p64(
u64(st.lastTransaction())
+ 1)
>>> cn._storage._start == p64(
ltid
+ 1)
True
True
A connection's high-water mark is set to the transaction id taken from
A connection's high-water mark is set to the transaction id taken from
...
@@ -105,7 +106,7 @@ but that doesn't work unless an object is modified. sync() will abort
...
@@ -105,7 +106,7 @@ but that doesn't work unless an object is modified. sync() will abort
a transaction and process invalidations.
a transaction and process invalidations.
>>> cn.sync()
>>> cn.sync()
>>> cn._storage._start == p64(
u64(st.lastTransaction()) +
1)
>>> cn._storage._start == p64(
ltid + 20
1)
True
True
Basic functionality
Basic functionality
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment