Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Z
ZEO
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Kirill Smelkov
ZEO
Commits
6fbe47eb
Commit
6fbe47eb
authored
Jul 07, 2016
by
Jim Fulton
Browse files
Options
Browse Files
Download
Plain Diff
Merge remote-tracking branch 'origin/asyncio' into py2
Conflicts: setup.py src/ZEO/tests/protocols.test
parents
8705602c
1eb086da
Changes
13
Hide whitespace changes
Inline
Side-by-side
Showing
13 changed files
with
375 additions
and
493 deletions
+375
-493
setup.py
setup.py
+1
-1
src/ZEO/ClientStorage.py
src/ZEO/ClientStorage.py
+21
-8
src/ZEO/StorageServer.py
src/ZEO/StorageServer.py
+39
-118
src/ZEO/TransactionBuffer.py
src/ZEO/TransactionBuffer.py
+9
-10
src/ZEO/asyncio/server.py
src/ZEO/asyncio/server.py
+2
-3
src/ZEO/asyncio/tests.py
src/ZEO/asyncio/tests.py
+3
-3
src/ZEO/cache.py
src/ZEO/cache.py
+180
-166
src/ZEO/interfaces.py
src/ZEO/interfaces.py
+1
-2
src/ZEO/tests/ConnectionTests.py
src/ZEO/tests/ConnectionTests.py
+0
-84
src/ZEO/tests/forker.py
src/ZEO/tests/forker.py
+6
-2
src/ZEO/tests/protocols.test
src/ZEO/tests/protocols.test
+89
-80
src/ZEO/tests/testZEO.py
src/ZEO/tests/testZEO.py
+7
-8
src/ZEO/tests/testZEO2.py
src/ZEO/tests/testZEO2.py
+17
-8
No files found.
setup.py
View file @
6fbe47eb
...
@@ -26,7 +26,7 @@ if (3, 0) < sys.version_info < (3, 4):
...
@@ -26,7 +26,7 @@ if (3, 0) < sys.version_info < (3, 4):
sys
.
exit
(
0
)
sys
.
exit
(
0
)
install_requires
=
[
install_requires
=
[
'ZODB >= 5.0.0a
1
'
,
'ZODB >= 5.0.0a
5
'
,
'six'
,
'six'
,
'transaction >= 1.6.0'
,
'transaction >= 1.6.0'
,
'persistent >= 4.1.0'
,
'persistent >= 4.1.0'
,
...
...
src/ZEO/ClientStorage.py
View file @
6fbe47eb
...
@@ -53,10 +53,7 @@ logger = logging.getLogger(__name__)
...
@@ -53,10 +53,7 @@ logger = logging.getLogger(__name__)
# max signed 64-bit value ~ infinity :) Signed cuz LBTree and TimeStamp
# max signed 64-bit value ~ infinity :) Signed cuz LBTree and TimeStamp
m64
=
b'
\
x7f
\
xff
\
xff
\
xff
\
xff
\
xff
\
xff
\
xff
'
m64
=
b'
\
x7f
\
xff
\
xff
\
xff
\
xff
\
xff
\
xff
\
xff
'
try
:
from
ZODB.ConflictResolution
import
ResolvedSerial
from
ZODB.ConflictResolution
import
ResolvedSerial
except
ImportError
:
ResolvedSerial
=
'rs'
def
tid2time
(
tid
):
def
tid2time
(
tid
):
return
str
(
TimeStamp
(
tid
))
return
str
(
TimeStamp
(
tid
))
...
@@ -77,6 +74,7 @@ def get_timestamp(prev_ts=None):
...
@@ -77,6 +74,7 @@ def get_timestamp(prev_ts=None):
MB
=
1024
**
2
MB
=
1024
**
2
@
zope
.
interface
.
implementer
(
ZODB
.
interfaces
.
IMultiCommitStorage
)
class
ClientStorage
(
object
):
class
ClientStorage
(
object
):
"""A storage class that is a network client to a remote storage.
"""A storage class that is a network client to a remote storage.
...
@@ -724,18 +722,31 @@ class ClientStorage(object):
...
@@ -724,18 +722,31 @@ class ClientStorage(object):
"""
"""
tbuf
=
self
.
_check_trans
(
txn
,
'tpc_vote'
)
tbuf
=
self
.
_check_trans
(
txn
,
'tpc_vote'
)
try
:
try
:
self
.
_call
(
'vote'
,
id
(
txn
))
for
oid
in
self
.
_call
(
'vote'
,
id
(
txn
))
or
():
tbuf
.
serial
(
oid
,
ResolvedSerial
)
except
POSException
.
StorageTransactionError
:
except
POSException
.
StorageTransactionError
:
# Hm, we got disconnected and reconnected bwtween
# Hm, we got disconnected and reconnected bwtween
# _check_trans and voting. Let's chack the transaction again:
# _check_trans and voting. Let's chack the transaction again:
tbuf
=
self
.
_check_trans
(
txn
,
'tpc_vote'
)
self
.
_check_trans
(
txn
,
'tpc_vote'
)
raise
except
POSException
.
ConflictError
as
err
:
oid
=
getattr
(
err
,
'oid'
,
None
)
if
oid
is
not
None
:
# This is a band-aid to help recover from a situation
# that shouldn't happen. A Client somehow misses some
# invalidations and has out of date data in its
# cache. We need some whay to invalidate the cache
# entry without invalidations. So, if we see a
# (unresolved) conflict error, we assume that the
# cache entry is bad and invalidate it.
self
.
_cache
.
invalidate
(
oid
,
None
)
raise
raise
if
tbuf
.
exception
:
if
tbuf
.
exception
:
raise
tbuf
.
exception
raise
tbuf
.
exception
if
tbuf
.
serials
:
if
tbuf
.
resolved
:
return
list
(
tbuf
.
serials
.
items
()
)
return
list
(
tbuf
.
resolved
)
else
:
else
:
return
None
return
None
...
@@ -830,6 +841,8 @@ class ClientStorage(object):
...
@@ -830,6 +841,8 @@ class ClientStorage(object):
self
.
_update_blob_cache
(
tbuf
,
tid
)
self
.
_update_blob_cache
(
tbuf
,
tid
)
return
tid
def
_update_blob_cache
(
self
,
tbuf
,
tid
):
def
_update_blob_cache
(
self
,
tbuf
,
tid
):
"""Internal helper move blobs updated by a transaction to the cache.
"""Internal helper move blobs updated by a transaction to the cache.
"""
"""
...
...
src/ZEO/StorageServer.py
View file @
6fbe47eb
...
@@ -84,7 +84,7 @@ class ZEOStorage:
...
@@ -84,7 +84,7 @@ class ZEOStorage:
blob_tempfile
=
None
blob_tempfile
=
None
log_label
=
'unconnected'
log_label
=
'unconnected'
locked
=
False
# Don't have storage lock
locked
=
False
# Don't have storage lock
verifying
=
store_failed
=
0
verifying
=
0
def
__init__
(
self
,
server
,
read_only
=
0
):
def
__init__
(
self
,
server
,
read_only
=
0
):
self
.
server
=
server
self
.
server
=
server
...
@@ -338,7 +338,6 @@ class ZEOStorage:
...
@@ -338,7 +338,6 @@ class ZEOStorage:
self
.
blob_log
=
[]
self
.
blob_log
=
[]
self
.
tid
=
tid
self
.
tid
=
tid
self
.
status
=
status
self
.
status
=
status
self
.
store_failed
=
0
self
.
stats
.
active_txns
+=
1
self
.
stats
.
active_txns
+=
1
# Assign the transaction attribute last. This is so we don't
# Assign the transaction attribute last. This is so we don't
...
@@ -426,38 +425,40 @@ class ZEOStorage:
...
@@ -426,38 +425,40 @@ class ZEOStorage:
self
.
storage
.
tpc_begin
(
self
.
transaction
)
self
.
storage
.
tpc_begin
(
self
.
transaction
)
for
op
,
args
in
self
.
txnlog
:
for
op
,
args
in
self
.
txnlog
:
if
not
getattr
(
self
,
op
)(
*
args
):
getattr
(
self
,
op
)(
*
args
)
break
# Blob support
# Blob support
while
self
.
blob_log
and
not
self
.
store_failed
:
while
self
.
blob_log
:
oid
,
oldserial
,
data
,
blobfilename
=
self
.
blob_log
.
pop
()
oid
,
oldserial
,
data
,
blobfilename
=
self
.
blob_log
.
pop
()
self
.
_store
(
oid
,
oldserial
,
data
,
blobfilename
)
self
.
_store
(
oid
,
oldserial
,
data
,
blobfilename
)
if
not
self
.
store_failed
:
serials
=
self
.
storage
.
tpc_vote
(
self
.
transaction
)
# Only call tpc_vote of no store call failed,
if
serials
:
# otherwise the serialnos() call will deliver an
if
not
isinstance
(
serials
[
0
],
bytes
):
# exception that will be handled by the client in
serials
=
(
oid
for
(
oid
,
serial
)
in
serials
# its tpc_vote() method.
if
serial
==
ResolvedSerial
)
serials
=
self
.
storage
.
tpc_vote
(
self
.
transaction
)
if
serials
:
self
.
serials
.
extend
(
serials
)
self
.
connection
.
async
(
'serialnos'
,
self
.
serials
)
self
.
serials
.
extend
(
serials
)
except
Exception
:
except
Exception
as
err
:
self
.
storage
.
tpc_abort
(
self
.
transaction
)
self
.
storage
.
tpc_abort
(
self
.
transaction
)
self
.
_clear_transaction
()
self
.
_clear_transaction
()
if
isinstance
(
err
,
ConflictError
):
self
.
stats
.
conflicts
+=
1
self
.
log
(
"conflict error %s"
%
err
,
BLATHER
)
if
not
isinstance
(
err
,
TransactionError
):
logger
.
exception
(
"While voting"
)
if
delay
is
not
None
:
if
delay
is
not
None
:
delay
.
error
(
sys
.
exc_info
())
delay
.
error
(
sys
.
exc_info
())
else
:
else
:
raise
raise
else
:
else
:
if
delay
is
not
None
:
if
delay
is
not
None
:
delay
.
reply
(
None
)
delay
.
reply
(
self
.
serials
)
else
:
else
:
return
None
return
self
.
serials
else
:
else
:
return
delay
return
delay
...
@@ -549,120 +550,45 @@ class ZEOStorage:
...
@@ -549,120 +550,45 @@ class ZEOStorage:
self
.
_check_tid
(
tid
,
exc
=
StorageTransactionError
)
self
.
_check_tid
(
tid
,
exc
=
StorageTransactionError
)
self
.
txnlog
.
undo
(
trans_id
)
self
.
txnlog
.
undo
(
trans_id
)
def
_op_error
(
self
,
oid
,
err
,
op
):
self
.
store_failed
=
1
if
isinstance
(
err
,
ConflictError
):
self
.
stats
.
conflicts
+=
1
self
.
log
(
"conflict error oid=%s msg=%s"
%
(
oid_repr
(
oid
),
str
(
err
)),
BLATHER
)
if
not
isinstance
(
err
,
TransactionError
):
# Unexpected errors are logged and passed to the client
self
.
log
(
"%s error: %s, %s"
%
((
op
,)
+
sys
.
exc_info
()[:
2
]),
logging
.
ERROR
,
exc_info
=
True
)
err
=
self
.
_marshal_error
(
err
)
# The exception is reported back as newserial for this oid
self
.
serials
.
append
((
oid
,
err
))
def
_delete
(
self
,
oid
,
serial
):
def
_delete
(
self
,
oid
,
serial
):
err
=
None
self
.
storage
.
deleteObject
(
oid
,
serial
,
self
.
transaction
)
try
:
self
.
storage
.
deleteObject
(
oid
,
serial
,
self
.
transaction
)
except
(
SystemExit
,
KeyboardInterrupt
):
raise
except
Exception
as
e
:
err
=
e
self
.
_op_error
(
oid
,
err
,
'delete'
)
return
err
is
None
def
_checkread
(
self
,
oid
,
serial
):
def
_checkread
(
self
,
oid
,
serial
):
err
=
None
self
.
storage
.
checkCurrentSerialInTransaction
(
try
:
oid
,
serial
,
self
.
transaction
)
self
.
storage
.
checkCurrentSerialInTransaction
(
oid
,
serial
,
self
.
transaction
)
except
(
SystemExit
,
KeyboardInterrupt
):
raise
except
Exception
as
e
:
err
=
e
self
.
_op_error
(
oid
,
err
,
'checkCurrentSerialInTransaction'
)
return
err
is
None
def
_store
(
self
,
oid
,
serial
,
data
,
blobfile
=
None
):
def
_store
(
self
,
oid
,
serial
,
data
,
blobfile
=
None
):
err
=
None
if
blobfile
is
None
:
try
:
newserial
=
self
.
storage
.
store
(
if
blobfile
is
None
:
oid
,
serial
,
data
,
''
,
self
.
transaction
)
newserial
=
self
.
storage
.
store
(
oid
,
serial
,
data
,
''
,
self
.
transaction
)
else
:
newserial
=
self
.
storage
.
storeBlob
(
oid
,
serial
,
data
,
blobfile
,
''
,
self
.
transaction
)
except
(
SystemExit
,
KeyboardInterrupt
):
raise
except
Exception
as
error
:
self
.
_op_error
(
oid
,
error
,
'store'
)
err
=
error
else
:
else
:
if
serial
!=
b"
\
0
\
0
\
0
\
0
\
0
\
0
\
0
\
0
"
:
newserial
=
self
.
storage
.
storeBlob
(
self
.
invalidated
.
append
(
oid
)
oid
,
serial
,
data
,
blobfile
,
''
,
self
.
transaction
)
if
serial
!=
b"
\
0
\
0
\
0
\
0
\
0
\
0
\
0
\
0
"
:
self
.
invalidated
.
append
(
oid
)
if
newserial
:
if
isinstance
(
newserial
,
bytes
):
if
isinstance
(
newserial
,
bytes
):
newserial
=
[(
oid
,
newserial
)]
newserial
=
[(
oid
,
newserial
)]
for
oid
,
s
in
newserial
or
()
:
for
oid
,
s
in
newserial
:
if
s
==
ResolvedSerial
:
if
s
==
ResolvedSerial
:
self
.
stats
.
conflicts_resolved
+=
1
self
.
stats
.
conflicts_resolved
+=
1
self
.
log
(
"conflict resolved oid=%s"
self
.
log
(
"conflict resolved oid=%s"
%
oid_repr
(
oid
),
BLATHER
)
%
oid_repr
(
oid
),
BLATHER
)
self
.
serials
.
append
(
oid
)
self
.
serials
.
append
((
oid
,
s
))
return
err
is
None
def
_restore
(
self
,
oid
,
serial
,
data
,
prev_txn
):
def
_restore
(
self
,
oid
,
serial
,
data
,
prev_txn
):
err
=
None
self
.
storage
.
restore
(
oid
,
serial
,
data
,
''
,
prev_txn
,
try
:
self
.
transaction
)
self
.
storage
.
restore
(
oid
,
serial
,
data
,
''
,
prev_txn
,
self
.
transaction
)
except
(
SystemExit
,
KeyboardInterrupt
):
raise
except
Exception
as
err
:
self
.
_op_error
(
oid
,
err
,
'restore'
)
return
err
is
None
def
_undo
(
self
,
trans_id
):
def
_undo
(
self
,
trans_id
):
err
=
None
tid
,
oids
=
self
.
storage
.
undo
(
trans_id
,
self
.
transaction
)
try
:
self
.
invalidated
.
extend
(
oids
)
tid
,
oids
=
self
.
storage
.
undo
(
trans_id
,
self
.
transaction
)
self
.
serials
.
extend
(
oids
)
except
(
SystemExit
,
KeyboardInterrupt
):
raise
except
Exception
as
e
:
err
=
e
self
.
_op_error
(
z64
,
err
,
'undo'
)
else
:
self
.
invalidated
.
extend
(
oids
)
self
.
serials
.
extend
((
oid
,
ResolvedSerial
)
for
oid
in
oids
)
return
err
is
None
def
_marshal_error
(
self
,
error
):
# Try to pickle the exception. If it can't be pickled,
# the RPC response would fail, so use something that can be pickled.
if
PY3
:
pickler
=
Pickler
(
BytesIO
(),
3
)
else
:
# The pure-python version requires at least one argument (PyPy)
pickler
=
Pickler
(
0
)
pickler
.
fast
=
1
try
:
pickler
.
dump
(
error
)
except
:
msg
=
"Couldn't pickle storage exception: %s"
%
repr
(
error
)
self
.
log
(
msg
,
logging
.
ERROR
)
error
=
StorageServerError
(
msg
)
return
error
# IStorageIteration support
# IStorageIteration support
...
@@ -1381,8 +1307,3 @@ class Serving(ServerEvent):
...
@@ -1381,8 +1307,3 @@ class Serving(ServerEvent):
class
Closed
(
ServerEvent
):
class
Closed
(
ServerEvent
):
pass
pass
default_cert_authenticate
=
'SIGNED'
def
ssl_config
(
section
):
from
.sslconfig
import
ssl_config
return
ssl_config
(
section
,
True
)
src/ZEO/TransactionBuffer.py
View file @
6fbe47eb
...
@@ -46,7 +46,7 @@ class TransactionBuffer:
...
@@ -46,7 +46,7 @@ class TransactionBuffer:
# stored are builtin types -- strings or None.
# stored are builtin types -- strings or None.
self
.
pickler
=
Pickler
(
self
.
file
,
1
)
self
.
pickler
=
Pickler
(
self
.
file
,
1
)
self
.
pickler
.
fast
=
1
self
.
pickler
.
fast
=
1
self
.
serials
=
{}
# processed { oid -> serial
}
self
.
resolved
=
set
()
# {oid
}
self
.
exception
=
None
self
.
exception
=
None
def
close
(
self
):
def
close
(
self
):
...
@@ -61,10 +61,9 @@ class TransactionBuffer:
...
@@ -61,10 +61,9 @@ class TransactionBuffer:
def
serial
(
self
,
oid
,
serial
):
def
serial
(
self
,
oid
,
serial
):
if
isinstance
(
serial
,
Exception
):
if
isinstance
(
serial
,
Exception
):
self
.
exception
=
serial
self
.
exception
=
serial
# This transaction will never be committed
self
.
serials
[
oid
]
=
None
elif
serial
==
ResolvedSerial
:
else
:
self
.
resolved
.
add
(
oid
)
self
.
serials
[
oid
]
=
serial
def
storeBlob
(
self
,
oid
,
blobfilename
):
def
storeBlob
(
self
,
oid
,
blobfilename
):
self
.
blobs
.
append
((
oid
,
blobfilename
))
self
.
blobs
.
append
((
oid
,
blobfilename
))
...
@@ -72,7 +71,7 @@ class TransactionBuffer:
...
@@ -72,7 +71,7 @@ class TransactionBuffer:
def
__iter__
(
self
):
def
__iter__
(
self
):
self
.
file
.
seek
(
0
)
self
.
file
.
seek
(
0
)
unpickler
=
Unpickler
(
self
.
file
)
unpickler
=
Unpickler
(
self
.
file
)
serials
=
self
.
serials
resolved
=
self
.
resolved
# Gaaaa, this is awkward. There can be entries in serials that
# Gaaaa, this is awkward. There can be entries in serials that
# aren't in the buffer, because undo. Entries can be repeated
# aren't in the buffer, because undo. Entries can be repeated
...
@@ -83,9 +82,9 @@ class TransactionBuffer:
...
@@ -83,9 +82,9 @@ class TransactionBuffer:
for
i
in
range
(
self
.
count
):
for
i
in
range
(
self
.
count
):
oid
,
data
=
unpickler
.
load
()
oid
,
data
=
unpickler
.
load
()
seen
.
add
(
oid
)
seen
.
add
(
oid
)
yield
oid
,
data
,
serials
.
get
(
oid
)
==
ResolvedSerial
yield
oid
,
data
,
oid
in
resolved
# We may have leftover
serial
s because undo
# We may have leftover
oid
s because undo
for
oid
,
serial
in
serials
.
items
()
:
for
oid
in
resolved
:
if
oid
not
in
seen
:
if
oid
not
in
seen
:
yield
oid
,
None
,
serial
==
ResolvedSerial
yield
oid
,
None
,
True
src/ZEO/asyncio/server.py
View file @
6fbe47eb
...
@@ -23,7 +23,7 @@ class ServerProtocol(base.Protocol):
...
@@ -23,7 +23,7 @@ class ServerProtocol(base.Protocol):
"""asyncio low-level ZEO server interface
"""asyncio low-level ZEO server interface
"""
"""
protocols
=
b'Z4'
,
b'Z5'
protocols
=
(
b'Z5'
,
)
name
=
'server protocol'
name
=
'server protocol'
methods
=
set
((
'register'
,
))
methods
=
set
((
'register'
,
))
...
@@ -169,7 +169,7 @@ class Delay(object):
...
@@ -169,7 +169,7 @@ class Delay(object):
def
error
(
self
,
exc_info
):
def
error
(
self
,
exc_info
):
self
.
sent
=
'error'
self
.
sent
=
'error'
log
(
"Error raised in delayed method"
,
logging
.
ERROR
,
exc_info
=
exc_info
)
log
ger
.
error
(
"Error raised in delayed method"
,
exc_info
=
exc_info
)
self
.
protocol
.
send_error
(
self
.
msgid
,
exc_info
[
1
])
self
.
protocol
.
send_error
(
self
.
msgid
,
exc_info
[
1
])
def
__repr__
(
self
):
def
__repr__
(
self
):
...
@@ -206,7 +206,6 @@ class MTDelay(Delay):
...
@@ -206,7 +206,6 @@ class MTDelay(Delay):
def
error
(
self
,
exc_info
):
def
error
(
self
,
exc_info
):
self
.
ready
.
wait
()
self
.
ready
.
wait
()
log
(
"Error raised in delayed method"
,
logging
.
ERROR
,
exc_info
=
exc_info
)
self
.
protocol
.
call_soon_threadsafe
(
Delay
.
error
,
self
,
exc_info
)
self
.
protocol
.
call_soon_threadsafe
(
Delay
.
error
,
self
,
exc_info
)
...
...
src/ZEO/asyncio/tests.py
View file @
6fbe47eb
...
@@ -757,7 +757,7 @@ class ServerTests(Base, setupstack.TestCase):
...
@@ -757,7 +757,7 @@ class ServerTests(Base, setupstack.TestCase):
self
.
target
=
protocol
.
zeo_storage
self
.
target
=
protocol
.
zeo_storage
if
finish
:
if
finish
:
self
.
assertEqual
(
self
.
pop
(
parse
=
False
),
best_protocol_version
)
self
.
assertEqual
(
self
.
pop
(
parse
=
False
),
best_protocol_version
)
protocol
.
data_received
(
sized
(
b'Z
4
'
))
protocol
.
data_received
(
sized
(
b'Z
5
'
))
return
protocol
return
protocol
message_id
=
0
message_id
=
0
...
@@ -795,9 +795,9 @@ class ServerTests(Base, setupstack.TestCase):
...
@@ -795,9 +795,9 @@ class ServerTests(Base, setupstack.TestCase):
self
.
assertEqual
(
self
.
pop
(
parse
=
False
),
best_protocol_version
)
self
.
assertEqual
(
self
.
pop
(
parse
=
False
),
best_protocol_version
)
# The client sends it's protocol:
# The client sends it's protocol:
protocol
.
data_received
(
sized
(
b'Z
4
'
))
protocol
.
data_received
(
sized
(
b'Z
5
'
))
self
.
assertEqual
(
protocol
.
protocol_version
,
b'Z
4
'
)
self
.
assertEqual
(
protocol
.
protocol_version
,
b'Z
5
'
)
protocol
.
zeo_storage
.
notify_connected
.
assert_called_once_with
(
protocol
)
protocol
.
zeo_storage
.
notify_connected
.
assert_called_once_with
(
protocol
)
...
...
src/ZEO/cache.py
View file @
6fbe47eb
...
@@ -33,7 +33,7 @@ import time
...
@@ -33,7 +33,7 @@ import time
import
ZODB.fsIndex
import
ZODB.fsIndex
import
zc.lockfile
import
zc.lockfile
from
ZODB.utils
import
p64
,
u64
,
z64
from
ZODB.utils
import
p64
,
u64
,
z64
,
RLock
import
six
import
six
from
._compat
import
PYPY
from
._compat
import
PYPY
...
@@ -182,6 +182,8 @@ class ClientCache(object):
...
@@ -182,6 +182,8 @@ class ClientCache(object):
# currentofs.
# currentofs.
self
.
currentofs
=
ZEC_HEADER_SIZE
self
.
currentofs
=
ZEC_HEADER_SIZE
self
.
_lock
=
RLock
()
# self.f is the open file object.
# self.f is the open file object.
# When we're not reusing an existing file, self.f is left None
# When we're not reusing an existing file, self.f is left None
# here -- the scan() method must be called then to open the file
# here -- the scan() method must be called then to open the file
...
@@ -239,9 +241,10 @@ class ClientCache(object):
...
@@ -239,9 +241,10 @@ class ClientCache(object):
return
self
return
self
def
clear
(
self
):
def
clear
(
self
):
self
.
f
.
seek
(
ZEC_HEADER_SIZE
)
with
self
.
_lock
:
self
.
f
.
truncate
()
self
.
f
.
seek
(
ZEC_HEADER_SIZE
)
self
.
_initfile
(
ZEC_HEADER_SIZE
)
self
.
f
.
truncate
()
self
.
_initfile
(
ZEC_HEADER_SIZE
)
##
##
# Scan the current contents of the cache file, calling `install`
# Scan the current contents of the cache file, calling `install`
...
@@ -451,26 +454,28 @@ class ClientCache(object):
...
@@ -451,26 +454,28 @@ class ClientCache(object):
# new tid must be strictly greater than our current idea of the most
# new tid must be strictly greater than our current idea of the most
# recent tid.
# recent tid.
def
setLastTid
(
self
,
tid
):
def
setLastTid
(
self
,
tid
):
if
(
not
tid
)
or
(
tid
==
z64
):
with
self
.
_lock
:
return
if
(
not
tid
)
or
(
tid
==
z64
):
if
(
tid
<=
self
.
tid
)
and
self
.
_len
:
return
if
tid
==
self
.
tid
:
if
(
tid
<=
self
.
tid
)
and
self
.
_len
:
return
# Be a little forgiving
if
tid
==
self
.
tid
:
raise
ValueError
(
"new last tid (%s) must be greater than "
return
# Be a little forgiving
"previous one (%s)"
raise
ValueError
(
"new last tid (%s) must be greater than "
%
(
u64
(
tid
),
u64
(
self
.
tid
)))
"previous one (%s)"
assert
isinstance
(
tid
,
bytes
)
and
len
(
tid
)
==
8
,
tid
%
(
u64
(
tid
),
u64
(
self
.
tid
)))
self
.
tid
=
tid
assert
isinstance
(
tid
,
bytes
)
and
len
(
tid
)
==
8
,
tid
self
.
f
.
seek
(
len
(
magic
))
self
.
tid
=
tid
self
.
f
.
write
(
tid
)
self
.
f
.
seek
(
len
(
magic
))
self
.
f
.
flush
()
self
.
f
.
write
(
tid
)
self
.
f
.
flush
()
##
##
# Return the last transaction seen by the cache.
# Return the last transaction seen by the cache.
# @return a transaction id
# @return a transaction id
# @defreturn string, or 8 nulls if no transaction is yet known
# @defreturn string, or 8 nulls if no transaction is yet known
def
getLastTid
(
self
):
def
getLastTid
(
self
):
return
self
.
tid
with
self
.
_lock
:
return
self
.
tid
##
##
# Return the current data record for oid.
# Return the current data record for oid.
...
@@ -479,52 +484,54 @@ class ClientCache(object):
...
@@ -479,52 +484,54 @@ class ClientCache(object):
# in the cache
# in the cache
# @defreturn 3-tuple: (string, string, string)
# @defreturn 3-tuple: (string, string, string)
def
load
(
self
,
oid
,
before_tid
=
None
):
def
load
(
self
,
oid
,
before_tid
=
None
):
ofs
=
self
.
current
.
get
(
oid
)
with
self
.
_lock
:
if
ofs
is
None
:
ofs
=
self
.
current
.
get
(
oid
)
self
.
_trace
(
0x20
,
oid
)
if
ofs
is
None
:
return
None
self
.
_trace
(
0x20
,
oid
)
self
.
f
.
seek
(
ofs
)
return
None
read
=
self
.
f
.
read
status
=
read
(
1
)
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
size
,
saved_oid
,
tid
,
end_tid
,
lver
,
ldata
=
unpack
(
">I8s8s8sHI"
,
read
(
34
))
assert
saved_oid
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
assert
end_tid
==
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
tid
,
end_tid
)
assert
lver
==
0
,
"Versions aren't supported"
if
before_tid
and
tid
>=
before_tid
:
return
None
data
=
read
(
ldata
)
assert
len
(
data
)
==
ldata
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
len
(
data
),
ldata
)
# WARNING: The following assert changes the file position.
# We must not depend on this below or we'll fail in optimized mode.
assert
read
(
8
)
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
self
.
_n_accesses
+=
1
self
.
_trace
(
0x22
,
oid
,
tid
,
end_tid
,
ldata
)
ofsofs
=
self
.
currentofs
-
ofs
if
ofsofs
<
0
:
ofsofs
+=
self
.
maxsize
if
(
ofsofs
>
self
.
rearrange
and
self
.
maxsize
>
10
*
len
(
data
)
and
size
>
4
):
# The record is far back and might get evicted, but it's
# valuable, so move it forward.
# Remove fromn old loc:
del
self
.
current
[
oid
]
self
.
f
.
seek
(
ofs
)
self
.
f
.
seek
(
ofs
)
self
.
f
.
write
(
b'f'
+
pack
(
">I"
,
size
))
read
=
self
.
f
.
read
status
=
read
(
1
)
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
size
,
saved_oid
,
tid
,
end_tid
,
lver
,
ldata
=
unpack
(
">I8s8s8sHI"
,
read
(
34
))
assert
saved_oid
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
assert
end_tid
==
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
tid
,
end_tid
)
assert
lver
==
0
,
"Versions aren't supported"
if
before_tid
and
tid
>=
before_tid
:
return
None
data
=
read
(
ldata
)
assert
len
(
data
)
==
ldata
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
len
(
data
),
ldata
)
# WARNING: The following assert changes the file position.
# We must not depend on this below or we'll fail in optimized mode.
assert
read
(
8
)
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
self
.
_n_accesses
+=
1
self
.
_trace
(
0x22
,
oid
,
tid
,
end_tid
,
ldata
)
# Write to new location:
ofsofs
=
self
.
currentofs
-
ofs
self
.
_store
(
oid
,
tid
,
None
,
data
,
size
)
if
ofsofs
<
0
:
ofsofs
+=
self
.
maxsize
return
data
,
tid
if
(
ofsofs
>
self
.
rearrange
and
self
.
maxsize
>
10
*
len
(
data
)
and
size
>
4
):
# The record is far back and might get evicted, but it's
# valuable, so move it forward.
# Remove fromn old loc:
del
self
.
current
[
oid
]
self
.
f
.
seek
(
ofs
)
self
.
f
.
write
(
b'f'
+
pack
(
">I"
,
size
))
# Write to new location:
self
.
_store
(
oid
,
tid
,
None
,
data
,
size
)
return
data
,
tid
##
##
# Return a non-current revision of oid that was current before tid.
# Return a non-current revision of oid that was current before tid.
...
@@ -533,54 +540,56 @@ class ClientCache(object):
...
@@ -533,54 +540,56 @@ class ClientCache(object):
# @return data record, serial number, start tid, and end tid
# @return data record, serial number, start tid, and end tid
# @defreturn 4-tuple: (string, string, string, string)
# @defreturn 4-tuple: (string, string, string, string)
def
loadBefore
(
self
,
oid
,
before_tid
):
def
loadBefore
(
self
,
oid
,
before_tid
):
noncurrent_for_oid
=
self
.
noncurrent
.
get
(
u64
(
oid
))
with
self
.
_lock
:
if
noncurrent_for_oid
is
None
:
noncurrent_for_oid
=
self
.
noncurrent
.
get
(
u64
(
oid
))
result
=
self
.
load
(
oid
,
before_tid
)
if
noncurrent_for_oid
is
None
:
if
result
:
result
=
self
.
load
(
oid
,
before_tid
)
return
result
[
0
],
result
[
1
],
None
if
result
:
else
:
return
result
[
0
],
result
[
1
],
None
self
.
_trace
(
0x24
,
oid
,
""
,
before_tid
)
else
:
return
result
self
.
_trace
(
0x24
,
oid
,
""
,
before_tid
)
return
result
items
=
noncurrent_for_oid
.
items
(
None
,
u64
(
before_tid
)
-
1
)
if
not
items
:
items
=
noncurrent_for_oid
.
items
(
None
,
u64
(
before_tid
)
-
1
)
result
=
self
.
load
(
oid
,
before_tid
)
if
not
items
:
if
result
:
result
=
self
.
load
(
oid
,
before_tid
)
return
result
[
0
],
result
[
1
],
None
if
result
:
else
:
return
result
[
0
],
result
[
1
],
None
self
.
_trace
(
0x24
,
oid
,
""
,
before_tid
)
else
:
return
result
self
.
_trace
(
0x24
,
oid
,
""
,
before_tid
)
return
result
tid
,
ofs
=
items
[
-
1
]
tid
,
ofs
=
items
[
-
1
]
self
.
f
.
seek
(
ofs
)
self
.
f
.
seek
(
ofs
)
read
=
self
.
f
.
read
read
=
self
.
f
.
read
status
=
read
(
1
)
status
=
read
(
1
)
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
before_tid
)
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
before_tid
)
size
,
saved_oid
,
saved_tid
,
end_tid
,
lver
,
ldata
=
unpack
(
size
,
saved_oid
,
saved_tid
,
end_tid
,
lver
,
ldata
=
unpack
(
">I8s8s8sHI"
,
read
(
34
))
">I8s8s8sHI"
,
read
(
34
))
assert
saved_oid
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
assert
saved_oid
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
assert
saved_tid
==
p64
(
tid
),
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_tid
,
tid
)
assert
saved_tid
==
p64
(
tid
),
(
assert
end_tid
!=
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
ofs
,
self
.
f
.
tell
(),
oid
,
saved_tid
,
tid
)
assert
lver
==
0
,
"Versions aren't supported"
assert
end_tid
!=
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
data
=
read
(
ldata
)
assert
lver
==
0
,
"Versions aren't supported"
assert
len
(
data
)
==
ldata
,
(
ofs
,
self
.
f
.
tell
())
data
=
read
(
ldata
)
assert
len
(
data
)
==
ldata
,
(
ofs
,
self
.
f
.
tell
())
# WARNING: The following assert changes the file position.
# We must not depend on this below or we'll fail in optimized mode.
# WARNING: The following assert changes the file position.
assert
read
(
8
)
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
# We must not depend on this below or we'll fail in optimized mode.
assert
read
(
8
)
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
if
end_tid
<
before_tid
:
result
=
self
.
load
(
oid
,
before_tid
)
if
end_tid
<
before_tid
:
if
result
:
result
=
self
.
load
(
oid
,
before_tid
)
return
result
[
0
],
result
[
1
],
None
if
result
:
else
:
return
result
[
0
],
result
[
1
],
None
self
.
_trace
(
0x24
,
oid
,
""
,
before_tid
)
else
:
return
result
self
.
_trace
(
0x24
,
oid
,
""
,
before_tid
)
return
result
self
.
_n_accesses
+=
1
self
.
_n_accesses
+=
1
self
.
_trace
(
0x26
,
oid
,
""
,
saved_tid
)
self
.
_trace
(
0x26
,
oid
,
""
,
saved_tid
)
return
data
,
saved_tid
,
end_tid
return
data
,
saved_tid
,
end_tid
##
##
# Store a new data record in the cache.
# Store a new data record in the cache.
...
@@ -591,45 +600,48 @@ class ClientCache(object):
...
@@ -591,45 +600,48 @@ class ClientCache(object):
# current.
# current.
# @param data the actual data
# @param data the actual data
def
store
(
self
,
oid
,
start_tid
,
end_tid
,
data
):
def
store
(
self
,
oid
,
start_tid
,
end_tid
,
data
):
seek
=
self
.
f
.
seek
with
self
.
_lock
:
if
end_tid
is
None
:
seek
=
self
.
f
.
seek
ofs
=
self
.
current
.
get
(
oid
)
if
end_tid
is
None
:
if
ofs
:
ofs
=
self
.
current
.
get
(
oid
)
seek
(
ofs
)
if
ofs
:
read
=
self
.
f
.
read
seek
(
ofs
)
status
=
read
(
1
)
read
=
self
.
f
.
read
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
status
=
read
(
1
)
size
,
saved_oid
,
saved_tid
,
end_tid
=
unpack
(
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
">I8s8s8s"
,
read
(
28
))
size
,
saved_oid
,
saved_tid
,
end_tid
=
unpack
(
assert
saved_oid
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
">I8s8s8s"
,
read
(
28
))
assert
end_tid
==
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
assert
saved_oid
==
oid
,
(
if
saved_tid
==
start_tid
:
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
assert
end_tid
==
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
if
saved_tid
==
start_tid
:
return
raise
ValueError
(
"already have current data for oid"
)
else
:
noncurrent_for_oid
=
self
.
noncurrent
.
get
(
u64
(
oid
))
if
noncurrent_for_oid
and
(
u64
(
start_tid
)
in
noncurrent_for_oid
):
return
return
raise
ValueError
(
"already have current data for oid"
)
else
:
noncurrent_for_oid
=
self
.
noncurrent
.
get
(
u64
(
oid
))
if
noncurrent_for_oid
and
(
u64
(
start_tid
)
in
noncurrent_for_oid
):
return
size
=
allocated_record_overhead
+
len
(
data
)
size
=
allocated_record_overhead
+
len
(
data
)
# A number of cache simulation experiments all concluded that the
# A number of cache simulation experiments all concluded that the
# 2nd-level ZEO cache got a much higher hit rate if "very large"
# 2nd-level ZEO cache got a much higher hit rate if "very large"
# objects simply weren't cached. For now, we ignore the request
# objects simply weren't cached. For now, we ignore the request
# only if the entire cache file is too small to hold the object.
# only if the entire cache file is too small to hold the object.
if
size
>=
min
(
max_block_size
,
self
.
maxsize
-
ZEC_HEADER_SIZE
):
if
size
>=
min
(
max_block_size
,
self
.
maxsize
-
ZEC_HEADER_SIZE
):
return
return
self
.
_n_adds
+=
1
self
.
_n_adds
+=
1
self
.
_n_added_bytes
+=
size
self
.
_n_added_bytes
+=
size
self
.
_len
+=
1
self
.
_len
+=
1
self
.
_store
(
oid
,
start_tid
,
end_tid
,
data
,
size
)
self
.
_store
(
oid
,
start_tid
,
end_tid
,
data
,
size
)
if
end_tid
:
if
end_tid
:
self
.
_trace
(
0x54
,
oid
,
start_tid
,
end_tid
,
dlen
=
len
(
data
))
self
.
_trace
(
0x54
,
oid
,
start_tid
,
end_tid
,
dlen
=
len
(
data
))
else
:
else
:
self
.
_trace
(
0x52
,
oid
,
start_tid
,
dlen
=
len
(
data
))
self
.
_trace
(
0x52
,
oid
,
start_tid
,
dlen
=
len
(
data
))
def
_store
(
self
,
oid
,
start_tid
,
end_tid
,
data
,
size
):
def
_store
(
self
,
oid
,
start_tid
,
end_tid
,
data
,
size
):
# Low-level store used by store and load
# Low-level store used by store and load
...
@@ -696,35 +708,37 @@ class ClientCache(object):
...
@@ -696,35 +708,37 @@ class ClientCache(object):
# - tid the id of the transaction that wrote a new revision of oid,
# - tid the id of the transaction that wrote a new revision of oid,
# or None to forget all cached info about oid.
# or None to forget all cached info about oid.
def
invalidate
(
self
,
oid
,
tid
):
def
invalidate
(
self
,
oid
,
tid
):
ofs
=
self
.
current
.
get
(
oid
)
with
self
.
_lock
:
if
ofs
is
None
:
ofs
=
self
.
current
.
get
(
oid
)
# 0x10 == invalidate (miss)
if
ofs
is
None
:
self
.
_trace
(
0x10
,
oid
,
tid
)
# 0x10 == invalidate (miss)
return
self
.
_trace
(
0x10
,
oid
,
tid
)
return
self
.
f
.
seek
(
ofs
)
read
=
self
.
f
.
read
status
=
read
(
1
)
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
size
,
saved_oid
,
saved_tid
,
end_tid
=
unpack
(
">I8s8s8s"
,
read
(
28
))
assert
saved_oid
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
assert
end_tid
==
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
del
self
.
current
[
oid
]
if
tid
is
None
:
self
.
f
.
seek
(
ofs
)
self
.
f
.
seek
(
ofs
)
self
.
f
.
write
(
b'f'
+
pack
(
">I"
,
size
))
read
=
self
.
f
.
read
# 0x1E = invalidate (hit, discarding current or non-current)
status
=
read
(
1
)
self
.
_trace
(
0x1E
,
oid
,
tid
)
assert
status
==
b'a'
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
self
.
_len
-=
1
size
,
saved_oid
,
saved_tid
,
end_tid
=
unpack
(
">I8s8s8s"
,
read
(
28
))
else
:
assert
saved_oid
==
oid
,
(
ofs
,
self
.
f
.
tell
(),
oid
,
saved_oid
)
if
tid
==
saved_tid
:
assert
end_tid
==
z64
,
(
ofs
,
self
.
f
.
tell
(),
oid
)
logger
.
warning
(
"Ignoring invalidation with same tid as current"
)
del
self
.
current
[
oid
]
return
if
tid
is
None
:
self
.
f
.
seek
(
ofs
+
21
)
self
.
f
.
seek
(
ofs
)
self
.
f
.
write
(
tid
)
self
.
f
.
write
(
b'f'
+
pack
(
">I"
,
size
))
self
.
_set_noncurrent
(
oid
,
saved_tid
,
ofs
)
# 0x1E = invalidate (hit, discarding current or non-current)
# 0x1C = invalidate (hit, saving non-current)
self
.
_trace
(
0x1E
,
oid
,
tid
)
self
.
_trace
(
0x1C
,
oid
,
tid
)
self
.
_len
-=
1
else
:
if
tid
==
saved_tid
:
logger
.
warning
(
"Ignoring invalidation with same tid as current"
)
return
self
.
f
.
seek
(
ofs
+
21
)
self
.
f
.
write
(
tid
)
self
.
_set_noncurrent
(
oid
,
saved_tid
,
ofs
)
# 0x1C = invalidate (hit, saving non-current)
self
.
_trace
(
0x1C
,
oid
,
tid
)
##
##
# Generates (oid, serial) oairs for all objects in the
# Generates (oid, serial) oairs for all objects in the
...
...
src/ZEO/interfaces.py
View file @
6fbe47eb
...
@@ -24,8 +24,7 @@ class StaleCache(object):
...
@@ -24,8 +24,7 @@ class StaleCache(object):
class
IClientCache
(
zope
.
interface
.
Interface
):
class
IClientCache
(
zope
.
interface
.
Interface
):
"""Client cache interface.
"""Client cache interface.
Note that caches need not be thread safe, fpr the most part,
Note that caches need to be thread safe.
except for getLastTid, which may be called from multiple threads.
"""
"""
def
close
():
def
close
():
...
...
src/ZEO/tests/ConnectionTests.py
View file @
6fbe47eb
...
@@ -1018,90 +1018,6 @@ class TimeoutTests(CommonSetupTearDown):
...
@@ -1018,90 +1018,6 @@ class TimeoutTests(CommonSetupTearDown):
# or the server.
# or the server.
self
.
assertRaises
(
KeyError
,
storage
.
load
,
oid
,
''
)
self
.
assertRaises
(
KeyError
,
storage
.
load
,
oid
,
''
)
def
checkTimeoutProvokingConflicts
(
self
):
self
.
_storage
=
storage
=
self
.
openClientStorage
()
# Assert that the zeo cache is empty.
self
.
assert_
(
not
list
(
storage
.
_cache
.
contents
()))
# Create the object
oid
=
storage
.
new_oid
()
obj
=
MinPO
(
7
)
# We need to successfully commit an object now so we have something to
# conflict about.
t
=
Transaction
()
storage
.
tpc_begin
(
t
)
revid1a
=
storage
.
store
(
oid
,
ZERO
,
zodb_pickle
(
obj
),
''
,
t
)
revid1b
=
storage
.
tpc_vote
(
t
)
revid1
=
handle_serials
(
oid
,
revid1a
,
revid1b
)
storage
.
tpc_finish
(
t
)
# Now do a store, sleeping before the finish so as to cause a timeout.
obj
.
value
=
8
t
=
Transaction
()
old_connection_count
=
storage
.
connection_count_for_tests
storage
.
tpc_begin
(
t
)
revid2a
=
storage
.
store
(
oid
,
revid1
,
zodb_pickle
(
obj
),
''
,
t
)
revid2b
=
storage
.
tpc_vote
(
t
)
revid2
=
handle_serials
(
oid
,
revid2a
,
revid2b
)
# Now sleep long enough for the storage to time out.
# This used to sleep for 3 seconds, and sometimes (but very rarely)
# failed then. Now we try for a minute. It typically succeeds
# on the second time thru the loop, and, since self.timeout is 1,
# it's typically faster now (2/1.8 ~= 1.11 seconds sleeping instead
# of 3).
deadline
=
time
.
time
()
+
60
# wait up to a minute
while
time
.
time
()
<
deadline
:
if
(
storage
.
is_connected
()
and
(
storage
.
connection_count_for_tests
==
old_connection_count
)
):
time
.
sleep
(
self
.
timeout
/
1.8
)
else
:
break
self
.
assert_
(
(
not
storage
.
is_connected
())
or
(
storage
.
connection_count_for_tests
>
old_connection_count
)
)
storage
.
_wait
()
self
.
assert_
(
storage
.
is_connected
())
# We expect finish to fail.
self
.
assertRaises
(
ClientDisconnected
,
storage
.
tpc_finish
,
t
)
storage
.
tpc_abort
(
t
)
# Now we think we've committed the second transaction, but we really
# haven't. A third one should produce a POSKeyError on the server,
# which manifests as a ConflictError on the client.
obj
.
value
=
9
t
=
Transaction
()
storage
.
tpc_begin
(
t
)
storage
.
store
(
oid
,
revid2
,
zodb_pickle
(
obj
),
''
,
t
)
self
.
assertRaises
(
ConflictError
,
storage
.
tpc_vote
,
t
)
# Even aborting won't help.
storage
.
tpc_abort
(
t
)
self
.
assertRaises
(
ZODB
.
POSException
.
StorageTransactionError
,
storage
.
tpc_finish
,
t
)
# Try again.
obj
.
value
=
10
t
=
Transaction
()
storage
.
tpc_begin
(
t
)
storage
.
store
(
oid
,
revid2
,
zodb_pickle
(
obj
),
''
,
t
)
# Even aborting won't help.
self
.
assertRaises
(
ConflictError
,
storage
.
tpc_vote
,
t
)
# Abort this one and try a transaction that should succeed.
storage
.
tpc_abort
(
t
)
# Now do a store.
obj
.
value
=
11
t
=
Transaction
()
storage
.
tpc_begin
(
t
)
revid2a
=
storage
.
store
(
oid
,
revid1
,
zodb_pickle
(
obj
),
''
,
t
)
revid2b
=
storage
.
tpc_vote
(
t
)
revid2
=
handle_serials
(
oid
,
revid2a
,
revid2b
)
storage
.
tpc_finish
(
t
)
# Now load the object and verify that it has a value of 11.
data
,
revid
=
storage
.
load
(
oid
,
''
)
self
.
assertEqual
(
zodb_unpickle
(
data
),
MinPO
(
11
))
self
.
assertEqual
(
revid
,
revid2
)
class
MSTThread
(
threading
.
Thread
):
class
MSTThread
(
threading
.
Thread
):
__super_init
=
threading
.
Thread
.
__init__
__super_init
=
threading
.
Thread
.
__init__
...
...
src/ZEO/tests/forker.py
View file @
6fbe47eb
...
@@ -95,6 +95,10 @@ def runner(config, qin, qout, timeout=None,
...
@@ -95,6 +95,10 @@ def runner(config, qin, qout, timeout=None,
import
ZEO.asyncio.server
import
ZEO.asyncio.server
old_protocol
=
ZEO
.
asyncio
.
server
.
best_protocol_version
old_protocol
=
ZEO
.
asyncio
.
server
.
best_protocol_version
ZEO
.
asyncio
.
server
.
best_protocol_version
=
protocol
ZEO
.
asyncio
.
server
.
best_protocol_version
=
protocol
old_protocols
=
ZEO
.
asyncio
.
server
.
ServerProtocol
.
protocols
ZEO
.
asyncio
.
server
.
ServerProtocol
.
protocols
=
tuple
(
sorted
(
set
(
old_protocols
)
|
set
([
protocol
])
))
try
:
try
:
import
ZEO.runzeo
,
threading
import
ZEO.runzeo
,
threading
...
@@ -142,8 +146,8 @@ def runner(config, qin, qout, timeout=None,
...
@@ -142,8 +146,8 @@ def runner(config, qin, qout, timeout=None,
finally
:
finally
:
if
old_protocol
:
if
old_protocol
:
ZEO
.
asyncio
.
server
.
best_protocol_version
=
protocol
ZEO
.
asyncio
.
server
.
best_protocol_version
=
old_
protocol
ZEO
.
asyncio
.
server
.
ServerProtocol
.
protocols
=
old_protocols
def
stop_runner
(
thread
,
config
,
qin
,
qout
,
stop_timeout
=
9
,
pid
=
None
):
def
stop_runner
(
thread
,
config
,
qin
,
qout
,
stop_timeout
=
9
,
pid
=
None
):
qin
.
put
(
'stop'
)
qin
.
put
(
'stop'
)
...
...
src/ZEO/tests/protocols.test
View file @
6fbe47eb
...
@@ -5,7 +5,7 @@ A full test of all protocols isn't practical. But we'll do a limited
...
@@ -5,7 +5,7 @@ A full test of all protocols isn't practical. But we'll do a limited
test
that
at
least
the
current
and
previous
protocols
are
supported
in
test
that
at
least
the
current
and
previous
protocols
are
supported
in
both
directions
.
both
directions
.
Let
's start a Z
309
server
Let
's start a Z
4
server
>>> storage_conf = '''
>>> storage_conf = '''
...
<
blobstorage
>
...
<
blobstorage
>
...
@@ -94,82 +94,91 @@ A current client should be able to connect to a old server:
...
@@ -94,82 +94,91 @@ A current client should be able to connect to a old server:
>>> zope.testing.setupstack.rmtree('
blobs
')
>>> zope.testing.setupstack.rmtree('
blobs
')
>>> zope.testing.setupstack.rmtree('
server
-
blobs
')
>>> zope.testing.setupstack.rmtree('
server
-
blobs
')
And the other way around:
#############################################################################
# Note that the ZEO 5.0 server only supports clients that use the Z5 protocol
>>> addr, _ = start_server(storage_conf, dict(invalidation_queue_size=5))
# And the other way around:
Note that we'
ll
have
to
pull
some
hijinks
:
# >>> addr, _ = start_server(storage_conf, dict(invalidation_queue_size=5))
>>>
import
ZEO
.
asyncio
.
client
>>>
old_protocols
=
ZEO
.
asyncio
.
client
.
Protocol
.
protocols
# Note that we'
ll
have
to
pull
some
hijinks
:
>>>
ZEO
.
asyncio
.
client
.
Protocol
.
protocols
=
[
b
'Z4'
]
# >>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>>
db
=
ZEO
.
DB
(
addr
,
client
=
'client'
,
blob_dir
=
'blobs'
)
# >>> str(db.storage.protocol_version.decode('ascii'))
>>>
str
(
db
.
storage
.
protocol_version
.
decode
(
'ascii'
))
# 'Z4'
'Z4'
# >>> wait_connected(db.storage)
>>>
wait_connected
(
db
.
storage
)
# >>> conn = db.open()
>>>
conn
=
db
.
open
()
# >>> conn.root().x = 0
>>>
conn
.
root
()
.
x
=
0
# >>> transaction.commit()
>>>
transaction
.
commit
()
# >>> len(db.history(conn.root()._p_oid, 99))
>>>
len
(
db
.
history
(
conn
.
root
()
.
_p_oid
,
99
))
# 2
2
# >>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
>>>
conn
.
root
()[
'blob1'
]
=
ZODB
.
blob
.
Blob
()
# >>> db.storage.protocol_version
>>>
with
conn
.
root
()[
'blob1'
]
.
open
(
'w'
)
as
f
:
# b'Z4'
...
r
=
f
.
write
(
b
'blob data 1'
)
# >>> wait_connected(db.storage)
>>>
transaction
.
commit
()
# >>> conn = db.open()
# >>> conn.root().x = 0
>>>
db2
=
ZEO
.
DB
(
addr
,
blob_dir
=
'server-blobs'
,
shared_blob_dir
=
True
)
# >>> transaction.commit()
>>>
wait_connected
(
db2
.
storage
)
# >>> len(db.history(conn.root()._p_oid, 99))
>>>
conn2
=
db2
.
open
()
# 2
>>>
for
i
in
range
(
5
)
:
...
conn2
.
root
()
.
x
+=
1
# >>> conn.root()['blob1'] = ZODB.blob.Blob()
...
transaction
.
commit
()
# >>> with conn.root()['blob1'].open('w') as f:
>>>
conn2
.
root
()[
'blob2'
]
=
ZODB
.
blob
.
Blob
()
# ... r = f.write(b'blob data 1')
>>>
with
conn2
.
root
()[
'blob2'
]
.
open
(
'w'
)
as
f
:
# >>> transaction.commit()
...
r
=
f
.
write
(
b
'blob data 2'
)
>>>
transaction
.
commit
()
# >>> db2 = ZEO.DB(addr, blob_dir='server-blobs', shared_blob_dir=True)
# >>> wait_connected(db2.storage)
# >>> conn2 = db2.open()
>>>
@
wait_until
()
# >>> for i in range(5):
...
def
x_to_be_5
()
:
# ... conn2.root().x += 1
...
conn
.
sync
()
# ... transaction.commit()
...
return
conn
.
root
()
.
x
==
5
# >>> conn2.root()['blob2'] = ZODB.blob.Blob()
# >>> with conn2.root()['blob2'].open('w') as f:
>>>
db
.
close
()
# ... r = f.write(b'blob data 2')
# >>> transaction.commit()
>>>
for
i
in
range
(
2
)
:
...
conn2
.
root
()
.
x
+=
1
...
transaction
.
commit
()
# >>> @wait_until()
# ... def x_to_be_5():
>>>
db
=
ZEO
.
DB
(
addr
,
client
=
'client'
,
blob_dir
=
'blobs'
)
# ... conn.sync()
>>>
wait_connected
(
db
.
storage
)
# ... return conn.root().x == 5
>>>
conn
=
db
.
open
()
>>>
conn
.
root
()
.
x
# >>> db.close()
7
# >>> for i in range(2):
>>>
db
.
close
()
# ... conn2.root().x += 1
# ... transaction.commit()
>>>
for
i
in
range
(
10
)
:
...
conn2
.
root
()
.
x
+=
1
# >>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
...
transaction
.
commit
()
# >>> wait_connected(db.storage)
# >>> conn = db.open()
>>>
db
=
ZEO
.
DB
(
addr
,
client
=
'client'
,
blob_dir
=
'blobs'
)
# >>> conn.root().x
>>>
wait_connected
(
db
.
storage
)
# 7
>>>
conn
=
db
.
open
()
>>>
conn
.
root
()
.
x
# >>> db.close()
17
# >>> for i in range(10):
>>>
with
conn
.
root
()[
'blob1'
]
.
open
()
as
f
:
# ... conn2.root().x += 1
...
f
.
read
()
# ... transaction.commit()
b
'blob data 1'
>>>
with
conn
.
root
()[
'blob2'
]
.
open
()
as
f
:
# >>> db = ZEO.DB(addr, client='client', blob_dir='blobs')
...
f
.
read
()
# >>> wait_connected(db.storage)
b
'blob data 2'
# >>> conn = db.open()
# >>> conn.root().x
>>>
db2
.
close
()
# 17
>>>
db
.
close
()
# >>> with conn.root()['blob1'].open() as f:
Undo
the
hijinks
:
# ... f.read()
# b'blob data 1'
>>>
ZEO
.
asyncio
.
client
.
Protocol
.
protocols
=
old_protocols
# >>> with conn.root()['blob2'].open() as f:
# ... f.read()
# b'blob data 2'
# >>> db2.close()
# >>> db.close()
# Undo the hijinks:
# >>> ZEO.asyncio.client.Protocol.protocols = old_protocols
src/ZEO/tests/testZEO.py
View file @
6fbe47eb
...
@@ -745,24 +745,23 @@ class StorageServerWrapper:
...
@@ -745,24 +745,23 @@ class StorageServerWrapper:
self
.
server
.
tpc_begin
(
id
(
transaction
),
''
,
''
,
{},
None
,
' '
)
self
.
server
.
tpc_begin
(
id
(
transaction
),
''
,
''
,
{},
None
,
' '
)
def
tpc_vote
(
self
,
transaction
):
def
tpc_vote
(
self
,
transaction
):
vote_result
=
self
.
server
.
vote
(
id
(
transaction
))
result
=
self
.
server
.
vote
(
id
(
transaction
))
assert
vote_result
is
None
assert
result
==
self
.
server
.
connection
.
serials
[:]
result
=
self
.
server
.
connection
.
serials
[:]
del
self
.
server
.
connection
.
serials
[:]
del
self
.
server
.
connection
.
serials
[:]
return
result
return
result
def
store
(
self
,
oid
,
serial
,
data
,
version_ignored
,
transaction
):
def
store
(
self
,
oid
,
serial
,
data
,
version_ignored
,
transaction
):
self
.
server
.
storea
(
oid
,
serial
,
data
,
id
(
transaction
))
self
.
server
.
storea
(
oid
,
serial
,
data
,
id
(
transaction
))
def
send_reply
(
self
,
*
args
):
# Masquerade as conn
def
send_reply
(
self
,
_
,
result
):
# Masquerade as conn
pass
self
.
_result
=
result
def
tpc_abort
(
self
,
transaction
):
def
tpc_abort
(
self
,
transaction
):
self
.
server
.
tpc_abort
(
id
(
transaction
))
self
.
server
.
tpc_abort
(
id
(
transaction
))
def
tpc_finish
(
self
,
transaction
,
func
=
lambda
:
None
):
def
tpc_finish
(
self
,
transaction
,
func
=
lambda
:
None
):
self
.
server
.
tpc_finish
(
id
(
transaction
)).
set_sender
(
0
,
self
)
self
.
server
.
tpc_finish
(
id
(
transaction
)).
set_sender
(
0
,
self
)
return
self
.
_result
def
multiple_storages_invalidation_queue_is_not_insane
():
def
multiple_storages_invalidation_queue_is_not_insane
():
"""
"""
...
@@ -929,14 +928,14 @@ def tpc_finish_error():
...
@@ -929,14 +928,14 @@ def tpc_finish_error():
buffer, sadly, using implementation details:
buffer, sadly, using implementation details:
>>> tbuf = t.data(client)
>>> tbuf = t.data(client)
>>> tbuf.
serials
= None
>>> tbuf.
resolved
= None
tpc_finish will fail:
tpc_finish will fail:
>>> client.tpc_finish(t) # doctest: +ELLIPSIS
>>> client.tpc_finish(t) # doctest: +ELLIPSIS
Traceback (most recent call last):
Traceback (most recent call last):
...
...
Attribut
eError: ...
Typ
eError: ...
>>> client.tpc_abort(t)
>>> client.tpc_abort(t)
>>> t.abort()
>>> t.abort()
...
...
src/ZEO/tests/testZEO2.py
View file @
6fbe47eb
...
@@ -78,6 +78,8 @@ will conflict. It will be blocked at the vote call.
...
@@ -78,6 +78,8 @@ will conflict. It will be blocked at the vote call.
>>> class Sender:
>>> class Sender:
... def send_reply(self, id, reply):
... def send_reply(self, id, reply):
... print('reply', id, reply)
... print('reply', id, reply)
... def send_error(self, id, err):
... print('error', id, err)
>>> delay.set_sender(1, Sender())
>>> delay.set_sender(1, Sender())
>>> logger = logging.getLogger('ZEO')
>>> logger = logging.getLogger('ZEO')
...
@@ -87,13 +89,20 @@ will conflict. It will be blocked at the vote call.
...
@@ -87,13 +89,20 @@ will conflict. It will be blocked at the vote call.
Now, when we abort the transaction for the first client. The second
Now, when we abort the transaction for the first client. The second
client will be restarted. It will get a conflict error, that is
client will be restarted. It will get a conflict error, that is
handled correctly
:
raised to the client
:
>>> zs1.tpc_abort('0') # doctest: +ELLIPSIS
>>> zs1.tpc_abort('0') # doctest: +ELLIPSIS
reply 1 None
Error raised in delayed method
Traceback (most recent call last):
...
ZODB.POSException.ConflictError: ...
error 1 database conflict error ...
The transaction is aborted by the server:
>>> fs.tpc_transaction() is
not
None
>>> fs.tpc_transaction() is None
True
True
>>> zs2.connected
>>> zs2.connected
True
True
...
@@ -116,7 +125,7 @@ And an initial client.
...
@@ -116,7 +125,7 @@ And an initial client.
>>> zs1 = ZEO.tests.servertesting.client(server, 1)
>>> zs1 = ZEO.tests.servertesting.client(server, 1)
>>> zs1.tpc_begin('0', '', '', {})
>>> zs1.tpc_begin('0', '', '', {})
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '0')
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64,
b
'x', '0')
Intentionally break zs1:
Intentionally break zs1:
...
@@ -135,7 +144,7 @@ We can start another client and get the storage lock.
...
@@ -135,7 +144,7 @@ We can start another client and get the storage lock.
>>> zs1 = ZEO.tests.servertesting.client(server, 1)
>>> zs1 = ZEO.tests.servertesting.client(server, 1)
>>> zs1.tpc_begin('1', '', '', {})
>>> zs1.tpc_begin('1', '', '', {})
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64, 'x', '1')
>>> zs1.storea(ZODB.utils.p64(99), ZODB.utils.z64,
b
'x', '1')
>>> _ = zs1.vote('1') # doctest: +ELLIPSIS
>>> _ = zs1.vote('1') # doctest: +ELLIPSIS
>>> zs1.tpc_finish('1').set_sender(0, zs1.connection)
>>> zs1.tpc_finish('1').set_sender(0, zs1.connection)
...
@@ -220,7 +229,7 @@ We start a transaction and vote, this leads to getting the lock.
...
@@ -220,7 +229,7 @@ We start a transaction and vote, this leads to getting the lock.
ZEO.asyncio.server INFO
ZEO.asyncio.server INFO
received handshake 'Z5'
received handshake 'Z5'
>>> tid1 = start_trans(zs1)
>>> tid1 = start_trans(zs1)
>>> zs1.vote(tid1) # doctest: +ELLIPSIS
>>>
resolved1 =
zs1.vote(tid1) # doctest: +ELLIPSIS
ZEO.StorageServer DEBUG
ZEO.StorageServer DEBUG
(test-addr-1) ('1') lock: transactions waiting: 0
(test-addr-1) ('1') lock: transactions waiting: 0
ZEO.StorageServer BLATHER
ZEO.StorageServer BLATHER
...
@@ -477,7 +486,7 @@ ZEOStorage as closed and see if trying to get a lock cleans it up:
...
@@ -477,7 +486,7 @@ ZEOStorage as closed and see if trying to get a lock cleans it up:
ZEO.asyncio.server INFO
ZEO.asyncio.server INFO
received handshake 'Z5'
received handshake 'Z5'
>>> tid1 = start_trans(zs1)
>>> tid1 = start_trans(zs1)
>>> zs1.vote(tid1) # doctest: +ELLIPSIS
>>>
resolved1 =
zs1.vote(tid1) # doctest: +ELLIPSIS
ZEO.StorageServer DEBUG
ZEO.StorageServer DEBUG
(test-addr-1) ('1') lock: transactions waiting: 0
(test-addr-1) ('1') lock: transactions waiting: 0
ZEO.StorageServer BLATHER
ZEO.StorageServer BLATHER
...
@@ -493,7 +502,7 @@ ZEOStorage as closed and see if trying to get a lock cleans it up:
...
@@ -493,7 +502,7 @@ ZEOStorage as closed and see if trying to get a lock cleans it up:
ZEO.asyncio.server INFO
ZEO.asyncio.server INFO
received handshake 'Z5'
received handshake 'Z5'
>>> tid2 = start_trans(zs2)
>>> tid2 = start_trans(zs2)
>>> zs2.vote(tid2) # doctest: +ELLIPSIS
>>>
resolved2 =
zs2.vote(tid2) # doctest: +ELLIPSIS
ZEO.StorageServer DEBUG
ZEO.StorageServer DEBUG
(test-addr-2) ('1') lock: transactions waiting: 0
(test-addr-2) ('1') lock: transactions waiting: 0
ZEO.StorageServer BLATHER
ZEO.StorageServer BLATHER
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment