Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Z
ZEO
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Kirill Smelkov
ZEO
Commits
ac13c2a8
Commit
ac13c2a8
authored
8 years ago
by
Jim Fulton
Committed by
GitHub
8 years ago
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #80 from zopefoundation/msgpack
Msgpack
parents
c0667b8c
dad1a01f
Changes
19
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
19 changed files
with
233 additions
and
96 deletions
+233
-96
.travis.yml
.travis.yml
+4
-1
CHANGES.rst
CHANGES.rst
+7
-0
README.rst
README.rst
+11
-0
setup.py
setup.py
+6
-2
src/ZEO/ClientStorage.py
src/ZEO/ClientStorage.py
+1
-1
src/ZEO/StorageServer.py
src/ZEO/StorageServer.py
+3
-2
src/ZEO/asyncio/base.py
src/ZEO/asyncio/base.py
+0
-3
src/ZEO/asyncio/client.py
src/ZEO/asyncio/client.py
+11
-7
src/ZEO/asyncio/marshal.py
src/ZEO/asyncio/marshal.py
+44
-4
src/ZEO/asyncio/mtacceptor.py
src/ZEO/asyncio/mtacceptor.py
+3
-2
src/ZEO/asyncio/server.py
src/ZEO/asyncio/server.py
+20
-14
src/ZEO/asyncio/tests.py
src/ZEO/asyncio/tests.py
+65
-52
src/ZEO/runzeo.py
src/ZEO/runzeo.py
+3
-0
src/ZEO/server.xml
src/ZEO/server.xml
+13
-0
src/ZEO/tests/forker.py
src/ZEO/tests/forker.py
+7
-3
src/ZEO/tests/protocols.test
src/ZEO/tests/protocols.test
+3
-3
src/ZEO/tests/servertesting.py
src/ZEO/tests/servertesting.py
+1
-1
src/ZEO/tests/testZEO.py
src/ZEO/tests/testZEO.py
+30
-0
src/ZEO/tests/utils.py
src/ZEO/tests/utils.py
+1
-1
No files found.
.travis.yml
View file @
ac13c2a8
...
@@ -13,7 +13,10 @@ matrix:
...
@@ -13,7 +13,10 @@ matrix:
env
:
ZEO_MTACCEPTOR=1
env
:
ZEO_MTACCEPTOR=1
-
os
:
linux
-
os
:
linux
python
:
3.5
python
:
3.5
env
:
ZEO_MTACCEPTOR=1
env
:
ZEO_MSGPACK=1 ZEO_MTACCEPTOR=1
-
os
:
linux
python
:
2.7
env
:
ZEO_MSGPACK=1
-
os
:
linux
-
os
:
linux
python
:
2.7
python
:
2.7
env
:
ZEO4_SERVER=1
env
:
ZEO4_SERVER=1
...
...
This diff is collapsed.
Click to expand it.
CHANGES.rst
View file @
ac13c2a8
Changelog
Changelog
=========
=========
- Added support for serializing ZEO messages using `msgpack
<http://msgpack.org/index.html>`_ rather than pickle. This helps
pave the way to supporting `byteserver
<https://github.com/jimfulton/byteserver>`_, but it also allows ZEO
servers to support Python 2 or 3 clients (but not both at the same
time) and may provide a small performance improvement.
5.0.4 (2016-11-18)
5.0.4 (2016-11-18)
------------------
------------------
...
...
This diff is collapsed.
Click to expand it.
README.rst
View file @
ac13c2a8
...
@@ -289,6 +289,17 @@ client-conflict-resolution
...
@@ -289,6 +289,17 @@ client-conflict-resolution
Flag indicating that clients should perform conflict
Flag indicating that clients should perform conflict
resolution. This option defaults to false.
resolution. This option defaults to false.
msgpack
Use `msgpack <http://msgpack.org/index.html>`_ to serialize
and de-serialize ZEO protocol messages.
An advantage of using msgpack for ZEO communication is that
it's a tiny bit faster and a ZEO server can support Python 2
or Python 3 clients (but not both).
msgpack can also be enabled by setting the ``ZEO_MSGPACK``
environment to a non-empty string.
Server SSL configuration
Server SSL configuration
~~~~~~~~~~~~~~~~~~~~~~~~
~~~~~~~~~~~~~~~~~~~~~~~~
...
...
This diff is collapsed.
Click to expand it.
setup.py
View file @
ac13c2a8
...
@@ -36,7 +36,7 @@ install_requires = [
...
@@ -36,7 +36,7 @@ install_requires = [
'zope.interface'
,
'zope.interface'
,
]
]
tests_require
=
[
'zope.testing'
,
'manuel'
,
'random2'
,
'mock'
]
tests_require
=
[
'zope.testing'
,
'manuel'
,
'random2'
,
'mock'
,
'msgpack-python'
]
if
sys
.
version_info
[:
2
]
<
(
3
,
):
if
sys
.
version_info
[:
2
]
<
(
3
,
):
install_requires
.
extend
((
'futures'
,
'trollius'
))
install_requires
.
extend
((
'futures'
,
'trollius'
))
...
@@ -128,7 +128,11 @@ setup(name="ZEO",
...
@@ -128,7 +128,11 @@ setup(name="ZEO",
classifiers
=
classifiers
,
classifiers
=
classifiers
,
test_suite
=
"__main__.alltests"
,
# to support "setup.py test"
test_suite
=
"__main__.alltests"
,
# to support "setup.py test"
tests_require
=
tests_require
,
tests_require
=
tests_require
,
extras_require
=
dict
(
test
=
tests_require
,
uvloop
=
[
'uvloop >=0.5.1'
]),
extras_require
=
dict
(
test
=
tests_require
,
uvloop
=
[
'uvloop >=0.5.1'
],
msgpack
=
[
'msgpack-python'
],
),
install_requires
=
install_requires
,
install_requires
=
install_requires
,
zip_safe
=
False
,
zip_safe
=
False
,
entry_points
=
"""
entry_points
=
"""
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/ClientStorage.py
View file @
ac13c2a8
...
@@ -383,7 +383,7 @@ class ClientStorage(ZODB.ConflictResolution.ConflictResolvingStorage):
...
@@ -383,7 +383,7 @@ class ClientStorage(ZODB.ConflictResolution.ConflictResolvingStorage):
'interfaces'
,
()):
'interfaces'
,
()):
zope
.
interface
.
alsoProvides
(
self
,
iface
)
zope
.
interface
.
alsoProvides
(
self
,
iface
)
if
self
.
protocol_version
>=
b'Z
5'
:
if
self
.
protocol_version
[
1
:]
>=
b'
5'
:
self
.
ping
=
lambda
:
self
.
_call
(
'ping'
)
self
.
ping
=
lambda
:
self
.
_call
(
'ping'
)
else
:
else
:
self
.
ping
=
lambda
:
self
.
_call
(
'lastTransaction'
)
self
.
ping
=
lambda
:
self
.
_call
(
'lastTransaction'
)
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/StorageServer.py
View file @
ac13c2a8
...
@@ -228,7 +228,7 @@ class ZEOStorage:
...
@@ -228,7 +228,7 @@ class ZEOStorage:
storage
=
self
.
storage
storage
=
self
.
storage
supportsUndo
=
(
getattr
(
storage
,
'supportsUndo'
,
lambda
:
False
)()
supportsUndo
=
(
getattr
(
storage
,
'supportsUndo'
,
lambda
:
False
)()
and
self
.
connection
.
protocol_version
>=
b'Z
310'
)
and
self
.
connection
.
protocol_version
[
1
:]
>=
b'
310'
)
# Communicate the backend storage interfaces to the client
# Communicate the backend storage interfaces to the client
storage_provides
=
zope
.
interface
.
providedBy
(
storage
)
storage_provides
=
zope
.
interface
.
providedBy
(
storage
)
...
@@ -660,6 +660,7 @@ class StorageServer:
...
@@ -660,6 +660,7 @@ class StorageServer:
ssl
=
None
,
ssl
=
None
,
client_conflict_resolution
=
False
,
client_conflict_resolution
=
False
,
Acceptor
=
Acceptor
,
Acceptor
=
Acceptor
,
msgpack
=
False
,
):
):
"""StorageServer constructor.
"""StorageServer constructor.
...
@@ -754,7 +755,7 @@ class StorageServer:
...
@@ -754,7 +755,7 @@ class StorageServer:
self
.
client_conflict_resolution
=
client_conflict_resolution
self
.
client_conflict_resolution
=
client_conflict_resolution
if
addr
is
not
None
:
if
addr
is
not
None
:
self
.
acceptor
=
Acceptor
(
self
,
addr
,
ssl
)
self
.
acceptor
=
Acceptor
(
self
,
addr
,
ssl
,
msgpack
)
if
isinstance
(
addr
,
tuple
)
and
addr
[
0
]:
if
isinstance
(
addr
,
tuple
)
and
addr
[
0
]:
self
.
addr
=
self
.
acceptor
.
addr
self
.
addr
=
self
.
acceptor
.
addr
else
:
else
:
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/asyncio/base.py
View file @
ac13c2a8
...
@@ -10,8 +10,6 @@ import socket
...
@@ -10,8 +10,6 @@ import socket
from
struct
import
unpack
from
struct
import
unpack
import
sys
import
sys
from
.marshal
import
encoder
logger
=
logging
.
getLogger
(
__name__
)
logger
=
logging
.
getLogger
(
__name__
)
INET_FAMILIES
=
socket
.
AF_INET
,
socket
.
AF_INET6
INET_FAMILIES
=
socket
.
AF_INET
,
socket
.
AF_INET6
...
@@ -135,7 +133,6 @@ class Protocol(asyncio.Protocol):
...
@@ -135,7 +133,6 @@ class Protocol(asyncio.Protocol):
def
first_message_received
(
self
,
protocol_version
):
def
first_message_received
(
self
,
protocol_version
):
# Handler for first/handshake message, set up in __init__
# Handler for first/handshake message, set up in __init__
del
self
.
message_received
# use default handler from here on
del
self
.
message_received
# use default handler from here on
self
.
encode
=
encoder
()
self
.
finish_connect
(
protocol_version
)
self
.
finish_connect
(
protocol_version
)
def
call_async
(
self
,
method
,
args
):
def
call_async
(
self
,
method
,
args
):
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/asyncio/client.py
View file @
ac13c2a8
...
@@ -13,7 +13,7 @@ import ZEO.interfaces
...
@@ -13,7 +13,7 @@ import ZEO.interfaces
from
.
import
base
from
.
import
base
from
.compat
import
asyncio
,
new_event_loop
from
.compat
import
asyncio
,
new_event_loop
from
.marshal
import
decode
from
.marshal
import
encoder
,
decoder
logger
=
logging
.
getLogger
(
__name__
)
logger
=
logging
.
getLogger
(
__name__
)
...
@@ -63,7 +63,7 @@ class Protocol(base.Protocol):
...
@@ -63,7 +63,7 @@ class Protocol(base.Protocol):
# One place where special care was required was in cache setup on
# One place where special care was required was in cache setup on
# connect. See finish connect below.
# connect. See finish connect below.
protocols
=
b'
Z309'
,
b'Z310'
,
b'Z3101'
,
b'Z4'
,
b'Z
5'
protocols
=
b'
309'
,
b'310'
,
b'3101'
,
b'4'
,
b'
5'
def
__init__
(
self
,
loop
,
def
__init__
(
self
,
loop
,
addr
,
client
,
storage_key
,
read_only
,
connect_poll
=
1
,
addr
,
client
,
storage_key
,
read_only
,
connect_poll
=
1
,
...
@@ -165,13 +165,17 @@ class Protocol(base.Protocol):
...
@@ -165,13 +165,17 @@ class Protocol(base.Protocol):
# lastTid before processing (and possibly missing) subsequent
# lastTid before processing (and possibly missing) subsequent
# invalidations.
# invalidations.
self
.
protocol_version
=
min
(
protocol_version
,
self
.
protocols
[
-
1
])
version
=
min
(
protocol_version
[
1
:],
self
.
protocols
[
-
1
])
if
version
not
in
self
.
protocols
:
if
self
.
protocol_version
not
in
self
.
protocols
:
self
.
client
.
register_failed
(
self
.
client
.
register_failed
(
self
,
ZEO
.
Exceptions
.
ProtocolError
(
protocol_version
))
self
,
ZEO
.
Exceptions
.
ProtocolError
(
protocol_version
))
return
return
self
.
protocol_version
=
protocol_version
[:
1
]
+
version
self
.
encode
=
encoder
(
protocol_version
)
self
.
decode
=
decoder
(
protocol_version
)
self
.
heartbeat_bytes
=
self
.
encode
(
-
1
,
0
,
'.reply'
,
None
)
self
.
_write
(
self
.
protocol_version
)
self
.
_write
(
self
.
protocol_version
)
credentials
=
(
self
.
credentials
,)
if
self
.
credentials
else
()
credentials
=
(
self
.
credentials
,)
if
self
.
credentials
else
()
...
@@ -199,7 +203,7 @@ class Protocol(base.Protocol):
...
@@ -199,7 +203,7 @@ class Protocol(base.Protocol):
exception_type_type
=
type
(
Exception
)
exception_type_type
=
type
(
Exception
)
def
message_received
(
self
,
data
):
def
message_received
(
self
,
data
):
msgid
,
async
,
name
,
args
=
decode
(
data
)
msgid
,
async
,
name
,
args
=
self
.
decode
(
data
)
if
name
==
'.reply'
:
if
name
==
'.reply'
:
future
=
self
.
futures
.
pop
(
msgid
)
future
=
self
.
futures
.
pop
(
msgid
)
if
(
async
):
# ZEO 5 exception
if
(
async
):
# ZEO 5 exception
...
@@ -267,7 +271,7 @@ class Protocol(base.Protocol):
...
@@ -267,7 +271,7 @@ class Protocol(base.Protocol):
def
heartbeat
(
self
,
write
=
True
):
def
heartbeat
(
self
,
write
=
True
):
if
write
:
if
write
:
self
.
_write
(
b'(J
\
xff
\
xff
\
xff
\
xff
K
\
x00
U
\
x06
.replyNt.'
)
self
.
_write
(
self
.
heartbeat_bytes
)
self
.
heartbeat_handle
=
self
.
loop
.
call_later
(
self
.
heartbeat_handle
=
self
.
loop
.
call_later
(
self
.
heartbeat_interval
,
self
.
heartbeat
)
self
.
heartbeat_interval
,
self
.
heartbeat
)
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/asyncio/marshal.py
View file @
ac13c2a8
...
@@ -26,10 +26,21 @@ from ..shortrepr import short_repr
...
@@ -26,10 +26,21 @@ from ..shortrepr import short_repr
logger
=
logging
.
getLogger
(
__name__
)
logger
=
logging
.
getLogger
(
__name__
)
def
encoder
():
def
encoder
(
protocol
,
server
=
False
):
"""Return a non-thread-safe encoder
"""Return a non-thread-safe encoder
"""
"""
if
protocol
[:
1
]
==
b'M'
:
from
msgpack
import
packb
default
=
server_default
if
server
else
None
def
encode
(
*
args
):
return
packb
(
args
,
use_bin_type
=
True
,
default
=
default
)
return
encode
else
:
assert
protocol
[:
1
]
==
b'Z'
if
PY3
or
PYPY
:
if
PY3
or
PYPY
:
f
=
BytesIO
()
f
=
BytesIO
()
getvalue
=
f
.
getvalue
getvalue
=
f
.
getvalue
...
@@ -54,9 +65,20 @@ def encoder():
...
@@ -54,9 +65,20 @@ def encoder():
def
encode
(
*
args
):
def
encode
(
*
args
):
return
encoder
()(
*
args
)
return
encoder
(
b'Z'
)(
*
args
)
def
decode
(
msg
):
def
decoder
(
protocol
):
if
protocol
[:
1
]
==
b'M'
:
from
msgpack
import
unpackb
def
msgpack_decode
(
data
):
"""Decodes msg and returns its parts"""
return
unpackb
(
data
,
encoding
=
'utf-8'
,
use_list
=
False
)
return
msgpack_decode
else
:
assert
protocol
[:
1
]
==
b'Z'
return
pickle_decode
def
pickle_decode
(
msg
):
"""Decodes msg and returns its parts"""
"""Decodes msg and returns its parts"""
unpickler
=
Unpickler
(
BytesIO
(
msg
))
unpickler
=
Unpickler
(
BytesIO
(
msg
))
unpickler
.
find_global
=
find_global
unpickler
.
find_global
=
find_global
...
@@ -71,7 +93,14 @@ def decode(msg):
...
@@ -71,7 +93,14 @@ def decode(msg):
logger
.
error
(
"can't decode message: %s"
%
short_repr
(
msg
))
logger
.
error
(
"can't decode message: %s"
%
short_repr
(
msg
))
raise
raise
def
server_decode
(
msg
):
def
server_decoder
(
protocol
):
if
protocol
[:
1
]
==
b'M'
:
return
decoder
(
protocol
)
else
:
assert
protocol
[:
1
]
==
b'Z'
return
pickle_server_decode
def
pickle_server_decode
(
msg
):
"""Decodes msg and returns its parts"""
"""Decodes msg and returns its parts"""
unpickler
=
Unpickler
(
BytesIO
(
msg
))
unpickler
=
Unpickler
(
BytesIO
(
msg
))
unpickler
.
find_global
=
server_find_global
unpickler
.
find_global
=
server_find_global
...
@@ -87,6 +116,17 @@ def server_decode(msg):
...
@@ -87,6 +116,17 @@ def server_decode(msg):
logger
.
error
(
"can't decode message: %s"
%
short_repr
(
msg
))
logger
.
error
(
"can't decode message: %s"
%
short_repr
(
msg
))
raise
raise
def
server_default
(
obj
):
if
isinstance
(
obj
,
Exception
):
return
reduce_exception
(
obj
)
else
:
return
obj
def
reduce_exception
(
exc
):
class_
=
exc
.
__class__
class_
=
"%s.%s"
%
(
class_
.
__module__
,
class_
.
__name__
)
return
class_
,
exc
.
__dict__
or
exc
.
args
_globals
=
globals
()
_globals
=
globals
()
_silly
=
(
'__doc__'
,)
_silly
=
(
'__doc__'
,)
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/asyncio/mtacceptor.py
View file @
ac13c2a8
...
@@ -76,13 +76,14 @@ class Acceptor(asyncore.dispatcher):
...
@@ -76,13 +76,14 @@ class Acceptor(asyncore.dispatcher):
And creates a separate thread for each.
And creates a separate thread for each.
"""
"""
def
__init__
(
self
,
storage_server
,
addr
,
ssl
):
def
__init__
(
self
,
storage_server
,
addr
,
ssl
,
msgpack
):
self
.
storage_server
=
storage_server
self
.
storage_server
=
storage_server
self
.
addr
=
addr
self
.
addr
=
addr
self
.
__socket_map
=
{}
self
.
__socket_map
=
{}
asyncore
.
dispatcher
.
__init__
(
self
,
map
=
self
.
__socket_map
)
asyncore
.
dispatcher
.
__init__
(
self
,
map
=
self
.
__socket_map
)
self
.
ssl_context
=
ssl
self
.
ssl_context
=
ssl
self
.
msgpack
=
msgpack
self
.
_open_socket
()
self
.
_open_socket
()
def
_open_socket
(
self
):
def
_open_socket
(
self
):
...
@@ -165,7 +166,7 @@ class Acceptor(asyncore.dispatcher):
...
@@ -165,7 +166,7 @@ class Acceptor(asyncore.dispatcher):
def
run
():
def
run
():
loop
=
new_event_loop
()
loop
=
new_event_loop
()
zs
=
self
.
storage_server
.
create_client_handler
()
zs
=
self
.
storage_server
.
create_client_handler
()
protocol
=
ServerProtocol
(
loop
,
self
.
addr
,
zs
)
protocol
=
ServerProtocol
(
loop
,
self
.
addr
,
zs
,
self
.
msgpack
)
protocol
.
stop
=
loop
.
stop
protocol
.
stop
=
loop
.
stop
if
self
.
ssl_context
is
None
:
if
self
.
ssl_context
is
None
:
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/asyncio/server.py
View file @
ac13c2a8
...
@@ -11,13 +11,13 @@ from ..shortrepr import short_repr
...
@@ -11,13 +11,13 @@ from ..shortrepr import short_repr
from
.
import
base
from
.
import
base
from
.compat
import
asyncio
,
new_event_loop
from
.compat
import
asyncio
,
new_event_loop
from
.marshal
import
server_decode
from
.marshal
import
server_decode
r
,
encoder
,
reduce_exception
class
ServerProtocol
(
base
.
Protocol
):
class
ServerProtocol
(
base
.
Protocol
):
"""asyncio low-level ZEO server interface
"""asyncio low-level ZEO server interface
"""
"""
protocols
=
(
b'
Z
5'
,
)
protocols
=
(
b'5'
,
)
name
=
'server protocol'
name
=
'server protocol'
methods
=
set
((
'register'
,
))
methods
=
set
((
'register'
,
))
...
@@ -26,12 +26,16 @@ class ServerProtocol(base.Protocol):
...
@@ -26,12 +26,16 @@ class ServerProtocol(base.Protocol):
ZODB
.
POSException
.
POSKeyError
,
ZODB
.
POSException
.
POSKeyError
,
)
)
def
__init__
(
self
,
loop
,
addr
,
zeo_storage
):
def
__init__
(
self
,
loop
,
addr
,
zeo_storage
,
msgpack
):
"""Create a server's client interface
"""Create a server's client interface
"""
"""
super
(
ServerProtocol
,
self
).
__init__
(
loop
,
addr
)
super
(
ServerProtocol
,
self
).
__init__
(
loop
,
addr
)
self
.
zeo_storage
=
zeo_storage
self
.
zeo_storage
=
zeo_storage
self
.
announce_protocol
=
(
(
b'M'
if
msgpack
else
b'Z'
)
+
best_protocol_version
)
closed
=
False
closed
=
False
def
close
(
self
):
def
close
(
self
):
logger
.
debug
(
"Closing server protocol"
)
logger
.
debug
(
"Closing server protocol"
)
...
@@ -44,7 +48,7 @@ class ServerProtocol(base.Protocol):
...
@@ -44,7 +48,7 @@ class ServerProtocol(base.Protocol):
def
connection_made
(
self
,
transport
):
def
connection_made
(
self
,
transport
):
self
.
connected
=
True
self
.
connected
=
True
super
(
ServerProtocol
,
self
).
connection_made
(
transport
)
super
(
ServerProtocol
,
self
).
connection_made
(
transport
)
self
.
_write
(
best_protocol_version
)
self
.
_write
(
self
.
announce_protocol
)
def
connection_lost
(
self
,
exc
):
def
connection_lost
(
self
,
exc
):
self
.
connected
=
False
self
.
connected
=
False
...
@@ -61,10 +65,13 @@ class ServerProtocol(base.Protocol):
...
@@ -61,10 +65,13 @@ class ServerProtocol(base.Protocol):
self
.
_write
(
json
.
dumps
(
self
.
zeo_storage
.
ruok
()).
encode
(
"ascii"
))
self
.
_write
(
json
.
dumps
(
self
.
zeo_storage
.
ruok
()).
encode
(
"ascii"
))
self
.
close
()
self
.
close
()
else
:
else
:
if
protocol_version
in
self
.
protocols
:
version
=
protocol_version
[
1
:]
if
version
in
self
.
protocols
:
logger
.
info
(
"received handshake %r"
%
logger
.
info
(
"received handshake %r"
%
str
(
protocol_version
.
decode
(
'ascii'
)))
str
(
protocol_version
.
decode
(
'ascii'
)))
self
.
protocol_version
=
protocol_version
self
.
protocol_version
=
protocol_version
self
.
encode
=
encoder
(
protocol_version
,
True
)
self
.
decode
=
server_decoder
(
protocol_version
)
self
.
zeo_storage
.
notify_connected
(
self
)
self
.
zeo_storage
.
notify_connected
(
self
)
else
:
else
:
logger
.
error
(
"bad handshake %s"
%
short_repr
(
protocol_version
))
logger
.
error
(
"bad handshake %s"
%
short_repr
(
protocol_version
))
...
@@ -79,7 +86,7 @@ class ServerProtocol(base.Protocol):
...
@@ -79,7 +86,7 @@ class ServerProtocol(base.Protocol):
def
message_received
(
self
,
message
):
def
message_received
(
self
,
message
):
try
:
try
:
message_id
,
async
,
name
,
args
=
se
rver_
decode
(
message
)
message_id
,
async
,
name
,
args
=
se
lf
.
decode
(
message
)
except
Exception
:
except
Exception
:
logger
.
exception
(
"Can't deserialize message"
)
logger
.
exception
(
"Can't deserialize message"
)
self
.
close
()
self
.
close
()
...
@@ -128,10 +135,7 @@ class ServerProtocol(base.Protocol):
...
@@ -128,10 +135,7 @@ class ServerProtocol(base.Protocol):
def
send_error
(
self
,
message_id
,
exc
,
send_error
=
False
):
def
send_error
(
self
,
message_id
,
exc
,
send_error
=
False
):
"""Abstracting here so we can make this cleaner in the future
"""Abstracting here so we can make this cleaner in the future
"""
"""
class_
=
exc
.
__class__
self
.
send_reply
(
message_id
,
reduce_exception
(
exc
),
send_error
,
2
)
class_
=
"%s.%s"
%
(
class_
.
__module__
,
class_
.
__name__
)
args
=
class_
,
exc
.
__dict__
or
exc
.
args
self
.
send_reply
(
message_id
,
args
,
send_error
,
2
)
def
async
(
self
,
method
,
*
args
):
def
async
(
self
,
method
,
*
args
):
self
.
call_async
(
method
,
args
)
self
.
call_async
(
method
,
args
)
...
@@ -144,8 +148,8 @@ best_protocol_version = os.environ.get(
...
@@ -144,8 +148,8 @@ best_protocol_version = os.environ.get(
ServerProtocol
.
protocols
[
-
1
].
decode
(
'utf-8'
)).
encode
(
'utf-8'
)
ServerProtocol
.
protocols
[
-
1
].
decode
(
'utf-8'
)).
encode
(
'utf-8'
)
assert
best_protocol_version
in
ServerProtocol
.
protocols
assert
best_protocol_version
in
ServerProtocol
.
protocols
def
new_connection
(
loop
,
addr
,
socket
,
zeo_storage
):
def
new_connection
(
loop
,
addr
,
socket
,
zeo_storage
,
msgpack
):
protocol
=
ServerProtocol
(
loop
,
addr
,
zeo_storage
)
protocol
=
ServerProtocol
(
loop
,
addr
,
zeo_storage
,
msgpack
)
cr
=
loop
.
create_connection
((
lambda
:
protocol
),
sock
=
socket
)
cr
=
loop
.
create_connection
((
lambda
:
protocol
),
sock
=
socket
)
asyncio
.
async
(
cr
,
loop
=
loop
)
asyncio
.
async
(
cr
,
loop
=
loop
)
...
@@ -213,10 +217,11 @@ class MTDelay(Delay):
...
@@ -213,10 +217,11 @@ class MTDelay(Delay):
class
Acceptor
(
object
):
class
Acceptor
(
object
):
def
__init__
(
self
,
storage_server
,
addr
,
ssl
):
def
__init__
(
self
,
storage_server
,
addr
,
ssl
,
msgpack
):
self
.
storage_server
=
storage_server
self
.
storage_server
=
storage_server
self
.
addr
=
addr
self
.
addr
=
addr
self
.
ssl_context
=
ssl
self
.
ssl_context
=
ssl
self
.
msgpack
=
msgpack
self
.
event_loop
=
loop
=
new_event_loop
()
self
.
event_loop
=
loop
=
new_event_loop
()
if
isinstance
(
addr
,
tuple
):
if
isinstance
(
addr
,
tuple
):
...
@@ -243,7 +248,8 @@ class Acceptor(object):
...
@@ -243,7 +248,8 @@ class Acceptor(object):
try
:
try
:
logger
.
debug
(
"Accepted connection"
)
logger
.
debug
(
"Accepted connection"
)
zs
=
self
.
storage_server
.
create_client_handler
()
zs
=
self
.
storage_server
.
create_client_handler
()
protocol
=
ServerProtocol
(
self
.
event_loop
,
self
.
addr
,
zs
)
protocol
=
ServerProtocol
(
self
.
event_loop
,
self
.
addr
,
zs
,
self
.
msgpack
)
except
Exception
:
except
Exception
:
logger
.
exception
(
"Failure in protocol factory"
)
logger
.
exception
(
"Failure in protocol factory"
)
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/asyncio/tests.py
View file @
ac13c2a8
This diff is collapsed.
Click to expand it.
src/ZEO/runzeo.py
View file @
ac13c2a8
...
@@ -100,6 +100,7 @@ class ZEOOptionsMixin:
...
@@ -100,6 +100,7 @@ class ZEOOptionsMixin:
self
.
add
(
"client_conflict_resolution"
,
self
.
add
(
"client_conflict_resolution"
,
"zeo.client_conflict_resolution"
,
"zeo.client_conflict_resolution"
,
default
=
0
)
default
=
0
)
self
.
add
(
"msgpack"
,
"zeo.msgpack"
,
default
=
0
)
self
.
add
(
"invalidation_queue_size"
,
"zeo.invalidation_queue_size"
,
self
.
add
(
"invalidation_queue_size"
,
"zeo.invalidation_queue_size"
,
default
=
100
)
default
=
100
)
self
.
add
(
"invalidation_age"
,
"zeo.invalidation_age"
)
self
.
add
(
"invalidation_age"
,
"zeo.invalidation_age"
)
...
@@ -347,6 +348,8 @@ def create_server(storages, options):
...
@@ -347,6 +348,8 @@ def create_server(storages, options):
storages
,
storages
,
read_only
=
options
.
read_only
,
read_only
=
options
.
read_only
,
client_conflict_resolution
=
options
.
client_conflict_resolution
,
client_conflict_resolution
=
options
.
client_conflict_resolution
,
msgpack
=
(
options
.
msgpack
if
isinstance
(
options
.
msgpack
,
bool
)
else
os
.
environ
.
get
(
'ZEO_MSGPACK'
)),
invalidation_queue_size
=
options
.
invalidation_queue_size
,
invalidation_queue_size
=
options
.
invalidation_queue_size
,
invalidation_age
=
options
.
invalidation_age
,
invalidation_age
=
options
.
invalidation_age
,
transaction_timeout
=
options
.
transaction_timeout
,
transaction_timeout
=
options
.
transaction_timeout
,
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/server.xml
View file @
ac13c2a8
...
@@ -115,6 +115,19 @@
...
@@ -115,6 +115,19 @@
</description>
</description>
</key>
</key>
<key
name=
"msgpack"
datatype=
"boolean"
required=
"no"
>
<description>
Use msgpack to serialize and de-serialize ZEO protocol messages.
An advantage of using msgpack for ZEO communication is that
it's a tiny bit faster and a ZEO server can support Python 2
or Python 3 clients (but not both).
msgpack can also be enabled by setting the ``ZEO_MSGPACK``
environment to a non-empty string.
</description>
</key>
</sectiontype>
</sectiontype>
</component>
</component>
This diff is collapsed.
Click to expand it.
src/ZEO/tests/forker.py
View file @
ac13c2a8
...
@@ -70,7 +70,7 @@ class ZEOConfig:
...
@@ -70,7 +70,7 @@ class ZEOConfig:
for
name
in
(
for
name
in
(
'invalidation_queue_size'
,
'invalidation_age'
,
'invalidation_queue_size'
,
'invalidation_age'
,
'transaction_timeout'
,
'pid_filename'
,
'transaction_timeout'
,
'pid_filename'
,
'msgpack'
,
'ssl_certificate'
,
'ssl_key'
,
'client_conflict_resolution'
,
'ssl_certificate'
,
'ssl_key'
,
'client_conflict_resolution'
,
):
):
v
=
getattr
(
self
,
name
,
None
)
v
=
getattr
(
self
,
name
,
None
)
...
@@ -200,6 +200,7 @@ def start_zeo_server(storage_conf=None, zeo_conf=None, port=None, keep=False,
...
@@ -200,6 +200,7 @@ def start_zeo_server(storage_conf=None, zeo_conf=None, port=None, keep=False,
path
=
'Data.fs'
,
protocol
=
None
,
blob_dir
=
None
,
path
=
'Data.fs'
,
protocol
=
None
,
blob_dir
=
None
,
suicide
=
True
,
debug
=
False
,
suicide
=
True
,
debug
=
False
,
threaded
=
False
,
start_timeout
=
33
,
name
=
None
,
log
=
None
,
threaded
=
False
,
start_timeout
=
33
,
name
=
None
,
log
=
None
,
show_config
=
False
):
):
"""Start a ZEO server in a separate process.
"""Start a ZEO server in a separate process.
...
@@ -231,11 +232,14 @@ def start_zeo_server(storage_conf=None, zeo_conf=None, port=None, keep=False,
...
@@ -231,11 +232,14 @@ def start_zeo_server(storage_conf=None, zeo_conf=None, port=None, keep=False,
z
.
__dict__
.
update
(
zeo_conf
)
z
.
__dict__
.
update
(
zeo_conf
)
zeo_conf
=
str
(
z
)
zeo_conf
=
str
(
z
)
zeo_conf
=
str
(
zeo_conf
)
+
'
\
n
\
n
'
+
storage_conf
if
show_config
:
print
(
zeo_conf
)
# Store the config info in a temp file.
# Store the config info in a temp file.
tmpfile
=
tempfile
.
mktemp
(
".conf"
,
dir
=
os
.
getcwd
())
tmpfile
=
tempfile
.
mktemp
(
".conf"
,
dir
=
os
.
getcwd
())
fp
=
open
(
tmpfile
,
'w'
)
fp
=
open
(
tmpfile
,
'w'
)
fp
.
write
(
str
(
zeo_conf
)
+
'
\
n
\
n
'
)
fp
.
write
(
zeo_conf
)
fp
.
write
(
storage_conf
)
fp
.
close
()
fp
.
close
()
if
threaded
:
if
threaded
:
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/tests/protocols.test
View file @
ac13c2a8
...
@@ -17,15 +17,15 @@ Let's start a Z4 server
...
@@ -17,15 +17,15 @@ Let's start a Z4 server
...
'''
...
'''
>>> addr, stop = start_server(
>>> addr, stop = start_server(
... storage_conf, dict(invalidation_queue_size=5), protocol=b'
Z
4
')
... storage_conf, dict(invalidation_queue_size=5), protocol=b'
4
')
A current client should be able to connect to a old server:
A current client should be able to connect to a old server:
>>> import ZEO, ZODB.blob, transaction
>>> import ZEO, ZODB.blob, transaction
>>> db = ZEO.DB(addr, client='
client
', blob_dir='
blobs
')
>>> db = ZEO.DB(addr, client='
client
', blob_dir='
blobs
')
>>> wait_connected(db.storage)
>>> wait_connected(db.storage)
>>> str(db.storage.protocol_version.decode('
ascii
'))
>>> str(db.storage.protocol_version.decode('
ascii
'))
[1:]
'
Z
4
'
'
4
'
>>> conn = db.open()
>>> conn = db.open()
>>> conn.root().x = 0
>>> conn.root().x = 0
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/tests/servertesting.py
View file @
ac13c2a8
...
@@ -44,7 +44,7 @@ class StorageServer(ZEO.StorageServer.StorageServer):
...
@@ -44,7 +44,7 @@ class StorageServer(ZEO.StorageServer.StorageServer):
def
client
(
server
,
name
=
'client'
):
def
client
(
server
,
name
=
'client'
):
zs
=
ZEO
.
StorageServer
.
ZEOStorage
(
server
)
zs
=
ZEO
.
StorageServer
.
ZEOStorage
(
server
)
protocol
=
ZEO
.
asyncio
.
tests
.
server_protocol
(
protocol
=
ZEO
.
asyncio
.
tests
.
server_protocol
(
zs
,
protocol_version
=
b'Z5'
,
addr
=
'test-addr-%s'
%
name
)
False
,
zs
,
protocol_version
=
b'Z5'
,
addr
=
'test-addr-%s'
%
name
)
zs
.
notify_connected
(
protocol
)
zs
.
notify_connected
(
protocol
)
zs
.
register
(
'1'
,
0
)
zs
.
register
(
'1'
,
0
)
return
zs
return
zs
This diff is collapsed.
Click to expand it.
src/ZEO/tests/testZEO.py
View file @
ac13c2a8
...
@@ -1465,6 +1465,36 @@ def ClientDisconnected_errors_are_TransientErrors():
...
@@ -1465,6 +1465,36 @@ def ClientDisconnected_errors_are_TransientErrors():
True
True
"""
"""
if
not
os
.
environ
.
get
(
'ZEO4_SERVER'
):
if
os
.
environ
.
get
(
'ZEO_MSGPACK'
):
def
test_runzeo_msgpack_support
():
"""
>>> import ZEO
>>> a, s = ZEO.server(threaded=False)
>>> conn = ZEO.connection(a)
>>> str(conn.db().storage.protocol_version.decode('ascii'))
'M5'
>>> conn.close(); s()
"""
else
:
def
test_runzeo_msgpack_support
():
"""
>>> import ZEO
>>> a, s = ZEO.server(threaded=False)
>>> conn = ZEO.connection(a)
>>> str(conn.db().storage.protocol_version.decode('ascii'))
'Z5'
>>> conn.close(); s()
>>> a, s = ZEO.server(zeo_conf=dict(msgpack=True), threaded=False)
>>> conn = ZEO.connection(a)
>>> str(conn.db().storage.protocol_version.decode('ascii'))
'M5'
>>> conn.close(); s()
"""
if
sys
.
platform
.
startswith
(
'win'
):
if
sys
.
platform
.
startswith
(
'win'
):
del
runzeo_logrotate_on_sigusr2
del
runzeo_logrotate_on_sigusr2
del
unix_domain_sockets
del
unix_domain_sockets
...
...
This diff is collapsed.
Click to expand it.
src/ZEO/tests/utils.py
View file @
ac13c2a8
...
@@ -39,7 +39,7 @@ class StorageServer:
...
@@ -39,7 +39,7 @@ class StorageServer:
"""
"""
def
__init__
(
self
,
test
,
storage
,
def
__init__
(
self
,
test
,
storage
,
protocol_version
=
best_protocol_version
,
protocol_version
=
b
'Z'
+
b
est_protocol_version
,
**
kw
):
**
kw
):
self
.
test
=
test
self
.
test
=
test
self
.
storage_server
=
ZEO
.
StorageServer
.
StorageServer
(
self
.
storage_server
=
ZEO
.
StorageServer
.
StorageServer
(
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment