protocol: switch to msgpack for packet serialization
Not only for performance reasons (which is significant in the case of replication; tools/matrix is ~3% faster) but also because of several ugly things in the way packets were defined: - packet field names, which are only documentary; for roots fields, they even just duplicate the packet names - a lot of repetitions for packet names, and even confusion between the name of the packet definition and the name of the actual notify/request packet - the need to implement field types for anything, like PByte to support new compression formats, since PBoolean is not enough neo/lib/protocol.py is now much smaller.
| ... | @@ -16,12 +16,19 @@ | ... | @@ -16,12 +16,19 @@ |
| from functools import wraps | from functools import wraps | ||
| from time import time | from time import time | ||
| import msgpack | |||
| from msgpack.exceptions import UnpackValueError | |||
| from . import attributeTracker, logging | from . import attributeTracker, logging | ||
| from .connector import ConnectorException, ConnectorDelayedConnection | from .connector import ConnectorException, ConnectorDelayedConnection | ||
| from .locking import RLock | from .locking import RLock | ||
| from .protocol import uuid_str, Errors, PacketMalformedError, Packets | from .protocol import uuid_str, Errors, PacketMalformedError, Packets, \ | ||
| from .util import dummy_read_buffer, ReadBuffer | Unpacker | ||
| @apply | |||
| class dummy_read_buffer(msgpack.Unpacker): | |||
| def feed(self, _): | |||
| pass | |||
| class ConnectionClosed(Exception): | class ConnectionClosed(Exception): | ||
| pass | pass | ||
| ... | @@ -310,12 +317,12 @@ class Connection(BaseConnection): | ... | @@ -310,12 +317,12 @@ class Connection(BaseConnection): |
| client = False | client = False | ||
| server = False | server = False | ||
| peer_id = None | peer_id = None | ||
| _parser_state = None | _total_unpacked = 0 | ||
| _timeout = None | _timeout = None | ||
| def __init__(self, event_manager, *args, **kw): | def __init__(self, event_manager, *args, **kw): | ||
| BaseConnection.__init__(self, event_manager, *args, **kw) | BaseConnection.__init__(self, event_manager, *args, **kw) | ||
| self.read_buf = ReadBuffer() | self.read_buf = Unpacker() | ||
| self.cur_id = 0 | self.cur_id = 0 | ||
| self.aborted = False | self.aborted = False | ||
| self.uuid = None | self.uuid = None | ||
| ... | @@ -425,42 +432,36 @@ class Connection(BaseConnection): | ... | @@ -425,42 +432,36 @@ class Connection(BaseConnection): |
| self._closure() | self._closure() | ||
| def _parse(self): | def _parse(self): | ||
| read = self.read_buf.read | from .protocol import ENCODED_VERSION, Packets | ||
| version = read(4) | read_buf = self.read_buf | ||
| if version is None: | version = read_buf.read_bytes(4) | ||
| return | |||
| from .protocol import (ENCODED_VERSION, MAX_PACKET_SIZE, | |||
| PACKET_HEADER_FORMAT, Packets) | |||
| if version != ENCODED_VERSION: | if version != ENCODED_VERSION: | ||
| if len(version) < 4: # unlikely so tested last | |||
| # Not enough data and there's no API to know it in advance. | |||
| # Put it back. | |||
| read_buf.feed(version) | |||
| return | |||
| logging.warning('Protocol version mismatch with %r', self) | logging.warning('Protocol version mismatch with %r', self) | ||
| raise ConnectorException | raise ConnectorException | ||
| header_size = PACKET_HEADER_FORMAT.size | read_next = read_buf.next | ||
| unpack = PACKET_HEADER_FORMAT.unpack | read_pos = read_buf.tell | ||
| def parse(): | def parse(): | ||
| state = self._parser_state | try: | ||
| if state is None: | msg_id, msg_type, args = read_next() | ||
| header = read(header_size) | except StopIteration: | ||
| if header is None: | return | ||
| return | except UnpackValueError as e: | ||
| msg_id, msg_type, msg_len = unpack(header) | raise PacketMalformedError(str(e)) | ||
| try: | try: | ||
| packet_klass = Packets[msg_type] | packet_klass = Packets[msg_type] | ||
| except KeyError: | except KeyError: | ||
| raise PacketMalformedError('Unknown packet type') | raise PacketMalformedError('Unknown packet type') | ||
| if msg_len > MAX_PACKET_SIZE: | pos = read_pos() | ||
|
|||
| raise PacketMalformedError('message too big (%d)' % msg_len) | packet = packet_klass(*args) | ||
| else: | packet.setId(msg_id) | ||
| msg_id, packet_klass, msg_len = state | packet.size = pos - self._total_unpacked | ||
| data = read(msg_len) | self._total_unpacked = pos | ||
| if data is None: | return packet | ||
| # Not enough. | |||
| if state is None: | |||
| self._parser_state = msg_id, packet_klass, msg_len | |||
| else: | |||
| self._parser_state = None | |||
| packet = packet_klass() | |||
| packet.setContent(msg_id, data) | |||
| return packet | |||
| self._parse = parse | self._parse = parse | ||
| return parse() | return parse() | ||
| ... | @@ -513,7 +514,7 @@ class Connection(BaseConnection): | ... | @@ -513,7 +514,7 @@ class Connection(BaseConnection): |
| def close(self): | def close(self): | ||
| if self.connector is None: | if self.connector is None: | ||
| assert self._on_close is None | assert self._on_close is None | ||
| assert not self.read_buf | assert not self.read_buf.read_bytes(1) | ||
| assert not self.isPending() | assert not self.isPending() | ||
| return | return | ||
| # process the network events with the last registered handler to | # process the network events with the last registered handler to | ||
| ... | @@ -524,7 +525,7 @@ class Connection(BaseConnection): | ... | @@ -524,7 +525,7 @@ class Connection(BaseConnection): |
| if self._on_close is not None: | if self._on_close is not None: | ||
| self._on_close() | self._on_close() | ||
| self._on_close = None | self._on_close = None | ||
| self.read_buf.clear() | self.read_buf = dummy_read_buffer | ||
| try: | try: | ||
| if self.connecting: | if self.connecting: | ||
| handler.connectionFailed(self) | handler.connectionFailed(self) | ||
| ... | ... | ||