Merge pull request #110 from zopefoundation/issue104

Rename all 'async' to 'async_' for compatibility with Python 3.7.

.travis.yml

@@ -13,10 +13,10 @@ matrix:
         - os: linux
           python: 3.6
         - os: linux
-          python: 3.4
+          python: 3.6
           env: ZEO_MTACCEPTOR=1
         - os: linux
-          python: 3.5
+          python: 3.6
           env: ZEO_MSGPACK=1 ZEO_MTACCEPTOR=1
         - os: linux
           python: 2.7

CHANGES.rst

@@ -12,6 +12,14 @@ Changelog
   aren't supported on Windows. See `issue 107
   <https://github.com/zopefoundation/ZEO/issues/107>`_.
 
+- Renamed all ``async`` attributes to ``async_`` for compatibility
+  with Python 3.7. See `issue 104
+  <https://github.com/zopefoundation/ZEO/issues/104>`_.
+
+- Fix: Client-side updates for ZODB 5.4.0 or databases that already
+  had ``zodbpickle.binary`` OIDs. See `issue 113
+  <https://github.com/zopefoundation/ZEO/issues/113>`_.
+
 5.1.2 (2018-03-27)
 ------------------
 
@@ -21,7 +29,6 @@ Changelog
   necessary for compatibility with ZODB 5.4.0 on Python 2. See `issue
   107 <https://github.com/zopefoundation/ZEO/issues/107>`_.)
 
-
 5.1.1 (2017-12-18)
 ------------------
 

setup.py

@@ -33,6 +33,7 @@ tests_require = [
     'random2',
     'mock',
     'msgpack-python',
+    'zope.testrunner',
 ]
 
 classifiers = """

src/ZEO/ClientStorage.py

@@ -271,7 +271,7 @@ class ClientStorage(ZODB.ConflictResolution.ConflictResolvingStorage):
             credentials=credentials,
             )
         self._call = self._server.call
-        self._async = self._server.async
+        self._async = self._server.async_
         self._async_iter = self._server.async_iter
         self._wait = self._server.wait
 

src/ZEO/StorageServer.py

@@ -104,7 +104,7 @@ class ZEOStorage(object):
         self.connected = True
         assert conn.protocol_version is not None
         self.log_label = _addr_label(conn.addr)
-        self.async = conn.async
+        self.async_ = conn.async_
         self.async_threadsafe = conn.async_threadsafe
 
     def notify_disconnected(self):
@@ -337,7 +337,7 @@ class ZEOStorage(object):
 
         self.stats.commits += 1
         self.storage.tpc_finish(self.transaction, self._invalidate)
-        self.async('info', self.get_size_info())
+        self.async_('info', self.get_size_info())
         # Note that the tid is still current because we still hold the
         # commit lock. We'll relinquish it in _clear_transaction.
         tid = self.storage.lastTransaction()

src/ZEO/asyncio/client.py

@@ -203,10 +203,10 @@ class Protocol(base.Protocol):
 
     exception_type_type = type(Exception)
     def message_received(self, data):
-        msgid, async, name, args = self.decode(data)
+        msgid, async_, name, args = self.decode(data)
         if name == '.reply':
             future = self.futures.pop(msgid)
-            if (async): # ZEO 5 exception
+            if async_: # ZEO 5 exception
                 class_, args = args
                 factory = exc_factories.get(class_)
                 if factory:
@@ -231,7 +231,7 @@ class Protocol(base.Protocol):
             else:
                 future.set_result(args)
         else:
-            assert async # clients only get async calls
+            assert async_ # clients only get async calls
             if name in self.client_methods:
                 getattr(self.client, name)(*args)
             else:
@@ -770,7 +770,7 @@ class ClientRunner(object):
             self.call_threadsafe, result, True, method, args)
         return result
 
-    def async(self, method, *args):
+    def async_(self, method, *args):
         return self.__call(self.call_async_threadsafe, method, args)
 
     def async_iter(self, it):

src/ZEO/asyncio/marshal.py

@@ -24,6 +24,7 @@ import logging
 from .._compat import Unpickler, Pickler, BytesIO, PY3, PYPY
 from ..shortrepr import short_repr
 
+PY2 = not PY3
 logger = logging.getLogger(__name__)
 
 def encoder(protocol, server=False):
@@ -132,6 +133,8 @@ _silly = ('__doc__',)
 
 exception_type_type = type(Exception)
 
+_SAFE_MODULE_NAMES = ('ZopeUndo.Prefix', 'copy_reg', '__builtin__', 'zodbpickle')
+
 def find_global(module, name):
     """Helper for message unpickler"""
     try:
@@ -144,7 +147,7 @@ def find_global(module, name):
     except AttributeError:
         raise ImportError("module %s has no global %s" % (module, name))
 
-    safe = getattr(r, '__no_side_effects__', 0)
+    safe = getattr(r, '__no_side_effects__', 0) or (PY2 and module in _SAFE_MODULE_NAMES)
     if safe:
         return r
 
@@ -156,7 +159,7 @@ def find_global(module, name):
 
 def server_find_global(module, name):
     """Helper for message unpickler"""
-    if module not in ('ZopeUndo.Prefix', 'copy_reg', '__builtin__', 'zodbpickle'):
+    if module not in _SAFE_MODULE_NAMES:
         raise ImportError("Module not allowed: %s" % (module,))
 
     try:

src/ZEO/asyncio/server.py

@@ -86,7 +86,7 @@ class ServerProtocol(base.Protocol):
 
     def message_received(self, message):
         try:
-            message_id, async, name, args = self.decode(message)
+            message_id, async_, name, args = self.decode(message)
         except Exception:
             logger.exception("Can't deserialize message")
             self.close()
@@ -104,13 +104,13 @@ class ServerProtocol(base.Protocol):
         except Exception as exc:
             if not isinstance(exc, self.unlogged_exception_types):
                 logger.exception(
-                    "Bad %srequest, %r", 'async ' if async else '', name)
-            if async:
+                    "Bad %srequest, %r", 'async ' if async_ else '', name)
+            if async_:
                 return self.close() # No way to recover/cry for help
             else:
                 return self.send_error(message_id, exc)
 
-        if not async:
+        if not async_:
             self.send_reply(message_id, result)
 
     def send_reply(self, message_id, result, send_error=False, flag=0):
@@ -138,7 +138,7 @@ class ServerProtocol(base.Protocol):
         """
         self.send_reply(message_id, reduce_exception(exc), send_error, 2)
 
-    def async(self, method, *args):
+    def async_(self, method, *args):
         self.call_async(method, args)
 
     def async_threadsafe(self, method, *args):

src/ZEO/asyncio/testing.py

@@ -167,7 +167,10 @@ class ClientRunner(object):
     def call(self, method, *args, **kw):
         return getattr(self, method)(*args)
 
-    async = async_iter = call
+    async_ = async_iter = call
 
     def wait(self, timeout=None):
         pass
+
+    def close(self):
+        pass

src/ZEO/asyncio/tests.py

@@ -112,9 +112,9 @@ class ClientTests(Base, setupstack.TestCase, ClientRunner):
 
         return (wrapper, cache, self.loop, self.client, protocol, transport)
 
-    def respond(self, message_id, result, async=False):
+    def respond(self, message_id, result, async_=False):
         self.loop.protocol.data_received(
-            sized(self.encode(message_id, async, '.reply', result)))
+            sized(self.encode(message_id, async_, '.reply', result)))
 
     def wait_for_result(self, future, timeout):
         if future.done() and future.exception() is not None:
@@ -160,7 +160,7 @@ class ClientTests(Base, setupstack.TestCase, ClientRunner):
         self.assertFalse(f1.done())
 
         # If we try to make an async call, we get an immediate error:
-        self.assertRaises(ClientDisconnected, self.async, 'bar', 3, 4)
+        self.assertRaises(ClientDisconnected, self.async_, 'bar', 3, 4)
 
         # The wrapper object (ClientStorage) hasn't been notified:
         self.assertFalse(wrapper.notify_connected.called)
@@ -191,7 +191,7 @@ class ClientTests(Base, setupstack.TestCase, ClientRunner):
         self.assertEqual(f1.result(), 42)
 
         # Now we can make async calls:
-        f2 = self.async('bar', 3, 4)
+        f2 = self.async_('bar', 3, 4)
         self.assertTrue(f2.done() and f2.exception() is None)
         self.assertEqual(self.pop(), (0, True, 'bar', (3, 4)))
 
@@ -581,10 +581,10 @@ class ClientTests(Base, setupstack.TestCase, ClientRunner):
 
         # Give the transport a small capacity:
         transport.capacity = 2
-        self.async('foo')
-        self.async('bar')
-        self.async('baz')
-        self.async('splat')
+        self.async_('foo')
+        self.async_('bar')
+        self.async_('baz')
+        self.async_('splat')
 
         # The first 2 were sent, but the remaining were queued.
         self.assertEqual(self.pop(),

src/ZEO/tests/ConnectionTests.py

@@ -965,13 +965,14 @@ class TimeoutTests(CommonSetupTearDown):
             self.assertRaises(ClientDisconnected, storage.tpc_finish, txn)
 
         # Make sure it's logged as CRITICAL
-        for line in open("server.log"):
-            if (('Transaction timeout after' in line) and
-                ('CRITICAL ZEO.StorageServer' in line)
+        with open("server.log") as f:
+            for line in f:
+                if (('Transaction timeout after' in line) and
+                    ('CRITICAL ZEO.StorageServer' in line)
                 ):
-                break
-        else:
-            self.assertTrue(False, 'bad logging')
+                    break
+            else:
+                self.fail('bad logging')
 
         storage.close()
 

src/ZEO/tests/ZEO4/zrpc/connection.py

@@ -418,10 +418,10 @@ class Connection(smac.SizedMessageAsyncConnection, object):
         # will raise an exception.  The exception will ultimately
         # result in asycnore calling handle_error(), which will
         # close the connection.
-        msgid, async, name, args = self.decode(message)
+        msgid, async_, name, args = self.decode(message)
 
         if debug_zrpc:
-            self.log("recv msg: %s, %s, %s, %s" % (msgid, async, name,
+            self.log("recv msg: %s, %s, %s, %s" % (msgid, async_, name,
                                                    short_repr(args)),
                      level=TRACE)
 
@@ -446,12 +446,12 @@ class Connection(smac.SizedMessageAsyncConnection, object):
                     self.send_reply(msgid, ret)
 
         elif name == REPLY:
-            assert not async
+            assert not async_
             self.handle_reply(msgid, args)
         else:
-            self.handle_request(msgid, async, name, args)
+            self.handle_request(msgid, async_, name, args)
 
-    def handle_request(self, msgid, async, name, args):
+    def handle_request(self, msgid, async_, name, args):
         obj = self.obj
 
         if name.startswith('_') or not hasattr(obj, name):
@@ -482,14 +482,14 @@ class Connection(smac.SizedMessageAsyncConnection, object):
                 self.log("%s() raised exception: %s" % (name, msg),
                          logging.ERROR, exc_info=True)
             error = sys.exc_info()[:2]
-            if async:
+            if async_:
                 self.log("Asynchronous call raised exception: %s" % self,
                          level=logging.ERROR, exc_info=True)
             else:
                 self.return_error(msgid, *error)
             return
 
-        if async:
+        if async_:
             if ret is not None:
                 raise ZRPCError("async method %s returned value %s" %
                                 (name, short_repr(ret)))
@@ -543,17 +543,17 @@ class Connection(smac.SizedMessageAsyncConnection, object):
         else:
             self.__super_setSessionKey(key)
 
-    def send_call(self, method, args, async=False):
+    def send_call(self, method, args, async_=False):
         # send a message and return its msgid
-        if async:
+        if async_:
             msgid = 0
         else:
             msgid = self._new_msgid()
 
         if debug_zrpc:
-            self.log("send msg: %d, %d, %s, ..." % (msgid, async, method),
+            self.log("send msg: %d, %d, %s, ..." % (msgid, async_, method),
                      level=TRACE)
-        buf = self.encode(msgid, async, method, args)
+        buf = self.encode(msgid, async_, method, args)
         self.message_output(buf)
         return msgid
 

src/ZEO/tests/ZEO4/zrpc/marshal.py

@@ -17,6 +17,8 @@ from ZEO._compat import Unpickler, Pickler, BytesIO, PY3, PYPY
 from .error import ZRPCError
 from .log import log, short_repr
 
+PY2 = not PY3
+
 def encode(*args): # args: (msgid, flags, name, args)
     # (We used to have a global pickler, but that's not thread-safe. :-( )
 
@@ -106,6 +108,8 @@ _silly = ('__doc__',)
 
 exception_type_type = type(Exception)
 
+_SAFE_MODULE_NAMES = ('ZopeUndo.Prefix', 'copy_reg', '__builtin__', 'zodbpickle')
+
 def find_global(module, name):
     """Helper for message unpickler"""
     try:
@@ -118,7 +122,7 @@ def find_global(module, name):
     except AttributeError:
         raise ZRPCError("module %s has no global %s" % (module, name))
 
-    safe = getattr(r, '__no_side_effects__', 0)
+    safe = getattr(r, '__no_side_effects__', 0) or (PY2 and module in _SAFE_MODULE_NAMES)
     if safe:
         return r
 
@@ -130,9 +134,10 @@ def find_global(module, name):
 
 def server_find_global(module, name):
     """Helper for message unpickler"""
+    if module not in _SAFE_MODULE_NAMES:
+        raise ImportError("Module not allowed: %s" % (module,))
+
     try:
-        if module != 'ZopeUndo.Prefix':
-            raise ImportError
         m = __import__(module, _globals, _globals, _silly)
     except ImportError as msg:
         raise ZRPCError("import error %s: %s" % (module, msg))

src/ZEO/tests/testConversionSupport.py

@@ -60,7 +60,7 @@ class FakeConnection(object):
     addr = 'test'
 
     call_soon_threadsafe = lambda f, *a: f(*a)
-    async = async_threadsafe = None
+    async_ = async_threadsafe = None
 
 def test_server_record_iternext():
     """
@@ -123,8 +123,6 @@ First, fake out the connection manager so we can make a connection:
     ...
     ...        return oid, oid*8, 'data ' + oid, next
     ...
-    ...    def close(self):
-    ...        pass
 
     >>> client = ZEO.client(
     ...     '', wait=False, _client_factory=Client)

src/ZEO/tests/testTransactionBuffer.py

@@ -40,6 +40,7 @@ class TransBufTests(unittest.TestCase):
         store(tbuf)
         for o in tbuf:
             pass
+        tbuf.close()
 
     def checkOrderPreserved(self):
         tbuf = TransactionBuffer(0)

src/ZEO/tests/testZEO.py

@@ -801,11 +801,11 @@ class FauxConn(object):
     peer_protocol_version = protocol_version
 
     serials = []
-    def async(self, method, *args):
+    def async_(self, method, *args):
         if method == 'serialnos':
             self.serials.extend(args[0])
 
-    call_soon_threadsafe = async_threadsafe = async
+    call_soon_threadsafe = async_threadsafe = async_
 
 class StorageServerWrapper(object):
 
@@ -1353,10 +1353,11 @@ You can specify the client label via a configuration file as well:
     >>> db.close()
     >>> @wait_until
     ... def check_for_test_label_2():
-    ...     for line in open('server.log'):
-    ...         if 'test-label-2' in line:
-    ...             print(line.split()[1:4])
-    ...             return True
+    ...     with open('server.log') as f:
+    ...         for line in f:
+    ...             if 'test-label-2' in line:
+    ...                 print(line.split()[1:4])
+    ...                 return True
     ['INFO', 'ZEO.StorageServer', '(test-label-2']
 
     """

src/ZEO/tests/testZEO2.py

@@ -500,8 +500,8 @@ def test_suite():
         doctest.DocTestSuite(
             setUp=ZODB.tests.util.setUp, tearDown=setupstack.tearDown,
             checker=renormalizing.RENormalizing([
-                (re.compile('\d+/test-addr'), ''),
-                (re.compile("'lock_time': \d+.\d+"), 'lock_time'),
+                (re.compile(r'\d+/test-addr'), ''),
+                (re.compile(r"'lock_time': \d+.\d+"), 'lock_time'),
                 (re.compile(r"'start': '[^\n]+'"), 'start'),
                 (re.compile('ZODB.POSException.StorageTransactionError'),
                  'StorageTransactionError'),

src/ZEO/tests/utils.py

@@ -25,10 +25,10 @@ class ServerProtocol(object):
     def call_soon_threadsafe(self, func, *args):
         func(*args)
 
-    def async(self, *args):
+    def async_(self, *args):
         self.calls.append(args)
 
-    async_threadsafe = async
+    async_threadsafe = async_
 
 class StorageServer(object):
     """Create a client interface to a StorageServer.