Merge remote-tracking branch 'origin/asyncio' into simplify-server-commit-lock-management

Conflicts:
	src/ZEO/StorageServer.py

CHANGES.rst

 Changelog
 =========
 
+- Fixed: SSL clients of servers with signed certs didn't load default
+  certs and were unable to connect.
+
+
 5.0.0a0 (2016-07-08)
 --------------------
 

src/ZEO/ClientStorage.py

@@ -490,7 +490,7 @@ class ClientStorage(ZODB.ConflictResolution.ConflictResolvingStorage):
         return self._call('record_iternext', next)
 
     def getTid(self, oid):
-        # XXX deprecated: used by storage server for full cache verification.
+        # XXX deprecated: but ZODB tests use this. They shouldn't
         return self._call('getTid', oid)
 
     def loadSerial(self, oid, serial):
@@ -504,8 +504,15 @@ class ClientStorage(ZODB.ConflictResolution.ConflictResolvingStorage):
         return result[:2]
 
     def loadBefore(self, oid, tid):
+        result = self._cache.loadBefore(oid, tid)
+        if result:
+            return result
+
         return self._server.load_before(oid, tid)
 
+    def prefetch(self, oids, tid):
+        self._server.prefetch(oids, tid)
+
     def new_oid(self):
         """Storage API: return a new object identifier.
         """

src/ZEO/StorageServer.py

@@ -76,10 +76,6 @@ registered_methods = set(( 'get_info', 'lastTransaction',
 class ZEOStorage:
     """Proxy to underlying storage for a single remote client."""
 
-    # A list of extension methods.  A subclass with extra methods
-    # should override.
-    extensions = []
-
     connected = connection = stats = storage = storage_id = transaction = None
     blob_tempfile = None
     log_label = 'unconnected'
@@ -91,10 +87,6 @@ class ZEOStorage:
         self.client_conflict_resolution = server.client_conflict_resolution
         # timeout and stats will be initialized in register()
         self.read_only = read_only
-        # The authentication protocol may define extra methods.
-        self._extensions = {}
-        for func in self.extensions:
-            self._extensions[func.__name__] = None
         self._iterators = {}
         self._iterator_ids = itertools.count()
         # Stores the last item that was handed out for a
@@ -149,23 +141,13 @@ class ZEOStorage:
         if not info['supportsUndo']:
             self.undoLog = self.undoInfo = lambda *a,**k: ()
 
+        # XXX deprecated: but ZODB tests use getTid. They shouldn't
         self.getTid = storage.getTid
-        self.load = storage.load
+
         self.loadSerial = storage.loadSerial
         record_iternext = getattr(storage, 'record_iternext', None)
         if record_iternext is not None:
             self.record_iternext = record_iternext
-
-        try:
-            fn = storage.getExtensionMethods
-        except AttributeError:
-            pass # no extension methods
-        else:
-            d = fn()
-            self._extensions.update(d)
-            for name in d:
-                assert not hasattr(self, name)
-                setattr(self, name, getattr(storage, name))
         self.lastTransaction = storage.lastTransaction
 
         try:
@@ -252,7 +234,6 @@ class ZEOStorage:
                 'size': storage.getSize(),
                 'name': storage.getName(),
                 'supportsUndo': supportsUndo,
-                'extensionMethods': self.getExtensionMethods(),
                 'supports_record_iternext': hasattr(self, 'record_iternext'),
                 'interfaces': tuple(interfaces),
                 }
@@ -262,13 +243,6 @@ class ZEOStorage:
                 'size': self.storage.getSize(),
                 }
 
-    def getExtensionMethods(self):
-        return self._extensions
-
-    def loadEx(self, oid):
-        self.stats.loads += 1
-        return self.storage.load(oid, '')
-
     def loadBefore(self, oid, tid):
         self.stats.loads += 1
         return self.storage.loadBefore(oid, tid)
@@ -737,6 +711,7 @@ class StorageServer:
 
 
         self._lock = Lock()
+        self.ssl = ssl # For dev convenience
 
         self.read_only = read_only
         self.database = None

src/ZEO/interfaces.py

@@ -90,14 +90,6 @@ class IServeable(zope.interface.Interface):
     """Interface provided by storages that can be served by ZEO
     """
 
-    def getTid(oid):
-        """The last transaction to change an object
-
-        Return the transaction id of the last transaction that committed a
-        change to an object with the given object id.
-
-        """
-
     def tpc_transaction():
         """The current transaction being committed.
 

src/ZEO/tests/forker.py

@@ -110,6 +110,7 @@ def runner(config, qin, qout, timeout=None,
         options = ZEO.runzeo.ZEOOptions()
         options.realize(['-C', config])
         server = ZEO.runzeo.ZEOServer(options)
+        globals()[(name if name else 'last') + '_server'] = server
         server.open_storages()
         server.clear_socket()
         server.create_server()

src/ZEO/tests/testZEO2.py

@@ -492,6 +492,50 @@ ZEOStorage as closed and see if trying to get a lock cleans it up:
     >>> logging.getLogger('ZEO').removeHandler(handler)
     """
 
+def test_prefetch(self):
+    """The client storage prefetch method pre-fetches from the server
+
+    >>> count = 999
+
+    >>> import ZEO
+    >>> addr, stop = ZEO.server()
+    >>> conn = ZEO.connection(addr)
+    >>> root = conn.root()
+    >>> cls = root.__class__
+    >>> for i in range(count):
+    ...     root[i] = cls()
+    >>> conn.transaction_manager.commit()
+    >>> oids = [root[i]._p_oid for i in range(count)]
+    >>> conn.close()
+    >>> conn = ZEO.connection(addr)
+    >>> storage = conn.db().storage
+    >>> len(storage._cache)
+    1
+    >>> storage.prefetch(oids, conn._storage._start)
+
+    The prefetch returns before the cache is filled:
+
+    >>> len(storage._cache) < count
+    True
+
+    But it is filled eventually:
+
+    >>> from zope.testing.wait import wait
+    >>> wait(lambda : len(storage._cache) > count)
+
+    >>> loads = storage.server_status()['loads']
+
+    Now if we reload the data, it will be satisfied from the cache:
+
+    >>> for oid in oids:
+    ...     _ = conn._storage.load(oid)
+
+    >>> storage.server_status()['loads'] == loads
+    True
+
+    >>> conn.close()
+    >>> stop()
+    """
 
 def test_suite():
     return unittest.TestSuite((

src/ZEO/tests/testssl.py

@@ -195,6 +195,8 @@ class SSLConfigTestMockiavellian(ZEOConfigTestBase):
             factory, context, (client_cert, client_key, None),
             check_hostname=True)
 
+        context.load_default_certs.assert_called_with()
+
     @mock.patch('ssl.create_default_context')
     @mock.patch('ZEO.ClientStorage.ClientStorage')
     def test_ssl_mockiavellian_client_auth_dir(
@@ -210,6 +212,7 @@ class SSLConfigTestMockiavellian(ZEOConfigTestBase):
             capath=here,
             check_hostname=True,
             )
+        context.load_default_certs.assert_not_called()
 
     @mock.patch('ssl.create_default_context')
     @mock.patch('ZEO.ClientStorage.ClientStorage')
@@ -226,6 +229,7 @@ class SSLConfigTestMockiavellian(ZEOConfigTestBase):
             cafile=server_cert,
             check_hostname=True,
             )
+        context.load_default_certs.assert_not_called()
 
     @mock.patch('ssl.create_default_context')
     @mock.patch('ZEO.ClientStorage.ClientStorage')
@@ -345,7 +349,10 @@ server_config = """
     </zeo>
     """.format(server_cert, server_key, client_cert)
 
-def client_ssl():
+def client_ssl(cafile=server_key,
+               client_cert=client_cert,
+               client_key=client_key,
+               ):
     context = ssl.create_default_context(
         ssl.Purpose.CLIENT_AUTH, cafile=server_cert)
 
@@ -353,3 +360,7 @@ def client_ssl():
     context.verify_mode = ssl.CERT_REQUIRED
     context.check_hostname = False
     return context
+
+# Here's a command to create a cert/key pair:
+# openssl req -x509 -newkey rsa:2048 -keyout key.pem -out cert.pem \
+#             -days 999999 -nodes -batch

src/ZEO/zconfig.py

@@ -11,12 +11,16 @@ def ssl_config(section, server):
     if auth:
         if os.path.isdir(auth):
             capath=auth
-        else:
+        elif auth != 'DYNAMIC':
             cafile=auth
 
     context = ssl.create_default_context(
         ssl.Purpose.CLIENT_AUTH, cafile=cafile, capath=capath)
 
+    if not auth:
+        assert not server
+        context.load_default_certs()
+
     if section.certificate:
         password = section.password_function
         if password: