Fix the SSL tests for Python 3.6

Also update the SSL test for Python 2.7 to test the fix for Python bug
 no. 23804. This may need some tweaking for the 2.7.8 environment to
disable some tests. We'll let Travis run 2.7.8 to find out.

changelog.rst

@@ -58,7 +58,14 @@ Platforms
 ---------
 
 - As mentioned above, Python 2.6 is no longer supported.
-- Python 3.6 is now tested on POSIX platforms.
+- Python 3.6 is now tested on POSIX platforms. This includes a few
+  notable changes:
+
+  * SSLContext.wrap_socket accepts the ``session`` parameter, though
+    this parameter isn't useful prior to 3.6.
+  * SSLSocket.recv(0) or read(0) returns an empty byte string. This is
+    a fix for `Python bug #23804 <http://bugs.python.org/issue23804>`_
+    which has been merged into Python 2.7 and Python 3.
 
 Stdlib Compatibility
 --------------------

src/gevent/_ssl3.py

@@ -47,12 +47,15 @@ class SSLContext(orig_SSLContext):
     def wrap_socket(self, sock, server_side=False,
                     do_handshake_on_connect=True,
                     suppress_ragged_eofs=True,
-                    server_hostname=None):
+                    server_hostname=None,
+                    session=None): # 3.6
+        # Sadly, using *args and **kwargs doesn't work
         return SSLSocket(sock=sock, server_side=server_side,
                          do_handshake_on_connect=do_handshake_on_connect,
                          suppress_ragged_eofs=suppress_ragged_eofs,
                          server_hostname=server_hostname,
-                         _context=self)
+                         _context=self,
+                         _session=session)
 
     if not hasattr(orig_SSLContext, 'check_hostname'):
         # Python 3.3 lacks this
@@ -98,6 +101,15 @@ class _contextawaresock(socket._gevent_sock_class): # Python 2: pylint:disable=s
     def context(self, ctx):
         self._sslsock().context = ctx
 
+    @property
+    def session(self):
+        """The SSLSession for client socket."""
+        return self._sslsock().session
+
+    @session.setter
+    def session(self, session):
+        self._sslsock().session = session
+
     def __getattr__(self, name):
         try:
             return getattr(self._sslsock(), name)
@@ -125,6 +137,7 @@ class SSLSocket(socket):
                  family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None,
                  suppress_ragged_eofs=True, npn_protocols=None, ciphers=None,
                  server_hostname=None,
+                 _session=None, # 3.6
                  _context=None):
         # pylint:disable=too-many-locals,too-many-statements,too-many-branches
         if _context:
@@ -157,12 +170,16 @@ class SSLSocket(socket):
         # mixed in.
         if sock.getsockopt(SOL_SOCKET, SO_TYPE) != SOCK_STREAM:
             raise NotImplementedError("only stream sockets are supported")
-        if server_side and server_hostname:
+        if server_side:
+            if server_hostname:
                 raise ValueError("server_hostname can only be specified "
                                  "in client mode")
+            if _session is not None:
+                raise ValueError("session can only be specified "
+                                 "in client mode")
         if self._context.check_hostname and not server_hostname:
             raise ValueError("check_hostname requires server_hostname")
-
+        self._session = _session
         self.server_side = server_side
         self.server_hostname = server_hostname
         self.do_handshake_on_connect = do_handshake_on_connect
@@ -198,6 +215,8 @@ class SSLSocket(socket):
             try:
                 self._sslobj = self._context._wrap_socket(self._sock, server_side,
                                                           server_hostname)
+                if _session is not None: # 3.6
+                    self._sslobj = SSLObject(self._sslobj, owner=self, session=self._session)
                 if do_handshake_on_connect:
                     timeout = self.gettimeout()
                     if timeout == 0.0:
@@ -218,6 +237,24 @@ class SSLSocket(socket):
         self._context = ctx
         self._sslobj.context = ctx
 
+    @property
+    def session(self):
+        """The SSLSession for client socket."""
+        if self._sslobj is not None:
+            return self._sslobj.session
+
+    @session.setter
+    def session(self, session):
+        self._session = session
+        if self._sslobj is not None:
+            self._sslobj.session = session
+
+    @property
+    def session_reused(self):
+        """Was the client session reused during handshake"""
+        if self._sslobj is not None:
+            return self._sslobj.session_reused
+
     def dup(self):
         raise NotImplementedError("Can't dup() %s instances" %
                                   self.__class__.__name__)
@@ -243,7 +280,8 @@ class SSLSocket(socket):
         while True:
             if not self._sslobj:
                 raise ValueError("Read on closed or unwrapped SSL socket.")
-
+            if len == 0:
+                return b'' if buffer is None else 0
             try:
                 if buffer is not None:
                     return self._sslobj.read(len, buffer)
@@ -298,7 +336,13 @@ class SSLSocket(socket):
 
         self._checkClosed()
         self._check_connected()
-        return self._sslobj.peer_certificate(binary_form)
+        try:
+            c = self._sslobj.peer_certificate
+        except AttributeError:
+            # 3.6
+            c = self._sslobj.getpeercert
+
+        return c(binary_form)
 
     def selected_npn_protocol(self):
         self._checkClosed()
@@ -407,6 +451,10 @@ class SSLSocket(socket):
                 raise ValueError(
                     "non-zero flags not allowed in calls to recv() on %s" %
                     self.__class__)
+            if buflen == 0:
+                # https://github.com/python/cpython/commit/00915577dd84ba75016400793bf547666e6b29b5
+                # Python #23804
+                return b''
             return self.read(buflen)
         else:
             return socket.recv(self, buflen, flags)
@@ -521,6 +569,8 @@ class SSLSocket(socket):
         if self._connected:
             raise ValueError("attempt to connect already-connected SSLSocket!")
         self._sslobj = self._context._wrap_socket(self._sock, False, self.server_hostname)
+        if self._session is not None: # 3.6
+            self._sslobj = SSLObject(self._sslobj, owner=self, session=self._session)
         try:
             if connect_ex:
                 rc = socket.connect_ex(self, addr)

src/gevent/_sslgte279.py

@@ -296,7 +296,7 @@ class SSLSocket(socket):
             # EAGAIN.
             self.getpeername()
 
-    def read(self, len=0, buffer=None):
+    def read(self, len=1024, buffer=None):
         """Read up to LEN bytes and return them.
         Return zero-length string on EOF."""
         self._checkClosed()
@@ -304,7 +304,8 @@ class SSLSocket(socket):
         while 1:
             if not self._sslobj:
                 raise ValueError("Read on closed or unwrapped SSL socket.")
-
+            if len == 0:
+                return b'' if buffer is None else 0
             try:
                 if buffer is not None:
                     return self._sslobj.read(len, buffer)
@@ -456,13 +457,17 @@ class SSLSocket(socket):
                 raise ValueError(
                     "non-zero flags not allowed in calls to recv() on %s" %
                     self.__class__)
+            if buflen == 0:
+                return b''
             return self.read(buflen)
         else:
             return socket.recv(self, buflen, flags)
 
     def recv_into(self, buffer, nbytes=None, flags=0):
         self._checkClosed()
-        if buffer and (nbytes is None):
+        if buffer is not None and (nbytes is None):
+            # Fix for python bug #23804: bool(bytearray()) is False,
+            # but we should read 0 bytes.
             nbytes = len(buffer)
         elif nbytes is None:
             nbytes = 1024

src/greentest/2.7/test_ssl.py

@@ -4,6 +4,7 @@
 import sys
 import unittest
 from test import test_support as support
+from test.script_helper import assert_python_ok
 import asyncore
 import socket
 import select
@@ -991,7 +992,7 @@ class ContextTests(unittest.TestCase):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         ctx.set_default_verify_paths()
 
-    @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") # gevent run on 2.7.8
+    @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build")
     def test_set_ecdh_curve(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         ctx.set_ecdh_curve("prime256v1")
@@ -1174,6 +1175,57 @@ class ContextTests(unittest.TestCase):
         self.assertEqual(ctx.verify_mode, ssl.CERT_NONE)
         self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2)
 
+    def test__https_verify_certificates(self):
+        # Unit test to check the contect factory mapping
+        # The factories themselves are tested above
+        # This test will fail by design if run under PYTHONHTTPSVERIFY=0
+        # (as will various test_httplib tests)
+
+        # Uses a fresh SSL module to avoid affecting the real one
+        local_ssl = support.import_fresh_module("ssl")
+        # Certificate verification is enabled by default
+        self.assertIs(local_ssl._create_default_https_context,
+                      local_ssl.create_default_context)
+        # Turn default verification off
+        local_ssl._https_verify_certificates(enable=False)
+        self.assertIs(local_ssl._create_default_https_context,
+                      local_ssl._create_unverified_context)
+        # And back on
+        local_ssl._https_verify_certificates(enable=True)
+        self.assertIs(local_ssl._create_default_https_context,
+                      local_ssl.create_default_context)
+        # The default behaviour is to enable
+        local_ssl._https_verify_certificates(enable=False)
+        local_ssl._https_verify_certificates()
+        self.assertIs(local_ssl._create_default_https_context,
+                      local_ssl.create_default_context)
+
+    def test__https_verify_envvar(self):
+        # Unit test to check the PYTHONHTTPSVERIFY handling
+        # Need to use a subprocess so it can still be run under -E
+        https_is_verified = """import ssl, sys; \
+            status = "Error: _create_default_https_context does not verify certs" \
+                       if ssl._create_default_https_context is \
+                          ssl._create_unverified_context \
+                     else None; \
+            sys.exit(status)"""
+        https_is_not_verified = """import ssl, sys; \
+            status = "Error: _create_default_https_context verifies certs" \
+                       if ssl._create_default_https_context is \
+                          ssl.create_default_context \
+                     else None; \
+            sys.exit(status)"""
+        extra_env = {}
+        # Omitting it leaves verification on
+        assert_python_ok("-c", https_is_verified, **extra_env)
+        # Setting it to zero turns verification off
+        extra_env[ssl._https_verify_envvar] = "0"
+        assert_python_ok("-c", https_is_not_verified, **extra_env)
+        # Any other value should also leave it on
+        for setting in ("", "1", "enabled", "foo"):
+            extra_env[ssl._https_verify_envvar] = setting
+            assert_python_ok("-c", https_is_verified, **extra_env)
+
     def test_check_hostname(self):
         ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         self.assertFalse(ctx.check_hostname)
@@ -2570,9 +2622,40 @@ else:
                         # consume data
                         s.read()
 
+                # read(-1, buffer) is supported, even though read(-1) is not
+                data = b"data"
+                s.send(data)
+                buffer = bytearray(len(data))
+                self.assertEqual(s.read(-1, buffer), len(data))
+                self.assertEqual(buffer, data)
+
                 s.write(b"over\n")
+
+                self.assertRaises(ValueError, s.recv, -1)
+                self.assertRaises(ValueError, s.read, -1)
+
                 s.close()
 
+        def test_recv_zero(self):
+            server = ThreadedEchoServer(CERTFILE)
+            server.__enter__()
+            self.addCleanup(server.__exit__, None, None)
+            s = socket.create_connection((HOST, server.port))
+            self.addCleanup(s.close)
+            s = ssl.wrap_socket(s, suppress_ragged_eofs=False)
+            self.addCleanup(s.close)
+
+            # recv/read(0) should return no data
+            s.send(b"data")
+            self.assertEqual(s.recv(0), b"")
+            self.assertEqual(s.read(0), b"")
+            self.assertEqual(s.read(), b"data")
+
+            # Should not block if the other end sends no data
+            s.setblocking(False)
+            self.assertEqual(s.recv(0), b"")
+            self.assertEqual(s.recv_into(bytearray()), 0)
+
         def test_handshake_timeout(self):
             # Issue #5103: SSL handshake must respect the socket timeout
             server = socket.socket(socket.AF_INET)

src/greentest/3.6/allsans.pem

+-----BEGIN PRIVATE KEY-----
+MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOoy7/QOtTjQ0niE
+6uDcTwtkC0R2Tvy1AjVnXohCntZfdzbTGDoYTgXSOLsP8A697jUiJ8VCePGH50xG
+Z4DKnAF3a9O3a9nr2pLXb0iY3XOMv+YEBii7CfI+3oxFYgCl0sMgHzDD2ZTVYAsm
+DWgLUVsE2gHEccRwrM2tPf2EgR+FAgMBAAECgYEA3qyfyYVSeTrTYxO93x6ZaVMu
+A2IZp9zSxMQL9bKiI2GRj+cV2ebSCGbg2btFnD6qBor7FWsmYz+8g6FNN/9sY4az
+61rMqMtQvLBe+7L8w70FeTze4qQ4Y1oQri0qD6tBWhDVlpnbI5Py9bkZKD67yVUk
+elcEA/5x4PrYXkuqsAECQQD80NjT0mDvaY0JOOaQFSEpMv6QiUA8GGX8Xli7IoKb
+tAolPG8rQBa+qSpcWfDMTrWw/aWHuMEEQoP/bVDH9W4FAkEA7SYQbBAKnojZ5A3G
+kOHdV7aeivRQxQk/JN8Fb8oKB9Csvpv/BsuGxPKXHdhFa6CBTTsNRtHQw/szPo4l
+xMIjgQJAPoMxqibR+0EBM6+TKzteSL6oPXsCnBl4Vk/J5vPgkbmR7KUl4+7j8N8J
+b2554TrxKEN/w7CGYZRE6UrRd7ATNQJAWD7Yz41sli+wfPdPU2xo1BHljyl4wMk/
+EPZYbI/PCbdyAH/F935WyQTIjNeEhZc1Zkq6FwdOWw8ns3hrv3rKgQJAHXv1BqUa
+czGPIFxX2TNoqtcl6/En4vrxVB1wzsfzkkDAg98kBl7qsF+S3qujSzKikjeaVbI2
+/CyWR2P3yLtOmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIIDcjCCAtugAwIBAgIJAN5dc9TOWjB7MA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV
+BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u
+IFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2FsbHNhbnMwHhcNMTYwODA1
+MTAyMTExWhcNMjYwODAzMTAyMTExWjBdMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwO
+Q2FzdGxlIEFudGhyYXgxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0
+aW9uMRAwDgYDVQQDDAdhbGxzYW5zMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB
+gQDqMu/0DrU40NJ4hOrg3E8LZAtEdk78tQI1Z16IQp7WX3c20xg6GE4F0ji7D/AO
+ve41IifFQnjxh+dMRmeAypwBd2vTt2vZ69qS129ImN1zjL/mBAYouwnyPt6MRWIA
+pdLDIB8ww9mU1WALJg1oC1FbBNoBxHHEcKzNrT39hIEfhQIDAQABo4IBODCCATQw
+ggEwBgNVHREEggEnMIIBI4IHYWxsc2Fuc6AeBgMqAwSgFwwVc29tZSBvdGhlciBp
+ZGVudGlmaWVyoDUGBisGAQUCAqArMCmgEBsOS0VSQkVST1MuUkVBTE2hFTAToAMC
+AQGhDDAKGwh1c2VybmFtZYEQdXNlckBleGFtcGxlLm9yZ4IPd3d3LmV4YW1wbGUu
+b3JnpGcwZTELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMw
+IQYDVQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEYMBYGA1UEAwwPZGly
+bmFtZSBleGFtcGxlhhdodHRwczovL3d3dy5weXRob24ub3JnL4cEfwAAAYcQAAAA
+AAAAAAAAAAAAAAAAAYgEKgMEBTANBgkqhkiG9w0BAQsFAAOBgQAy16h+F+nOmeiT
+VWR0fc8F/j6FcadbLseAUaogcC15OGxCl4UYpLV88HBkABOoGCpP155qwWTwOrdG
+iYPGJSusf1OnJEbvzFejZf6u078bPd9/ZL4VWLjv+FPGkjd+N+/OaqMvgj8Lu99f
+3Y/C4S7YbHxxwff6C6l2Xli+q6gnuQ==
+-----END CERTIFICATE-----