Merge pull request #557 from gevent/py34-test

Add Python 3.4 to test set. Fixes #408. Fixes #487. The build is green for PyPy, Py26 and Py34. Py27 intermittently displays a timing issue; Py33 intermittently displays a refcount "leak".

.travis.yml

@@ -3,6 +3,7 @@ python:
  - "2.6"
  - "2.7"
  - "3.3"
+ - "3.4"
  - "pypy"
 script:
  - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then NWORKERS=4 PYTHON=pypy make travis_pypy; fi

gevent/_socket3.py

@@ -44,9 +44,11 @@ class socket(_socket.socket):
         self._write_event = io_class(fileno, 2)
         self.timeout = _socket.getdefaulttimeout()
 
-    @property
-    def type(self):
-        return _socket.socket.type.__get__(self) & ~_socket.SOCK_NONBLOCK
+    if hasattr(_socket, 'SOCK_NONBLOCK'):
+        # Only defined under Linux
+        @property
+        def type(self):
+            return _socket.socket.type.__get__(self) & ~_socket.SOCK_NONBLOCK
 
     def __enter__(self):
         return self

gevent/_ssl3.py

@@ -60,6 +60,7 @@ class SSLSocket(socket):
                  suppress_ragged_eofs=True, npn_protocols=None, ciphers=None,
                  server_hostname=None,
                  _context=None):
+
         if _context:
             self.context = _context
         else:

gevent/event.py

@@ -153,7 +153,7 @@ class AsyncResult(object):
         >>> try:
         ...     result.get()
         ... except ZeroDivisionError:
-        ...     print 'ZeroDivisionError'
+        ...     print('ZeroDivisionError')
         ZeroDivisionError
     """
     def __init__(self):

gevent/threading.py

@@ -44,3 +44,40 @@ if PYPY:
         k,v = __threading__._active.items()[0]
         del __threading__._active[k]
         __threading__._active[_get_ident()] = v
+
+import sys
+if sys.version_info[:2] >= (3, 4):
+    # XXX: Issue 18808 breaks us on Python 3.4.
+    # Thread objects now expect a callback from the interpreter itself
+    # (threadmodule.c:release_sentinel). Because this never happens
+    # when a greenlet exits, join() and friends will block forever.
+    # The solution below involves capturing the greenlet when it is
+    # started and deferring the known broken methods to it.
+
+    class Thread(__threading__.Thread):
+        _greenlet = None
+
+        def is_alive(self):
+            return bool(self._greenlet)
+
+        isAlive = is_alive
+
+        def _set_tstate_lock(self):
+            self._greenlet = getcurrent()
+
+        def run(self):
+            try:
+                super(Thread, self).run()
+            finally:
+                del self._greenlet # avoid ref cycles
+                self._stop() # mark as finished
+
+        def join(self, timeout=None):
+            if self._greenlet is None:
+                return
+            self._greenlet.join(timeout=timeout)
+
+        def _wait_for_tstate_lock(self, *args, **kwargs):
+            raise NotImplementedError()
+
+    __implements__.append('Thread')

gevent/timeout.py

@@ -33,7 +33,7 @@ class Timeout(BaseException):
 
         >>> import gevent
         >>> gevent.Timeout(0.1).start()
-        >>> gevent.sleep(0.2)
+        >>> gevent.sleep(0.2)  #doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
          ...
         Timeout: 0.1 seconds
@@ -132,7 +132,7 @@ class Timeout(BaseException):
 
     def __str__(self):
         """
-        >>> raise Timeout
+        >>> raise Timeout #doctest: +IGNORE_EXCEPTION_DETAIL
         Traceback (most recent call last):
             ...
         Timeout

greentest/test__ares_host_result.py

@@ -21,20 +21,10 @@ class TestPickle(greentest.TestCase):
         assert r == loaded, (r, loaded)
         assert r.family == loaded.family, (r, loaded)
 
-    def test0(self):
-        return self._test(0)
-
-    def test1(self):
-        return self._test(1)
-
-    def test2(self):
-        return self._test(2)
-
-    if pickle.HIGHEST_PROTOCOL == 3:
-        def test3(self):
-            return self._test(3)
-    else:
-        assert pickle.HIGHEST_PROTOCOL == 2, pickle.HIGHEST_PROTOCOL
+for i in range(0, pickle.HIGHEST_PROTOCOL):
+    def make_test(j):
+        return lambda self: self._test(j)
+    setattr(TestPickle, 'test' + str(i), make_test(i) )
 
 
 if __name__ == '__main__':

greentest/test__socket.py

@@ -40,6 +40,8 @@ class TestTCP(greentest.TestCase):
     __timeout__ = None
     TIMEOUT_ERROR = socket.timeout
     long_data = ", ".join([str(x) for x in range(20000)])
+    if six.PY3:
+        long_data = long_data.encode('ascii')
 
     def setUp(self):
         greentest.TestCase.setUp(self)
@@ -49,6 +51,10 @@ class TestTCP(greentest.TestCase):
         self.port = listener.getsockname()[1]
 
     def cleanup(self):
+        try:
+            self.listener.close()
+        except:
+            pass
         del self.listener
 
     def create_connection(self):
@@ -62,7 +68,11 @@ class TestTCP(greentest.TestCase):
 
         def accept_and_read():
             try:
-                read_data.append(self.listener.accept()[0].makefile().read())
+                conn, _ = self.listener.accept()
+                r = conn.makefile(mode='rb')
+                read_data.append(r.read())
+                r.close()
+                conn.close()
             except:
                 traceback.print_exc()
                 os._exit(1)
@@ -72,13 +82,14 @@ class TestTCP(greentest.TestCase):
         client.sendall(data)
         client.close()
         server.join()
-        assert read_data[0] == self.long_data, read_data
+        self.assertEqual(read_data[0], self.long_data)
 
     def test_sendall_str(self):
         self._test_sendall(self.long_data)
 
-    def test_sendall_unicode(self):
-        self._test_sendall(six.text_type(self.long_data))
+    if not six.PY3:
+        def test_sendall_unicode(self):
+            self._test_sendall(six.text_type(self.long_data))
 
     def test_sendall_array(self):
         data = array.array("B", self.long_data)
@@ -89,25 +100,28 @@ class TestTCP(greentest.TestCase):
         N = 100000
 
         def server():
-            (client, addr) = self.listener.accept()
+            (remote_client, _) = self.listener.accept()
             # start reading, then, while reading, start writing. the reader should not hang forever
 
             def sendall():
-                client.sendall('t' * N)
+                remote_client.sendall(b't' * N)
 
             sender = Thread(target=sendall)
-            result = client.recv(1000)
-            self.assertEqual(result, 'hello world')
+            result = remote_client.recv(1000)
+            self.assertEqual(result, b'hello world')
             sender.join()
+            remote_client.close()
 
         server_thread = Thread(target=server)
         client = self.create_connection()
-        client_reader = Thread(target=client.makefile().read, args=(N, ))
+        client_file = client.makefile()
+        client_reader = Thread(target=client_file.read, args=(N, ))
         time.sleep(0.1)
-        client.send('hello world')
+        client.sendall(b'hello world')
         time.sleep(0.1)
 
         # close() used to hang
+        client_file.close()
         client.close()
 
         # this tests "full duplex" bug;
@@ -125,6 +139,8 @@ class TestTCP(greentest.TestCase):
         took = time.time() - start
         assert 1 - 0.1 <= took <= 1 + 0.1, (time.time() - start)
         acceptor.join()
+        client.close()
+        client_sock[0][0].close()
 
     # On Windows send() accepts whatever is thrown at it
     if sys.platform != 'win32':
@@ -136,28 +152,30 @@ class TestTCP(greentest.TestCase):
             time.sleep(0.1)
             assert client_sock
             client.settimeout(0.1)
-            data_sent = 'h' * 1000000
+            data_sent = b'h' * 1000000
             start = time.time()
             self.assertRaises(self.TIMEOUT_ERROR, client.sendall, data_sent)
             took = time.time() - start
             assert 0.1 - 0.01 <= took <= 0.1 + 0.1, took
             acceptor.join()
+            client.close()
+            client_sock[0][0].close()
 
     def test_makefile(self):
 
         def accept_once():
             conn, addr = self.listener.accept()
-            fd = conn.makefile(mode='w')
-            fd.write('hello\n')
+            fd = conn.makefile(mode='wb')
+            fd.write(b'hello\n')
             fd.close()
             conn.close()  # for pypy
 
         acceptor = Thread(target=accept_once)
         client = self.create_connection()
-        fd = client.makefile()
+        fd = client.makefile(mode='rb')
         client.close()
-        assert fd.readline() == 'hello\n'
-        assert fd.read() == ''
+        assert fd.readline() == b'hello\n'
+        assert fd.read() == b''
         fd.close()
         acceptor.join()
 

greentest/test__ssl.py

@@ -10,12 +10,14 @@ class TestSSL(TestTCP):
 
     certfile = os.path.join(os.path.dirname(__file__), 'test_server.crt')
     privfile = os.path.join(os.path.dirname(__file__), 'test_server.key')
-    TIMEOUT_ERROR = socket.sslerror
+    # Python 2.x has socket.sslerror, which we need to be sure is an alias for
+    # ssl.SSLError. That's gone in Py3 though.
+    TIMEOUT_ERROR = getattr(socket, 'sslerror', ssl.SSLError)
 
     def setUp(self):
         greentest.TestCase.setUp(self)
-        self.listener, r = ssl_listener(('127.0.0.1', 0), self.privfile, self.certfile)
-        self.port = r.getsockname()[1]
+        self.listener, raw_listener = ssl_listener(('127.0.0.1', 0), self.privfile, self.certfile)
+        self.port = self.listener.getsockname()[1]
 
     def create_connection(self):
         return ssl.wrap_socket(super(TestSSL, self).create_connection())
@@ -27,10 +29,10 @@ del TestTCP
 
 
 def ssl_listener(address, private_key, certificate):
-    r = socket.socket()
-    greentest.bind_and_listen(r, address)
-    sock = ssl.wrap_socket(r, private_key, certificate)
-    return sock, r
+    raw_listener = socket.socket()
+    greentest.bind_and_listen(raw_listener, address)
+    sock = ssl.wrap_socket(raw_listener, private_key, certificate)
+    return sock, raw_listener
 
 
 if __name__ == '__main__':

greentest/testrunner.py

@@ -122,7 +122,13 @@ def discover(tests=None, ignore=None):
     default_options = {'timeout': TIMEOUT}
 
     for filename in tests:
-        if 'TESTRUNNER' in open(filename).read():
+        with open(filename, 'rb') as f:
+            # Some of the test files (e.g., test__socket_dns) are
+            # UTF8 encoded. Depending on the environment, Python 3 may
+            # try to decode those as ASCII, which fails with UnicodeDecodeError.
+            # Thus, be sure to open and compare in binary mode.
+            contents = f.read()
+        if b'TESTRUNNER' in contents:
             module = __import__(filename.rsplit('.', 1)[0])
             for cmd, options in module.TESTRUNNER():
                 if remove_options(cmd)[-1] in ignore:

known_failures.py

@@ -92,7 +92,6 @@ test__os.py
 test__backdoor.py
 test_threading_2.py
 test__refcount.py
-test__socket.py
 test__subprocess.py
 test__all__.py
 test__fileobject.py
@@ -100,7 +99,6 @@ test__pywsgi.py
 test__socket_ex.py
 test__example_echoserver.py
 test__subprocess_poll.py
-test__ssl.py
 test__makefile_ref.py
 test__socketpair.py
 test__server_pywsgi.py
@@ -109,6 +107,7 @@ test__server.py
 test__example_portforwarder.py
 test__execmodules.py
 FLAKY test__greenio.py
+FLAKY test__socket_dns.py
 '''.strip().split('\n')
 
     if os.environ.get('GEVENT_RESOLVER') == 'ares':
@@ -132,6 +131,7 @@ FLAKY test__greenio.py
             test__socket_close.py
             test__select.py
             test__greenlet.py
+            FLAKY test__socket.py
 '''.strip().split()