Fix test__pywsgi.py under Python 3. fix lint.

.pep8

 [pep8]
-ignore=E702,E265,E402,E731,E266,E261,W503
+ignore=E702,E265,E402,E731,E266,E261,W503,E129
 max_line_length=160
 exclude=.tox,.git,build,2.6,2.7,2.7pypy,3.3,test_support.py,test_queue.py,patched_tests_setup.py,test_threading_2.py,lock_tests.py,_sslgte279.py

gevent/_socket3.py

@@ -27,6 +27,7 @@ def _get_memory(string, offset):
 
 timeout_default = object()
 
+
 class _wrefsocket(_socket.socket):
     # Plain stdlib socket.socket objects subclass _socket.socket
     # and add weakref ability. The ssl module, for one, counts on this.
@@ -34,11 +35,12 @@ class _wrefsocket(_socket.socket):
     # monkey patched to be the object from this module), but we still
     # need to make sure what we do create can be weakrefd.
 
-    __slots__ = ["__weakref__",]
+    __slots__ = ["__weakref__", ]
 
 _closedsocket = _wrefsocket()
 _closedsocket.close()
 
+
 class socket(object):
 
     def __init__(self, family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None):
@@ -416,6 +418,7 @@ if hasattr(_socket, "socketpair"):
         b = socket(family, type, proto, b.detach())
         return a, b
 
+
 # PyPy needs drop and reuse
 def _do_reuse_or_drop(socket, methname):
     try:
@@ -427,6 +430,7 @@ def _do_reuse_or_drop(socket, methname):
 
 from io import BytesIO
 
+
 class _fileobject(object):
     """Faux file object attached to a socket object."""
 
@@ -500,7 +504,7 @@ class _fileobject(object):
             view = memoryview(data)
             try:
                 while write_offset < data_size:
-                    self._sock.sendall(view[write_offset:write_offset+buffer_size])
+                    self._sock.sendall(view[write_offset:write_offset + buffer_size])
                     write_offset += buffer_size
             finally:
                 if write_offset < data_size:
@@ -519,8 +523,7 @@ class _fileobject(object):
             return
         self._wbuf.append(data)
         self._wbuf_len += len(data)
-        if (self._wbufsize == 0 or
-            (self._wbufsize == 1 and b'\n' in data) or
+        if (self._wbufsize == 0 or (self._wbufsize == 1 and b'\n' in data) or
             (self._wbufsize > 1 and self._wbuf_len >= self._wbufsize)):
             self.flush()
 
@@ -530,8 +533,7 @@ class _fileobject(object):
         lines = filter(None, map(str, list))
         self._wbuf_len += sum(map(len, lines))
         self._wbuf.extend(lines)
-        if (self._wbufsize <= 1 or
-            self._wbuf_len >= self._wbufsize):
+        if (self._wbufsize <= 1 or self._wbuf_len >= self._wbufsize):
             self.flush()
 
     def read(self, size=-1):

gevent/monkey.py

@@ -154,7 +154,6 @@ def patch_thread(threading=True, _threading_local=True, Event=False):
             def join(timeout=None):
                 if threading.current_thread() is main_thread:
                     raise RuntimeError("Cannot join current thread")
-                self = _greenlet
                 if _greenlet.dead or not main_thread.is_alive():
                     return
                 elif timeout:

gevent/pywsgi.py

@@ -128,7 +128,7 @@ class Input(object):
                     length -= datalen
                     if length == 0:
                         break
-                if use_readline and data[-1] == b"\n":
+                if use_readline and data[-1] == b"\n"[0]:
                     break
             else:
                 line = rfile.readline()
@@ -339,9 +339,11 @@ class WSGIHandler(object):
     def handle_one_request(self):
         if self.rfile.closed:
             return
-
         try:
             self.requestline = self.read_requestline()
+            # Account for old subclasses that haven't done this
+            if PY3 and isinstance(self.requestline, bytes):
+                self.requestline = self.requestline.decode('latin-1')
         except socket.error:
             # "Connection reset by peer" or other socket errors aren't interesting here
             return

greentest/test__backdoor.py

@@ -24,12 +24,14 @@ def create_connection(address):
     conn.connect(address)
     return conn
 
+
 def readline(conn):
     f = conn.makefile()
     line = f.readline()
     f.close()
     return line
 
+
 class Test(greentest.TestCase):
 
     def test(self):

greentest/test__greenio.py

@@ -26,6 +26,7 @@ import sys
 PYPY = hasattr(sys, 'pypy_version_info')
 PY3 = sys.version_info[0] >= 3
 
+
 def _write_to_closed(f, s):
     try:
         r = f.write(s)

greentest/test__issue6.py

 from __future__ import print_function
 import sys
 
-
 if not sys.argv[1:]:
     from subprocess import Popen, PIPE
     p = Popen([sys.executable, __file__, 'subprocess'], stdin=PIPE, stdout=PIPE, stderr=PIPE)

greentest/test__makefile_ref.py

@@ -16,6 +16,7 @@ lsof_command = 'lsof -p %s > %s' % (pid, tmpname)
 import sys
 PY3 = sys.version_info[0] >= 3
 
+
 def get_open_files():
     if os.system(lsof_command):
         raise OSError('lsof failed')

greentest/test__pywsgi.py

@@ -28,6 +28,7 @@ try:
     from StringIO import StringIO
 except ImportError:
     from io import BytesIO as StringIO
+import weakref
 try:
     from wsgiref.validate import validator
 except ImportError:
@@ -56,7 +57,7 @@ try:
 except ImportError:
     pass
 
-if greentest.PYPY:
+if greentest.PYPY or PY3:
     from errno import ECONNRESET
     CONN_ABORTED_ERRORS.append(ECONNRESET)
 
@@ -197,8 +198,12 @@ class Response(object):
 
 read_http = Response.read
 
-
-class DebugFileObject(object):
+if not PY3:
+    # Under Python 3, socket.makefile does not use
+    # socket._fileobject; instead it uses the io package.
+    # We don't want to artificially interfere with that because
+    # then we won't be testing the actual code that's in use.
+    class DebugFileObject(object):
 
         def __init__(self, obj):
             self.obj = obj
@@ -219,11 +224,10 @@ class DebugFileObject(object):
             assert item != 'obj'
             return getattr(self.obj, item)
 
-
-def makefile(self, mode='r', bufsize=-1):
+    def makefile(self, mode='r', bufsize=-1):
         return DebugFileObject(socket._fileobject(self.dup(), mode, bufsize))
 
-socket.socket.makefile = makefile
+    socket.socket.makefile = makefile
 
 
 class TestCase(greentest.TestCase):
@@ -241,9 +245,20 @@ class TestCase(greentest.TestCase):
         self.init_server(application)
         self.server.start()
         self.port = self.server.server_port
+        # We keep a list of sockets/files we need to close so we
+        # don't get ResourceWarnings under Py3
+        self.connected = list()
         greentest.TestCase.setUp(self)
 
+    def close_opened(self):
+        for x in self.connected:
+            x = x()
+            if x is not None:
+                x.close()
+        self.connected = list()
+
     def tearDown(self):
+        self.close_opened()
         greentest.TestCase.tearDown(self)
         timeout = gevent.Timeout.start_new(0.5)
         try:
@@ -254,33 +269,32 @@ class TestCase(greentest.TestCase):
 
     def connect(self):
         conn = socket.create_connection(('127.0.0.1', self.port))
+        self.connected.append(weakref.ref(conn))
         result = conn
         if PY3:
             conn_makefile = conn.makefile
+
             def makefile(*args, **kwargs):
+                if 'bufsize' in kwargs:
+                    kwargs['buffering'] = kwargs.pop('bufsize')
+
                 if 'mode' in kwargs:
                     return conn_makefile(*args, **kwargs)
+
                 # Under Python3, you can't read and write to the same
-                # makefile() opened in r, and r+ is not allowed
-                kwargs['mode'] = 'rb'
+                # makefile() opened in (default) r, and r+ is not allowed
+                kwargs['mode'] = 'rwb'
                 rconn = conn_makefile(*args, **kwargs)
-                kwargs['mode'] = 'wb'
-                wconn = conn_makefile(**kwargs)
-                rconn._sock = conn
-                _rconn_close = rconn.close
+                _rconn_write = rconn.write
+
                 def write(data):
                     if isinstance(data, str):
                         data = data.encode('ascii')
-                    return wconn.write(data)
-                def flush():
-                    return wconn.flush()
-                def close():
-                    _rconn_close()
-                    wconn.close()
+                    return _rconn_write(data)
                 rconn.write = write
-                rconn.flush = flush
-                rconn.close = close
+                self.connected.append(weakref.ref(rconn))
                 return rconn
+
             class proxy(object):
                 def __getattribute__(self, name):
                     if name == 'makefile':
@@ -532,26 +546,26 @@ class TestChunkedPost(TestCase):
             data = env['wsgi.input'].read(6)
             return [data]
         elif env['PATH_INFO'] == '/b':
-            return [x for x in iter(lambda: env['wsgi.input'].read(6), '')]
+            lines = [x for x in iter(lambda: env['wsgi.input'].read(6), b'')]
+            return lines
         elif env['PATH_INFO'] == '/c':
-            return [x for x in iter(lambda: env['wsgi.input'].read(1), '')]
+            return [x for x in iter(lambda: env['wsgi.input'].read(1), b'')]
 
     def test_014_chunked_post(self):
         fd = self.makefile()
-        data = ('POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n'
-                'Transfer-Encoding: chunked\r\n\r\n'
-                '2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n')
+        data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n'
+                b'Transfer-Encoding: chunked\r\n\r\n'
+                b'2\r\noh\r\n4\r\n hai\r\n0\r\n\r\n')
         fd.write(data)
         read_http(fd, body='oh hai')
+        self.close_opened()
 
-        if not PY3:
-            # XXX: Problem with the chunked input or validater?
         fd = self.makefile()
-            fd.write(data.replace('/a', '/b'))
+        fd.write(data.replace(b'/a', b'/b'))
         read_http(fd, body='oh hai')
 
         fd = self.makefile()
-            fd.write(data.replace('/a', '/c'))
+        fd.write(data.replace(b'/a', b'/c'))
         read_http(fd, body='oh hai')
 
 
@@ -970,12 +984,14 @@ class ChunkedInputTests(TestCase):
     def test_short_read_with_content_length(self):
         body = self.body()
         req = b"POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\nContent-Length:1000\r\n\r\n" + body
-
-        fd = self.connect().makefile(bufsize=1)
+        conn = self.connect()
+        fd = conn.makefile(bufsize=1)
         fd.write(req)
         read_http(fd, body="this is ch")
 
         self.ping_if_possible(fd)
+        fd.close()
+        conn.close()
 
     def test_short_read_with_zero_content_length(self):
         body = self.body()
@@ -1017,7 +1033,9 @@ class ChunkedInputTests(TestCase):
 
     def test_chunked_readline(self):
         body = self.body()
-        req = "POST /lines HTTP/1.1\r\nContent-Length: %s\r\ntransfer-encoding: Chunked\r\n\r\n%s" % (len(body), body)
+        req = "POST /lines HTTP/1.1\r\nContent-Length: %s\r\ntransfer-encoding: Chunked\r\n\r\n" % (len(body))
+        req = req.encode('latin-1')
+        req += body
 
         fd = self.connect().makefile(bufsize=1)
         fd.write(req)
@@ -1028,9 +1046,21 @@ class ChunkedInputTests(TestCase):
             self.expect_one_error()
         body = b'4\r\nthi'
         req = b"POST /short-read HTTP/1.1\r\ntransfer-encoding: Chunked\r\n\r\n" + body
-        fd = self.connect().makefile(bufsize=1)
+        sock = self.connect()
+        fd = sock.makefile(bufsize=1, mode='wb')
         fd.write(req)
         fd.close()
+        if PY3:
+            # Python 3 keeps the socket open even though the only
+            # makefile is gone; python 2 closed them both (because there were
+            # no outstanding references to the socket). Closing is essential for the server
+            # to get the message that the read will fail. It's better to be explicit
+            # to avoid a ResourceWarning
+            sock.close()
+        else:
+            # Under Py2 it still needs to go away, which was implicit before
+            del sock
+
         gevent.sleep(0.01) # timing needed for cpython
 
         if server_implements_chunked:
@@ -1117,15 +1147,15 @@ class TestLeakInput(TestCase):
 
     def test_connection_close_leak_simple(self):
         fd = self.connect().makefile(bufsize=1)
-        fd.write("GET / HTTP/1.0\r\nConnection: close\r\n\r\n")
+        fd.write(b"GET / HTTP/1.0\r\nConnection: close\r\n\r\n")
         d = fd.read()
-        assert d.startswith("HTTP/1.1 200 OK"), "bad response: %r" % d
+        assert d.startswith(b"HTTP/1.1 200 OK"), "bad response: %r" % d
 
     def test_connection_close_leak_frame(self):
         fd = self.connect().makefile(bufsize=1)
-        fd.write("GET /leak-frame HTTP/1.0\r\nConnection: close\r\n\r\n")
+        fd.write(b"GET /leak-frame HTTP/1.0\r\nConnection: close\r\n\r\n")
         d = fd.read()
-        assert d.startswith("HTTP/1.1 200 OK"), "bad response: %r" % d
+        assert d.startswith(b"HTTP/1.1 200 OK"), "bad response: %r" % d
         self._leak_environ.pop('_leak')
 
 
@@ -1154,11 +1184,11 @@ class Handler(pywsgi.WSGIHandler):
 
     def read_requestline(self):
         data = self.rfile.read(7)
-        if data[0] == '<':
+        if data[0] == b'<'[0]:
             try:
                 data += self.rfile.read(15)
-                if data.lower() == '<policy-file-request/>':
-                    self.socket.sendall('HELLO')
+                if data.lower() == b'<policy-file-request/>':
+                    self.socket.sendall(b'HELLO')
                 else:
                     self.log_error('Invalid request: %r', data)
             finally:
@@ -1182,9 +1212,9 @@ class TestSubclass1(TestCase):
 
     def test(self):
         fd = self.makefile()
-        fd.write('<policy-file-request/>\x00')
+        fd.write(b'<policy-file-request/>\x00')
         fd.flush() # flush() is needed on PyPy, apparently it buffers slightly differently
-        self.assertEqual(fd.read(), 'HELLO')
+        self.assertEqual(fd.read(), b'HELLO')
 
         fd = self.makefile()
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
@@ -1194,7 +1224,7 @@ class TestSubclass1(TestCase):
         fd = self.makefile()
         fd.write('<policy-file-XXXuest/>\x00')
         fd.flush()
-        self.assertEqual(fd.read(), '')
+        self.assertEqual(fd.read(), b'')
 
 
 class TestErrorAfterChunk(TestCase):
@@ -1238,7 +1268,7 @@ class TestInputRaw(greentest.BaseTestCase):
         def assertEqual(self, data, expected, *args):
             if isinstance(expected, str):
                 expected = expected.encode('ascii')
-            super(TestInputRaw,self).assertEqual(data, expected, *args)
+            super(TestInputRaw, self).assertEqual(data, expected, *args)
 
     def test_short_post(self):
         i = self.make_input("1", content_length=2)

greentest/test__server.py

@@ -85,37 +85,22 @@ class TestCase(greentest.TestCase):
         sock.connect((self.server.server_host, self.server.server_port))
 
         if PY3:
-            kwargs = {'buffering': bufsize, 'mode': 'rb'}
+            # Under Python3, you can't read and write to the same
+            # makefile() opened in r, and r+ is not allowed
+            kwargs = {'buffering': bufsize, 'mode': 'rwb'}
         else:
             kwargs = {'bufsize': bufsize}
 
         rconn = sock.makefile(**kwargs)
         if PY3:
-            # Under Python3, you can't read and write to the same
-            # makefile() opened in r, and r+ is not allowed
-            kwargs['mode'] = 'wb'
-            wconn = sock.makefile(**kwargs)
             rconn._sock = sock
-            _rconn_close = rconn.close
-            def write(data):
-                if isinstance(data, str):
-                    data = data.encode('ascii')
-                return wconn.write(data)
-            def flush():
-                return wconn.flush()
-            def close():
-                _rconn_close()
-                wconn.close()
-            rconn.write = write
-            rconn.flush = flush
-            rconn.close = close
         rconn._sock.settimeout(timeout)
         sock.close()
         return rconn
 
     def send_request(self, url='/', timeout=0.1, bufsize=1):
         conn = self.makefile(timeout=timeout, bufsize=bufsize)
-        conn.write('GET %s HTTP/1.0\r\n\r\n' % url)
+        conn.write(('GET %s HTTP/1.0\r\n\r\n' % url).encode('latin-1'))
         conn.flush()
         return conn
 
@@ -141,7 +126,7 @@ class TestCase(greentest.TestCase):
 
     def assertNotAccepted(self):
         conn = self.makefile()
-        conn.write('GET / HTTP/1.0\r\n\r\n')
+        conn.write(b'GET / HTTP/1.0\r\n\r\n')
         conn.flush()
         result = ''
         try:

greentest/test__server_pywsgi.py

@@ -51,7 +51,7 @@ class Settings:
     @staticmethod
     def assert500(self):
         conn = self.makefile()
-        conn.write('GET / HTTP/1.0\r\n\r\n')
+        conn.write(b'GET / HTTP/1.0\r\n\r\n')
         result = conn.read()
         assert result.startswith(internal_error_start), (result, internal_error_start)
         assert result.endswith(internal_error_end), (result, internal_error_end)
@@ -61,7 +61,7 @@ class Settings:
     @staticmethod
     def assert503(self):
         conn = self.makefile()
-        conn.write('GET / HTTP/1.0\r\n\r\n')
+        conn.write(b'GET / HTTP/1.0\r\n\r\n')
         result = conn.read()
         assert result == internal_error503, (result, internal_error503)
 

greentest/test_issue112.py

@@ -5,5 +5,4 @@ if sys.version_info[0] == 2:
     gevent.monkey.patch_all()
     import gevent
 
-
     assert threading._sleep is gevent.sleep, threading._sleep

known_failures.py

@@ -81,7 +81,6 @@ if PYPY:
 if PY3:
     # No idea / TODO
     FAILING_TESTS += '''
-test__pywsgi.py
 FLAKY test__socket_dns.py
 '''.strip().split('\n')