Commit d07e4bb6 authored by Jason Madden's avatar Jason Madden

Merge pull request #581 from gevent/py3fileobject

Fix Python 3 test failures. Thanks to the contributors of the following issues (at least): Fixes #412. Fixes #413. Fixes #416. Fixes #421. Fixes #462. Fixes #409.
parents 81846faf 668765cb
[pep8]
ignore=E702,E265,E402,E731,E266,E261,W503
ignore=E702,E265,E402,E731,E266,E261,W503,E129
max_line_length=160
exclude=.tox,.git,build,2.6,2.7,2.7pypy,3.3,test_support.py,test_queue.py,patched_tests_setup.py,test_threading_2.py,lock_tests.py,_sslgte279.py
......@@ -13,21 +13,20 @@ from gevent.server import StreamServer
# this handler will be run for each incoming connection in a dedicated greenlet
def echo(socket, address):
print('New connection from %s:%s' % address)
socket.sendall('Welcome to the echo server! Type quit to exit.\r\n')
socket.sendall(b'Welcome to the echo server! Type quit to exit.\r\n')
# using a makefile because we want to use readline()
fileobj = socket.makefile()
rfileobj = socket.makefile(mode='rb')
while True:
line = fileobj.readline()
line = rfileobj.readline()
if not line:
print("client disconnected")
break
if line.strip().lower() == 'quit':
if line.strip().lower() == b'quit':
print("client quit")
break
fileobj.write(line)
fileobj.flush()
socket.sendall(line)
print("echoed %r" % line)
rfileobj.close()
if __name__ == '__main__':
# to make the server use SSL, pass certfile and keyfile arguments to the constructor
......
......@@ -13,7 +13,7 @@ class EchoServer(DatagramServer):
def handle(self, data, address):
print('%s: got %r' % (address[0], data))
self.socket.sendto('Received %s bytes' % len(data), address)
self.socket.sendto(('Received %s bytes' % len(data)).encode('utf-8'), address)
if __name__ == '__main__':
......
......@@ -105,8 +105,10 @@ def fix_links(data, proxy_url, host_url):
result = m.group('before') + '"' + join(proxy_url, host_url, url) + '"'
#print('replaced %r -> %r' % (m.group(0), result))
return result
data = data.decode('latin-1') # XXX Assuming charset. Can regexes work with bytes data?
data = _link_re_1.sub(fix_link_cb, data)
data = _link_re_2.sub(fix_link_cb, data)
data = data.encode('latin-1')
return data
_link_re_1 = re.compile('''(?P<before>(href|src|action)\s*=\s*)(?P<quote>['"])(?P<url>[^#].*?)(?P=quote)''')
......@@ -114,7 +116,7 @@ _link_re_2 = re.compile('''(?P<before>(href|src|action)\s*=\s*)(?P<url>[^'"#>][^
drop_headers = ['transfer-encoding', 'set-cookie']
FORM = """<html><head>
FORM = b"""<html><head>
<title>Web Proxy - gevent example</title></head><body>
<table width=60% height=100% align=center>
<tr height=30%><td align=center valign=bottom>Type in URL you want to visit and press Enter</td></tr>
......
......@@ -7,10 +7,10 @@ from gevent.pywsgi import WSGIServer
def application(env, start_response):
if env['PATH_INFO'] == '/':
start_response('200 OK', [('Content-Type', 'text/html')])
return ["<b>hello world</b>"]
return [b"<b>hello world</b>"]
else:
start_response('404 Not Found', [('Content-Type', 'text/html')])
return ['<h1>Not Found</h1>']
return [b'<h1>Not Found</h1>']
if __name__ == '__main__':
......
......@@ -8,10 +8,10 @@ from gevent import pywsgi
def hello_world(env, start_response):
if env['PATH_INFO'] == '/':
start_response('200 OK', [('Content-Type', 'text/html')])
return ["<b>hello world</b>"]
return [b"<b>hello world</b>"]
else:
start_response('404 Not Found', [('Content-Type', 'text/html')])
return ['<h1>Not Found</h1>']
return [b'<h1>Not Found</h1>']
print('Serving on https://127.0.0.1:8443')
server = pywsgi.WSGIServer(('0.0.0.0', 8443), hello_world, keyfile='server.key', certfile='server.crt')
......
from __future__ import absolute_import
import os
import sys
from types import UnboundMethodType
from gevent._fileobjectcommon import cancel_wait_ex
from gevent._fileobjectcommon import FileObjectClosed
from gevent._socket2 import _fileobject
from gevent._socket2 import _get_memory
from gevent.hub import get_hub, PYPY, integer_types
from gevent.os import _read
from gevent.os import _write
from gevent.os import ignored_errors
from gevent.os import make_nonblocking
from gevent.socket import EBADF
try:
from gevent._util import SocketAdapter__del__, noop
except ImportError:
SocketAdapter__del__ = None
noop = None
class NA(object):
def __repr__(self):
return 'N/A'
NA = NA()
class SocketAdapter(object):
"""Socket-like API on top of a file descriptor.
The main purpose of it is to re-use _fileobject to create proper cooperative file objects
from file descriptors on POSIX platforms.
"""
def __init__(self, fileno, mode=None, close=True):
if not isinstance(fileno, integer_types):
raise TypeError('fileno must be int: %r' % fileno)
self._fileno = fileno
self._mode = mode or 'rb'
self._close = close
self._translate = 'U' in self._mode
make_nonblocking(fileno)
self._eat_newline = False
self.hub = get_hub()
io = self.hub.loop.io
self._read_event = io(fileno, 1)
self._write_event = io(fileno, 2)
self._refcount = 1
def __repr__(self):
if self._fileno is None:
return '<%s at 0x%x closed>' % (self.__class__.__name__, id(self))
else:
args = (self.__class__.__name__, id(self), getattr(self, '_fileno', NA), getattr(self, '_mode', NA))
return '<%s at 0x%x (%r, %r)>' % args
def makefile(self, *args, **kwargs):
return _fileobject(self, *args, **kwargs)
def fileno(self):
result = self._fileno
if result is None:
raise IOError(EBADF, 'Bad file descriptor (%s object is closed)' % self.__class__.__name__)
return result
def detach(self):
x = self._fileno
self._fileno = None
return x
def _reuse(self):
self._refcount += 1
def _drop(self):
self._refcount -= 1
if self._refcount <= 0:
self._realclose()
def close(self):
self._drop()
def _realclose(self):
self.hub.cancel_wait(self._read_event, cancel_wait_ex)
self.hub.cancel_wait(self._write_event, cancel_wait_ex)
fileno = self._fileno
if fileno is not None:
self._fileno = None
if self._close:
os.close(fileno)
def sendall(self, data):
fileno = self.fileno()
bytes_total = len(data)
bytes_written = 0
while True:
try:
bytes_written += _write(fileno, _get_memory(data, bytes_written))
except (IOError, OSError) as ex:
code = ex.args[0]
if code not in ignored_errors:
raise
sys.exc_clear()
if bytes_written >= bytes_total:
return
self.hub.wait(self._write_event)
def recv(self, size):
while True:
try:
data = _read(self.fileno(), size)
except (IOError, OSError) as ex:
code = ex.args[0]
if code not in ignored_errors:
raise
sys.exc_clear()
else:
if not self._translate or not data:
return data
if self._eat_newline:
self._eat_newline = False
if data.startswith(b'\n'):
data = data[1:]
if not data:
return self.recv(size)
if data.endswith(b'\r'):
self._eat_newline = True
return self._translate_newlines(data)
self.hub.wait(self._read_event)
def _translate_newlines(self, data):
data = data.replace(b"\r\n", b"\n")
data = data.replace(b"\r", b"\n")
return data
if not SocketAdapter__del__:
def __del__(self, close=os.close):
fileno = self._fileno
if fileno is not None:
close(fileno)
if SocketAdapter__del__:
SocketAdapter.__del__ = UnboundMethodType(SocketAdapter__del__, None, SocketAdapter)
class FileObjectPosix(_fileobject):
def __init__(self, fobj=None, mode='rb', bufsize=-1, close=True):
if isinstance(fobj, integer_types):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
sock = SocketAdapter(fileno, mode, close=close)
self._fobj = fobj
self._closed = False
_fileobject.__init__(self, sock, mode=mode, bufsize=bufsize, close=close)
if PYPY:
sock._drop()
def __repr__(self):
if self._sock is None:
return '<%s closed>' % self.__class__.__name__
elif self._fobj is None:
return '<%s %s>' % (self.__class__.__name__, self._sock)
else:
return '<%s %s _fobj=%r>' % (self.__class__.__name__, self._sock, self._fobj)
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
# cannot rely on self._sock for this because we need to keep that until flush() is done
return
self._closed = True
sock = self._sock
if sock is None:
return
try:
self.flush()
finally:
if self._fobj is not None or not self._close:
sock.detach()
else:
sock._drop()
self._sock = None
self._fobj = None
def __getattr__(self, item):
assert item != '_fobj'
if self._fobj is None:
raise FileObjectClosed
return getattr(self._fobj, item)
if not noop:
def __del__(self):
# disable _fileobject's __del__
pass
if noop:
FileObjectPosix.__del__ = UnboundMethodType(FileObjectPosix, None, noop)
import os
import io
from io import BufferedRandom
from io import BufferedReader
from io import BufferedWriter
from io import BytesIO
from io import DEFAULT_BUFFER_SIZE
from io import RawIOBase
from io import TextIOWrapper
from io import UnsupportedOperation
from gevent._fileobjectcommon import cancel_wait_ex
from gevent.hub import get_hub
from gevent.os import _read
from gevent.os import _write
from gevent.os import ignored_errors
from gevent.os import make_nonblocking
class GreenFileDescriptorIO(RawIOBase):
def __init__(self, fileno, mode='r', closefd=True):
RawIOBase.__init__(self)
self._closed = False
self._closefd = closefd
self._fileno = fileno
make_nonblocking(fileno)
self._readable = 'r' in mode
self._writable = 'w' in mode
self.hub = get_hub()
io = self.hub.loop.io
if self._readable:
self._read_event = io(fileno, 1)
else:
self._read_event = None
if self._writable:
self._write_event = io(fileno, 2)
else:
self._write_event = None
def readable(self):
return self._readable
def writable(self):
return self._writable
def fileno(self):
return self._fileno
@property
def closed(self):
return self._closed
def close(self):
if self._closed:
return
self.flush()
self._closed = True
if self._readable:
self.hub.cancel_wait(self._read_event, cancel_wait_ex)
if self._writable:
self.hub.cancel_wait(self._write_event, cancel_wait_ex)
fileno = self._fileno
if self._closefd:
self._fileno = None
os.close(fileno)
def read(self, n=1):
if not self._readable:
raise UnsupportedOperation('readinto')
while True:
try:
return _read(self._fileno, n)
except (IOError, OSError) as ex:
if ex.args[0] not in ignored_errors:
raise
self.hub.wait(self._read_event)
def readall(self):
ret = BytesIO()
while True:
data = self.read(DEFAULT_BUFFER_SIZE)
if not data:
break
ret.write(data)
return ret.getvalue()
def readinto(self, b):
data = self.read(len(b))
n = len(data)
try:
b[:n] = data
except TypeError as err:
import array
if not isinstance(b, array.array):
raise err
b[:n] = array.array(b'b', data)
return n
def write(self, b):
if not self._writable:
raise UnsupportedOperation('write')
while True:
try:
return _write(self._fileno, b)
except (IOError, OSError) as ex:
if ex.args[0] not in ignored_errors:
raise
self.hub.wait(self._write_event)
class FileObjectPosix:
default_bufsize = io.DEFAULT_BUFFER_SIZE
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
if isinstance(fobj, int):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
if not isinstance(fileno, int):
raise TypeError('fileno must be int: %r' % fileno)
mode = (mode or 'rb').replace('b', '')
if 'U' in mode:
self._translate = True
mode = mode.replace('U', '')
else:
self._translate = False
assert len(mode) == 1, 'mode can only be [rb, rU, wb]'
self._fobj = fobj
self._closed = False
self._close = close
self.fileio = GreenFileDescriptorIO(fileno, mode, closefd=close)
if bufsize < 0:
bufsize = self.default_bufsize
if mode == 'r':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedReader(self.fileio, bufsize)
elif mode == 'w':
if bufsize == 0:
bufsize = 1
elif bufsize == 1:
bufsize = self.default_bufsize
self.io = BufferedWriter(self.fileio, bufsize)
else:
# QQQ: not used
self.io = BufferedRandom(self.fileio, bufsize)
if self._translate:
self.io = TextIOWrapper(self.io)
@property
def closed(self):
"""True if the file is cloed"""
return self._closed
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
return
self._closed = True
try:
self.io.close()
self.fileio.close()
finally:
self._fobj = None
def flush(self):
self.io.flush()
def fileno(self):
return self.io.fileno()
def write(self, data):
self.io.write(data)
def writelines(self, list):
self.io.writelines(list)
def read(self, size=-1):
return self.io.read(size)
def readline(self, size=-1):
return self.io.readline(size)
def readlines(self, sizehint=0):
return self.io.readlines(sizehint)
def __iter__(self):
return self.io
from gevent._socketcommon import EBADF
cancel_wait_ex = IOError(EBADF, 'File descriptor was closed in another greenlet')
FileObjectClosed = IOError(EBADF, 'Bad file descriptor (FileObject was closed)')
This diff is collapsed.
......@@ -42,7 +42,17 @@ __imports__ = ['error',
'getdefaulttimeout',
'setdefaulttimeout',
# Windows:
'errorTab']
'errorTab',
# Python 3
'AddressFamily',
'SocketKind',
'CMSG_LEN',
'CMSG_SPACE',
'dup',
'if_indextoname',
'if_nameindex',
'if_nametoindex',
'sethostname']
import sys
......@@ -212,7 +222,10 @@ def getfqdn(name=''):
else:
aliases.insert(0, hostname)
for name in aliases:
if '.' in name:
if isinstance(name, bytes):
if b'.' in name:
break
elif '.' in name:
break
else:
name = hostname
......
......@@ -22,6 +22,7 @@ __implements__ = ['SSLContext',
'wrap_socket',
'get_server_certificate']
__imports__ = []
for name in dir(__ssl__):
if name in __implements__:
......@@ -30,10 +31,11 @@ for name in dir(__ssl__):
continue
value = getattr(__ssl__, name)
globals()[name] = value
__imports__.append(name)
del name, value
__all__ = __implements__ + __imports__
orig_SSLContext = __ssl__.SSLContext
......@@ -126,7 +128,7 @@ class SSLSocket(socket):
if connected:
# create the SSL object
try:
self._sslobj = self.context._wrap_socket(self, server_side,
self._sslobj = self.context._wrap_socket(getattr(self, '_sock', self), server_side,
server_hostname)
if do_handshake_on_connect:
timeout = self.gettimeout()
......
......@@ -72,6 +72,7 @@ class BackdoorServer(StreamServer):
except ImportError:
import builtins
console.locals["builtins"] = builtins
console.locals['__builtins__'] = builtins
console.interact(banner=self.banner)
except SystemExit: # raised by quit()
if not PY3:
......@@ -89,8 +90,14 @@ class BackdoorServer(StreamServer):
class _fileobject(socket._fileobject):
def write(self, data):
self._sock.sendall(data)
if not PY3:
def write(self, data):
self._sock.sendall(data)
else:
def write(self, data):
if isinstance(data, str):
data = data.encode('utf-8')
self._sock.sendall(data)
def isatty(self):
return True
......@@ -98,8 +105,15 @@ class _fileobject(socket._fileobject):
def flush(self):
pass
def readline(self, *a):
return socket._fileobject.readline(self, *a).replace("\r\n", "\n")
def _readline(self, *a):
return socket._fileobject.readline(self, *a).replace(b"\r\n", b"\n")
if not PY3:
readline = _readline
else:
def readline(self, *a):
line = self._readline(*a)
return line.decode('utf-8')
if __name__ == '__main__':
if not sys.argv[1:]:
......
......@@ -496,7 +496,7 @@ cdef public class loop [object PyGeventLoopObject, type PyGeventLoop_Type]:
#endif
def stat(self, bytes path, float interval=0.0, ref=True, priority=None):
def stat(self, str path, float interval=0.0, ref=True, priority=None):
return stat(self, path, interval, ref, priority)
def run_callback(self, func, *args):
......@@ -1036,11 +1036,21 @@ cdef public class child(watcher) [object PyGeventChildObject, type PyGeventChild
cdef public class stat(watcher) [object PyGeventStatObject, type PyGeventStat_Type]:
WATCHER(stat)
cdef readonly bytes path
cdef readonly str path
cdef readonly bytes _paths
def __init__(self, loop loop, bytes path, float interval=0.0, ref=True, priority=None):
def __init__(self, loop loop, str path, float interval=0.0, ref=True, priority=None):
self.path = path
libev.ev_stat_init(&self._watcher, <void *>gevent_callback_stat, <char*>self.path, interval)
cdef bytes paths
if isinstance(path, unicode):
# the famous Python3 filesystem encoding debacle hits us here. Can we do better?
# We must keep a reference to the encoded string so that its bytes don't get freed
# and overwritten, leading to strange errors from libev ("no such file or directory")
paths = (<unicode>path).encode(sys.getfilesystemencoding())
self._paths = paths
else:
paths = <bytes>path
libev.ev_stat_init(&self._watcher, <void *>gevent_callback_stat, <char*>paths, interval)
self.loop = loop
if ref:
self._flags = 0
......
from __future__ import absolute_import
import sys
import os
from gevent._fileobjectcommon import FileObjectClosed
from gevent.hub import get_hub
from gevent.hub import integer_types
from gevent.socket import EBADF
from gevent.os import _read, _write, ignored_errors
from gevent.hub import PY3
from gevent.lock import Semaphore, DummySemaphore
PYPY = hasattr(sys, 'pypy_version_info')
if hasattr(sys, 'exc_clear'):
def _exc_clear():
sys.exc_clear()
else:
def _exc_clear():
return
try:
from fcntl import fcntl
......@@ -28,198 +34,10 @@ if fcntl is None:
else:
from gevent.socket import _fileobject, _get_memory
cancel_wait_ex = IOError(EBADF, 'File descriptor was closed in another greenlet')
from gevent.os import make_nonblocking
try:
from gevent._util import SocketAdapter__del__, noop
except ImportError:
SocketAdapter__del__ = None
noop = None
from types import UnboundMethodType
class NA(object):
def __repr__(self):
return 'N/A'
NA = NA()
class SocketAdapter(object):
"""Socket-like API on top of a file descriptor.
The main purpose of it is to re-use _fileobject to create proper cooperative file objects
from file descriptors on POSIX platforms.
"""
def __init__(self, fileno, mode=None, close=True):
if not isinstance(fileno, integer_types):
raise TypeError('fileno must be int: %r' % fileno)
self._fileno = fileno
self._mode = mode or 'rb'
self._close = close
self._translate = 'U' in self._mode
make_nonblocking(fileno)
self._eat_newline = False
self.hub = get_hub()
io = self.hub.loop.io
self._read_event = io(fileno, 1)
self._write_event = io(fileno, 2)
self._refcount = 1
def __repr__(self):
if self._fileno is None:
return '<%s at 0x%x closed>' % (self.__class__.__name__, id(self))
else:
args = (self.__class__.__name__, id(self), getattr(self, '_fileno', NA), getattr(self, '_mode', NA))
return '<%s at 0x%x (%r, %r)>' % args
def makefile(self, *args, **kwargs):
return _fileobject(self, *args, **kwargs)
def fileno(self):
result = self._fileno
if result is None:
raise IOError(EBADF, 'Bad file descriptor (%s object is closed)' % self.__class__.__name)
return result
def detach(self):
x = self._fileno
self._fileno = None
return x
def _reuse(self):
self._refcount += 1
def _drop(self):
self._refcount -= 1
if self._refcount <= 0:
self._realclose()
def close(self):
self._drop()
def _realclose(self):
self.hub.cancel_wait(self._read_event, cancel_wait_ex)
self.hub.cancel_wait(self._write_event, cancel_wait_ex)
fileno = self._fileno
if fileno is not None:
self._fileno = None
if self._close:
os.close(fileno)
def sendall(self, data):
fileno = self.fileno()
bytes_total = len(data)
bytes_written = 0
while True:
try:
bytes_written += _write(fileno, _get_memory(data, bytes_written))
except (IOError, OSError) as ex:
code = ex.args[0]
if code not in ignored_errors:
raise
sys.exc_clear()
if bytes_written >= bytes_total:
return
self.hub.wait(self._write_event)
def recv(self, size):
while True:
try:
data = _read(self.fileno(), size)
except (IOError, OSError) as ex:
code = ex.args[0]
if code not in ignored_errors:
raise
sys.exc_clear()
else:
if not self._translate or not data:
return data
if self._eat_newline:
self._eat_newline = False
if data.startswith('\n'):
data = data[1:]
if not data:
return self.recv(size)
if data.endswith('\r'):
self._eat_newline = True
return self._translate_newlines(data)
self.hub.wait(self._read_event)
def _translate_newlines(self, data):
data = data.replace("\r\n", "\n")
data = data.replace("\r", "\n")
return data
if not SocketAdapter__del__:
def __del__(self, close=os.close):
fileno = self._fileno
if fileno is not None:
close(fileno)
if SocketAdapter__del__:
SocketAdapter.__del__ = UnboundMethodType(SocketAdapter__del__, None, SocketAdapter)
class FileObjectPosix(_fileobject):
def __init__(self, fobj, mode='rb', bufsize=-1, close=True):
if isinstance(fobj, integer_types):
fileno = fobj
fobj = None
else:
fileno = fobj.fileno()
sock = SocketAdapter(fileno, mode, close=close)
self._fobj = fobj
self._closed = False
_fileobject.__init__(self, sock, mode=mode, bufsize=bufsize, close=close)
if PYPY:
sock._drop()
def __repr__(self):
if self._sock is None:
return '<%s closed>' % self.__class__.__name__
elif self._fobj is None:
return '<%s %s>' % (self.__class__.__name__, self._sock)
else:
return '<%s %s _fobj=%r>' % (self.__class__.__name__, self._sock, self._fobj)
def close(self):
if self._closed:
# make sure close() is only ran once when called concurrently
# cannot rely on self._sock for this because we need to keep that until flush() is done
return
self._closed = True
sock = self._sock
if sock is None:
return
try:
self.flush()
finally:
if self._fobj is not None or not self._close:
sock.detach()
else:
sock._drop()
self._sock = None
self._fobj = None
def __getattr__(self, item):
assert item != '_fobj'
if self._fobj is None:
raise FileObjectClosed
return getattr(self._fobj, item)
if not noop:
def __del__(self):
# disable _fileobject's __del__
pass
if noop:
FileObjectPosix.__del__ = UnboundMethodType(FileObjectPosix, None, noop)
if PY3:
from gevent._fileobject3 import FileObjectPosix
else:
from gevent._fileobject2 import FileObjectPosix
class FileObjectThread(object):
......@@ -295,9 +113,7 @@ class FileObjectThread(object):
if line:
return line
raise StopIteration
FileObjectClosed = IOError(EBADF, 'Bad file descriptor (FileObject was closed)')
__next__ = next
try:
......
......@@ -17,9 +17,11 @@ __all__ = ['patch_all',
if sys.version_info[0] >= 3:
string_types = str,
PY3 = True
else:
import __builtin__
string_types = __builtin__.basestring
PY3 = False
# maps module name -> attribute name -> original item
......@@ -85,6 +87,20 @@ def _patch_sys_std(name):
def patch_sys(stdin=True, stdout=True, stderr=True):
"""Patch sys.std[in,out,err] to use a cooperative IO via a threadpool.
This is relatively dangerous and can have unintended consequences such as hanging
the process or misinterpreting control keys.
This method does nothing on Python 3. The Python 3 interpreter wants to flush
the TextIOWrapper objects that make up stderr/stdout at shutdown time, but
using a threadpool at that time leads to a hang.
"""
# test__issue6.py demonstrates the hang if these lines are removed;
# strangely enough that test passes even without monkey-patching sys
if PY3:
return
if stdin:
_patch_sys_std('stdin')
if stdout:
......@@ -122,6 +138,38 @@ def patch_thread(threading=True, _threading_local=True, Event=False):
from gevent.local import local
_threading_local.local = local
if sys.version_info[:2] >= (3, 4):
# Issue 18808 changes the nature of Thread.join() to use
# locks. This means that a greenlet spawned in the main thread
# (which is already running) cannot wait for the main thread---it
# hangs forever. We patch around this if possible. See also
# gevent.threading.
threading = __import__('threading')
greenlet = __import__('greenlet')
if threading.current_thread() == threading.main_thread():
main_thread = threading.main_thread()
_greenlet = main_thread._greenlet = greenlet.getcurrent()
from .hub import sleep
def join(timeout=None):
if threading.current_thread() is main_thread:
raise RuntimeError("Cannot join current thread")
if _greenlet.dead or not main_thread.is_alive():
return
elif timeout:
raise ValueError("Cannot use a timeout to join the main thread")
# XXX: Make that work
else:
while main_thread.is_alive():
sleep(0.01)
main_thread.join = join
else:
# TODO: Can we use warnings here or does that mess up monkey patching?
print("Monkey-patching not on the main thread; "
"threading.main_thread().join() will hang from a greenlet",
file=sys.stderr)
def patch_socket(dns=True, aggressive=True):
"""Replace the standard socket object with gevent's cooperative sockets.
......@@ -169,6 +217,18 @@ def patch_select(aggressive=True):
remove_item(select, 'kqueue')
remove_item(select, 'kevent')
if sys.version_info[:2] >= (3, 4):
# Python 3 wants to use `select.select` as a member function,
# leading to this error in selectors.py
# r, w, _ = self._select(self._readers, self._writers, [], timeout)
# TypeError: select() takes from 3 to 4 positional arguments but 5 were given
select = __import__('select')
selectors = __import__('selectors')
if selectors.SelectSelector._select is select.select:
def _select(self, *args, **kwargs):
return select.select(*args, **kwargs)
selectors.SelectSelector._select = _select
def patch_subprocess():
patch_module('subprocess')
......
......@@ -27,13 +27,13 @@ _MONTHNAME = [None, # Dummy so we can use 1-based month numbers
"Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
_INTERNAL_ERROR_STATUS = '500 Internal Server Error'
_INTERNAL_ERROR_BODY = 'Internal Server Error'
_INTERNAL_ERROR_BODY = b'Internal Server Error'
_INTERNAL_ERROR_HEADERS = [('Content-Type', 'text/plain'),
('Connection', 'close'),
('Content-Length', str(len(_INTERNAL_ERROR_BODY)))]
_REQUEST_TOO_LONG_RESPONSE = "HTTP/1.1 414 Request URI Too Long\r\nConnection: close\r\nContent-length: 0\r\n\r\n"
_BAD_REQUEST_RESPONSE = "HTTP/1.1 400 Bad Request\r\nConnection: close\r\nContent-length: 0\r\n\r\n"
_CONTINUE_RESPONSE = "HTTP/1.1 100 Continue\r\n\r\n"
_REQUEST_TOO_LONG_RESPONSE = b"HTTP/1.1 414 Request URI Too Long\r\nConnection: close\r\nContent-length: 0\r\n\r\n"
_BAD_REQUEST_RESPONSE = b"HTTP/1.1 400 Bad Request\r\nConnection: close\r\nContent-length: 0\r\n\r\n"
_CONTINUE_RESPONSE = b"HTTP/1.1 100 Continue\r\n\r\n"
def format_date_time(timestamp):
......@@ -73,7 +73,7 @@ class Input(object):
if content_length is None:
# Either Content-Length or "Transfer-Encoding: chunked" must be present in a request with a body
# if it was chunked, then this function would have not been called
return ''
return b''
self._send_100_continue()
left = content_length - self.position
if length is None:
......@@ -81,11 +81,11 @@ class Input(object):
elif length > left:
length = left
if not length:
return ''
return b''
read = reader(length)
self.position += len(read)
if len(read) < length:
if (use_readline and not read.endswith("\n")) or not use_readline:
if (use_readline and not read.endswith(b"\n")) or not use_readline:
raise IOError("unexpected end of file while reading request at position %s" % (self.position,))
return read
......@@ -95,9 +95,9 @@ class Input(object):
self._send_100_continue()
if length == 0:
return ""
return b""
if length < 0:
if length is not None and length < 0:
length = None
if use_readline:
......@@ -128,18 +128,19 @@ class Input(object):
length -= datalen
if length == 0:
break
if use_readline and data[-1] == "\n":
if use_readline and data[-1] == b"\n"[0]:
break
else:
line = rfile.readline()
if not line.endswith("\n"):
if not line.endswith(b"\n"):
self.chunk_length = 0
raise IOError("unexpected end of file while reading chunked data header")
self.chunk_length = int(line.split(";", 1)[0], 16)
self.chunk_length = int(line.split(b";", 1)[0], 16)
self.position = 0
if self.chunk_length == 0:
rfile.readline()
return ''.join(response)
return b''.join(response)
def read(self, length=None):
if self.chunked_input:
......@@ -163,6 +164,7 @@ class Input(object):
if not line:
raise StopIteration
return line
__next__ = next
try:
......@@ -200,7 +202,13 @@ except ImportError:
class WSGIHandler(object):
protocol_version = 'HTTP/1.1'
MessageClass = headers_factory
if PY3:
# if we do like Py2, then headers_factory unconditionally
# becomes a bound method, meaning the fp argument becomes WSGIHandler
def MessageClass(self, *args):
return headers_factory(*args)
else:
MessageClass = headers_factory
def __init__(self, socket, address, server, rfile=None):
self.socket = socket
......@@ -229,13 +237,16 @@ class WSGIHandler(object):
break
finally:
if self.socket is not None:
_sock = getattr(self.socket, '_sock', None) # Python 3
try:
# read out request data to prevent error: [Errno 104] Connection reset by peer
try:
self.socket._sock.recv(16384)
finally:
self.socket._sock.close() # do not rely on garbage collection
self.socket.close()
if _sock:
try:
# socket.recv would hang
_sock.recv(16384)
finally:
_sock.close()
self.socket.close()
except socket.error:
pass
self.__dict__.pop('socket', None)
......@@ -270,6 +281,7 @@ class WSGIHandler(object):
return
self.headers = self.MessageClass(self.rfile, 0)
if self.headers.status:
self.log_error('Invalid headers status: %r', self.headers.status)
return
......@@ -319,14 +331,19 @@ class WSGIHandler(object):
traceback.print_exc()
def read_requestline(self):
return self.rfile.readline(MAX_REQUEST_LINE)
line = self.rfile.readline(MAX_REQUEST_LINE)
if PY3:
line = line.decode('latin-1')
return line
def handle_one_request(self):
if self.rfile.closed:
return
try:
self.requestline = self.read_requestline()
# Account for old subclasses that haven't done this
if PY3 and isinstance(self.requestline, bytes):
self.requestline = self.requestline.decode('latin-1')
except socket.error:
# "Connection reset by peer" or other socket errors aren't interesting here
return
......@@ -399,7 +416,7 @@ class WSGIHandler(object):
return
if self.response_use_chunked:
## Write the chunked encoding
data = "%x\r\n%s\r\n" % (len(data), data)
data = ("%x\r\n" % len(data)).encode('ascii') + data + b'\r\n'
self._sendall(data)
def write(self, data):
......@@ -418,17 +435,22 @@ class WSGIHandler(object):
self.headers_sent = True
self.finalize_headers()
towrite.extend('HTTP/1.1 %s\r\n' % self.status)
towrite.extend(('HTTP/1.1 %s\r\n' % self.status).encode('latin-1'))
for header in self.response_headers:
towrite.extend('%s: %s\r\n' % header)
towrite.extend(('%s: %s\r\n' % header).encode('latin-1'))
towrite.extend('\r\n')
towrite.extend(b'\r\n')
if data:
if self.response_use_chunked:
## Write the chunked encoding
towrite.extend("%x\r\n%s\r\n" % (len(data), data))
else:
towrite.extend(("%x\r\n" % len(data)).encode('latin-1'))
towrite.extend(data)
towrite.extend(b"\r\n")
else:
try:
towrite.extend(data)
except TypeError:
raise TypeError("Not an bytestring", data)
self._sendall(towrite)
def start_response(self, status, headers, exc_info=None):
......@@ -466,6 +488,8 @@ class WSGIHandler(object):
if self.code in (304, 204):
if self.provided_content_length is not None and self.provided_content_length != '0':
msg = 'Invalid Content-Length for %s response: %r (must be absent or zero)' % (self.code, self.provided_content_length)
if PY3:
msg = msg.encode('latin-1')
raise AssertionError(msg)
return self.write
......@@ -496,9 +520,9 @@ class WSGIHandler(object):
if data:
self.write(data)
if self.status and not self.headers_sent:
self.write('')
self.write(b'')
if self.response_use_chunked:
self.socket.sendall('0\r\n\r\n')
self.socket.sendall(b'0\r\n\r\n')
self.response_length += 5
def run_application(self):
......@@ -658,6 +682,8 @@ class WSGIServer(StreamServer):
name = socket.getfqdn(address[0])
except socket.error:
name = str(address[0])
if PY3 and not isinstance(name, str):
name = name.decode('ascii')
self.environ['SERVER_NAME'] = name
self.environ.setdefault('SERVER_PORT', str(address[1]))
else:
......
......@@ -208,6 +208,8 @@ class Resolver(object):
if _ip_address == ip_address:
raise
waiter.clear()
if isinstance(_ip_address, text_type):
_ip_address = _ip_address.encode('ascii')
self.ares.gethostbyaddr(waiter, _ip_address)
return waiter.get()
......
......@@ -98,7 +98,7 @@ class StreamServer(BaseServer):
try:
fd, address = sock._accept()
except BlockingIOError:
if sock.timeout == 0.0:
if not sock.timeout:
return
raise
sock = socket(sock.family, sock.type, sock.proto, fileno=fd)
......
......@@ -56,6 +56,10 @@ __extra__ = ['MAXFD',
'INFINITE',
'TerminateProcess']
if sys.version_info[:2] >= (3, 3):
__imports__ += ['DEVNULL',
'getstatusoutput',
'getoutput']
for name in __imports__[:]:
try:
......@@ -142,14 +146,14 @@ def check_output(*popenargs, **kwargs):
The arguments are the same as for the Popen constructor. Example:
>>> print(check_output(["ls", "-1", "/dev/null"]))
>>> print(check_output(["ls", "-1", "/dev/null"]).decode('ascii'))
/dev/null
<BLANKLINE>
The stdout argument is not allowed as it is used internally.
To capture standard error in the result, use stderr=STDOUT.
>>> print(check_output(["/bin/sh", "-c", "echo hello world"], stderr=STDOUT))
>>> print(check_output(["/bin/sh", "-c", "echo hello world"], stderr=STDOUT).decode('ascii'))
hello world
<BLANKLINE>
"""
......
......@@ -23,6 +23,12 @@ if sys.version_info[0] <= 2:
else:
import _thread as __thread__
__target__ = '_thread'
__imports__ += ['RLock',
'TIMEOUT_MAX',
'allocate',
'exit_thread',
'interrupt_main',
'start_new']
error = __thread__.error
from gevent.hub import getcurrent, GreenletExit
from gevent.greenlet import Greenlet
......@@ -63,6 +69,12 @@ if hasattr(__thread__, 'stack_size'):
else:
__implements__.remove('stack_size')
for name in __imports__[:]:
try:
value = getattr(__thread__, name)
globals()[name] = value
except AttributeError:
__imports__.remove(name)
__all__ = __implements__ + __imports__
__all__.remove('_local')
......
......@@ -69,10 +69,14 @@ if sys.version_info[:2] >= (3, 4):
try:
super(Thread, self).run()
finally:
del self._greenlet # avoid ref cycles
# avoid ref cycles, but keep in __dict__ so we can
# distinguish the started/never-started case
self._greenlet = None
self._stop() # mark as finished
def join(self, timeout=None):
if '_greenlet' not in self.__dict__:
raise RuntimeError("Cannot join an inactive thread")
if self._greenlet is None:
return
self._greenlet.join(timeout=timeout)
......@@ -81,3 +85,9 @@ if sys.version_info[:2] >= (3, 4):
raise NotImplementedError()
__implements__.append('Thread')
if sys.version_info[:2] >= (3, 3):
__implements__.remove('_get_ident')
__implements__.append('get_ident')
get_ident = _get_ident
__implements__.remove('_sleep')
......@@ -423,7 +423,8 @@ class BaseSemaphoreTests(BaseTestCase):
def test_constructor(self):
self.assertRaises(ValueError, self.semtype, value = -1)
self.assertRaises(ValueError, self.semtype, value = -sys.maxint)
# Py3 doesn't have sys.maxint
self.assertRaises(ValueError, self.semtype, value = -getattr(sys, 'maxint', getattr(sys, 'maxsize', None)))
def test_acquire(self):
sem = self.semtype(1)
......
......@@ -14,7 +14,7 @@ import ssl
class Test_wsgiserver(util.TestServer):
server = 'wsgiserver.py'
URL = 'http://127.0.0.1:8088'
not_found_message = '<h1>Not Found</h1>'
not_found_message = b'<h1>Not Found</h1>'
ssl_ctx = None
def read(self, path='/'):
......@@ -35,7 +35,7 @@ class Test_wsgiserver(util.TestServer):
def _test_hello(self):
status, data = self.read('/')
self.assertEqual(status, '200 OK')
self.assertEqual(data, "<b>hello world</b>")
self.assertEqual(data, b"<b>hello world</b>")
def _test_not_found(self):
status, data = self.read('/xxx')
......@@ -59,10 +59,10 @@ class Test_webproxy(Test_wsgiserver):
def _run_all_tests(self):
status, data = self.read('/')
self.assertEqual(status, '200 OK')
assert "gevent example" in data, repr(data)
assert b"gevent example" in data, repr(data)
status, data = self.read('/http://www.google.com')
self.assertEqual(status, '200 OK')
assert 'google' in data.lower(), repr(data)
assert b'google' in data.lower(), repr(data)
# class Test_webpy(Test_wsgiserver):
......
......@@ -11,7 +11,7 @@ MAPPING = {'gevent.local': '_threading_local',
'gevent.socket': 'socket',
'gevent.select': 'select',
'gevent.ssl': 'ssl',
'gevent.thread': 'thread',
'gevent.thread': '_thread' if six.PY3 else 'thread',
'gevent.subprocess': 'subprocess',
'gevent.os': 'os',
'gevent.threading': 'threading'}
......@@ -33,7 +33,7 @@ NOT_IMPLEMENTED = {
COULD_BE_MISSING = {
'socket': ['create_connection', 'RAND_add', 'RAND_egd', 'RAND_status']}
NO_ALL = ['gevent.threading', 'gevent._util', 'gevent._socketcommon']
NO_ALL = ['gevent.threading', 'gevent._util', 'gevent._socketcommon', 'gevent._fileobjectcommon']
class Test(unittest.TestCase):
......
......@@ -6,12 +6,16 @@ from six import xrange
def read_until(conn, postfix):
read = ''
read = b''
if isinstance(postfix, str) and str != bytes:
postfix = postfix.encode('utf-8')
while not read.endswith(postfix):
result = conn.recv(1)
if not result:
raise AssertionError('Connection ended before %r. Data read:\n%r' % (postfix, read))
read += result
if str != bytes:
read = read.decode('utf-8')
return read
......@@ -21,19 +25,29 @@ def create_connection(address):
return conn
def readline(conn):
f = conn.makefile()
line = f.readline()
f.close()
return line
class Test(greentest.TestCase):
def test(self):
server = backdoor.BackdoorServer(('127.0.0.1', 0))
server.start()
def connect():
conn = create_connection(('127.0.0.1', server.server_port))
read_until(conn, '>>> ')
conn.sendall('2+2\r\n')
line = conn.makefile().readline()
assert line.strip() == '4', repr(line)
try:
read_until(conn, '>>> ')
conn.sendall(b'2+2\r\n')
line = readline(conn)
self.assertEqual(line.strip(), '4', repr(line))
finally:
conn.close()
server.start()
try:
jobs = [gevent.spawn(connect) for _ in xrange(10)]
gevent.joinall(jobs, raise_error=True)
......@@ -47,10 +61,11 @@ class Test(greentest.TestCase):
try:
conn = create_connection(('127.0.0.1', server.server_port))
read_until(conn, '>>> ')
conn.sendall('quit()\r\n')
line = conn.makefile().read()
conn.sendall(b'quit()\r\n')
line = readline(conn)
self.assertEqual(line, '')
finally:
conn.close()
server.stop()
def test_sys_exit(self):
......@@ -58,21 +73,23 @@ class Test(greentest.TestCase):
server.start()
try:
conn = create_connection(('127.0.0.1', server.server_port))
read_until(conn, '>>> ')
conn.sendall('import sys; sys.exit(0)\r\n')
line = conn.makefile().read()
read_until(conn, b'>>> ')
conn.sendall(b'import sys; sys.exit(0)\r\n')
line = readline(conn)
self.assertEqual(line, '')
finally:
conn.close()
server.stop()
def test_banner(self):
banner = "Welcome stranger!"
banner = "Welcome stranger!" # native string
server = backdoor.BackdoorServer(('127.0.0.1', 0), banner=banner)
server.start()
try:
conn = create_connection(('127.0.0.1', server.server_port))
response = read_until(conn, '>>> ')
self.assertEqual(response[:len(banner)], banner)
response = read_until(conn, b'>>> ')
self.assertEqual(response[:len(banner)], banner, response)
conn.close()
finally:
server.stop()
......@@ -81,11 +98,12 @@ class Test(greentest.TestCase):
server.start()
try:
conn = create_connection(('127.0.0.1', server.server_port))
read_until(conn, '>>> ')
conn.sendall('locals()["__builtins__"]\r\n')
read_until(conn, b'>>> ')
conn.sendall(b'locals()["__builtins__"]\r\n')
response = read_until(conn, '>>> ')
self.assertTrue(len(response) < 300, msg="locals() unusable: %s..." % response[:100])
self.assertTrue(len(response) < 300, msg="locals() unusable: %s..." % response)
finally:
conn.close()
server.stop()
......
......@@ -19,7 +19,7 @@ try:
def write():
f = open(filename, 'wb', buffering=0)
f.write('x')
f.write(b'x')
f.close()
start = time.time()
......
from gevent.socket import create_connection, timeout
from unittest import main
import gevent
from gevent.hub import PY3
import util
......@@ -10,16 +11,29 @@ class Test(util.TestServer):
def _run_all_tests(self):
def test_client(message):
conn = create_connection(('127.0.0.1', 6000)).makefile(bufsize=1)
welcome = conn.readline()
assert 'Welcome' in welcome, repr(welcome)
conn.write(message)
received = conn.read(len(message))
if PY3:
kwargs = {'buffering': 1}
else:
kwargs = {'bufsize': 1}
kwargs['mode'] = 'rb'
conn = create_connection(('127.0.0.1', 6000))
conn.settimeout(0.1)
rfile = conn.makefile(**kwargs)
welcome = rfile.readline()
assert b'Welcome' in welcome, repr(welcome)
conn.sendall(message)
received = rfile.read(len(message))
self.assertEqual(received, message)
conn._sock.settimeout(0.1)
self.assertRaises(timeout, conn.read, 1)
client1 = gevent.spawn(test_client, 'hello\r\n')
client2 = gevent.spawn(test_client, 'world\r\n')
self.assertRaises(timeout, conn.recv, 1)
rfile.close()
conn.close()
client1 = gevent.spawn(test_client, b'hello\r\n')
client2 = gevent.spawn(test_client, b'world\r\n')
gevent.joinall([client1, client2], raise_error=True)
......
......@@ -35,12 +35,12 @@ class Test(util.TestServer):
server.start()
try:
conn = socket.create_connection(('127.0.0.1', 10011))
conn.sendall('msg1')
conn.sendall(b'msg1')
sleep(0.1)
self.popen.send_signal(15)
sleep(0.1)
try:
conn.sendall('msg2')
conn.sendall(b'msg2')
conn.close()
except socket.error:
if sys.platform != 'win32':
......@@ -55,9 +55,9 @@ class Test(util.TestServer):
server.close()
if sys.platform == 'win32':
self.assertEqual(['msg1'], log)
self.assertEqual([b'msg1'], log)
else:
self.assertEqual(['msg1', 'msg2'], log)
self.assertEqual([b'msg1', b'msg2'], log)
if __name__ == '__main__':
......
......@@ -9,9 +9,10 @@ class Test(util.TestServer):
def _run_all_tests(self):
sock = socket.socket(type=socket.SOCK_DGRAM)
sock.connect(('127.0.0.1', 9000))
sock.send('Test udp_server')
sock.send(b'Test udp_server')
data, address = sock.recvfrom(8192)
self.assertEqual(data, 'Received 15 bytes')
self.assertEqual(data, b'Received 15 bytes')
sock.close()
if __name__ == '__main__':
......
......@@ -13,7 +13,7 @@ class Test(greentest.TestCase):
def _test_del(self, **kwargs):
r, w = os.pipe()
s = FileObject(w, 'wb')
s.write('x')
s.write(b'x')
s.flush()
if PYPY:
s.close()
......@@ -25,7 +25,7 @@ class Test(greentest.TestCase):
pass # expected, because SocketAdapter already closed it
else:
raise AssertionError('os.close(%r) must not succeed' % w)
self.assertEqual(FileObject(r).read(), 'x')
self.assertEqual(FileObject(r).read(), b'x')
def test_del(self):
self._test_del()
......@@ -38,18 +38,18 @@ class Test(greentest.TestCase):
def test_del_noclose(self):
r, w = os.pipe()
s = FileObject(w, 'wb', close=False)
s.write('x')
s.write(b'x')
s.flush()
if PYPY:
s.close()
else:
del s
os.close(w)
self.assertEqual(FileObject(r).read(), 'x')
self.assertEqual(FileObject(r).read(), b'x')
def test_newlines(self):
r, w = os.pipe()
lines = ['line1\n', 'line2\r', 'line3\r\n', 'line4\r\nline5', '\nline6']
lines = [b'line1\n', b'line2\r', b'line3\r\n', b'line4\r\nline5', b'\nline6']
g = gevent.spawn(writer, FileObject(w, 'wb'), lines)
try:
result = FileObject(r, 'rU').read()
......@@ -76,7 +76,7 @@ else:
def _test_del(self, **kwargs):
r, w = os.pipe()
s = SocketAdapter(w)
s.sendall('x')
s.sendall(b'x')
if PYPY:
s.close()
else:
......@@ -87,7 +87,7 @@ else:
pass # expected, because SocketAdapter already closed it
else:
raise AssertionError('os.close(%r) must not succeed' % w)
self.assertEqual(FileObject(r).read(), 'x')
self.assertEqual(FileObject(r).read(), b'x')
def test_del(self):
self._test_del()
......@@ -98,10 +98,10 @@ else:
def test_del_noclose(self):
r, w = os.pipe()
s = SocketAdapter(w, close=False)
s.sendall('x')
s.sendall(b'x')
del s
os.close(w)
self.assertEqual(FileObject(r).read(), 'x')
self.assertEqual(FileObject(r).read(), b'x')
if __name__ == '__main__':
......
......@@ -24,6 +24,16 @@ import sys
PYPY = hasattr(sys, 'pypy_version_info')
PY3 = sys.version_info[0] >= 3
def _write_to_closed(f, s):
try:
r = f.write(s)
except ValueError:
assert PY3
else:
assert r is None, r
class TestGreenIo(TestCase):
......@@ -35,13 +45,12 @@ class TestGreenIo(TestCase):
# by closing the socket prior to using the made file
try:
conn, addr = listener.accept()
fd = conn.makefile()
fd = conn.makefile(mode='wb')
conn.close()
fd.write('hello\n')
fd.write(b'hello\n')
fd.close()
r = fd.write('a')
assert r is None, r
self.assertRaises(socket.error, conn.send, 'b')
_write_to_closed(fd, b'a')
self.assertRaises(socket.error, conn.send, b'b')
finally:
listener.close()
......@@ -50,23 +59,22 @@ class TestGreenIo(TestCase):
# by closing the made file and then sending a character
try:
conn, addr = listener.accept()
fd = conn.makefile()
fd.write('hello')
fd = conn.makefile(mode='wb')
fd.write(b'hello')
fd.close()
conn.send('\n')
conn.send(b'\n')
conn.close()
r = fd.write('a')
assert r is None, r
self.assertRaises(socket.error, conn.send, 'b')
_write_to_closed(fd, b'a')
self.assertRaises(socket.error, conn.send, b'b')
finally:
listener.close()
def did_it_work(server):
client = socket.create_connection(('127.0.0.1', server.getsockname()[1]))
fd = client.makefile()
fd = client.makefile(mode='rb')
client.close()
assert fd.readline() == 'hello\n'
assert fd.read() == ''
assert fd.readline() == b'hello\n'
assert fd.read() == b''
fd.close()
server = tcp_listener(('0.0.0.0', 0))
......@@ -90,11 +98,10 @@ class TestGreenIo(TestCase):
# closing the file object should close everything
try:
conn, addr = listener.accept()
conn = conn.makefile()
conn.write('hello\n')
conn = conn.makefile(mode='wb')
conn.write(b'hello\n')
conn.close()
r = conn.write('a')
assert r is None, r
_write_to_closed(conn, b'a')
finally:
listener.close()
......
from __future__ import print_function
import sys
if not sys.argv[1:]:
from subprocess import Popen, PIPE
p = Popen([sys.executable, __file__, 'subprocess'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
out, err = p.communicate('hello world\n')
out, err = p.communicate(b'hello world\n')
code = p.poll()
assert p.poll() == 0, (out, err, code)
assert out.strip() == '11 chars.', (out, err, code)
assert err == '', (out, err, code)
assert out.strip() == b'11 chars.', (out, err, code)
assert err == b'', (out, err, code)
elif sys.argv[1:] == ['subprocess']:
import gevent
......
......@@ -13,6 +13,9 @@ pid = os.getpid()
tmpname = '/tmp/test__makefile_ref.lsof.%s' % pid
lsof_command = 'lsof -p %s > %s' % (pid, tmpname)
import sys
PY3 = sys.version_info[0] >= 3
def get_open_files():
if os.system(lsof_command):
......@@ -78,7 +81,12 @@ class Test(unittest.TestCase):
if isinstance(sock, int):
self.assert_fd_closed(sock)
else:
self.assert_raises_EBADF(sock.fileno)
# Under Python3, the socket module returns -1 for a fileno
# of a closed socket; under Py2 it raises
if PY3:
self.assertEqual(sock.fileno(), -1)
else:
self.assert_raises_EBADF(sock.fileno)
self.assert_raises_EBADF(sock.getsockname)
self.assert_raises_EBADF(sock.accept)
if rest:
......@@ -103,9 +111,13 @@ class TestSocket(Test):
f = s.makefile()
self.assert_open(s, fileno)
s.close()
# this closes socket wrapper object but not the file descriptor
self.assert_closed(s)
self.assert_open(fileno)
# Under python 2, this closes socket wrapper object but not the file descriptor;
# under python 3, both stay open
if PY3:
self.assert_open(s, fileno)
else:
self.assert_closed(s)
self.assert_open(fileno)
f.close()
self.assert_closed(s)
self.assert_closed(fileno)
......@@ -169,8 +181,13 @@ class TestSocket(Test):
f = client_socket.makefile()
self.assert_open(client_socket, fileno)
client_socket.close()
self.assert_closed(client_socket)
self.assert_open(fileno)
# Under python 2, this closes socket wrapper object but not the file descriptor;
# under python 3, both stay open
if PY3:
self.assert_open(client_socket, fileno)
else:
self.assert_closed(client_socket)
self.assert_open(fileno)
f.close()
self.assert_closed(client_socket, fileno)
finally:
......@@ -306,6 +323,7 @@ class TestSSL(Test):
self.assert_closed(client_socket, fileno)
finally:
t.join()
connector.close()
def test_server_makefile2(self):
listener = socket.socket()
......@@ -337,6 +355,7 @@ class TestSSL(Test):
finally:
t.join()
listener.close()
connector.close()
def test_serverssl_makefile1(self):
listener = socket.socket()
......@@ -368,6 +387,7 @@ class TestSSL(Test):
finally:
t.join()
listener.close()
connector.close()
def test_serverssl_makefile2(self):
listener = socket.socket()
......@@ -381,7 +401,7 @@ class TestSSL(Test):
def connect():
connector.connect(('127.0.0.1', port))
s = ssl.wrap_socket(connector)
s.sendall('test_serverssl_makefile2')
s.sendall(b'test_serverssl_makefile2')
s.close()
connector.close()
......
from gevent import monkey
monkey.patch_all()
import sys
import time
assert 'built-in' not in repr(time.sleep), repr(time.sleep)
......@@ -11,7 +12,8 @@ except ImportError:
import threading
assert 'built-in' not in repr(thread.start_new_thread), repr(thread.start_new_thread)
assert 'built-in' not in repr(threading._start_new_thread), repr(threading._start_new_thread)
assert 'built-in' not in repr(threading._sleep), repr(threading._sleep)
if sys.version_info[0] == 2:
assert 'built-in' not in repr(threading._sleep), repr(threading._sleep)
import socket
from gevent import socket as gevent_socket
......
......@@ -32,7 +32,7 @@ class TestOS_tp(TestCase):
r, w = self.pipe()
# set nbytes such that for sure it is > maximum pipe buffer
nbytes = 1000000
block = 'x' * 4096
block = b'x' * 4096
buf = buffer_class(block)
# Lack of "nonlocal" keyword in Python 2.x:
bytesread = [0]
......
......@@ -463,18 +463,18 @@ class TestErrorInHandler(greentest.TestCase):
def test_imap(self):
p = pool.Pool(1)
it = p.imap(divide_by, [1, 0, 2])
self.assertEqual(it.next(), 1.0)
self.assertRaises(ZeroDivisionError, it.next)
self.assertEqual(it.next(), 0.5)
self.assertRaises(StopIteration, it.next)
self.assertEqual(next(it), 1.0)
self.assertRaises(ZeroDivisionError, next, it)
self.assertEqual(next(it), 0.5)
self.assertRaises(StopIteration, next, it)
def test_imap_unordered(self):
p = pool.Pool(1)
it = p.imap_unordered(divide_by, [1, 0, 2])
self.assertEqual(it.next(), 1.0)
self.assertRaises(ZeroDivisionError, it.next)
self.assertEqual(it.next(), 0.5)
self.assertRaises(StopIteration, it.next)
self.assertEqual(next(it), 1.0)
self.assertRaises(ZeroDivisionError, next, it)
self.assertEqual(next(it), 0.5)
self.assertRaises(StopIteration, next, it)
if __name__ == '__main__':
......
This diff is collapsed.
......@@ -24,10 +24,18 @@ are not leaked by the hub.
"""
from __future__ import print_function
from _socket import socket
class Socket(socket):
"Something we can have a weakref to"
import sys
if sys.version_info[0] >= 3:
# Python3 enforces that __weakref__ appears only once,
# and not when a slotted class inherits from an unslotted class.
# We mess around with the class MRO below and violate that rule
# (because socket.socket defines __slots__ with __weakref__),
# so import socket.socket before that can happen.
__import__('socket')
Socket = socket
else:
class Socket(socket):
"Something we can have a weakref to"
import _socket
_socket.socket = Socket
......@@ -69,9 +77,9 @@ def handle_request(s, raise_on_timeout):
return
#print('handle_request - accepted')
res = conn.recv(100)
assert res == 'hello', repr(res)
assert res == b'hello', repr(res)
#print('handle_request - recvd %r' % res)
res = conn.send('bye')
res = conn.send(b'bye')
#print('handle_request - sent %r' % res)
#print('handle_request - conn refcount: %s' % sys.getrefcount(conn))
#conn.close()
......@@ -82,10 +90,10 @@ def make_request(port):
s = socket.socket()
s.connect(('127.0.0.1', port))
#print('make_request - connected')
res = s.send('hello')
res = s.send(b'hello')
#print('make_request - sent %s' % res)
res = s.recv(100)
assert res == 'bye', repr(res)
assert res == b'bye', repr(res)
#print('make_request - recvd %r' % res)
#s.close()
......@@ -99,7 +107,9 @@ def run_interaction(run_client):
sleep(0.1 + SOCKET_TIMEOUT)
#print(sys.getrefcount(s._sock))
#s.close()
return weakref.ref(s._sock)
w = weakref.ref(s._sock)
s.close()
return w
def run_and_check(run_client):
......
from __future__ import print_function
import greentest
from gevent.hub import PY3
from gevent import socket
import gevent
from gevent.server import StreamServer
......@@ -21,15 +22,15 @@ class SimpleStreamServer(StreamServer):
print('Failed to parse request line: %r' % (request_line, ))
raise
if path == '/ping':
client_socket.sendall('HTTP/1.0 200 OK\r\n\r\nPONG')
client_socket.sendall(b'HTTP/1.0 200 OK\r\n\r\nPONG')
elif path in ['/long', '/short']:
client_socket.sendall('hello')
client_socket.sendall(b'hello')
while True:
data = client_socket.recv(1)
if not data:
break
else:
client_socket.sendall('HTTP/1.0 404 WTF?\r\n\r\n')
client_socket.sendall(b'HTTP/1.0 404 WTF?\r\n\r\n')
finally:
fd.close()
......@@ -82,14 +83,24 @@ class TestCase(greentest.TestCase):
def makefile(self, timeout=0.1, bufsize=1):
sock = socket.socket()
sock.connect((self.server.server_host, self.server.server_port))
fobj = sock.makefile(bufsize=bufsize)
fobj._sock.settimeout(timeout)
if PY3:
# Under Python3, you can't read and write to the same
# makefile() opened in r, and r+ is not allowed
kwargs = {'buffering': bufsize, 'mode': 'rwb'}
else:
kwargs = {'bufsize': bufsize}
rconn = sock.makefile(**kwargs)
if PY3:
rconn._sock = sock
rconn._sock.settimeout(timeout)
sock.close()
return fobj
return rconn
def send_request(self, url='/', timeout=0.1, bufsize=1):
conn = self.makefile(timeout=timeout, bufsize=bufsize)
conn.write('GET %s HTTP/1.0\r\n\r\n' % url)
conn.write(('GET %s HTTP/1.0\r\n\r\n' % url).encode('latin-1'))
conn.flush()
return conn
......@@ -115,7 +126,7 @@ class TestCase(greentest.TestCase):
def assertNotAccepted(self):
conn = self.makefile()
conn.write('GET / HTTP/1.0\r\n\r\n')
conn.write(b'GET / HTTP/1.0\r\n\r\n')
conn.flush()
result = ''
try:
......@@ -128,12 +139,14 @@ class TestCase(greentest.TestCase):
assert not result, repr(result)
return
assert result.startswith('HTTP/1.0 500 Internal Server Error'), repr(result)
conn.close()
def assertRequestSucceeded(self, timeout=0.1):
conn = self.makefile(timeout=timeout)
conn.write('GET /ping HTTP/1.0\r\n\r\n')
conn.write(b'GET /ping HTTP/1.0\r\n\r\n')
result = conn.read()
assert result.endswith('\r\n\r\nPONG'), repr(result)
conn.close()
assert result.endswith(b'\r\n\r\nPONG'), repr(result)
def start_server(self):
self.server.start()
......@@ -270,6 +283,7 @@ class TestDefaultSpawn(TestCase):
raise
finally:
timeout.cancel()
conn.close()
self.stop_server()
def init_server(self):
......@@ -319,6 +333,10 @@ class TestPoolSpawn(TestDefaultSpawn):
self.assertPoolFull()
self.assertPoolFull()
short_request._sock.close()
if PY3:
# We use two makefiles to simulate reading/writing
# under py3
short_request.close()
# gevent.http and gevent.wsgi cannot detect socket close, so sleep a little
# to let /short request finish
gevent.sleep(0.1)
......
......@@ -9,10 +9,10 @@ from test__server import Settings as server_Settings
def application(self, environ, start_response):
if environ['PATH_INFO'] == '/':
start_response("200 OK", [])
return ["PONG"]
return [b"PONG"]
if environ['PATH_INFO'] == '/ping':
start_response("200 OK", [])
return ["PONG"]
return [b"PONG"]
elif environ['PATH_INFO'] == '/short':
gevent.sleep(0.5)
start_response("200 OK", [])
......@@ -30,15 +30,15 @@ class SimpleWSGIServer(pywsgi.WSGIServer):
application = application
internal_error_start = 'HTTP/1.1 500 Internal Server Error\n'.replace('\n', '\r\n')
internal_error_end = '\n\nInternal Server Error'.replace('\n', '\r\n')
internal_error_start = b'HTTP/1.1 500 Internal Server Error\n'.replace(b'\n', b'\r\n')
internal_error_end = b'\n\nInternal Server Error'.replace(b'\n', b'\r\n')
internal_error503 = '''HTTP/1.1 503 Service Unavailable
internal_error503 = b'''HTTP/1.1 503 Service Unavailable
Connection: close
Content-type: text/plain
Content-length: 31
Service Temporarily Unavailable'''.replace('\n', '\r\n')
Service Temporarily Unavailable'''.replace(b'\n', b'\r\n')
class Settings:
......@@ -51,7 +51,7 @@ class Settings:
@staticmethod
def assert500(self):
conn = self.makefile()
conn.write('GET / HTTP/1.0\r\n\r\n')
conn.write(b'GET / HTTP/1.0\r\n\r\n')
result = conn.read()
assert result.startswith(internal_error_start), (result, internal_error_start)
assert result.endswith(internal_error_end), (result, internal_error_end)
......@@ -61,7 +61,7 @@ class Settings:
@staticmethod
def assert503(self):
conn = self.makefile()
conn.write('GET / HTTP/1.0\r\n\r\n')
conn.write(b'GET / HTTP/1.0\r\n\r\n')
result = conn.read()
assert result == internal_error503, (result, internal_error503)
......
......@@ -10,7 +10,7 @@ class TestClosedSocket(greentest.TestCase):
sock = socket.socket()
sock.close()
try:
sock.send('a', timeout=1)
sock.send(b'a', timeout=1)
except socket.error as ex:
if ex.args[0] != 9:
raise
......
......@@ -6,15 +6,15 @@ import unittest
class Test(unittest.TestCase):
def test(self):
msg = 'hello world'
msg = b'hello world'
x, y = socket.socketpair()
x.sendall(msg)
x.close()
read = y.makefile().read()
read = y.makefile('rb').read()
self.assertEqual(msg, read)
def test_fromfd(self):
msg = 'hello world'
msg = b'hello world'
x, y = socket.socketpair()
xx = socket.fromfd(x.fileno(), x.family, socket.SOCK_STREAM)
x.close()
......@@ -23,7 +23,7 @@ class Test(unittest.TestCase):
xx.sendall(msg)
xx.close()
read = yy.makefile().read()
read = yy.makefile('rb').read()
self.assertEqual(msg, read)
......
......@@ -10,6 +10,7 @@ import gc
PYPY = hasattr(sys, 'pypy_version_info')
PY3 = sys.version_info[0] >= 3
if subprocess.mswindows:
......@@ -86,9 +87,15 @@ class Test(greentest.TestCase):
'sys.stdout.flush();'
'sys.stdout.write("\\nline6");'],
stdout=subprocess.PIPE,
universal_newlines=1)
universal_newlines=1,
bufsize=1)
try:
stdout = p.stdout.read()
if PY3 and isinstance(stdout, bytes):
# OS X gives us binary back from stdout.read, but linux (travis ci)
# gives us text...text is correct because we're in universal newline
# mode
stdout = stdout.decode('ascii')
if python_universal_newlines:
# Interpreter with universal newline support
self.assertEqual(stdout,
......@@ -96,7 +103,7 @@ class Test(greentest.TestCase):
else:
# Interpreter without universal newline support
self.assertEqual(stdout,
b"line1\nline2\rline3\r\nline4\r\nline5\nline6")
"line1\nline2\rline3\r\nline4\r\nline5\nline6")
finally:
p.stdout.close()
......@@ -113,9 +120,15 @@ class Test(greentest.TestCase):
'sys.stdout.flush();'
'sys.stdout.write("\\nline6");'],
stdout=subprocess.PIPE,
universal_newlines=1)
universal_newlines=1,
bufsize=1)
try:
stdout = p.stdout.read()
if PY3 and isinstance(stdout, bytes):
# OS X gives us binary back from stdout.read, but linux (travis ci)
# gives us text...text is correct because we're in universal newline
# mode
stdout = stdout.decode('ascii')
if python_universal_newlines:
# Interpreter with universal newline support
self.assertEqual(stdout,
......@@ -123,7 +136,7 @@ class Test(greentest.TestCase):
else:
# Interpreter without universal newline support
self.assertEqual(stdout,
b"line1\nline2\rline3\r\nline4\r\nline5\nline6")
"line1\nline2\rline3\r\nline4\r\nline5\nline6")
finally:
p.stdout.close()
......@@ -134,7 +147,12 @@ class Test(greentest.TestCase):
r, w = os.pipe()
p = subprocess.Popen(['grep', 'text'], stdin=subprocess.FileObject(r))
try:
os.close(w)
# Closing one half of the pipe causes Python 3 on OS X to terminate the
# child process; it exits with code 1 and the assert that p.poll is None
# fails. Removing the close lets it pass under both Python 3 and 2.7.
# If subprocess.Popen._remove_nonblock_flag is changed to a noop, then
# the test fails (as expected) even with the close removed
#os.close(w)
time.sleep(0.1)
self.assertEqual(p.poll(), None)
finally:
......@@ -165,9 +183,9 @@ class Test(greentest.TestCase):
p = subprocess.Popen([sys.executable, '-u', '-c',
'import sys; sys.stdout.write(sys.stdin.readline())'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
p.stdin.write('foobar\n')
p.stdin.write(b'foobar\n')
r = p.stdout.readline()
self.assertEqual(r, 'foobar\n')
self.assertEqual(r, b'foobar\n')
if __name__ == '__main__':
......
import threading
import gevent.monkey
gevent.monkey.patch_all()
import gevent
import sys
if sys.version_info[0] == 2:
import threading
import gevent.monkey
gevent.monkey.patch_all()
import gevent
assert threading._sleep is gevent.sleep, threading._sleep
assert threading._sleep is gevent.sleep, threading._sleep
......@@ -20,7 +20,7 @@ if not hasattr(threading.Thread, 'is_alive'):
threading.Thread.is_alive = threading.Thread.isAlive
if not hasattr(threading.Thread, 'daemon'):
threading.Thread.daemon = property(threading.Thread.isDaemon, threading.Thread.setDaemon)
if not hasattr(threading._Condition, 'notify_all'):
if hasattr(threading, '_Condition') and not hasattr(threading._Condition, 'notify_all'):
threading._Condition.notify_all = threading._Condition.notifyAll
'''
......@@ -305,7 +305,11 @@ class ThreadTests(unittest.TestCase):
import subprocess
rc = subprocess.call([sys.executable, "-c", """if 1:
%s
import ctypes, sys, time, thread
import ctypes, sys, time
try:
import thread
except ImportError:
import _thread as thread # Py3
# This lock is used as a simple event variable.
ready = thread.allocate_lock()
......@@ -354,6 +358,8 @@ class ThreadTests(unittest.TestCase):
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
stdout = stdout.strip()
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
assert re.match('^Woke up, sleep function is: <.*?sleep.*?>$', stdout), repr(stdout)
stderr = re.sub(r"^\[\d+ refs\]", "", stderr, re.MULTILINE).strip()
self.assertEqual(stderr, "")
......@@ -425,10 +431,10 @@ class ThreadJoinOnShutdown(unittest.TestCase):
import subprocess
p = subprocess.Popen([sys.executable, "-c", script], stdout=subprocess.PIPE)
rc = p.wait()
data = p.stdout.read().replace('\r', '')
self.assertEqual(data, "end of main\nend of thread\n")
self.failIf(rc == 2, "interpreter was blocked")
self.failUnless(rc == 0, "Unexpected error")
data = p.stdout.read().replace(b'\r', b'')
self.assertEqual(data, b"end of main\nend of thread\n")
self.failIf(rc == 2, b"interpreter was blocked")
self.failUnless(rc == 0, b"Unexpected error")
def test_1_join_on_shutdown(self):
# The usual case: on exit, wait for a non-daemon thread
......
......@@ -5,14 +5,12 @@ import os
import sys
CPYTHON_DBG = hasattr(sys, 'gettotalrefcount')
LEAKTEST = os.getenv('GEVENTTEST_LEAKCHECK')
PYPY = hasattr(sys, 'pypy_version_info')
PY3 = sys.version_info[0] >= 3
FAILING_TESTS = [
# needs investigating
'FLAKY test__issue6.py',
# Sometimes fails with AssertionError: ...\nIOError: close() called during concurrent operation on the same file object.\n'
# Sometimes it contains "\nUnhandled exception in thread started by \nsys.excepthook is missing\nlost sys.stderr\n"
......@@ -20,7 +18,7 @@ FAILING_TESTS = [
]
if os.environ.get('GEVENT_RESOLVER') == 'ares' or CPYTHON_DBG:
if os.environ.get('GEVENT_RESOLVER') == 'ares' or LEAKTEST:
# XXX fix this
FAILING_TESTS += [
'FLAKY test__socket_dns.py',
......@@ -49,7 +47,7 @@ if sys.platform == 'win32':
]
if CPYTHON_DBG:
if LEAKTEST:
FAILING_TESTS += ['FLAKY test__backdoor.py']
FAILING_TESTS += ['FLAKY test__os.py']
......@@ -83,38 +81,10 @@ if PYPY:
if PY3:
# No idea / TODO
FAILING_TESTS += '''
test__example_udp_server.py
test__examples.py
test__pool.py
FLAKY test___example_servers.py
test__example_udp_client.py
test__os.py
test__backdoor.py
test_threading_2.py
test__refcount.py
test__subprocess.py
test__all__.py
test__fileobject.py
test__pywsgi.py
test__socket_ex.py
test__example_echoserver.py
test__subprocess_poll.py
test__makefile_ref.py
test__socketpair.py
test__server_pywsgi.py
test__core_stat.py
test__server.py
test__example_portforwarder.py
test__execmodules.py
FLAKY test__greenio.py
FLAKY test__socket_dns.py
'''.strip().split('\n')
if os.environ.get('GEVENT_RESOLVER') == 'ares':
FAILING_TESTS += [
'test__greenness.py']
if CPYTHON_DBG:
if LEAKTEST:
FAILING_TESTS += ['FLAKY test__threadpool.py']
# refcount problems:
FAILING_TESTS += '''
......
......@@ -77,7 +77,7 @@ def pyflakes(args):
pyflakes('examples/ greentest/*.py util/ *.py')
if sys.version_info[0] == 3:
ignored_files = ['gevent/_util_py2.py', 'gevent/_socket2.py']
ignored_files = ['gevent/_util_py2.py', 'gevent/_socket2.py', 'gevent/_fileobject2.py']
else:
ignored_files = ['gevent/_socket3.py']
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment