Skip to content

G
gevent
Commit 79021f13 authored by Jason Madden's avatar Jason Madden
Browse files
Options

Update documentation.

parent 0dc161ef
Hide whitespace changes
Inline Side-by-side
Showing with 89 additions and 88 deletions
CHANGES.rst
View file @ 79021f13
......@@ -15,7 +15,8 @@
consistently text and binary modes. If neither 'b' nor 't' is given
in the mode, they will read and write native strings. If 't' is
given, they will always work with unicode strings, and 'b' will
always work with byte strings. See :issue:`1441`.
always work with byte strings. (FileObjectPosix already worked this
way.) See :issue:`1441`.
- The file objects accept *encoding*, *errors* and *newline*
arguments. On Python 2, these are only used if 't' is in the mode.
......
src/gevent/_fileobjectcommon.py
View file @ 79021f13
......@@ -77,7 +77,7 @@ class FlushingBufferedWriter(io.BufferedWriter):
class OpenDescriptor(object): # pylint:disable=too-many-instance-attributes
"""
Interprets the arguments to `open`.
Interprets the arguments to `open`. Internal use only.
Originally based on code in the stdlib's _pyio.py (Python implementation of
the :mod:`io` module), but modified for gevent:
......@@ -137,8 +137,10 @@ class OpenDescriptor(object): # pylint:disable=too-many-instance-attributes
binary = "b" in modes
universal = 'U' in modes
can_write = creating or writing or appending or updating
if universal:
if creating or writing or appending or updating:
if can_write:
raise ValueError("mode U cannot be combined with 'x', 'w', 'a', or '+'")
import warnings
warnings.warn("'U' mode is deprecated",
......@@ -179,6 +181,8 @@ class OpenDescriptor(object): # pylint:disable=too-many-instance-attributes
self.updating = updating
self.text = text
self.binary = binary
self.can_write = can_write
self.can_read = reading or updating
self.native = (
not self.text and not self.binary # Neither t nor b given.
and not encoding and not errors # And no encoding or error handling either.
......@@ -309,9 +313,6 @@ class FileObjectBase(object):
_io = None
def __init__(self, fobj, closefd):
"""
:param fobj: An io.IOBase-like object.
"""
self._io = fobj
# We don't actually use this property ourself, but we save it (and
# pass it along) for compatibility.
......@@ -395,6 +396,13 @@ class FileObjectBase(object):
class FileObjectBlock(FileObjectBase):
"""
FileObjectBlock()
A simple synchronous wrapper around a file object.
Adds no concurrency or gevent compatibility.
"""
def __init__(self, fobj, *args, **kwargs):
descriptor = OpenDescriptor(fobj, *args, **kwargs)
......@@ -406,6 +414,8 @@ class FileObjectBlock(FileObjectBase):
class FileObjectThread(FileObjectBase):
"""
FileObjectThread()
A file-like object wrapping another file-like object, performing all blocking
operations on that object in a background thread.
......@@ -416,21 +426,11 @@ class FileObjectThread(FileObjectBase):
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
.. versionchanged:: 1.5
Accept str and ``PathLike`` objects for *fobj* on all versions of Python.
.. versionchanged:: 1.5
Add *encoding*, *errors* and *newline* arguments.
.. versionchanged:: 1.5
Accept *closefd* and *buffering* instead of *close* and *bufsize* arguments.
The latter remain for backwards compatibility.
"""
def __init__(self, *args, **kwargs):
"""
:param fobj: The underlying file-like object to wrap, or something
acceptable to :func:`io.open` (along with *mode* and *buffering*)
:keyword bool lock: If True (the default) then all operations will
be performed one-by-one. Note that this does not guarantee that, if using
this file object from multiple threads/greenlets, operations will be performed
......
src/gevent/_fileobjectposix.py
View file @ 79021f13
......@@ -23,6 +23,9 @@ from gevent.os import make_nonblocking
class GreenFileDescriptorIO(RawIOBase):
# Internal, undocumented, class. All that's documented is that this
# is a IOBase object. Constructor is private.
# Note that RawIOBase has a __del__ method that calls
# self.close(). (In C implementations like CPython, this is
# the type's tp_dealloc slot; prior to Python 3, the object doesn't
......@@ -34,15 +37,15 @@ class GreenFileDescriptorIO(RawIOBase):
_seekable = None
_keep_alive = None # An object that needs to live as long as we do.
def __init__(self, fileno, mode='r', closefd=True):
def __init__(self, fileno, open_descriptor, closefd=True):
RawIOBase.__init__(self)
self._closefd = closefd
self._fileno = fileno
self.mode = mode
self.mode = open_descriptor.fileio_mode
make_nonblocking(fileno)
readable = 'r' in mode or '+' in mode
writable = 'w' in mode or '+' in mode
readable = open_descriptor.can_read
writable = open_descriptor.can_write
self.hub = get_hub()
io_watcher = self.hub.loop.io
......@@ -70,6 +73,7 @@ class GreenFileDescriptorIO(RawIOBase):
raise
def isatty(self):
# TODO: Couldn't we just subclass FileIO?
f = FileIO(self._fileno, 'r', False)
try:
return f.isatty()
......@@ -206,7 +210,7 @@ class GreenOpenDescriptor(OpenDescriptor):
def open_raw(self):
if self.is_fd():
fileio = GreenFileDescriptorIO(self.fobj, self.fileio_mode, closefd=self.closefd)
fileio = GreenFileDescriptorIO(self.fobj, self, closefd=self.closefd)
else:
closefd = False
# Either an existing file object or a path string (which
......@@ -219,13 +223,15 @@ class GreenOpenDescriptor(OpenDescriptor):
raw = OpenDescriptor.open_raw(self)
fileno = raw.fileno()
fileio = GreenFileDescriptorIO(fileno, self.fileio_mode, closefd=closefd)
fileio = GreenFileDescriptorIO(fileno, self, closefd=closefd)
fileio._keep_alive = raw
return fileio
class FileObjectPosix(FileObjectBase):
"""
FileObjectPosix()
A file-like object that operates on non-blocking files but
provides a synchronous, cooperative interface.
......@@ -250,11 +256,6 @@ class FileObjectPosix(FileObjectBase):
:func:`~gevent.os.tp_read` and :func:`~gevent.os.tp_write` to bypass this
concern.
.. note::
Random read/write (e.g., ``mode='rwb'``) is not supported.
For that, use :class:`io.BufferedRWPair` around two instance of this
class.
.. tip::
Although this object provides a :meth:`fileno` method and so
can itself be passed to :func:`fcntl.fcntl`, setting the
......@@ -275,61 +276,32 @@ class FileObjectPosix(FileObjectBase):
better file-like semantics (and portability to Python 3).
.. versionchanged:: 1.2a1
Document the ``fileio`` attribute for non-blocking reads.
.. versionchanged:: 1.2a1
A bufsize of 0 in write mode is no longer forced to be 1.
Instead, the underlying buffer is flushed after every write
operation to simulate a bufsize of 0. In gevent 1.0, a
bufsize of 0 was flushed when a newline was written, while
in gevent 1.1 it was flushed when more than one byte was
written. Note that this may have performance impacts.
.. versionchanged:: 1.3a1
On Python 2, enabling universal newlines no longer forces unicode
IO.
.. versionchanged:: 1.5
The default value for *mode* was changed from ``rb`` to ``r``. This is consistent
with :func:`open`, :func:`io.open`, and :class:`~.FileObjectThread`, which is the
default ``FileObject`` on some platforms.
.. versionchanged:: 1.5
Support strings and ``PathLike`` objects for ``fobj`` on all versions
of Python. Note that caution above.
.. versionchanged:: 1.5
Add *encoding*, *errors* and *newline* argument.
.. versionchanged:: 1.5
Accept *closefd* and *buffering* instead of *close* and *bufsize* arguments.
The latter remain for backwards compatibility.
.. versionchanged:: 1.5
.. versionchanged:: 1.5
Stop forcing buffering. Previously, given a ``buffering=0`` argument,
*buffering8 would be set to 1, and ``buffering=1`` would be forced to
*buffering* would be set to 1, and ``buffering=1`` would be forced to
the default buffer size. This was a workaround for a long-standing concurrency
issue. Now the *buffering* argument is interpreted as intended.
"""
#: platform specific default for the *bufsize* parameter
default_bufsize = DEFAULT_BUFFER_SIZE
def __init__(self, *args, **kwargs):
# pylint:disable=too-many-locals
"""
:param fobj: Either an integer fileno, or an object supporting the
usual :meth:`socket.fileno` method. The file *will* be
put in non-blocking mode using :func:`gevent.os.make_nonblocking`.
:keyword str mode: The manner of access to the file, one of "rb", "rU" or "wb"
(where the "b" or "U" can be omitted).
If "U" is part of the mode, universal newlines will be used. On Python 2,
if 't' is not in the mode, this will result in returning byte (native) strings;
putting 't' in the mode will return text (unicode) strings. This may cause
:exc:`UnicodeDecodeError` to be raised.
:keyword int buffering: If given, the size of the buffer to use. The default
value means to use a platform-specific default
Other values are interpreted as for the :mod:`io` package.
Buffering is ignored in text mode.
.. versionchanged:: 1.3a1
On Python 2, enabling universal newlines no longer forces unicode
IO.
.. versionchanged:: 1.2a1
A bufsize of 0 in write mode is no longer forced to be 1.
Instead, the underlying buffer is flushed after every write
operation to simulate a bufsize of 0. In gevent 1.0, a
bufsize of 0 was flushed when a newline was written, while
in gevent 1.1 it was flushed when more than one byte was
written. Note that this may have performance impacts.
"""
descriptor = GreenOpenDescriptor(*args, **kwargs)
# This attribute is documented as available for non-blocking reads.
self.fileio, buffered_fobj = descriptor.open_raw_and_wrapped()
super(FileObjectPosix, self).__init__(buffered_fobj, descriptor.closefd)
......
src/gevent/_socket3.py
View file @ 79021f13
......@@ -261,7 +261,9 @@ class socket(object):
except the only mode characters supported are 'r', 'w' and 'b'.
The semantics are similar too.
"""
# (XXX refactor to share code?)
# XXX refactor to share code? We ought to be able to use our FileObject,
# adding the appropriate amount of refcounting. At the very least we can use our
# OpenDescriptor to handle the parsing.
for c in mode:
if c not in {"r", "w", "b"}:
raise ValueError("invalid mode %r (only r, w, b allowed)")
......
src/gevent/_sslgte279.py
View file @ 79021f13
......@@ -56,14 +56,16 @@ if 'namedtuple' in __all__:
# See notes in _socket2.py. Python 3 returns much nicer
# `io` object wrapped around a SocketIO class.
assert not hasattr(__ssl__._fileobject, '__enter__') # pylint:disable=no-member
if hasattr(__ssl__, '_fileobject'):
assert not hasattr(__ssl__._fileobject, '__enter__') # pylint:disable=no-member
class _fileobject(__ssl__._fileobject): # pylint:disable=no-member
class _fileobject(getattr(__ssl__, '_fileobject', object)): # pylint:disable=no-member
def __enter__(self):
return self
def __exit__(self, *args):
# pylint:disable=no-member
if not self.closed:
self.close()
......@@ -96,14 +98,14 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None,
context = SSLContext(PROTOCOL_SSLv23)
# SSLv2 considered harmful.
context.options |= OP_NO_SSLv2
context.options |= OP_NO_SSLv2 # pylint:disable=no-member
# SSLv3 has problematic security and is only required for really old
# clients such as IE6 on Windows XP
context.options |= OP_NO_SSLv3
context.options |= OP_NO_SSLv3 # pylint:disable=no-member
# disable compression to prevent CRIME attacks (OpenSSL 1.0+)
context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0)
context.options |= getattr(_ssl, "OP_NO_COMPRESSION", 0) # pylint:disable=no-member
if purpose == Purpose.SERVER_AUTH:
# verify certs and host name in client mode
......@@ -112,11 +114,11 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None,
elif purpose == Purpose.CLIENT_AUTH:
# Prefer the server's ciphers by default so that we get stronger
# encryption
context.options |= getattr(_ssl, "OP_CIPHER_SERVER_PREFERENCE", 0)
context.options |= getattr(_ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) # pylint:disable=no-member
# Use single use keys in order to improve forward secrecy
context.options |= getattr(_ssl, "OP_SINGLE_DH_USE", 0)
context.options |= getattr(_ssl, "OP_SINGLE_ECDH_USE", 0)
context.options |= getattr(_ssl, "OP_SINGLE_DH_USE", 0) # pylint:disable=no-member
context.options |= getattr(_ssl, "OP_SINGLE_ECDH_USE", 0) # pylint:disable=no-member
# disallow ciphers with known vulnerabilities
context.set_ciphers(_RESTRICTED_SERVER_CIPHERS)
......@@ -146,10 +148,10 @@ def _create_unverified_context(protocol=PROTOCOL_SSLv23, cert_reqs=None,
context = SSLContext(protocol)
# SSLv2 considered harmful.
context.options |= OP_NO_SSLv2
context.options |= OP_NO_SSLv2 # pylint:disable=no-member
# SSLv3 has problematic security and is only required for really old
# clients such as IE6 on Windows XP
context.options |= OP_NO_SSLv3
context.options |= OP_NO_SSLv3 # pylint:disable=no-member
if cert_reqs is not None:
context.verify_mode = cert_reqs
......
src/gevent/fileobject.py
View file @ 79021f13
"""
Wrappers to make file-like objects cooperative.
.. class:: FileObject
.. class:: FileObject(fobj, mode='r', buffering=-1, closefd=True, encoding=None, errors=None, newline=None)
The main entry point to the file-like gevent-compatible behaviour. It will be defined
to be the best available implementation.
The main entry point to the file-like gevent-compatible behaviour. It
will be defined to be the best available implementation.
All the parameters are as for :func:`io.open`.
:param fobj: Usually a file descriptor of a socket. Can also be
another object with a ``fileno()`` method, or an object that can
be passed to ``io.open()`` (e.g., a file system path). If the object
is not a socket, the results will vary based on the platform and the
type of object being opened.
All supported versions of Python allow :class:`os.PathLike` objects.
.. versionchanged:: 1.5
Accept str and ``PathLike`` objects for *fobj* on all versions of Python.
.. versionchanged:: 1.5
Add *encoding*, *errors* and *newline* arguments.
.. versionchanged:: 1.5
Accept *closefd* and *buffering* instead of *close* and *bufsize* arguments.
The latter remain for backwards compatibility.
There are two main implementations of ``FileObject``. On all systems,
there is :class:`FileObjectThread` which uses the built-in native
......@@ -12,9 +30,11 @@ threadpool to avoid blocking the entire interpreter. On UNIX systems
(those that support the :mod:`fcntl` module), there is also
:class:`FileObjectPosix` which uses native non-blocking semantics.
A third class, :class:`FileObjectBlock`, is simply a wrapper that executes everything
synchronously (and so is not gevent-compatible). It is provided for testing and debugging
purposes.
A third class, :class:`FileObjectBlock`, is simply a wrapper that
executes everything synchronously (and so is not gevent-compatible).
It is provided for testing and debugging purposes.
All classes have the same signature; some may accept extra keyword arguments.
Configuration
=============
......@@ -27,8 +47,10 @@ You may also set it to the fully qualified class name of another
object that implements the file interface to use one of your own
objects.
.. note:: The environment variable must be set at the time this module
is first imported.
.. note::
The environment variable must be set at the time this module
is first imported.
Classes
=======
......@@ -58,4 +80,6 @@ from gevent._fileobjectcommon import FileObjectBlock
# None of the possible objects can live in this module because
# we would get an import cycle and the config couldn't be set from code.
# TODO: zope.hookable would be great for allowing this to be imported
# without requiring configuration but still being very fast.
FileObject = config.fileobject
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7