Commit e093d31d authored by Jason Madden's avatar Jason Madden

Centralize configuration in a ``gevent.config`` object.

This lets us document things more clearly, and, with care, allows
configuring gevent from Python code without any environment varibles.

All the fileobject implementations needed to be moved out of the
``gevent.fileobject`` module so as not to introduce import cycles that
made configuring from code impossible.

Fixes #1090
parent 654e8302
...@@ -46,6 +46,18 @@ ...@@ -46,6 +46,18 @@
:mod:`time`, much like :mod:`gevent.socket` can be imported instead :mod:`time`, much like :mod:`gevent.socket` can be imported instead
of :mod:`socket`. of :mod:`socket`.
- Centralize all gevent configuration in an object at
``gevent.config``, allowing for gevent to be configured through code
and not *necessarily* environment variables, and also provide a
centralized place for documentation. See :issue:`1090`.
- The new ``GEVENT_CORE_CFFI_ONLY`` environment variable has been
replaced with the pre-existing ``GEVENT_LOOP`` environment
variable. That variable may take the values ``libev-cext``,
``libev-cffi``, or ``libuv-cffi``, (or be a list in preference
order, or be a dotted name; it may also be assigned to an
object in Python code at ``gevent.config.loop``).
1.3a1 (2018-01-27) 1.3a1 (2018-01-27)
================== ==================
......
...@@ -53,9 +53,6 @@ test_prelim: ...@@ -53,9 +53,6 @@ test_prelim:
make bench make bench
# Folding from https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh#L38-L44 # Folding from https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh#L38-L44
# echo -e "travis_fold:start:${GEVENT_CORE_CFFI_ONLY}\033[33;1m${GEVENT_CORE_CFFI_ONLY}\033[0m"
# Make calls /bin/echo, which doesn't support the -e option, which is part of the bash builtin.
# we need a python script to do this, or possible the GNU make shell function
basictest: test_prelim basictest: test_prelim
${PYTHON} scripts/travis.py fold_start basictest "Running basic tests" ${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
...@@ -83,7 +80,7 @@ threadfiletest: ...@@ -83,7 +80,7 @@ threadfiletest:
allbackendtest: allbackendtest:
${PYTHON} scripts/travis.py fold_start default "Testing default backend" ${PYTHON} scripts/travis.py fold_start default "Testing default backend"
GEVENT_CORE_CFFI_ONLY= GEVENTTEST_COVERAGE=1 make alltest GEVENTTEST_COVERAGE=1 make alltest
${PYTHON} scripts/travis.py fold_end default ${PYTHON} scripts/travis.py fold_end default
GEVENTTEST_COVERAGE=1 make cffibackendtest GEVENTTEST_COVERAGE=1 make cffibackendtest
# because we set parallel=true, each run produces new and different coverage files; they all need # because we set parallel=true, each run produces new and different coverage files; they all need
...@@ -93,10 +90,10 @@ allbackendtest: ...@@ -93,10 +90,10 @@ allbackendtest:
cffibackendtest: cffibackendtest:
${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend" ${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
GEVENT_CORE_CFFI_ONLY=libuv GEVENTTEST_COVERAGE=1 make alltest GEVENT_LOOP=libuv GEVENTTEST_COVERAGE=1 make alltest
${PYTHON} scripts/travis.py fold_end libuv ${PYTHON} scripts/travis.py fold_end libuv
${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend" ${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
GEVENT_CORE_CFFI_ONLY=libev make alltest GEVENT_LOOP=libev-cffi make alltest
${PYTHON} scripts/travis.py fold_end libev ${PYTHON} scripts/travis.py fold_end libev
leaktest: test_prelim leaktest: test_prelim
...@@ -203,4 +200,4 @@ test-py27-noembed: $(PY27) ...@@ -203,4 +200,4 @@ test-py27-noembed: $(PY27)
cd deps/libev && ./configure --disable-dependency-tracking && make cd deps/libev && ./configure --disable-dependency-tracking && make
cd deps/c-ares && ./configure --disable-dependency-tracking && make cd deps/c-ares && ./configure --disable-dependency-tracking && make
cd deps/libuv && ./autogen.sh && ./configure --disable-static && make cd deps/libuv && ./autogen.sh && ./configure --disable-static && make
CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_CORE_CEXT_ONLY=1 PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_LOOP=libev-cext PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest
...@@ -26,7 +26,7 @@ environment: ...@@ -26,7 +26,7 @@ environment:
PYTHON_VERSION: "3.6.x" # currently 3.6.0 PYTHON_VERSION: "3.6.x" # currently 3.6.0
PYTHON_ARCH: "64" PYTHON_ARCH: "64"
PYTHON_EXE: python PYTHON_EXE: python
GEVENT_CORE_CFFI_ONLY: libuv GEVENT_LOOP: libuv
- PYTHON: "C:\\Python27-x64" - PYTHON: "C:\\Python27-x64"
PYTHON_VERSION: "2.7.x" # currently 2.7.13 PYTHON_VERSION: "2.7.x" # currently 2.7.13
......
...@@ -87,7 +87,7 @@ release = version ...@@ -87,7 +87,7 @@ release = version
exclude_trees = ['_build'] exclude_trees = ['_build']
# The reST default role (used for this markup: `text`) to use for all documents. # The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None default_role = 'obj'
# If true, '()' will be appended to :func: etc. cross-reference text. # If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True add_function_parentheses = True
......
...@@ -191,3 +191,9 @@ Timeouts ...@@ -191,3 +191,9 @@ Timeouts
:special-members: __enter__, __exit__ :special-members: __enter__, __exit__
.. autofunction:: with_timeout .. autofunction:: with_timeout
Configuration
=============
.. autoclass:: gevent._config.Config
...@@ -4,6 +4,9 @@ gevent is a coroutine-based Python networking library that uses greenlet ...@@ -4,6 +4,9 @@ gevent is a coroutine-based Python networking library that uses greenlet
to provide a high-level synchronous API on top of libev event loop. to provide a high-level synchronous API on top of libev event loop.
See http://www.gevent.org/ for the documentation. See http://www.gevent.org/ for the documentation.
.. versionchanged:: 1.3a2
Add the `config` object.
""" """
from __future__ import absolute_import from __future__ import absolute_import
...@@ -43,6 +46,8 @@ __all__ = [ ...@@ -43,6 +46,8 @@ __all__ = [
'reinit', 'reinit',
'getswitchinterval', 'getswitchinterval',
'setswitchinterval', 'setswitchinterval',
# Added in 1.3a2
'config',
] ]
...@@ -72,11 +77,14 @@ except AttributeError: ...@@ -72,11 +77,14 @@ except AttributeError:
global _switchinterval global _switchinterval
_switchinterval = interval _switchinterval = interval
from gevent._config import config
from gevent.hub import get_hub, iwait, wait from gevent.hub import get_hub, iwait, wait
from gevent.greenlet import Greenlet, joinall, killall from gevent.greenlet import Greenlet, joinall, killall
joinall = joinall # export for pylint joinall = joinall # export for pylint
spawn = Greenlet.spawn spawn = Greenlet.spawn
spawn_later = Greenlet.spawn_later spawn_later = Greenlet.spawn_later
#: The singleton configuration object for gevent.
config = config
from gevent.timeout import Timeout, with_timeout from gevent.timeout import Timeout, with_timeout
from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit
......
This diff is collapsed.
...@@ -10,14 +10,17 @@ import signal as signalmodule ...@@ -10,14 +10,17 @@ import signal as signalmodule
import functools import functools
import warnings import warnings
from gevent._config import config
try: try:
from tracemalloc import get_object_traceback from tracemalloc import get_object_traceback
def tracemalloc(init): def tracemalloc(init):
# PYTHONTRACEMALLOC env var controls this. # PYTHONTRACEMALLOC env var controls this on Python 3.
return init return init
except ImportError: # Python < 3.4 except ImportError: # Python < 3.4
if os.getenv('PYTHONTRACEMALLOC'):
if config.trace_malloc:
# Use the same env var to turn this on for Python 2 # Use the same env var to turn this on for Python 2
import traceback import traceback
......
from __future__ import absolute_import, print_function, division
try: try:
from errno import EBADF from errno import EBADF
except ImportError: except ImportError:
EBADF = 9 EBADF = 9
import os
from io import TextIOWrapper from io import TextIOWrapper
import functools
import sys
from gevent.hub import get_hub
from gevent._compat import integer_types
from gevent._compat import reraise
from gevent.lock import Semaphore, DummySemaphore
class cancel_wait_ex(IOError): class cancel_wait_ex(IOError):
...@@ -136,3 +146,130 @@ class FileObjectBase(object): ...@@ -136,3 +146,130 @@ class FileObjectBase(object):
def __exit__(self, *args): def __exit__(self, *args):
self.close() self.close()
class FileObjectBlock(FileObjectBase):
def __init__(self, fobj, *args, **kwargs):
closefd = kwargs.pop('close', True)
if kwargs:
raise TypeError('Unexpected arguments: %r' % kwargs.keys())
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectBlock does not support close=False on an fd.')
fobj = os.fdopen(fobj, *args)
super(FileObjectBlock, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
fobj.close()
class FileObjectThread(FileObjectBase):
"""
A file-like object wrapping another file-like object, performing all blocking
operations on that object in a background thread.
.. caution::
Attempting to change the threadpool or lock of an existing FileObjectThread
has undefined consequences.
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
"""
:param fobj: The underlying file-like object to wrap, or an integer fileno
that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
:keyword bool lock: If True (the default) then all operations will
be performed one-by-one. Note that this does not guarantee that, if using
this file object from multiple threads/greenlets, operations will be performed
in any particular order, only that no two operations will be attempted at the
same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
file operations with an external resource.
:keyword bool close: If True (the default) then when this object is closed,
the underlying object is closed as well.
"""
closefd = close
self.threadpool = threadpool or get_hub().threadpool
self.lock = lock
if self.lock is True:
self.lock = Semaphore()
elif not self.lock:
self.lock = DummySemaphore()
if not hasattr(self.lock, '__enter__'):
raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectThread does not support close=False on an fd.')
if mode is None:
assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
fobj = os.fdopen(fobj)
else:
fobj = os.fdopen(fobj, mode, bufsize)
self.__io_holder = [fobj] # signal for _wrap_method
super(FileObjectThread, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
self.__io_holder[0] = None # for _wrap_method
try:
with self.lock:
self.threadpool.apply(fobj.flush)
finally:
if closefd:
# Note that we're not taking the lock; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged by the pool
def close():
try:
fobj.close()
except: # pylint:disable=bare-except
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def _do_delegate_methods(self):
super(FileObjectThread, self)._do_delegate_methods()
if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
self.read1 = self.read
self.__io_holder[0] = self._io
def _extra_repr(self):
return ' threadpool=%r' % (self.threadpool,)
def __iter__(self):
return self
def next(self):
line = self.readline()
if line:
return line
raise StopIteration
__next__ = next
def _wrap_method(self, method):
# NOTE: We are careful to avoid introducing a refcycle
# within self. Our wrapper cannot refer to self.
io_holder = self.__io_holder
lock = self.lock
threadpool = self.threadpool
@functools.wraps(method)
def thread_method(*args, **kwargs):
if io_holder[0] is None:
# This is different than FileObjectPosix, etc,
# because we want to save the expensive trip through
# the threadpool.
raise FileObjectClosed()
with lock:
return threadpool.apply(method, args, kwargs)
return thread_method
# Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details. # Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details.
"""
Deprecated; this does not reflect all the possible options
and its interface varies.
.. versionchanged:: 1.3a2
Deprecated.
"""
from __future__ import absolute_import from __future__ import absolute_import
import os import sys
from gevent._config import config
from gevent._util import copy_globals from gevent._util import copy_globals
try: _core = sys.modules[config.loop.__module__]
if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
raise ImportError("Not attempting corecext")
from gevent.libev import corecext as _core
except ImportError:
if os.environ.get('GEVENT_CORE_CEXT_ONLY'):
raise
# CFFI/PyPy
lib = os.environ.get('GEVENT_CORE_CFFI_ONLY')
if lib == 'libuv':
from gevent.libuv import loop as _core
else:
try:
from gevent.libev import corecffi as _core
except ImportError:
from gevent.libuv import loop as _core
copy_globals(_core, globals()) copy_globals(_core, globals())
......
...@@ -35,31 +35,12 @@ Classes ...@@ -35,31 +35,12 @@ Classes
""" """
from __future__ import absolute_import from __future__ import absolute_import
import functools from gevent._config import config
import sys
import os
from gevent._fileobjectcommon import FileObjectClosed
from gevent._fileobjectcommon import FileObjectBase
from gevent.hub import get_hub
from gevent._compat import integer_types
from gevent._compat import reraise
from gevent.lock import Semaphore, DummySemaphore
PYPY = hasattr(sys, 'pypy_version_info')
if hasattr(sys, 'exc_clear'):
def _exc_clear():
sys.exc_clear()
else:
def _exc_clear():
return
__all__ = [ __all__ = [
'FileObjectPosix', 'FileObjectPosix',
'FileObjectThread', 'FileObjectThread',
'FileObjectBlock',
'FileObject', 'FileObject',
] ]
...@@ -71,149 +52,10 @@ else: ...@@ -71,149 +52,10 @@ else:
del fcntl del fcntl
from gevent._fileobjectposix import FileObjectPosix from gevent._fileobjectposix import FileObjectPosix
from gevent._fileobjectcommon import FileObjectThread
from gevent._fileobjectcommon import FileObjectBlock
class FileObjectThread(FileObjectBase):
"""
A file-like object wrapping another file-like object, performing all blocking
operations on that object in a background thread.
.. caution::
Attempting to change the threadpool or lock of an existing FileObjectThread
has undefined consequences.
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
"""
:param fobj: The underlying file-like object to wrap, or an integer fileno
that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
:keyword bool lock: If True (the default) then all operations will
be performed one-by-one. Note that this does not guarantee that, if using
this file object from multiple threads/greenlets, operations will be performed
in any particular order, only that no two operations will be attempted at the
same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
file operations with an external resource.
:keyword bool close: If True (the default) then when this object is closed,
the underlying object is closed as well.
"""
closefd = close
self.threadpool = threadpool or get_hub().threadpool
self.lock = lock
if self.lock is True:
self.lock = Semaphore()
elif not self.lock:
self.lock = DummySemaphore()
if not hasattr(self.lock, '__enter__'):
raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectThread does not support close=False on an fd.')
if mode is None:
assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
fobj = os.fdopen(fobj)
else:
fobj = os.fdopen(fobj, mode, bufsize)
self.__io_holder = [fobj] # signal for _wrap_method
super(FileObjectThread, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
self.__io_holder[0] = None # for _wrap_method
try:
with self.lock:
self.threadpool.apply(fobj.flush)
finally:
if closefd:
# Note that we're not taking the lock; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged by the pool
def close():
try:
fobj.close()
except: # pylint:disable=bare-except
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def _do_delegate_methods(self):
super(FileObjectThread, self)._do_delegate_methods()
if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
self.read1 = self.read
self.__io_holder[0] = self._io
def _extra_repr(self):
return ' threadpool=%r' % (self.threadpool,)
def __iter__(self):
return self
def next(self):
line = self.readline()
if line:
return line
raise StopIteration
__next__ = next
def _wrap_method(self, method):
# NOTE: We are careful to avoid introducing a refcycle
# within self. Our wrapper cannot refer to self.
io_holder = self.__io_holder
lock = self.lock
threadpool = self.threadpool
@functools.wraps(method)
def thread_method(*args, **kwargs):
if io_holder[0] is None:
# This is different than FileObjectPosix, etc,
# because we want to save the expensive trip through
# the threadpool.
raise FileObjectClosed()
with lock:
return threadpool.apply(method, args, kwargs)
return thread_method
try:
FileObject = FileObjectPosix
except NameError:
FileObject = FileObjectThread
class FileObjectBlock(FileObjectBase):
def __init__(self, fobj, *args, **kwargs):
closefd = kwargs.pop('close', True)
if kwargs:
raise TypeError('Unexpected arguments: %r' % kwargs.keys())
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectBlock does not support close=False on an fd.')
fobj = os.fdopen(fobj, *args)
super(FileObjectBlock, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
fobj.close()
config = os.environ.get('GEVENT_FILE') # None of the possible objects can live in this module because
if config: # we would get an import cycle and the config couldn't be set from code.
klass = {'thread': 'gevent.fileobject.FileObjectThread', FileObject = config.fileobject
'posix': 'gevent.fileobject.FileObjectPosix',
'block': 'gevent.fileobject.FileObjectBlock'}.get(config, config)
if klass.startswith('gevent.fileobject.'):
FileObject = globals()[klass.split('.', 2)[-1]]
else:
from gevent.hub import _import
FileObject = _import(klass)
del klass
...@@ -26,6 +26,7 @@ __all__ = [ ...@@ -26,6 +26,7 @@ __all__ = [
'Waiter', 'Waiter',
] ]
from gevent._config import config
from gevent._compat import string_types from gevent._compat import string_types
from gevent._compat import xrange from gevent._compat import xrange
from gevent._util import _NONE from gevent._util import _NONE
...@@ -389,89 +390,29 @@ def set_hub(hub): ...@@ -389,89 +390,29 @@ def set_hub(hub):
_threadlocal.hub = hub _threadlocal.hub = hub
def _import(path):
# pylint:disable=too-many-branches
if isinstance(path, list):
if not path:
raise ImportError('Cannot import from empty list: %r' % (path, ))
for item in path[:-1]:
try:
return _import(item)
except ImportError:
pass
return _import(path[-1])
if not isinstance(path, string_types):
return path
if '.' not in path:
raise ImportError("Cannot import %r (required format: [path/][package.]module.class)" % path)
if '/' in path:
# This is dangerous, subject to race conditions, and
# may not work properly for things like namespace packages
import warnings
warnings.warn("Absolute paths are deprecated. Please put the package "
"on sys.path first",
DeprecationWarning)
package_path, path = path.rsplit('/', 1)
sys.path = [package_path] + sys.path
else:
package_path = None
try:
module, item = path.rsplit('.', 1)
x = __import__(module)
for attr in path.split('.')[1:]:
oldx = x
x = getattr(x, attr, _NONE)
if x is _NONE:
raise ImportError('Cannot import %r from %r' % (attr, oldx))
return x
finally:
if '/' in path:
try:
sys.path.remove(package_path)
except ValueError:
pass
def config(default, envvar): def _config(default, envvar):
result = os.environ.get(envvar) or default # absolute import gets confused pylint: disable=no-member result = os.environ.get(envvar) or default # absolute import gets confused pylint: disable=no-member
if isinstance(result, string_types): if isinstance(result, string_types):
return result.split(',') return result.split(',')
return result return result
def resolver_config(default, envvar):
result = config(default, envvar)
return [_resolvers.get(x, x) for x in result]
_resolvers = {
'ares': 'gevent.resolver.ares.Resolver',
'thread': 'gevent.resolver.thread.Resolver',
'block': 'gevent.resolver.blocking.Resolver',
'dnspython': 'gevent.resolver.dnspython.Resolver',
}
_DEFAULT_LOOP_CLASS = 'gevent.core.loop'
class Hub(RawGreenlet): class Hub(RawGreenlet):
"""A greenlet that runs the event loop. """
A greenlet that runs the event loop.
It is created automatically by :func:`get_hub`. It is created automatically by :func:`get_hub`.
**Switching** .. rubric:: Switching
Every time this greenlet (i.e., the event loop) is switched *to*, if Every time this greenlet (i.e., the event loop) is switched *to*,
the current greenlet has a ``switch_out`` method, it will be called. This if the current greenlet has a ``switch_out`` method, it will be
allows a greenlet to take some cleanup actions before yielding control. This method called. This allows a greenlet to take some cleanup actions before
should not call any gevent blocking functions. yielding control. This method should not call any gevent blocking
functions.
""" """
#: If instances of these classes are raised into the event loop, #: If instances of these classes are raised into the event loop,
...@@ -483,37 +424,8 @@ class Hub(RawGreenlet): ...@@ -483,37 +424,8 @@ class Hub(RawGreenlet):
#: do not get logged/printed when raised by the event loop. #: do not get logged/printed when raised by the event loop.
NOT_ERROR = (GreenletExit, SystemExit) NOT_ERROR = (GreenletExit, SystemExit)
loop_class = config(_DEFAULT_LOOP_CLASS, 'GEVENT_LOOP')
# For the standard class, go ahead and import it when this class
# is defined. This is no loss of generality because the envvar is
# only read when this class is defined, and we know that the
# standard class will be available. This can solve problems with
# the class being imported from multiple threads at once, leading
# to one of the imports failing. Only do this for the object we
# need in the constructor, as the rest of the factories are
# themselves handled lazily. See #687. (People using a custom loop_class
# can probably manage to get_hub() from the main thread or otherwise import
# that loop_class themselves.)
if loop_class == [_DEFAULT_LOOP_CLASS]:
loop_class = [_import(loop_class)]
resolver_class = [
'gevent.resolver.thread.Resolver',
'gevent.resolver.dnspython.Resolver',
'gevent.resolver.ares.Resolver',
'gevent.resolver.blacking.Resolver',
]
#: The class or callable object, or the name of a factory function or class,
#: that will be used to create :attr:`resolver`. By default, configured according to
#: :doc:`dns`. If a list, a list of objects in preference order.
resolver_class = resolver_config(resolver_class, 'GEVENT_RESOLVER')
threadpool_class = config('gevent.threadpool.ThreadPool', 'GEVENT_THREADPOOL')
backend = config(None, 'GEVENT_BACKEND')
threadpool_size = 10
# using pprint.pformat can override custom __repr__ methods on dict/list threadpool_size = 10
# subclasses, which can be a security concern
format_context = 'pprint.saferepr'
def __init__(self, loop=None, default=None): def __init__(self, loop=None, default=None):
...@@ -530,13 +442,21 @@ class Hub(RawGreenlet): ...@@ -530,13 +442,21 @@ class Hub(RawGreenlet):
else: else:
if default is None and get_ident() != MAIN_THREAD: if default is None and get_ident() != MAIN_THREAD:
default = False default = False
loop_class = _import(self.loop_class)
if loop is None: if loop is None:
loop = self.backend loop = self.backend
self.loop = loop_class(flags=loop, default=default) self.loop = self.loop_class(flags=loop, default=default) # pylint:disable=not-callable
self._resolver = None self._resolver = None
self._threadpool = None self._threadpool = None
self.format_context = _import(self.format_context) self.format_context = config.format_context
@property
def loop_class(self):
return config.loop
@property
def backend(self):
return config.libev_backend
def __repr__(self): def __repr__(self):
if self.loop is None: if self.loop is None:
...@@ -787,11 +707,13 @@ class Hub(RawGreenlet): ...@@ -787,11 +707,13 @@ class Hub(RawGreenlet):
if _threadlocal.hub is self: if _threadlocal.hub is self:
_threadlocal.hub = None _threadlocal.hub = None
@property
def resolver_class(self):
return config.resolver
def _get_resolver(self): def _get_resolver(self):
if self._resolver is None: if self._resolver is None:
if self.resolver_class is not None: self._resolver = self.resolver_class(hub=self) # pylint:disable=not-callable
self.resolver_class = _import(self.resolver_class)
self._resolver = self.resolver_class(hub=self)
return self._resolver return self._resolver
def _set_resolver(self, value): def _set_resolver(self, value):
...@@ -802,11 +724,15 @@ class Hub(RawGreenlet): ...@@ -802,11 +724,15 @@ class Hub(RawGreenlet):
resolver = property(_get_resolver, _set_resolver, _del_resolver) resolver = property(_get_resolver, _set_resolver, _del_resolver)
@property
def threadpool_class(self):
return config.threadpool
def _get_threadpool(self): def _get_threadpool(self):
if self._threadpool is None: if self._threadpool is None:
if self.threadpool_class is not None: # pylint:disable=not-callable
self.threadpool_class = _import(self.threadpool_class) self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
return self._threadpool return self._threadpool
def _set_threadpool(self, value): def _set_threadpool(self, value):
......
...@@ -46,6 +46,7 @@ from __future__ import absolute_import ...@@ -46,6 +46,7 @@ from __future__ import absolute_import
import os import os
import sys import sys
from gevent.hub import get_hub, reinit from gevent.hub import get_hub, reinit
from gevent._config import config
from gevent._compat import PY3 from gevent._compat import PY3
from gevent._util import copy_globals from gevent._util import copy_globals
import errno import errno
...@@ -418,7 +419,7 @@ if hasattr(os, 'fork'): ...@@ -418,7 +419,7 @@ if hasattr(os, 'fork'):
__extensions__.append('forkpty_and_watch') __extensions__.append('forkpty_and_watch')
# Watch children by default # Watch children by default
if not os.getenv('GEVENT_NOWAITPID'): if not config.disable_watch_children:
# Broken out into separate functions instead of simple name aliases # Broken out into separate functions instead of simple name aliases
# for documentation purposes. # for documentation purposes.
def fork(*args, **kwargs): def fork(*args, **kwargs):
......
...@@ -27,6 +27,9 @@ from gevent.socket import SOCK_DGRAM ...@@ -27,6 +27,9 @@ from gevent.socket import SOCK_DGRAM
from gevent.socket import SOCK_RAW from gevent.socket import SOCK_RAW
from gevent.socket import AI_NUMERICHOST from gevent.socket import AI_NUMERICHOST
from gevent._config import config
from gevent._config import AresSettingMixin
from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
from . import _lookup_port as lookup_port from . import _lookup_port as lookup_port
from . import _resolve_special from . import _resolve_special
...@@ -84,12 +87,11 @@ class Resolver(AbstractResolver): ...@@ -84,12 +87,11 @@ class Resolver(AbstractResolver):
hub = get_hub() hub = get_hub()
self.hub = hub self.hub = hub
if use_environ: if use_environ:
for key in os.environ: for setting in config.settings.values():
if key.startswith('GEVENTARES_'): if isinstance(setting, AresSettingMixin):
name = key[11:].lower() value = setting.get()
if name: if value is not None:
value = os.environ[key] kwargs.setdefault(setting.kwarg_name, value)
kwargs.setdefault(name, value)
self.ares = self.ares_class(hub.loop, **kwargs) self.ares = self.ares_class(hub.loop, **kwargs)
self.pid = os.getpid() self.pid = os.getpid()
self.params = kwargs self.params = kwargs
......
...@@ -23,9 +23,11 @@ from greentest.sysinfo import PY3 ...@@ -23,9 +23,11 @@ from greentest.sysinfo import PY3
from greentest.sysinfo import PYPY from greentest.sysinfo import PYPY
from greentest.sysinfo import WIN from greentest.sysinfo import WIN
from greentest.sysinfo import LIBUV from greentest.sysinfo import LIBUV
from greentest.sysinfo import OSX
from greentest.sysinfo import RUNNING_ON_TRAVIS from greentest.sysinfo import RUNNING_ON_TRAVIS
from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION
from greentest.sysinfo import RESOLVER_ARES
# Travis is slow and overloaded; Appveyor used to be faster, but # Travis is slow and overloaded; Appveyor used to be faster, but
...@@ -60,6 +62,11 @@ if RUNNING_ON_TRAVIS: ...@@ -60,6 +62,11 @@ if RUNNING_ON_TRAVIS:
# connected to with the same error. # connected to with the same error.
DEFAULT_BIND_ADDR = '127.0.0.1' DEFAULT_BIND_ADDR = '127.0.0.1'
if RESOLVER_ARES and OSX:
# Ares likes to raise "malformed domain name" on '', at least
# on OS X
DEFAULT_BIND_ADDR = '127.0.0.1'
# For in-process sockets # For in-process sockets
DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0 DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0
......
...@@ -30,7 +30,7 @@ LINUX = sys.platform.startswith('linux') ...@@ -30,7 +30,7 @@ LINUX = sys.platform.startswith('linux')
OSX = sys.platform == 'darwin' OSX = sys.platform == 'darwin'
# XXX: Formalize this better # XXX: Formalize this better
LIBUV = os.getenv('GEVENT_CORE_CFFI_ONLY') == 'libuv' or (PYPY and WIN) or hasattr(gevent.core, 'libuv') LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
CFFI_BACKEND = bool(os.getenv('GEVENT_CORE_CFFI_ONLY')) or PYPY CFFI_BACKEND = bool(os.getenv('GEVENT_CORE_CFFI_ONLY')) or PYPY
if '--debug-greentest' in sys.argv: if '--debug-greentest' in sys.argv:
......
# Copyright 2018 gevent contributors. See LICENSE for details.
import os
import unittest
from gevent import _config
class TestResolver(unittest.TestCase):
old_resolver = None
def setUp(self):
if 'GEVENT_RESOLVER' in os.environ:
self.old_resolver = os.environ['GEVENT_RESOLVER']
del os.environ['GEVENT_RESOLVER']
def tearDown(self):
if self.old_resolver:
os.environ['GEVENT_RESOLVER'] = self.old_resolver
def test_key(self):
self.assertEqual(_config.Resolver.environment_key, 'GEVENT_RESOLVER')
def test_default(self):
from gevent.resolver.thread import Resolver
conf = _config.Resolver()
self.assertEqual(conf.get(), Resolver)
def test_env(self):
from gevent.resolver.blocking import Resolver
os.environ['GEVENT_RESOLVER'] = 'foo,bar,block,dnspython'
conf = _config.Resolver()
self.assertEqual(conf.get(), Resolver)
os.environ['GEVENT_RESOLVER'] = 'dnspython'
# The existing value is unchanged
self.assertEqual(conf.get(), Resolver)
# A new object reflects it
conf = _config.Resolver()
from gevent.resolver.dnspython import Resolver as DResolver
self.assertEqual(conf.get(), DResolver)
def test_set_str_long(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set('gevent.resolver.blocking.Resolver')
self.assertEqual(conf.get(), Resolver)
def test_set_str_short(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set('block')
self.assertEqual(conf.get(), Resolver)
def test_set_class(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set(Resolver)
self.assertEqual(conf.get(), Resolver)
def test_set_through_config(self):
from gevent.resolver.thread import Resolver as Default
from gevent.resolver.blocking import Resolver
conf = _config.Config()
self.assertEqual(conf.resolver, Default)
conf.resolver = 'block'
self.assertEqual(conf.resolver, Resolver)
if __name__ == '__main__':
unittest.main()
...@@ -123,3 +123,5 @@ test_timeout.py ...@@ -123,3 +123,5 @@ test_timeout.py
# implicitly for localhost, which is covered well enough # implicitly for localhost, which is covered well enough
# elsewhere that we don't need to spend the 20s (*2) # elsewhere that we don't need to spend the 20s (*2)
test_asyncore.py test_asyncore.py
test___config.py
...@@ -36,7 +36,7 @@ commands = ...@@ -36,7 +36,7 @@ commands =
basepython = basepython =
python2.7 python2.7
setenv = setenv =
GEVENT_CORE_CFFI_ONLY=1 GEVENT_LOOP=libev-cffi
commands = commands =
make basictest make basictest
...@@ -44,7 +44,7 @@ commands = ...@@ -44,7 +44,7 @@ commands =
basepython = basepython =
python2.7 python2.7
setenv = setenv =
GEVENT_CORE_CFFI_ONLY=libuv GEVENT_LOOP=libuv-cffi
commands = commands =
make basictest make basictest
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment