Centralize configuration in a ``gevent.config`` object.

This lets us document things more clearly, and, with care, allows
configuring gevent from Python code without any environment varibles.

All the fileobject implementations needed to be moved out of the
``gevent.fileobject`` module so as not to introduce import cycles that
made configuring from code impossible.

Fixes #1090

CHANGES.rst

@@ -46,6 +46,18 @@
   :mod:`time`, much like :mod:`gevent.socket` can be imported instead
   of :mod:`socket`.
 
+- Centralize all gevent configuration in an object at
+  ``gevent.config``, allowing for gevent to be configured through code
+  and not *necessarily* environment variables, and also provide a
+  centralized place for documentation. See :issue:`1090`.
+
+  - The new ``GEVENT_CORE_CFFI_ONLY`` environment variable has been
+    replaced with the pre-existing ``GEVENT_LOOP`` environment
+    variable. That variable may take the values ``libev-cext``,
+    ``libev-cffi``, or ``libuv-cffi``, (or be a list in preference
+    order, or be a dotted name; it may also be assigned to an
+    object in Python code at ``gevent.config.loop``).
+
 1.3a1 (2018-01-27)
 ==================
 

Makefile

@@ -53,9 +53,6 @@ test_prelim:
 	make bench
 
 # Folding from https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh#L38-L44
-# 	echo -e "travis_fold:start:${GEVENT_CORE_CFFI_ONLY}\033[33;1m${GEVENT_CORE_CFFI_ONLY}\033[0m"
-# Make calls /bin/echo, which doesn't support the -e option, which is part of the bash builtin.
-# we need a python script to do this, or possible the GNU make shell function
 
 basictest: test_prelim
 	${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
@@ -83,7 +80,7 @@ threadfiletest:
 
 allbackendtest:
 	${PYTHON} scripts/travis.py fold_start default "Testing default backend"
-	GEVENT_CORE_CFFI_ONLY= GEVENTTEST_COVERAGE=1 make alltest
+	GEVENTTEST_COVERAGE=1 make alltest
 	${PYTHON} scripts/travis.py fold_end default
 	GEVENTTEST_COVERAGE=1 make cffibackendtest
 # because we set parallel=true, each run produces new and different coverage files; they all need
@@ -93,10 +90,10 @@ allbackendtest:
 
 cffibackendtest:
 	${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
-	GEVENT_CORE_CFFI_ONLY=libuv GEVENTTEST_COVERAGE=1 make alltest
+	GEVENT_LOOP=libuv GEVENTTEST_COVERAGE=1 make alltest
 	${PYTHON} scripts/travis.py fold_end libuv
 	${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
-	GEVENT_CORE_CFFI_ONLY=libev make alltest
+	GEVENT_LOOP=libev-cffi make alltest
 	${PYTHON} scripts/travis.py fold_end libev
 
 leaktest: test_prelim
@@ -203,4 +200,4 @@ test-py27-noembed: $(PY27)
 	cd deps/libev && ./configure --disable-dependency-tracking && make
 	cd deps/c-ares && ./configure --disable-dependency-tracking && make
 	cd deps/libuv && ./autogen.sh && ./configure --disable-static && make
-	CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_CORE_CEXT_ONLY=1 PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest
+	CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_LOOP=libev-cext PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest

appveyor.yml

@@ -26,7 +26,7 @@ environment:
       PYTHON_VERSION: "3.6.x" # currently 3.6.0
       PYTHON_ARCH: "64"
       PYTHON_EXE: python
-      GEVENT_CORE_CFFI_ONLY: libuv
+      GEVENT_LOOP: libuv
 
     - PYTHON: "C:\\Python27-x64"
       PYTHON_VERSION: "2.7.x" # currently 2.7.13

doc/conf.py

@@ -87,7 +87,7 @@ release = version
 exclude_trees = ['_build']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+default_role = 'obj'
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 add_function_parentheses = True

doc/gevent.rst

@@ -191,3 +191,9 @@ Timeouts
     :special-members: __enter__, __exit__
 
 .. autofunction:: with_timeout
+
+
+Configuration
+=============
+
+.. autoclass:: gevent._config.Config

src/gevent/__init__.py

@@ -4,6 +4,9 @@ gevent is a coroutine-based Python networking library that uses greenlet
 to provide a high-level synchronous API on top of libev event loop.
 
 See http://www.gevent.org/ for the documentation.
+
+.. versionchanged:: 1.3a2
+   Add the `config` object.
 """
 
 from __future__ import absolute_import
@@ -43,6 +46,8 @@ __all__ = [
     'reinit',
     'getswitchinterval',
     'setswitchinterval',
+    # Added in 1.3a2
+    'config',
 ]
 
 
@@ -72,11 +77,14 @@ except AttributeError:
             global _switchinterval
             _switchinterval = interval
 
+from gevent._config import config
 from gevent.hub import get_hub, iwait, wait
 from gevent.greenlet import Greenlet, joinall, killall
 joinall = joinall # export for pylint
 spawn = Greenlet.spawn
 spawn_later = Greenlet.spawn_later
+#: The singleton configuration object for gevent.
+config = config
 
 from gevent.timeout import Timeout, with_timeout
 from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit

src/gevent/_config.py

+# Copyright (c) 2018 gevent. See LICENSE for details.
+"""
+gevent tunables.
+
+This should be used as ``from gevent import config``. That variable
+is an object of :class:`Config`.
+
+.. versionadded:: 1.3a2
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import importlib
+import os
+import sys
+import textwrap
+
+from gevent._compat import string_types
+
+__all__ = [
+    'config',
+]
+
+ALL_SETTINGS = []
+
+class SettingType(type):
+    # pylint:disable=bad-mcs-classmethod-argument
+
+    def __new__(cls, name, bases, cls_dict):
+        if name == 'Setting':
+            return type.__new__(cls, name, bases, cls_dict)
+
+        cls_dict["order"] = len(ALL_SETTINGS)
+        if 'name' not in cls_dict:
+            cls_dict['name'] = name.lower()
+
+        if 'environment_key' not in cls_dict:
+            cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper()
+
+
+        new_class = type.__new__(cls, name, bases, cls_dict)
+        new_class.fmt_desc(cls_dict.get("desc", ""))
+        new_class.__doc__ = new_class.desc
+        ALL_SETTINGS.append(new_class)
+
+        if new_class.document:
+            setting_name = cls_dict['name']
+
+            def getter(self):
+                return self.settings[setting_name].get()
+
+            def setter(self, value):
+                self.settings[setting_name].set(value)
+
+            prop = property(getter, setter, doc=new_class.__doc__)
+
+            setattr(Config, cls_dict['name'], prop)
+        return new_class
+
+    def fmt_desc(cls, desc):
+        desc = textwrap.dedent(desc).strip()
+        if hasattr(cls, 'shortname_map'):
+            desc += (
+                "\n\nThis is an importable value. It can be "
+                "given as a string naming an importable object, "
+                "or a list of strings in preference order and the first "
+                "successfully importable object will be used. (Separate values "
+                "in the environment variable with commas.) "
+                "It can also be given as the callable object itself (in code). "
+            )
+            if cls.shortname_map:
+                desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),)
+        if getattr(cls.validate, '__doc__'):
+            desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip()
+        if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'):
+            default = "`%s`" % (cls.default,)
+        else:
+            default = "`%r`" % (cls.default,)
+        desc += "\n\nThe default value is %s" % (default,)
+        desc += ("\n\nThe environment variable ``%s`` "
+                 "can be used to control this." % (cls.environment_key,))
+        setattr(cls, "desc", desc)
+        return desc
+
+def validate_invalid(value):
+    raise ValueError("Not a valid value: %r" % (value,))
+
+def validate_bool(value):
+    """
+    This is a boolean value.
+
+    In the environment variable, it may be given as ``1``, ``true``,
+    ``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or
+    ``no`` for `False`.
+    """
+    if isinstance(value, string_types):
+        value = value.lower().strip()
+        if value in ('1', 'true', 'on', 'yes'):
+            value = True
+        elif value in ('0', 'false', 'off', 'no') or not value:
+            value = False
+        else:
+            raise ValueError("Invalid boolean string: %r" % (value,))
+    return bool(value)
+
+def validate_anything(value):
+    return value
+
+class Setting(object):
+    name = None
+    value = None
+    validate = staticmethod(validate_invalid)
+    default = None
+    environment_key = None
+    document = True
+
+    desc = """\
+
+    A long ReST description.
+
+    The first line should be a single sentence.
+
+    """
+
+    def _convert(self, value):
+        if isinstance(value, string_types):
+            return value.split(',')
+        return value
+
+    def _default(self):
+        result = os.environ.get(self.environment_key, self.default)
+        result = self._convert(result)
+        return result
+
+    def get(self):
+        # If we've been specifically set, return it
+        if 'value' in self.__dict__:
+            return self.value
+        # Otherwise, read from the environment and reify
+        # so we return consistent results.
+        self.value = self.validate(self._default())
+        return self.value
+
+    def set(self, val):
+        self.value = self.validate(self._convert(val))
+
+
+Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__))
+
+def make_settings():
+    """
+    Return fresh instances of all classes defined in `ALL_SETTINGS`.
+    """
+    settings = {}
+    for setting_kind in ALL_SETTINGS:
+        setting = setting_kind()
+        assert setting.name not in settings
+        settings[setting.name] = setting
+    return settings
+
+
+class Config(object):
+    """
+    Global configuration for gevent.
+
+    There is one instance of this object at ``gevent.config``. If you
+    are going to make changes in code, instead of using the documented
+    environment variables, you need to make the changes before using
+    any parts of gevent that might need those settings. For example::
+
+        >>> from gevent import config
+        >>> config.fileobject = 'thread'
+
+        >>> from gevent import fileobject
+        >>> fileobject.FileObject.__name__
+        'FileObjectThread'
+
+    .. versionadded:: 1.3a2
+
+    """
+
+    def __init__(self):
+        self.settings = make_settings()
+
+    def __getattr__(self, name):
+        if name not in self.settings:
+            raise AttributeError("No configuration setting for: %r" % name)
+        return self.settings[name].get()
+
+    def __setattr__(self, name, value):
+        if name != "settings" and name in self.settings:
+            self.set(name, value)
+        else:
+            super(Config, self).__setattr__(name, value)
+
+    def set(self, name, value):
+        if name not in self.settings:
+            raise AttributeError("No configuration setting for: %r" % name)
+        self.settings[name].set(value)
+
+    def __dir__(self):
+        return list(self.settings)
+
+
+class ImportableSetting(object):
+
+    def _import(self, path, _NONE=object):
+        # pylint:disable=too-many-branches
+        if isinstance(path, list):
+            if not path:
+                raise ImportError('Cannot import from empty list: %r' % (path, ))
+
+            for item in path[:-1]:
+                try:
+                    return self._import(item)
+                except ImportError:
+                    pass
+
+            return self._import(path[-1])
+
+        if not isinstance(path, string_types):
+            return path
+
+        if '.' not in path:
+            raise ImportError("Cannot import %r. "
+                              "Required format: [path/][package.]module.class. "
+                              "Or choice from %r"
+                              % (path, list(self.shortname_map)))
+
+        if '/' in path:
+            # This is dangerous, subject to race conditions, and
+            # may not work properly for things like namespace packages
+            import warnings
+            warnings.warn("Absolute paths are deprecated and will be removed in 1.4."
+                          "Please put the package on sys.path first",
+                          DeprecationWarning)
+            package_path, path = path.rsplit('/', 1)
+            sys.path = [package_path] + sys.path
+        else:
+            package_path = None
+
+        try:
+            module, item = path.rsplit('.', 1)
+            module = importlib.import_module(module)
+            x = getattr(module, item, _NONE)
+            if x is _NONE:
+                raise ImportError('Cannot import %r from %r' % (item, module))
+            return x
+        finally:
+            if '/' in path:
+                try:
+                    sys.path.remove(package_path)
+                except ValueError:
+                    pass
+
+    shortname_map = {}
+
+    def validate(self, value):
+        if isinstance(value, type):
+            return value
+        return self._import([self.shortname_map.get(x, x) for x in value])
+
+
+class Resolver(ImportableSetting, Setting):
+
+    desc = """\
+    The callable that will be used to create
+    :attr:`gevent.hub.Hub.resolver`.
+
+    See :doc:`dns` for more information.
+    """
+
+    default = [
+        'thread',
+        'dnspython',
+        'ares',
+        'block',
+    ]
+
+    shortname_map = {
+        'ares': 'gevent.resolver.ares.Resolver',
+        'thread': 'gevent.resolver.thread.Resolver',
+        'block': 'gevent.resolver.blocking.Resolver',
+        'dnspython': 'gevent.resolver.dnspython.Resolver',
+    }
+
+
+
+class Threadpool(ImportableSetting, Setting):
+
+    desc = """\
+    The kind of threadpool we use.
+    """
+
+    default = 'gevent.threadpool.ThreadPool'
+
+
+class Loop(ImportableSetting, Setting):
+
+    desc = """\
+    The kind of the loop we use.
+
+    """
+
+    default = [
+        'libev-cext',
+        'libev-cffi',
+        'libuv-cffi',
+    ]
+
+    shortname_map = {
+        'libev-cext': 'gevent.libev.corecext.loop',
+        'libev-cffi': 'gevent.libev.corecffi.loop',
+        'libuv-cffi': 'gevent.libuv.loop.loop',
+    }
+
+    shortname_map['libuv'] = shortname_map['libuv-cffi']
+
+
+class FormatContext(ImportableSetting, Setting):
+    name = 'format_context'
+
+    # using pprint.pformat can override custom __repr__ methods on dict/list
+    # subclasses, which can be a security concern
+    default = 'pprint.saferepr'
+
+
+class LibevBackend(Setting):
+    name = 'libev_backend'
+    environment_key = 'GEVENT_BACKEND'
+
+    desc = """\
+    The backend for libev, such as 'select'
+    """
+
+    default = None
+
+    validate = staticmethod(validate_anything)
+
+
+class FileObject(ImportableSetting, Setting):
+    desc = """\
+    The kind of ``FileObject`` we will use.
+
+    See :mod:`gevent.fileobject` for a detailed description.
+
+    """
+    environment_key = 'GEVENT_FILE'
+
+    default = [
+        'posix',
+        'thread',
+    ]
+
+    shortname_map = {
+        'thread': 'gevent._fileobjectcommon.FileObjectThread',
+        'posix': 'gevent._fileobjectposix.FileObjectPosix',
+        'block': 'gevent._fileobjectcommon.FileObjectBlock'
+    }
+
+
+class WatchChildren(Setting):
+    desc = """\
+    Should we *not* watch children with the event loop watchers?
+
+    This is an advanced setting.
+
+    See :mod:`gevent.os` for a detailed description.
+    """
+    name = 'disable_watch_children'
+    environment_key = 'GEVENT_NOWAITPID'
+    default = False
+
+    validate = staticmethod(validate_bool)
+
+
+class TraceMalloc(Setting):
+    name = 'trace_malloc'
+    environment_key = 'PYTHONTRACEMALLOC'
+    default = False
+    validate = staticmethod(validate_bool)
+
+    desc = """\
+    Should FFI objects track their allocation?
+
+    This is only useful for low-level debugging.
+
+    On Python 3, this environment variable is built in to the
+    interpreter, and it may also be set with the ``-X
+    tracemalloc`` command line argument.
+
+    On Python 2, gevent interprets this argument and adds extra
+    tracking information for FFI objects.
+    """
+
+# The ares settings are all interpreted by
+# gevent/resolver/ares.pyx, so we don't do
+# any validation here.
+
+class AresSettingMixin(object):
+
+    document = False
+
+    @property
+    def kwarg_name(self):
+        return self.name[5:]
+
+    validate = staticmethod(validate_anything)
+
+    _convert = staticmethod(validate_anything)
+
+class AresFlags(AresSettingMixin, Setting):
+    name = 'ares_flags'
+    default = None
+    environment_key = 'GEVENTARES_FLAGS'
+
+class AresTimeout(AresSettingMixin, Setting):
+    name = 'ares_timeout'
+    default = None
+    environment_key = 'GEVENTARES_TIMEOUT'
+
+class AresTries(AresSettingMixin, Setting):
+    name = 'ares_tries'
+    default = None
+    environment_key = 'GEVENTARES_TRIES'
+
+class AresNdots(AresSettingMixin, Setting):
+    name = 'ares_ndots'
+    default = None
+    environment_key = 'GEVENTARES_NDOTS'
+
+class AresUDPPort(AresSettingMixin, Setting):
+    name = 'ares_udp_port'
+    default = None
+    environment_key = 'GEVENTARES_UDP_PORT'
+
+class AresTCPPort(AresSettingMixin, Setting):
+    name = 'ares_tcp_port'
+    default = None
+    environment_key = 'GEVENTARES_TCP_PORT'
+
+class AresServers(AresSettingMixin, Setting):
+    document = True
+    name = 'ares_servers'
+    default = None
+    environment_key = 'GEVENTARES_SERVERS'
+
+
+config = Config()
+
+# Go ahead and attempt to import the loop when this class is
+# instantiated. The hub won't work if the loop can't be found. This
+# can solve problems with the class being imported from multiple
+# threads at once, leading to one of the imports failing.
+# factories are themselves handled lazily. See #687.
+
+# Don't cache it though, in case the user re-configures through the
+# API.
+
+try:
+    Loop().get()
+except ImportError: # pragma: no cover
+    pass

src/gevent/_ffi/watcher.py

@@ -10,14 +10,17 @@ import signal as signalmodule
 import functools
 import warnings
 
+from gevent._config import config
+
 try:
     from tracemalloc import get_object_traceback
 
     def tracemalloc(init):
-        # PYTHONTRACEMALLOC env var controls this.
+        # PYTHONTRACEMALLOC env var controls this on Python 3.
         return init
 except ImportError: # Python < 3.4
-    if os.getenv('PYTHONTRACEMALLOC'):
+
+    if config.trace_malloc:
         # Use the same env var to turn this on for Python 2
         import traceback
 

src/gevent/_fileobjectcommon.py

+from __future__ import absolute_import, print_function, division
 
 try:
     from errno import EBADF
 except ImportError:
     EBADF = 9
 
+import os
 from io import TextIOWrapper
+import functools
+import sys
+
+
+from gevent.hub import get_hub
+from gevent._compat import integer_types
+from gevent._compat import reraise
+from gevent.lock import Semaphore, DummySemaphore
 
 class cancel_wait_ex(IOError):
 
@@ -136,3 +146,130 @@ class FileObjectBase(object):
 
     def __exit__(self, *args):
         self.close()
+
+class FileObjectBlock(FileObjectBase):
+
+    def __init__(self, fobj, *args, **kwargs):
+        closefd = kwargs.pop('close', True)
+        if kwargs:
+            raise TypeError('Unexpected arguments: %r' % kwargs.keys())
+        if isinstance(fobj, integer_types):
+            if not closefd:
+                # we cannot do this, since fdopen object will close the descriptor
+                raise TypeError('FileObjectBlock does not support close=False on an fd.')
+            fobj = os.fdopen(fobj, *args)
+        super(FileObjectBlock, self).__init__(fobj, closefd)
+
+    def _do_close(self, fobj, closefd):
+        fobj.close()
+
+class FileObjectThread(FileObjectBase):
+    """
+    A file-like object wrapping another file-like object, performing all blocking
+    operations on that object in a background thread.
+
+    .. caution::
+        Attempting to change the threadpool or lock of an existing FileObjectThread
+        has undefined consequences.
+
+    .. versionchanged:: 1.1b1
+       The file object is closed using the threadpool. Note that whether or
+       not this action is synchronous or asynchronous is not documented.
+
+    """
+
+    def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
+        """
+        :param fobj: The underlying file-like object to wrap, or an integer fileno
+           that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
+        :keyword bool lock: If True (the default) then all operations will
+           be performed one-by-one. Note that this does not guarantee that, if using
+           this file object from multiple threads/greenlets, operations will be performed
+           in any particular order, only that no two operations will be attempted at the
+           same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
+           file operations with an external resource.
+        :keyword bool close: If True (the default) then when this object is closed,
+           the underlying object is closed as well.
+        """
+        closefd = close
+        self.threadpool = threadpool or get_hub().threadpool
+        self.lock = lock
+        if self.lock is True:
+            self.lock = Semaphore()
+        elif not self.lock:
+            self.lock = DummySemaphore()
+        if not hasattr(self.lock, '__enter__'):
+            raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
+        if isinstance(fobj, integer_types):
+            if not closefd:
+                # we cannot do this, since fdopen object will close the descriptor
+                raise TypeError('FileObjectThread does not support close=False on an fd.')
+            if mode is None:
+                assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
+                fobj = os.fdopen(fobj)
+            else:
+                fobj = os.fdopen(fobj, mode, bufsize)
+
+        self.__io_holder = [fobj] # signal for _wrap_method
+        super(FileObjectThread, self).__init__(fobj, closefd)
+
+    def _do_close(self, fobj, closefd):
+        self.__io_holder[0] = None # for _wrap_method
+        try:
+            with self.lock:
+                self.threadpool.apply(fobj.flush)
+        finally:
+            if closefd:
+                # Note that we're not taking the lock; older code
+                # did fobj.close() without going through the threadpool at all,
+                # so acquiring the lock could potentially introduce deadlocks
+                # that weren't present before. Avoiding the lock doesn't make
+                # the existing race condition any worse.
+                # We wrap the close in an exception handler and re-raise directly
+                # to avoid the (common, expected) IOError from being logged by the pool
+                def close():
+                    try:
+                        fobj.close()
+                    except: # pylint:disable=bare-except
+                        return sys.exc_info()
+                exc_info = self.threadpool.apply(close)
+                if exc_info:
+                    reraise(*exc_info)
+
+    def _do_delegate_methods(self):
+        super(FileObjectThread, self)._do_delegate_methods()
+        if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
+            self.read1 = self.read
+        self.__io_holder[0] = self._io
+
+    def _extra_repr(self):
+        return ' threadpool=%r' % (self.threadpool,)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        line = self.readline()
+        if line:
+            return line
+        raise StopIteration
+    __next__ = next
+
+    def _wrap_method(self, method):
+        # NOTE: We are careful to avoid introducing a refcycle
+        # within self. Our wrapper cannot refer to self.
+        io_holder = self.__io_holder
+        lock = self.lock
+        threadpool = self.threadpool
+
+        @functools.wraps(method)
+        def thread_method(*args, **kwargs):
+            if io_holder[0] is None:
+                # This is different than FileObjectPosix, etc,
+                # because we want to save the expensive trip through
+                # the threadpool.
+                raise FileObjectClosed()
+            with lock:
+                return threadpool.apply(method, args, kwargs)
+
+        return thread_method

src/gevent/core.py

 # Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details.
+"""
+Deprecated; this does not reflect all the possible options
+and its interface varies.
+
+.. versionchanged:: 1.3a2
+    Deprecated.
+"""
 from __future__ import absolute_import
 
-import os
+import sys
 
+from gevent._config import config
 from gevent._util import copy_globals
 
-try:
-    if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
-        raise ImportError("Not attempting corecext")
-
-    from gevent.libev import corecext as _core
-except ImportError:
-    if os.environ.get('GEVENT_CORE_CEXT_ONLY'):
-        raise
-
-    # CFFI/PyPy
-    lib = os.environ.get('GEVENT_CORE_CFFI_ONLY')
-    if lib == 'libuv':
-        from gevent.libuv import loop as _core
-    else:
-        try:
-            from gevent.libev import corecffi as _core
-        except ImportError:
-            from gevent.libuv import loop as _core
+_core = sys.modules[config.loop.__module__]
 
 copy_globals(_core, globals())
 

src/gevent/fileobject.py

@@ -35,31 +35,12 @@ Classes
 """
 from __future__ import absolute_import
 
-import functools
-import sys
-import os
-
-from gevent._fileobjectcommon import FileObjectClosed
-from gevent._fileobjectcommon import FileObjectBase
-from gevent.hub import get_hub
-from gevent._compat import integer_types
-from gevent._compat import reraise
-from gevent.lock import Semaphore, DummySemaphore
-
-
-PYPY = hasattr(sys, 'pypy_version_info')
-
-if hasattr(sys, 'exc_clear'):
-    def _exc_clear():
-        sys.exc_clear()
-else:
-    def _exc_clear():
-        return
-
+from gevent._config import config
 
 __all__ = [
     'FileObjectPosix',
     'FileObjectThread',
+    'FileObjectBlock',
     'FileObject',
 ]
 
@@ -71,149 +52,10 @@ else:
     del fcntl
     from gevent._fileobjectposix import FileObjectPosix
 
+from gevent._fileobjectcommon import FileObjectThread
+from gevent._fileobjectcommon import FileObjectBlock
 
-class FileObjectThread(FileObjectBase):
-    """
-    A file-like object wrapping another file-like object, performing all blocking
-    operations on that object in a background thread.
-
-    .. caution::
-        Attempting to change the threadpool or lock of an existing FileObjectThread
-        has undefined consequences.
-
-    .. versionchanged:: 1.1b1
-       The file object is closed using the threadpool. Note that whether or
-       not this action is synchronous or asynchronous is not documented.
-
-    """
-
-    def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
-        """
-        :param fobj: The underlying file-like object to wrap, or an integer fileno
-           that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
-        :keyword bool lock: If True (the default) then all operations will
-           be performed one-by-one. Note that this does not guarantee that, if using
-           this file object from multiple threads/greenlets, operations will be performed
-           in any particular order, only that no two operations will be attempted at the
-           same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
-           file operations with an external resource.
-        :keyword bool close: If True (the default) then when this object is closed,
-           the underlying object is closed as well.
-        """
-        closefd = close
-        self.threadpool = threadpool or get_hub().threadpool
-        self.lock = lock
-        if self.lock is True:
-            self.lock = Semaphore()
-        elif not self.lock:
-            self.lock = DummySemaphore()
-        if not hasattr(self.lock, '__enter__'):
-            raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
-        if isinstance(fobj, integer_types):
-            if not closefd:
-                # we cannot do this, since fdopen object will close the descriptor
-                raise TypeError('FileObjectThread does not support close=False on an fd.')
-            if mode is None:
-                assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
-                fobj = os.fdopen(fobj)
-            else:
-                fobj = os.fdopen(fobj, mode, bufsize)
-
-        self.__io_holder = [fobj] # signal for _wrap_method
-        super(FileObjectThread, self).__init__(fobj, closefd)
-
-    def _do_close(self, fobj, closefd):
-        self.__io_holder[0] = None # for _wrap_method
-        try:
-            with self.lock:
-                self.threadpool.apply(fobj.flush)
-        finally:
-            if closefd:
-                # Note that we're not taking the lock; older code
-                # did fobj.close() without going through the threadpool at all,
-                # so acquiring the lock could potentially introduce deadlocks
-                # that weren't present before. Avoiding the lock doesn't make
-                # the existing race condition any worse.
-                # We wrap the close in an exception handler and re-raise directly
-                # to avoid the (common, expected) IOError from being logged by the pool
-                def close():
-                    try:
-                        fobj.close()
-                    except: # pylint:disable=bare-except
-                        return sys.exc_info()
-                exc_info = self.threadpool.apply(close)
-                if exc_info:
-                    reraise(*exc_info)
-
-    def _do_delegate_methods(self):
-        super(FileObjectThread, self)._do_delegate_methods()
-        if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
-            self.read1 = self.read
-        self.__io_holder[0] = self._io
-
-    def _extra_repr(self):
-        return ' threadpool=%r' % (self.threadpool,)
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        line = self.readline()
-        if line:
-            return line
-        raise StopIteration
-    __next__ = next
-
-    def _wrap_method(self, method):
-        # NOTE: We are careful to avoid introducing a refcycle
-        # within self. Our wrapper cannot refer to self.
-        io_holder = self.__io_holder
-        lock = self.lock
-        threadpool = self.threadpool
-
-        @functools.wraps(method)
-        def thread_method(*args, **kwargs):
-            if io_holder[0] is None:
-                # This is different than FileObjectPosix, etc,
-                # because we want to save the expensive trip through
-                # the threadpool.
-                raise FileObjectClosed()
-            with lock:
-                return threadpool.apply(method, args, kwargs)
-
-        return thread_method
-
-
-try:
-    FileObject = FileObjectPosix
-except NameError:
-    FileObject = FileObjectThread
-
-
-class FileObjectBlock(FileObjectBase):
-
-    def __init__(self, fobj, *args, **kwargs):
-        closefd = kwargs.pop('close', True)
-        if kwargs:
-            raise TypeError('Unexpected arguments: %r' % kwargs.keys())
-        if isinstance(fobj, integer_types):
-            if not closefd:
-                # we cannot do this, since fdopen object will close the descriptor
-                raise TypeError('FileObjectBlock does not support close=False on an fd.')
-            fobj = os.fdopen(fobj, *args)
-        super(FileObjectBlock, self).__init__(fobj, closefd)
-
-    def _do_close(self, fobj, closefd):
-        fobj.close()
 
-config = os.environ.get('GEVENT_FILE')
-if config:
-    klass = {'thread': 'gevent.fileobject.FileObjectThread',
-             'posix': 'gevent.fileobject.FileObjectPosix',
-             'block': 'gevent.fileobject.FileObjectBlock'}.get(config, config)
-    if klass.startswith('gevent.fileobject.'):
-        FileObject = globals()[klass.split('.', 2)[-1]]
-    else:
-        from gevent.hub import _import
-        FileObject = _import(klass)
-    del klass
+# None of the possible objects can live in this module because
+# we would get an import cycle and the config couldn't be set from code.
+FileObject = config.fileobject

src/gevent/hub.py

@@ -26,6 +26,7 @@ __all__ = [
     'Waiter',
 ]
 
+from gevent._config import config
 from gevent._compat import string_types
 from gevent._compat import xrange
 from gevent._util import _NONE
@@ -389,89 +390,29 @@ def set_hub(hub):
     _threadlocal.hub = hub
 
 
-def _import(path):
-    # pylint:disable=too-many-branches
-    if isinstance(path, list):
-        if not path:
-            raise ImportError('Cannot import from empty list: %r' % (path, ))
-
-        for item in path[:-1]:
-            try:
-                return _import(item)
-            except ImportError:
-                pass
-
-        return _import(path[-1])
-
-    if not isinstance(path, string_types):
-        return path
-
-    if '.' not in path:
-        raise ImportError("Cannot import %r (required format: [path/][package.]module.class)" % path)
-
-    if '/' in path:
-        # This is dangerous, subject to race conditions, and
-        # may not work properly for things like namespace packages
-        import warnings
-        warnings.warn("Absolute paths are deprecated. Please put the package "
-                      "on sys.path first",
-                      DeprecationWarning)
-        package_path, path = path.rsplit('/', 1)
-        sys.path = [package_path] + sys.path
-    else:
-        package_path = None
-
-    try:
-        module, item = path.rsplit('.', 1)
-        x = __import__(module)
-        for attr in path.split('.')[1:]:
-            oldx = x
-            x = getattr(x, attr, _NONE)
-            if x is _NONE:
-                raise ImportError('Cannot import %r from %r' % (attr, oldx))
-        return x
-    finally:
-        if '/' in path:
-            try:
-                sys.path.remove(package_path)
-            except ValueError:
-                pass
 
 
-def config(default, envvar):
+def _config(default, envvar):
     result = os.environ.get(envvar) or default # absolute import gets confused pylint: disable=no-member
     if isinstance(result, string_types):
         return result.split(',')
     return result
 
 
-def resolver_config(default, envvar):
-    result = config(default, envvar)
-    return [_resolvers.get(x, x) for x in result]
-
-
-_resolvers = {
-    'ares': 'gevent.resolver.ares.Resolver',
-    'thread': 'gevent.resolver.thread.Resolver',
-    'block': 'gevent.resolver.blocking.Resolver',
-    'dnspython': 'gevent.resolver.dnspython.Resolver',
-}
-
-
-_DEFAULT_LOOP_CLASS = 'gevent.core.loop'
-
 
 class Hub(RawGreenlet):
-    """A greenlet that runs the event loop.
+    """
+    A greenlet that runs the event loop.
 
     It is created automatically by :func:`get_hub`.
 
-    **Switching**
+    .. rubric:: Switching
 
-    Every time this greenlet (i.e., the event loop) is switched *to*, if
-    the current greenlet has a ``switch_out`` method, it will be called. This
-    allows a greenlet to take some cleanup actions before yielding control. This method
-    should not call any gevent blocking functions.
+    Every time this greenlet (i.e., the event loop) is switched *to*,
+    if the current greenlet has a ``switch_out`` method, it will be
+    called. This allows a greenlet to take some cleanup actions before
+    yielding control. This method should not call any gevent blocking
+    functions.
     """
 
     #: If instances of these classes are raised into the event loop,
@@ -483,37 +424,8 @@ class Hub(RawGreenlet):
     #: do not get logged/printed when raised by the event loop.
     NOT_ERROR = (GreenletExit, SystemExit)
 
-    loop_class = config(_DEFAULT_LOOP_CLASS, 'GEVENT_LOOP')
-    # For the standard class, go ahead and import it when this class
-    # is defined. This is no loss of generality because the envvar is
-    # only read when this class is defined, and we know that the
-    # standard class will be available. This can solve problems with
-    # the class being imported from multiple threads at once, leading
-    # to one of the imports failing. Only do this for the object we
-    # need in the constructor, as the rest of the factories are
-    # themselves handled lazily. See #687. (People using a custom loop_class
-    # can probably manage to get_hub() from the main thread or otherwise import
-    # that loop_class themselves.)
-    if loop_class == [_DEFAULT_LOOP_CLASS]:
-        loop_class = [_import(loop_class)]
-
-    resolver_class = [
-        'gevent.resolver.thread.Resolver',
-        'gevent.resolver.dnspython.Resolver',
-        'gevent.resolver.ares.Resolver',
-        'gevent.resolver.blacking.Resolver',
-    ]
-    #: The class or callable object, or the name of a factory function or class,
-    #: that will be used to create :attr:`resolver`. By default, configured according to
-    #: :doc:`dns`. If a list, a list of objects in preference order.
-    resolver_class = resolver_config(resolver_class, 'GEVENT_RESOLVER')
-    threadpool_class = config('gevent.threadpool.ThreadPool', 'GEVENT_THREADPOOL')
-    backend = config(None, 'GEVENT_BACKEND')
-    threadpool_size = 10
 
-    # using pprint.pformat can override custom __repr__ methods on dict/list
-    # subclasses, which can be a security concern
-    format_context = 'pprint.saferepr'
+    threadpool_size = 10
 
 
     def __init__(self, loop=None, default=None):
@@ -530,13 +442,21 @@ class Hub(RawGreenlet):
         else:
             if default is None and get_ident() != MAIN_THREAD:
                 default = False
-            loop_class = _import(self.loop_class)
+
             if loop is None:
                 loop = self.backend
-            self.loop = loop_class(flags=loop, default=default)
+            self.loop = self.loop_class(flags=loop, default=default) # pylint:disable=not-callable
         self._resolver = None
         self._threadpool = None
-        self.format_context = _import(self.format_context)
+        self.format_context = config.format_context
+
+    @property
+    def loop_class(self):
+        return config.loop
+
+    @property
+    def backend(self):
+        return config.libev_backend
 
     def __repr__(self):
         if self.loop is None:
@@ -787,11 +707,13 @@ class Hub(RawGreenlet):
         if _threadlocal.hub is self:
             _threadlocal.hub = None
 
+    @property
+    def resolver_class(self):
+        return config.resolver
+
     def _get_resolver(self):
         if self._resolver is None:
-            if self.resolver_class is not None:
-                self.resolver_class = _import(self.resolver_class)
-                self._resolver = self.resolver_class(hub=self)
+            self._resolver = self.resolver_class(hub=self) # pylint:disable=not-callable
         return self._resolver
 
     def _set_resolver(self, value):
@@ -802,11 +724,15 @@ class Hub(RawGreenlet):
 
     resolver = property(_get_resolver, _set_resolver, _del_resolver)
 
+
+    @property
+    def threadpool_class(self):
+        return config.threadpool
+
     def _get_threadpool(self):
         if self._threadpool is None:
-            if self.threadpool_class is not None:
-                self.threadpool_class = _import(self.threadpool_class)
-                self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
+            # pylint:disable=not-callable
+            self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
         return self._threadpool
 
     def _set_threadpool(self, value):

src/gevent/os.py

@@ -46,6 +46,7 @@ from __future__ import absolute_import
 import os
 import sys
 from gevent.hub import get_hub, reinit
+from gevent._config import config
 from gevent._compat import PY3
 from gevent._util import copy_globals
 import errno
@@ -418,7 +419,7 @@ if hasattr(os, 'fork'):
             __extensions__.append('forkpty_and_watch')
 
         # Watch children by default
-        if not os.getenv('GEVENT_NOWAITPID'):
+        if not config.disable_watch_children:
             # Broken out into separate functions instead of simple name aliases
             # for documentation purposes.
             def fork(*args, **kwargs):

src/gevent/resolver/ares.py

@@ -27,6 +27,9 @@ from gevent.socket import SOCK_DGRAM
 from gevent.socket import SOCK_RAW
 from gevent.socket import AI_NUMERICHOST
 
+from gevent._config import config
+from gevent._config import AresSettingMixin
+
 from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
 from . import _lookup_port as lookup_port
 from . import _resolve_special
@@ -84,12 +87,11 @@ class Resolver(AbstractResolver):
             hub = get_hub()
         self.hub = hub
         if use_environ:
-            for key in os.environ:
-                if key.startswith('GEVENTARES_'):
-                    name = key[11:].lower()
-                    if name:
-                        value = os.environ[key]
-                        kwargs.setdefault(name, value)
+            for setting in config.settings.values():
+                if isinstance(setting, AresSettingMixin):
+                    value = setting.get()
+                    if value is not None:
+                        kwargs.setdefault(setting.kwarg_name, value)
         self.ares = self.ares_class(hub.loop, **kwargs)
         self.pid = os.getpid()
         self.params = kwargs

src/greentest/greentest/params.py

@@ -23,9 +23,11 @@ from greentest.sysinfo import PY3
 from greentest.sysinfo import PYPY
 from greentest.sysinfo import WIN
 from greentest.sysinfo import LIBUV
+from greentest.sysinfo import OSX
 
 from greentest.sysinfo import RUNNING_ON_TRAVIS
 from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION
+from greentest.sysinfo import RESOLVER_ARES
 
 
 # Travis is slow and overloaded; Appveyor used to be faster, but
@@ -60,6 +62,11 @@ if RUNNING_ON_TRAVIS:
     # connected to with the same error.
     DEFAULT_BIND_ADDR = '127.0.0.1'
 
+if RESOLVER_ARES and OSX:
+    # Ares likes to raise "malformed domain name" on '', at least
+    # on OS X
+    DEFAULT_BIND_ADDR = '127.0.0.1'
+
 # For in-process sockets
 DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0
 

src/greentest/greentest/sysinfo.py

@@ -30,7 +30,7 @@ LINUX = sys.platform.startswith('linux')
 OSX = sys.platform == 'darwin'
 
 # XXX: Formalize this better
-LIBUV = os.getenv('GEVENT_CORE_CFFI_ONLY') == 'libuv' or (PYPY and WIN) or hasattr(gevent.core, 'libuv')
+LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
 CFFI_BACKEND = bool(os.getenv('GEVENT_CORE_CFFI_ONLY')) or PYPY
 
 if '--debug-greentest' in sys.argv:

src/greentest/test___config.py

+# Copyright 2018 gevent contributors. See LICENSE for details.
+
+import os
+import unittest
+
+from gevent import _config
+
+class TestResolver(unittest.TestCase):
+
+    old_resolver = None
+
+    def setUp(self):
+        if 'GEVENT_RESOLVER' in os.environ:
+            self.old_resolver = os.environ['GEVENT_RESOLVER']
+            del os.environ['GEVENT_RESOLVER']
+
+    def tearDown(self):
+        if self.old_resolver:
+            os.environ['GEVENT_RESOLVER'] = self.old_resolver
+
+    def test_key(self):
+        self.assertEqual(_config.Resolver.environment_key, 'GEVENT_RESOLVER')
+
+    def test_default(self):
+        from gevent.resolver.thread import Resolver
+
+        conf = _config.Resolver()
+        self.assertEqual(conf.get(), Resolver)
+
+    def test_env(self):
+        from gevent.resolver.blocking import Resolver
+
+        os.environ['GEVENT_RESOLVER'] = 'foo,bar,block,dnspython'
+
+        conf = _config.Resolver()
+        self.assertEqual(conf.get(), Resolver)
+
+        os.environ['GEVENT_RESOLVER'] = 'dnspython'
+
+        # The existing value is unchanged
+        self.assertEqual(conf.get(), Resolver)
+
+        # A new object reflects it
+        conf = _config.Resolver()
+        from gevent.resolver.dnspython import Resolver as DResolver
+        self.assertEqual(conf.get(), DResolver)
+
+    def test_set_str_long(self):
+        from gevent.resolver.blocking import Resolver
+        conf = _config.Resolver()
+        conf.set('gevent.resolver.blocking.Resolver')
+
+        self.assertEqual(conf.get(), Resolver)
+
+    def test_set_str_short(self):
+        from gevent.resolver.blocking import Resolver
+        conf = _config.Resolver()
+        conf.set('block')
+
+        self.assertEqual(conf.get(), Resolver)
+
+    def test_set_class(self):
+        from gevent.resolver.blocking import Resolver
+        conf = _config.Resolver()
+        conf.set(Resolver)
+
+        self.assertEqual(conf.get(), Resolver)
+
+
+    def test_set_through_config(self):
+        from gevent.resolver.thread import Resolver as Default
+        from gevent.resolver.blocking import Resolver
+
+        conf = _config.Config()
+        self.assertEqual(conf.resolver, Default)
+
+        conf.resolver = 'block'
+        self.assertEqual(conf.resolver, Resolver)
+
+if __name__ == '__main__':
+    unittest.main()

src/greentest/tests_that_dont_use_resolver.txt

@@ -123,3 +123,5 @@ test_timeout.py
 # implicitly for localhost, which is covered well enough
 # elsewhere that we don't need to spend the 20s (*2)
 test_asyncore.py
+
+test___config.py

tox.ini

@@ -36,7 +36,7 @@ commands =
 basepython =
     python2.7
 setenv =
-	GEVENT_CORE_CFFI_ONLY=1
+	GEVENT_LOOP=libev-cffi
 commands =
     make basictest
 
@@ -44,7 +44,7 @@ commands =
 basepython =
     python2.7
 setenv =
-	GEVENT_CORE_CFFI_ONLY=libuv
+	GEVENT_LOOP=libuv-cffi
 commands =
     make basictest