Merge pull request #1387 from gevent/issue1366

Make the optional backends optional in test__all__ and test__execmodules

src/gevent/testing/modules.py

@@ -26,6 +26,7 @@ import warnings
 import gevent
 
 from . import sysinfo
+from . import util
 
 
 OPTIONAL_MODULES = [
@@ -33,10 +34,28 @@ OPTIONAL_MODULES = [
     'gevent.resolver.ares',
     'gevent.libev',
     'gevent.libev.watcher',
+    'gevent.libuv.loop',
+    'gevent.libuv.watcher',
 ]
 
 
-def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
+def walk_modules(
+        basedir=None,
+        modpath=None,
+        include_so=False,
+        recursive=False,
+        check_optional=True,
+):
+    """
+    Find gevent modules, yielding tuples of ``(path, importable_module_name)``.
+
+    :keyword bool check_optional: If true (the default), then if we discover a
+       module that is known to be optional on this system (such as a backend),
+       we will attempt to import it; if the import fails, it will not be returned.
+       If false, then we will not make such an attempt, the caller will need to be prepared
+       for an `ImportError`; the caller can examine *OPTIONAL_MODULES* against
+       the yielded *importable_module_name*.
+    """
     # pylint:disable=too-many-branches
     if sysinfo.PYPY:
         include_so = False
@@ -47,6 +66,7 @@ def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
     else:
         if modpath is None:
             modpath = ''
+
     for fn in sorted(os.listdir(basedir)):
         path = os.path.join(basedir, fn)
         if os.path.isdir(path):
@@ -57,9 +77,11 @@ def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
             pkg_init = os.path.join(path, '__init__.py')
             if os.path.exists(pkg_init):
                 yield pkg_init, modpath + fn
-                for p, m in walk_modules(path, modpath + fn + "."):
+                for p, m in walk_modules(path, modpath + fn + ".",
+                                         check_optional=check_optional):
                     yield p, m
             continue
+
         if fn.endswith('.py'):
             x = fn[:-3]
             if x.endswith('_d'):
@@ -68,12 +90,13 @@ def walk_modules(basedir=None, modpath=None, include_so=False, recursive=False):
                      'corecffi', '_corecffi', '_corecffi_build']:
                 continue
             modname = modpath + x
-            if modname in OPTIONAL_MODULES:
+            if check_optional and modname in OPTIONAL_MODULES:
                 try:
                     with warnings.catch_warnings():
                         warnings.simplefilter('ignore', DeprecationWarning)
                         importlib.import_module(modname)
                 except ImportError:
+                    util.debug("Unable to import optional module %s", modname)
                     continue
             yield path, modname
         elif include_so and fn.endswith(sysinfo.SHARED_OBJECT_EXTENSION):

src/gevent/testing/testrunner.py

@@ -12,7 +12,6 @@ from datetime import timedelta
 from multiprocessing.pool import ThreadPool
 from multiprocessing import cpu_count
 from . import util
-from .util import log
 from .sysinfo import RUNNING_ON_CI
 from .sysinfo import PYPY
 from .sysinfo import PY2
@@ -125,7 +124,7 @@ def run_many(tests,
 
     try:
         try:
-            log("Running tests in parallel with concurrency %s" % (NWORKERS,),)
+            util.log("Running tests in parallel with concurrency %s" % (NWORKERS,),)
             for cmd, options in tests:
                 total += 1
                 options = options or {}
@@ -136,20 +135,21 @@ def run_many(tests,
             pool.close()
             pool.join()
 
-            log("Running tests marked standalone")
-            for cmd, options in run_alone:
-                run_one(cmd, **options)
+            if run_alone:
+                util.log("Running tests marked standalone")
+                for cmd, options in run_alone:
+                    run_one(cmd, **options)
 
         except KeyboardInterrupt:
             try:
-                log('Waiting for currently running to finish...')
+                util.log('Waiting for currently running to finish...')
                 reap_all()
             except KeyboardInterrupt:
                 pool.terminate()
                 report(total, failed, passed, exit=False, took=time.time() - start,
                        configured_failing_tests=configured_failing_tests,
                        total_cases=total_cases[0], total_skipped=total_skipped[0])
-                log('(partial results)\n')
+                util.log('(partial results)\n')
                 raise
     except:
         traceback.print_exc()
@@ -203,8 +203,15 @@ def discover(
     to_import = []
 
     for filename in tests:
-        module_name = os.path.splitext(filename)[0]
-        qualified_name = package + '.' + module_name if package else module_name
+        # Support either 'gevent.tests.foo' or 'gevent/tests/foo.py'
+        if filename.startswith('gevent.tests'):
+            # XXX: How does this interact with 'package'? Probably not well
+            qualified_name = module_name = filename
+            filename = filename[len('gevent.tests') + 1:]
+            filename = filename.replace('.', os.sep) + '.py'
+        else:
+            module_name = os.path.splitext(filename)[0]
+            qualified_name = package + '.' + module_name if package else module_name
         with open(os.path.abspath(filename), 'rb') as f:
             # Some of the test files (e.g., test__socket_dns) are
             # UTF8 encoded. Depending on the environment, Python 3 may
@@ -291,12 +298,12 @@ def report(total, failed, passed, exit=True, took=None,
     # pylint:disable=redefined-builtin,too-many-branches,too-many-locals
     runtimelog = util.runtimelog
     if runtimelog:
-        log('\nLongest-running tests:')
+        util.log('\nLongest-running tests:')
         runtimelog.sort()
         length = len('%.1f' % -runtimelog[0][0])
         frmt = '%' + str(length) + '.1f seconds: %s'
         for delta, name in runtimelog[:5]:
-            log(frmt, -delta, name)
+            util.log(frmt, -delta, name)
     if took:
         took = ' in %s' % format_seconds(took)
     else:
@@ -311,11 +318,11 @@ def report(total, failed, passed, exit=True, took=None,
             passed_unexpected.append(name)
 
     if passed_unexpected:
-        log('\n%s/%s unexpected passes', len(passed_unexpected), total, color='error')
+        util.log('\n%s/%s unexpected passes', len(passed_unexpected), total, color='error')
         print_list(passed_unexpected)
 
     if failed:
-        log('\n%s/%s tests failed%s', len(failed), total, took)
+        util.log('\n%s/%s tests failed%s', len(failed), total, took)
 
         for name in failed:
             if matches(configured_failing_tests, name, include_flaky=True):
@@ -324,14 +331,14 @@ def report(total, failed, passed, exit=True, took=None,
                 failed_unexpected.append(name)
 
         if failed_expected:
-            log('\n%s/%s expected failures', len(failed_expected), total)
+            util.log('\n%s/%s expected failures', len(failed_expected), total)
             print_list(failed_expected)
 
         if failed_unexpected:
-            log('\n%s/%s unexpected failures', len(failed_unexpected), total, color='error')
+            util.log('\n%s/%s unexpected failures', len(failed_unexpected), total, color='error')
             print_list(failed_unexpected)
     else:
-        log(
+        util.log(
             '\nRan %s tests%s in %s files%s',
             total_cases,
             util._colorize('skipped', " (skipped=%d)" % total_skipped) if total_skipped else '',
@@ -350,7 +357,7 @@ def report(total, failed, passed, exit=True, took=None,
 
 def print_list(lst):
     for name in lst:
-        log(' - %s', name)
+        util.log(' - %s', name)
 
 def _setup_environ(debug=False):
     if ('PYTHONWARNINGS' not in os.environ
@@ -417,6 +424,14 @@ def main():
     parser.add_argument("--package", default="gevent.tests")
     parser.add_argument('tests', nargs='*')
     options = parser.parse_args()
+
+    # Set this before any test imports in case of 'from .util import QUIET';
+    # not that this matters much because we spawn tests in subprocesses,
+    # it's the environment setting that matters
+    util.QUIET = options.quiet
+    if 'GEVENTTEST_QUIET' not in os.environ:
+        os.environ['GEVENTTEST_QUIET'] = str(options.quiet)
+
     FAILING_TESTS = []
     IGNORED_TESTS = []
     RUN_ALONE = []

src/gevent/testing/util.py

@@ -2,19 +2,23 @@ from __future__ import print_function, absolute_import, division
 import re
 import sys
 import os
-from . import six
 import traceback
 import unittest
 import threading
 import subprocess
 import time
 
+from . import six
+from gevent._config import validate_bool
+
 # pylint: disable=broad-except,attribute-defined-outside-init
 
 runtimelog = []
 MIN_RUNTIME = 1.0
 BUFFER_OUTPUT = False
-QUIET = False
+# This is set by the testrunner, defaulting to true (be quiet)
+# But if we're run standalone, default to false
+QUIET = validate_bool(os.environ.get('GEVENTTEST_QUIET', '0'))
 
 
 class Popen(subprocess.Popen):
@@ -34,6 +38,7 @@ _colorscheme = {
     'normal': 'normal',
     'default': 'default',
     'info': 'normal',
+    'debug': 'cyan',
     'suboptimal-behaviour': 'magenta',
     'error': 'brightred',
     'number': 'green',
@@ -89,6 +94,11 @@ def _colorize(what, message, normal='normal'):
     return _color(what) + message + _color(normal)
 
 def log(message, *args, **kwargs):
+    """
+    Log a *message*
+
+    :keyword str color: One of the values from _colorscheme
+    """
     color = kwargs.pop('color', 'normal')
     try:
         if args:
@@ -110,6 +120,13 @@ def log(message, *args, **kwargs):
         string = _colorize(color, string)
         sys.stderr.write(string + '\n')
 
+def debug(message, *args, **kwargs):
+    """
+    Log the *message* only if we're not in quiet mode.
+    """
+    if not QUIET:
+        kwargs.setdefault('color', 'debug')
+        log(message, *args, **kwargs)
 
 def killpg(pid):
     if not hasattr(os, 'killpg'):
@@ -169,6 +186,18 @@ def kill(popen):
     except Exception:
         traceback.print_exc()
 
+# A set of environment keys we ignore for printing purposes
+IGNORED_GEVENT_ENV_KEYS = {
+    'GEVENTTEST_QUIET',
+    'GEVENT_DEBUG',
+}
+
+# A set of (name, value) pairs we ignore for printing purposes.
+# These should match the defaults.
+IGNORED_GEVENT_ENV_ITEMS = {
+    ('GEVENT_RESOLVER', 'thread'),
+    ('GEVENT_RESOLVER_NAMESERVERS', '8.8.8.8')
+}
 
 def getname(command, env=None, setenv=None):
     result = []
@@ -177,8 +206,13 @@ def getname(command, env=None, setenv=None):
     env.update(setenv or {})
 
     for key, value in sorted(env.items()):
-        if key.startswith('GEVENT'):
-            result.append('%s=%s' % (key, value))
+        if not key.startswith('GEVENT'):
+            continue
+        if key in IGNORED_GEVENT_ENV_KEYS:
+            continue
+        if (key, value) in IGNORED_GEVENT_ENV_ITEMS:
+            continue
+        result.append('%s=%s' % (key, value))
 
     if isinstance(command, six.string_types):
         result.append(command)

src/gevent/tests/test__all__.py

-"""Check __all__, __implements__, __extensions__, __imports__ of the modules"""
+# Check __all__, __implements__, __extensions__, __imports__ of the modules
+
 from __future__ import print_function
 
 import sys
@@ -8,8 +9,9 @@ import importlib
 import warnings
 
 from gevent.testing import six
-from gevent.testing.modules import walk_modules
+from gevent.testing import modules
 from gevent.testing.sysinfo import PLATFORM_SPECIFIC_SUFFIXES
+from gevent.testing.util import debug
 
 from gevent._patcher import MAPPING
 
@@ -61,6 +63,30 @@ if sys.platform.startswith('win'):
     EXTRA_EXTENSIONS.append('gevent.signal')
 
 
+
+_MISSING = '<marker object>'
+
+def _create_tests(cls):
+    path = modname = orig_modname = None
+
+    for path, modname in modules.walk_modules(include_so=False, recursive=True, check_optional=False):
+        orig_modname = modname
+        test_name = 'test_%s' % orig_modname.replace('.', '_')
+
+        modname = modname.replace('gevent.', '').split('.')[0]
+
+        fn = lambda self, n=orig_modname: self._test(n)
+
+        if not modname: # pragma: no cover
+            # With walk_modules, can we even get here?
+            fn = unittest.skip(
+                "No such module '%s' at '%s'" % (orig_modname, path))(fn)
+
+        setattr(cls, test_name, fn)
+
+    return cls
+
+@_create_tests
 class Test(unittest.TestCase):
 
     stdlib_has_all = False
@@ -68,10 +94,11 @@ class Test(unittest.TestCase):
     stdlib_name = None
     stdlib_module = None
     module = None
+    modname = None
     __implements__ = __extensions__ = __imports__ = ()
 
     def check_all(self):
-        "Check that __all__ is present and does not contain invalid entries"
+        # Check that __all__ is present and does not contain invalid entries
         if not hasattr(self.module, '__all__'):
             self.assertIn(self.modname, NO_ALL)
             return
@@ -81,12 +108,14 @@ class Test(unittest.TestCase):
         self.assertEqual(sorted(names), sorted(self.module.__all__))
 
     def check_all_formula(self):
-        "Check __all__ = __implements__ + __extensions__ + __imported__"
+        # Check __all__ = __implements__ + __extensions__ + __imported__
         all_calculated = self.__implements__ + self.__imports__ + self.__extensions__
         self.assertEqual(sorted(all_calculated), sorted(self.module.__all__))
 
     def check_implements_presence_justified(self):
-        "Check that __implements__ is present only if the module is modeled after a module from stdlib (like gevent.socket)."
+        # Check that __implements__ is present only if the module is modeled
+        # after a module from stdlib (like gevent.socket).
+
         if self.modname in ALLOW_IMPLEMENTS:
             return
         if self.__implements__ is not None and self.stdlib_module is None:
@@ -104,7 +133,8 @@ class Test(unittest.TestCase):
             self.stdlib_all = [name for name in self.stdlib_all if not isinstance(getattr(self.stdlib_module, name), types.ModuleType)]
 
     def check_implements_subset_of_stdlib_all(self):
-        "Check that __implements__ + __imports__ is a subset of the corresponding standard module __all__ or dir()"
+        # Check that __implements__ + __imports__ is a subset of the
+        # corresponding standard module __all__ or dir()
         for name in self.__implements__ + self.__imports__:
             if name in self.stdlib_all:
                 continue
@@ -115,7 +145,9 @@ class Test(unittest.TestCase):
             raise AssertionError('%r is not found in %r.__all__ nor in dir(%r)' % (name, self.stdlib_module, self.stdlib_module))
 
     def check_implements_actually_implements(self):
-        """Check that the module actually implements the entries from __implements__"""
+        # Check that the module actually implements the entries from
+        # __implements__
+
         for name in self.__implements__:
             item = getattr(self.module, name)
             try:
@@ -126,14 +158,17 @@ class Test(unittest.TestCase):
                     raise
 
     def check_imports_actually_imports(self):
-        """Check that the module actually imports the entries from __imports__"""
+        # Check that the module actually imports the entries from
+        # __imports__
         for name in self.__imports__:
             item = getattr(self.module, name)
             stdlib_item = getattr(self.stdlib_module, name)
             self.assertIs(item, stdlib_item)
 
     def check_extensions_actually_extend(self):
-        """Check that the module actually defines new entries in __extensions__"""
+        # Check that the module actually defines new entries in
+        # __extensions__
+
         if self.modname in EXTRA_EXTENSIONS:
             return
         for name in self.__extensions__:
@@ -141,7 +176,9 @@ class Test(unittest.TestCase):
                 raise AssertionError("'%r' is not an extension, it is found in %r" % (name, self.stdlib_module))
 
     def check_completeness(self): # pylint:disable=too-many-branches
-        """Check that __all__ (or dir()) of the corresponsing stdlib is a subset of __all__ of this module"""
+        # Check that __all__ (or dir()) of the corresponsing stdlib is
+        # a subset of __all__ of this module
+
         missed = []
         for name in self.stdlib_all:
             if name not in getattr(self.module, '__all__', []):
@@ -162,8 +199,8 @@ class Test(unittest.TestCase):
                     # We often don't want __all__ to be set because we wind up
                     # documenting things that we just copy in from the stdlib.
                     # But if we implement it, don't print a warning
-                    if getattr(self.module, name, self) is self:
-                        print('IncompleteImplWarning: %s.%s' % (self.modname, name))
+                    if getattr(self.module, name, _MISSING) is _MISSING:
+                        debug('IncompleteImplWarning: %s.%s' % (self.modname, name))
                 else:
                     result.append(name)
             missed = result
@@ -188,7 +225,12 @@ are missing from %r:
         self.modname = modname
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', DeprecationWarning)
-            self.module = importlib.import_module(modname)
+            try:
+                self.module = importlib.import_module(modname)
+            except ImportError:
+                if modname in modules.OPTIONAL_MODULES:
+                    raise unittest.SkipTest("Unable to import %s" % modname)
+                raise
 
         self.check_all()
 
@@ -221,20 +263,7 @@ are missing from %r:
         self.check_extensions_actually_extend()
         self.check_completeness()
 
-    path = modname = orig_modname = None
 
-    for path, modname in walk_modules(include_so=False, recursive=True):
-        orig_modname = modname
-        modname = modname.replace('gevent.', '').split('.')[0]
-        if not modname:
-            print("WARNING: No such module '%s' at '%s'" % (orig_modname, path),
-                  file=sys.stderr)
-            continue
-        exec(
-            '''def test_%s(self): self._test("%s")''' % (
-                orig_modname.replace('.', '_').replace('-', '_'), orig_modname)
-        )
-    del path, modname, orig_modname
 
 
 if __name__ == "__main__":

src/gevent/tests/test__execmodules.py

 import unittest
 import warnings
 
-from gevent.testing.modules import walk_modules
+from gevent.testing import modules
 from gevent.testing import main
 from gevent.testing.sysinfo import NON_APPLICABLE_SUFFIXES
-
-
 from gevent.testing import six
 
 
-class TestExec(unittest.TestCase):
-    pass
-
-
 def make_exec_test(path, module):
 
     def test(_):
@@ -20,20 +14,30 @@ def make_exec_test(path, module):
             src = f.read()
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', DeprecationWarning)
-            six.exec_(src, {'__file__': path})
+            try:
+                six.exec_(src, {'__file__': path})
+            except ImportError:
+                if module in modules.OPTIONAL_MODULES:
+                    raise unittest.SkipTest("Unable to import optional module %s" % module)
+                raise
 
     name = "test_" + module.replace(".", "_")
     test.__name__ = name
-    setattr(TestExec, name, test)
+    return test
 
-def make_all_tests():
-    for path, module in walk_modules(recursive=True):
+def make_all_tests(cls):
+    for path, module in modules.walk_modules(recursive=True, check_optional=False):
         if module.endswith(NON_APPLICABLE_SUFFIXES):
             continue
-        make_exec_test(path, module)
+        test = make_exec_test(path, module)
+        setattr(cls, test.__name__, test)
+    return cls
+
 
+@make_all_tests
+class Test(unittest.TestCase):
+    pass
 
-make_all_tests()
 
 if __name__ == '__main__':
     main()