Mark some flaky tests for CI. One is a race condition that needs investigated.

src/greentest/_six.py

@@ -38,3 +38,10 @@ else:
     xrange = builtins.xrange
     string_types = builtins.basestring,
     text_type = builtins.unicode
+
+    exec_("""def reraise(tp, value, tb=None):
+    try:
+        raise tp, value, tb
+    finally:
+        tb = None
+""")

src/greentest/greentest.py

@@ -47,7 +47,7 @@ LINUX = sys.platform.startswith('linux')
 
 # XXX: Formalize this better
 LIBUV = os.getenv('GEVENT_CORE_CFFI_ONLY') == 'libuv' or (PYPY and WIN) or hasattr(gevent.core, 'libuv')
-
+CFFI_BACKEND = bool(os.getenv('GEVENT_CORE_CFFI_ONLY')) or PYPY
 
 if '--debug-greentest' in sys.argv:
     sys.argv.remove('--debug-greentest')
@@ -158,8 +158,47 @@ else:
     skipOnLibuv = _do_not_skip
 
 class ExpectedException(Exception):
-    """An exception whose traceback should be ignored"""
+    """An exception whose traceback should be ignored by the hub"""
+
+
+# The next exceptions allow us to raise them in a highly
+# greppable way so that we can debug them later.
+
+class FlakyTest(unittest.SkipTest):
+    """
+    A unittest exception that causes the test to be skipped when raised.
+
+    Use this carefully, it is a code smell and indicates an undebugged problem.
+    """
+
+class FlakyTestRaceCondition(FlakyTest):
+    """
+    Use this when the flaky test is definitely caused by a race condition.
+    """
 
+class FlakyTestTimeout(FlakyTest):
+    """
+    Use this when the flaky test is definitely caused by an
+    unexpected timeout.
+    """
+
+
+if RUNNING_ON_CI:
+    def reraiseFlakyTestRaceCondition():
+        six.reraise(FlakyTestRaceCondition,
+                    FlakyTestRaceCondition('\n'.join(dump_stacks())),
+                    sys.exc_info()[2])
+
+    def reraiseFlakyTestTimeout():
+        six.reraise(FlakyTestTimeout,
+                    None,
+                    sys.exc_info()[2])
+
+else:
+    def reraiseFlakyTestRaceCondition():
+        six.reraise(*sys.exc_info())
+
+    reraiseFlakyTestTimeout = reraiseFlakyTestRaceCondition
 
 def wrap_switch_count_check(method):
     @wraps(method)
@@ -914,3 +953,41 @@ else:
 
     def getrefcount(*args):
         return sys.getrefcount(*args)
+
+def dump_stacks():
+    """
+    Request information about the running threads of the current process.
+
+    :return: A sequence of text lines detailing the stacks of running
+            threads and greenlets. (One greenlet will duplicate one thread,
+            the current thread and greenlet.)
+    """
+    dump = []
+
+    # threads
+    import threading  # Late import this stuff because it may get monkey-patched
+    import traceback
+    from greenlet import greenlet
+
+    threads = {th.ident: th.name for th in threading.enumerate()}
+
+    for thread, frame in sys._current_frames().items():
+        dump.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
+        dump.append(''.join(traceback.format_stack(frame)))
+        dump.append('\n')
+
+    # greenlets
+
+    # if greenlet is present, let's dump each greenlet stack
+    # Use the gc module to inspect all objects to find the greenlets
+    # since there isn't a global registry
+    for ob in gc.get_objects():
+        if not isinstance(ob, greenlet):
+            continue
+        if not ob:
+            continue  # not running anymore or not started
+        dump.append('Greenlet %s\n' % ob)
+        dump.append(''.join(traceback.format_stack(ob.gr_frame)))
+        dump.append('\n')
+
+    return dump

src/greentest/known_failures.py

@@ -20,6 +20,7 @@ PYGTE279 = (
     and sys.version_info[1] >= 7
     and sys.version_info[2] >= 9
 )
+RESOLVER_ARES = os.getenv('GEVENT_RESOLVER') == 'ares'
 LIBUV = os.getenv('GEVENT_CORE_CFFI_ONLY') == 'libuv' # XXX: Formalize this better
 
 
@@ -138,6 +139,18 @@ if PYPY:
         'FLAKY test__backdoor.py',
     ]
 
+    if RESOLVER_ARES:
+
+        FAILING_TESTS += [
+            # A few errors and differences:
+            # AssertionError: ('255.255.255.255', 'http') != gaierror(4, 'ARES_ENOTFOUND: Domain name not found')
+            # AssertionError: OverflowError('port must be 0-65535.',) != ('readthedocs.org', '65535')
+            # AssertionError: Lists differ:
+            #     (10, 1, 6, '', ('2607:f8b0:4004:810::200e', 80, 0L, 0L))
+            #     (10, 1, 6, '', ('2607:f8b0:4004:805::200e', 80, 0, 0))
+            'test__socket_dns.py',
+        ]
+
     if PY3 and TRAVIS:
         FAILING_TESTS += [
             ## ---

src/greentest/test__server.py

@@ -406,7 +406,11 @@ class TestPoolSpawn(TestDefaultSpawn):
         # XXX: This tends to timeout. Which is weird, because what would have
         # been the third call to assertPoolFull() DID NOT timeout, hence why it
         # was removed.
-        self.assertRequestSucceeded()
+        try:
+            self.assertRequestSucceeded()
+        except socket.timeout:
+            greentest.reraiseFlakyTestTimeout()
+
         del long_request
 
     test_pool_full.error_fatal = False

src/greentest/test__signal.py

@@ -45,7 +45,7 @@ if hasattr(signal, 'SIGALRM'):
                 sig.cancel()
 
 
-        @greentest.skipIf(greentest.PY3 and greentest.LIBUV and greentest.RUNNING_ON_TRAVIS,
+        @greentest.skipIf(greentest.PY3 and greentest.CFFI_BACKEND and greentest.RUNNING_ON_TRAVIS,
                           "Fails for unknown reason")
         @greentest.ignores_leakcheck
         def test_reload(self):
@@ -59,13 +59,16 @@ if hasattr(signal, 'SIGALRM'):
             # doesn't happen). See
             # https://github.com/gevent/gevent/issues/805
 
-            # This fails on Python 3.6 under linux (travis CI) but not
-            # locally on macOS with:
+            # This fails on Python 3.5 under linux (travis CI) but not
+            # locally on macOS with (for both libuv and libev cffi); sometimes it
+            # failed with libuv on Python 3.6 too, but not always:
             #   AttributeError: cffi library 'gevent.libuv._corecffi' has no function,
             #      constant or global variable named '__loader__'
             # which in turn leads to:
             #   SystemError: <built-in function getattr> returned a result with an error set
 
+            # It's not safe to continue after a SystemError, so we just skip the test there.
+
             import gevent.signal # make sure it's in sys.modules pylint:disable=redefined-outer-name
             assert gevent.signal
             import site

src/greentest/test__threading_monkey_in_thread.py

 # We can monkey-patch in a thread, but things don't work as expected.
+from __future__ import print_function
 import sys
 import threading
 from gevent import monkey
@@ -32,7 +33,11 @@ class Test(greentest.TestCase):
 
         thread = threading.Thread(target=target)
         thread.start()
-        thread.join()
+        try:
+            thread.join()
+        except:
+            # XXX: This can raise LoopExit in some cases.
+            greentest.reraiseFlakyTestRaceCondition()
 
         self.assertNotIsInstance(current, threading._DummyThread)
         self.assertIsInstance(current, monkey.get_original('threading', 'Thread'))