Make gevent.killall do what Greenlet.kill does.

And stop a fresh greenlet from even running.

Fixes #1473.

Free user-provided resources as soon as the greenlet is cancelled, if it's not running.

.pylintrc

@@ -101,10 +101,11 @@ dummy-variables-rgx=_.*
 generated-members=exc_clear
 
 # List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set). This supports can work
+# (useful for classes with attributes dynamically set). This can work
 # with qualified names.
 #ignored-classes=SSLContext, SSLSocket, greenlet, Greenlet, parent, dead
 
+
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It

CHANGES.rst

@@ -81,6 +81,19 @@
   raised an exception. This could happen if the hub's ``handle_error``
   function was poorly customized, for example. See :issue:`1482`
 
+- Make ``gevent.killall`` stop greenlets from running that hadn't been
+  run yet. This make it consistent with ``Greenlet.kill()``. See
+  :issue:`1473` reported by kochelmonster.
+
+- Make ``gevent.spawn_raw`` set the ``loop`` attribute on returned
+  greenlets. This lets them work with more gevent APIs, notably
+  ``gevent.killall()``. They already had dictionaries, but this may
+  make them slightly larger, depending on platform (on CPython 2.7
+  through 3.6 there is no apparent difference for one attribute but on
+  CPython 3.7 and 3.8 dictionaries are initially empty and only
+  allocate space once an attribute is added; they're still smaller
+  than on earlier versions though).
+
 1.5a2 (2019-10-21)
 ==================
 

pyproject.toml

@@ -12,9 +12,9 @@ requires = [
      # 0.28 is faster, and (important!) lets us specify the target module
      # name to be created so that we can have both foo.py and _foo.so
      # at the same time. 0.29 fixes some issues with Python 3.7,
-     # and adds the 3str mode for transition to Python 3. 0.29.12+ is
+     # and adds the 3str mode for transition to Python 3. 0.29.14+ is
      # required for Python 3.8
-     "Cython >= 0.29.13",
+     "Cython >= 0.29.14",
      # See version requirements in setup.py
      "cffi >= 1.12.3 ; platform_python_implementation == 'CPython'",
      # Python 3.7 requires at least 0.4.14, which is ABI incompatible with earlier

src/gevent/_ffi/watcher.py

@@ -10,6 +10,7 @@ import functools
 import warnings
 
 from gevent._config import config
+from gevent._util import LazyOnClass
 
 try:
     from tracemalloc import get_object_traceback
@@ -108,26 +109,6 @@ def only_if_watcher(func):
     return if_w
 
 
-class LazyOnClass(object):
-
-    @classmethod
-    def lazy(cls, cls_dict, func):
-        "Put a LazyOnClass object in *cls_dict* with the same name as *func*"
-        cls_dict[func.__name__] = cls(func)
-
-    def __init__(self, func, name=None):
-        self.name = name or func.__name__
-        self.func = func
-
-    def __get__(self, inst, klass):
-        if inst is None: # pragma: no cover
-            return self
-
-        val = self.func(inst)
-        setattr(klass, self.name, val)
-        return val
-
-
 class AbstractWatcherType(type):
     """
     Base metaclass for watchers.

src/gevent/_greenlet.pxd

@@ -133,6 +133,9 @@ cdef class Greenlet(greenlet):
     cpdef rawlink(self, object callback)
     cpdef str _formatinfo(self)
 
+    # Helper for killall()
+    cpdef bint _maybe_kill_before_start(self, exception)
+
     # This is a helper function for a @property getter;
     # defining locals() for a @property doesn't seem to work.
     @cython.locals(reg=IdentRegistry)
@@ -144,11 +147,13 @@ cdef class Greenlet(greenlet):
     cdef bint __never_started_or_killed(self)
     cdef bint __start_completed(self)
     cdef __handle_death_before_start(self, tuple args)
+    @cython.final
+    cdef inline void __free(self)
 
     cdef __cancel_start(self)
 
-    cdef _report_result(self, object result)
-    cdef _report_error(self, tuple exc_info)
+    cdef inline __report_result(self, object result)
+    cdef inline __report_error(self, tuple exc_info)
     # This is used as the target of a callback
     # from the loop, and so needs to be a cpdef
     cpdef _notify_links(self)

src/gevent/_util.py

@@ -119,7 +119,9 @@ class Lazy(object):
     A non-data descriptor used just like @property. The
     difference is the function value is assigned to the instance
     dict the first time it is accessed and then the function is never
-    called agoin.
+    called again.
+
+    Contrast with `readproperty`.
     """
     def __init__(self, func):
         self.data = (func, func.__name__)
@@ -136,9 +138,16 @@ class Lazy(object):
 
 class readproperty(object):
     """
-    A non-data descriptor like @property. The difference is that
-    when the property is assigned to, it is cached in the instance
-    and the function is not called on that instance again.
+    A non-data descriptor similar to :class:`property`.
+
+    The difference is that the property can be assigned to directly,
+    without invoking a setter function. When the property is assigned
+    to, it is cached in the instance and the function is not called on
+    that instance again.
+
+    Contrast with `Lazy`, which caches the result of the function in the
+    instance the first time it is called and never calls the function on that
+    instance again.
     """
 
     def __init__(self, func):
@@ -151,6 +160,35 @@ class readproperty(object):
 
         return self.func(inst)
 
+class LazyOnClass(object):
+    """
+    Similar to `Lazy`, but stores the value in the class.
+
+    This is useful when the getter is expensive and conceptually
+    a shared class value, but we don't want import-time side-effects
+    such as expensive imports because it may not always be used.
+
+    Probably doesn't mix well with inheritance?
+    """
+
+    @classmethod
+    def lazy(cls, cls_dict, func):
+        "Put a LazyOnClass object in *cls_dict* with the same name as *func*"
+        cls_dict[func.__name__] = cls(func)
+
+    def __init__(self, func, name=None):
+        self.name = name or func.__name__
+        self.func = func
+
+    def __get__(self, inst, klass):
+        if inst is None: # pragma: no cover
+            return self
+
+        val = self.func(inst)
+        setattr(klass, self.name, val)
+        return val
+
+
 def gmctime():
     """
     Returns the current time as a string in RFC3339 format.

src/gevent/hub.py

@@ -96,12 +96,16 @@ def spawn_raw(function, *args, **kwargs):
        if ``GEVENT_TRACK_GREENLET_TREE`` is enabled (the default). If not enabled,
        those attributes will not be set.
 
+    .. versionchanged:: 1.5a3
+       The returned greenlet always has a *loop* attribute matching the
+       current hub's loop. This helps it work better with more gevent APIs.
     """
     if not callable(function):
         raise TypeError("function must be callable")
 
     # The hub is always the parent.
     hub = _get_hub_noargs()
+    loop = hub.loop
 
     factory = TrackedRawGreenlet if GEVENT_CONFIG.track_greenlet_tree else RawGreenlet
 
@@ -111,11 +115,11 @@ def spawn_raw(function, *args, **kwargs):
     if kwargs:
         function = _functools_partial(function, *args, **kwargs)
         g = factory(function, hub)
-        hub.loop.run_callback(g.switch)
+        loop.run_callback(g.switch)
     else:
         g = factory(function, hub)
-        hub.loop.run_callback(g.switch, *args)
-
+        loop.run_callback(g.switch, *args)
+    g.loop = hub.loop
     return g
 
 

src/gevent/testing/testcase.py

@@ -27,6 +27,7 @@ from unittest import TestCase as BaseTestCase
 from functools import wraps
 
 import gevent
+from gevent._util import LazyOnClass
 
 from . import sysinfo
 from . import params
@@ -77,6 +78,66 @@ class TimeAssertMixin(object):
             fuzzy=(0.01 if not sysinfo.EXPECT_POOR_TIMER_RESOLUTION and not sysinfo.LIBUV else 1.0)):
         return self.runs_in_given_time(0.0, fuzzy)
 
+
+class GreenletAssertMixin(object):
+    """Assertions related to greenlets."""
+
+    def assert_greenlet_ready(self, g):
+        self.assertTrue(g.dead, g)
+        self.assertTrue(g.ready(), g)
+        self.assertFalse(g, g)
+
+    def assert_greenlet_not_ready(self, g):
+        self.assertFalse(g.dead, g)
+        self.assertFalse(g.ready(), g)
+
+    def assert_greenlet_spawned(self, g):
+        self.assertTrue(g.started, g)
+        self.assertFalse(g.dead, g)
+
+    # No difference between spawned and switched-to once
+    assert_greenlet_started = assert_greenlet_spawned
+
+    def assert_greenlet_finished(self, g):
+        self.assertFalse(g.started, g)
+        self.assertTrue(g.dead, g)
+
+
+class StringAssertMixin(object):
+    """
+    Assertions dealing with strings.
+    """
+
+    @LazyOnClass
+    def HEX_NUM_RE(self):
+        import re
+        return re.compile('-?0x[0123456789abcdef]+L?', re.I)
+
+    def normalize_addr(self, s, replace='X'):
+        # https://github.com/PyCQA/pylint/issues/1127
+        return self.HEX_NUM_RE.sub(replace, s) # pylint:disable=no-member
+
+    def normalize_module(self, s, module=None, replace='module'):
+        if module is None:
+            module = type(self).__module__
+
+        return s.replace(module, replace)
+
+    def normalize(self, s):
+        return self.normalize_module(self.normalize_addr(s))
+
+    def assert_nstr_endswith(self, o, val):
+        s = str(o)
+        n = self.normalize(s)
+        self.assertTrue(n.endswith(val), (s, n))
+
+    def assert_nstr_startswith(self, o, val):
+        s = str(o)
+        n = self.normalize(s)
+        self.assertTrue(n.startswith(val), (s, n))
+
+
+
 class TestTimeout(gevent.Timeout):
     _expire_info = ''
 
@@ -178,7 +239,11 @@ class SubscriberCleanupMixin(object):
 
 
 class TestCase(TestCaseMetaClass("NewBase",
-                                 (SubscriberCleanupMixin, TimeAssertMixin, BaseTestCase,),
+                                 (SubscriberCleanupMixin,
+                                  TimeAssertMixin,
+                                  GreenletAssertMixin,
+                                  StringAssertMixin,
+                                  BaseTestCase,),
                                  {})):
     __timeout__ = params.LOCAL_TIMEOUT if not sysinfo.RUNNING_ON_CI else params.CI_TIMEOUT
 

src/gevent/tests/test__greenletset.py

@@ -138,7 +138,7 @@ class Test(greentest.TestCase):
         for g in s:
             assert g.dead
 
-    def test_killall_iterable_argument_timeout(self):
+    def test_killall_iterable_argument_timeout_not_started(self):
         def f():
             try:
                 gevent.sleep(1.5)
@@ -149,6 +149,25 @@ class Test(greentest.TestCase):
         s = set()
         s.add(p1)
         s.add(p2)
+        gevent.killall(s, timeout=0.5)
+
+        for g in s:
+            self.assertTrue(g.dead, g)
+
+    def test_killall_iterable_argument_timeout_started(self):
+        def f():
+            try:
+                gevent.sleep(1.5)
+            except: # pylint:disable=bare-except
+                gevent.sleep(1)
+        p1 = GreenletSubclass.spawn(f)
+        p2 = GreenletSubclass.spawn(f)
+
+        s = set()
+        s.add(p1)
+        s.add(p2)
+        # Get them both running.
+        gevent.sleep(timing.SMALLEST_RELIABLE_DELAY)
         with self.assertRaises(Timeout):
             gevent.killall(s, timeout=0.5)
 

src/gevent/tests/test__hub.py

@@ -159,6 +159,7 @@ class TestPeriodicMonitoringThread(greentest.TestCase):
         gevent.config.monitor_thread = self.monitor_thread
         self.monitored_hubs = None
         self._reset_hub()
+        super(TestPeriodicMonitoringThread, self).tearDown()
 
     def _monitor(self, hub):
         self.monitor_fired += 1

src/gevent/tests/test__issue330.py

@@ -65,7 +65,8 @@ class TestSwitch(greentest.TestCase):
             self.caught = e
 
     def test_kill_exception(self):
-        # Killing with gevent.kill gets the right exception
+        # Killing with gevent.kill gets the right exception,
+        # and we can pass exception objects, not just exception classes.
 
         g = gevent.spawn(self.catcher)
         g.start()

src/gevent/tests/test__pywsgi.py

@@ -17,7 +17,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
-# pylint: disable=too-many-lines,unused-argument
+# pylint: disable=too-many-lines,unused-argument,too-many-ancestors
 from __future__ import print_function
 
 from gevent import monkey

src/gevent/tests/test__server_pywsgi.py

@@ -93,13 +93,13 @@ class TestDefaultSpawn(test__server.TestDefaultSpawn):
 class TestSSLSocketNotAllowed(test__server.TestSSLSocketNotAllowed):
     Settings = Settings
 
-class TestRawSpawn(test__server.TestRawSpawn):
+class TestRawSpawn(test__server.TestRawSpawn): # pylint:disable=too-many-ancestors
     Settings = Settings
 
 class TestSSLGetCertificate(test__server.TestSSLGetCertificate):
     Settings = Settings
 
-class TestPoolSpawn(test__server.TestPoolSpawn):
+class TestPoolSpawn(test__server.TestPoolSpawn): # pylint:disable=too-many-ancestors
     Settings = Settings
 
 if __name__ == '__main__':

src/gevent/tests/test__threadpool_executor_patched.py

@@ -9,7 +9,7 @@ import gevent.testing as greentest
 from . import test__threadpool
 
 
-class TestPatchedTPE(test__threadpool.TestTPE):
+class TestPatchedTPE(test__threadpool.TestTPE): # pylint:disable=too-many-ancestors
     MONKEY_PATCHED = True