Merge pull request #1374 from gevent/issue1371

Avoid unbounded memory usage in deep spawn trees. Fixes #1371.

CHANGES.rst

@@ -40,6 +40,9 @@
 - Python 2: Avoid a memory leak when an `io.BufferedWriter` is wrapped
   around a socket. Reported by Damien Tournoud in :issue:`1318`.
 
+- Avoid unbounded memory usage when creating very deep spawn trees.
+  Reported in :issue:`1371` by dmrlawson.
+
 1.4.0 (2019-01-04)
 ==================
 

src/gevent/_ffi/loop.py

@@ -98,7 +98,7 @@ class AbstractCallbacks(object):
         This function should never return 0, as that's the default value that
         Python exceptions will produce.
         """
-        #print("Running callback", handle)
+        #_dbg("Running callback", handle)
         orig_ffi_watcher = None
         try:
             # Even dereferencing the handle needs to be inside the try/except;
@@ -116,6 +116,7 @@ class AbstractCallbacks(object):
             the_watcher = self.from_handle(handle)
             orig_ffi_watcher = the_watcher._watcher
             args = the_watcher.args
+            #_dbg("Running callback", the_watcher, orig_ffi_watcher, args)
             if args is None:
                 # Legacy behaviour from corecext: convert None into ()
                 # See test__core_watcher.py
@@ -204,7 +205,7 @@ class AbstractCallbacks(object):
         # at least for signals under libuv, which are delivered at very odd times.
         # Hopefully the event still shows up when we poll the next time.
         watcher = None
-        handle = tb.tb_frame.f_locals['handle'] if tb is not None else None
+        handle = tb.tb_frame.f_locals.get('handle') if tb is not None else None
         if handle: # handle could be NULL
             watcher = self.from_handle(handle)
         if watcher is not None:

src/gevent/greenlet.py

@@ -256,7 +256,12 @@ class Greenlet(greenlet):
                     spawner.spawn_tree_locals = self.spawn_tree_locals
 
             self._spawning_stack_frames = _extract_stack(self.spawning_stack_limit)
-            self._spawning_stack_frames.extend(getattr(spawner, '_spawning_stack_frames', []))
+            # Don't copy the spawning greenlet's
+            # '_spawning_stack_frames' into ours. That's somewhat
+            # confusing, and, if we're not careful, a deep spawn tree
+            # can lead to excessive memory usage (an infinite spawning
+            # tree could lead to unbounded memory usage without care
+            # --- see https://github.com/gevent/gevent/issues/1371)
         else:
             # None is the default for all of these in Cython, but we
             # need to declare them for pure-Python mode.

src/gevent/tests/test__greenlet.py

@@ -755,6 +755,31 @@ class TestBasic(greentest.TestCase):
         finally:
             greenlet.sys_getframe = ogf
 
+    def test_spawn_length_nested_doesnt_grow(self):
+        # https://github.com/gevent/gevent/pull/1374
+        def doit1():
+            getcurrent()._spawning_stack_frames.append('not copied')
+            return gevent.spawn(doit2)
+
+        def doit2():
+            return gevent.spawn(doit3)
+
+        def doit3():
+            current = gevent.getcurrent()
+
+            return current
+
+
+        g1 = gevent.spawn(doit1)
+        g1.join()
+        g2 = g1.value
+        g2.join()
+
+        g3 = g2.value
+        self.assertNotIn('not copied', g3._spawning_stack_frames)
+
+
+
 
 class TestStart(greentest.TestCase):