Merge pull request #1121 from gevent/locals-debug

Include locals in the debugging output.

CHANGES.rst

@@ -117,6 +117,12 @@
   used when the environment variable ``PURE_PYTHON`` is set. This is
   not recommended except for debugging and testing. See :issue:`1118`.
 
+- Include the values of `gevent.local.local` objects associated with
+  each greenlet in `gevent.util.format_run_info`.
+
+- `gevent.Greenlet` objects now have a `gevent.Greenlet.name`
+  attribute that is included in the default repr.
+
 
 1.3a1 (2018-01-27)
 ==================

doc/gevent.rst

@@ -36,6 +36,7 @@ generated.
 
 .. autoattribute:: Greenlet.exception
 .. autoattribute:: Greenlet.minimal_ident
+.. autoattribute:: Greenlet.name
 
 .. rubric:: Methods
 
@@ -97,8 +98,8 @@ __ https://greenlet.readthedocs.io/en/latest/#instantiation
 Spawn helpers
 =============
 
-.. autofunction:: spawn(function, *args, **kwargs)
-.. autofunction:: spawn_later(seconds, function, *args, **kwargs)
+.. autofunction:: spawn
+.. autofunction:: spawn_later
 .. autofunction:: spawn_raw
 
 

src/gevent/_local.pxd

 # cython: auto_pickle=False
 
 cimport cython
+from gevent._greenlet cimport Greenlet
 
 cdef bint _PYPY
 cdef ref
 cdef copy
 
 cdef object _marker
+cdef str key_prefix
 cdef bint _greenlet_imported
 
 
@@ -56,8 +58,10 @@ cdef class _local_deleted:
 @cython.internal
 cdef class _localimpl:
     cdef str key
-    cdef _wrefdict dicts
+    cdef dict dicts
     cdef tuple localargs
+    cdef dict localkwargs
+    cdef tuple localtypeid
     cdef object __weakref__
 
 
@@ -67,13 +71,12 @@ cdef class _localimpl_dict_entry:
     cdef object wrgreenlet
     cdef dict localdict
 
-@cython.locals(localdict=dict, key=str, greenlet=greenlet,
+@cython.locals(localdict=dict, key=str,
                greenlet_deleted=_greenlet_deleted,
                local_deleted=_local_deleted)
-cdef dict _localimpl_create_dict(_localimpl self)
-
-@cython.locals(entry=_localimpl_dict_entry, greenlet=greenlet)
-cdef inline dict _localimpl_get_dict(_localimpl self)
+cdef dict _localimpl_create_dict(_localimpl self,
+                                 greenlet greenlet,
+                                 object idt)
 
 cdef set _local_attrs
 
@@ -86,11 +89,22 @@ cdef class local:
     cdef set _local_type_vars
     cdef type _local_type
 
+    @cython.locals(entry=_localimpl_dict_entry,
+                   dct=dict, duplicate=dict,
+                   instance=local)
+    cpdef local __copy__(local self)
+
 
-@cython.locals(impl=_localimpl,dct=dict)
+@cython.locals(impl=_localimpl,dct=dict,
+               dct=dict, entry=_localimpl_dict_entry)
 cdef inline dict _local_get_dict(local self)
 
 @cython.locals(entry=_localimpl_dict_entry)
 cdef _local__copy_dict_from(local self, _localimpl impl, dict duplicate)
 
 cdef tuple _local_find_descriptors(local self)
+
+@cython.locals(result=list, local_impl=_localimpl,
+               entry=_localimpl_dict_entry, k=str,
+               greenlet_dict=dict)
+cpdef all_local_dicts_for_greenlet(greenlet greenlet)

src/gevent/_util.py

@@ -5,6 +5,8 @@ internal gevent utilities, not for external use.
 
 from __future__ import print_function, absolute_import, division
 
+from functools import update_wrapper
+
 from gevent._compat import iteritems
 
 
@@ -110,6 +112,7 @@ class Lazy(object):
     """
     def __init__(self, func):
         self.data = (func, func.__name__)
+        update_wrapper(self, func)
 
     def __get__(self, inst, class_):
         if inst is None:
@@ -129,6 +132,7 @@ class readproperty(object):
 
     def __init__(self, func):
         self.func = func
+        update_wrapper(self, func)
 
     def __get__(self, inst, class_):
         if inst is None:

src/gevent/greenlet.py

@@ -21,6 +21,7 @@ from gevent.hub import iwait
 from gevent.hub import wait
 from gevent.timeout import Timeout
 from gevent._util import Lazy
+from gevent._util import readproperty
 
 
 __all__ = [
@@ -312,6 +313,17 @@ class Greenlet(greenlet):
             self._ident = self._get_minimal_ident()
         return self._ident
 
+    @readproperty
+    def name(self):
+        """
+        The greenlet name. By default, a unique name is constructed using
+        the :attr:`minimal_ident`. You can assign a string to this
+        value to change it. It is shown in the `repr` of this object.
+
+        .. versionadded:: 1.3a2
+        """
+        return 'Greenlet-%d' % (self.minimal_ident)
+
     def _raise_exception(self):
         reraise(*self.exc_info)
 
@@ -428,7 +440,7 @@ class Greenlet(greenlet):
 
     def __repr__(self):
         classname = self.__class__.__name__
-        result = '<%s at %s' % (classname, hex(id(self)))
+        result = '<%s "%s" at %s' % (classname, self.name, hex(id(self)))
         formatted = self._formatinfo()
         if formatted:
             result += ': ' + formatted
@@ -526,6 +538,8 @@ class Greenlet(greenlet):
     @classmethod
     def spawn(cls, *args, **kwargs):
         """
+        spawn(function, *args, **kwargs) -> Greenlet
+
         Create a new :class:`Greenlet` object and schedule it to run ``function(*args, **kwargs)``.
         This can be used as ``gevent.spawn`` or ``Greenlet.spawn``.
 
@@ -542,8 +556,10 @@ class Greenlet(greenlet):
     @classmethod
     def spawn_later(cls, seconds, *args, **kwargs):
         """
-        Create and return a new Greenlet object scheduled to run ``function(*args, **kwargs)``
-        in the future loop iteration *seconds* later. This can be used as ``Greenlet.spawn_later``
+        spawn_later(seconds, function, *args, **kwargs) -> Greenlet
+
+        Create and return a new `Greenlet` object scheduled to run ``function(*args, **kwargs)``
+        in a future loop iteration *seconds* later. This can be used as ``Greenlet.spawn_later``
         or ``gevent.spawn_later``.
 
         The arguments are passed to :meth:`Greenlet.__init__`.

src/gevent/local.py

@@ -158,14 +158,56 @@ _PYPY = hasattr(sys, 'pypy_version_info')
 from copy import copy
 from weakref import ref
 
+
 locals()['getcurrent'] = __import__('greenlet').getcurrent
 locals()['greenlet_init'] = lambda: None
 
-
 __all__ = [
     "local",
 ]
 
+# The key used in the Thread objects' attribute dicts.
+# We keep it a string for speed but make it unlikely to clash with
+# a "real" attribute.
+key_prefix = '_gevent_local_localimpl_'
+
+# The overall structure is as follows:
+# For each local() object:
+# greenlet.__dict__[key_prefix + str(id(local))]
+#    => _localimpl.dicts[id(greenlet)] => (ref(greenlet), {})
+
+# That final tuple is actually a localimpl_dict_entry object.
+
+def all_local_dicts_for_greenlet(greenlet):
+    """
+    Internal debug helper for getting the local values associated
+    with a greenlet. This is subject to change or removal at any time.
+
+    :return: A list of ((type, id), {}) pairs, where the first element
+      is the type and id of the local object and the second object is its
+      instance dictionary, as seen from this greenlet.
+
+    .. versionadded:: 1.3a2
+    """
+
+    result = []
+    id_greenlet = id(greenlet)
+    greenlet_dict = greenlet.__dict__
+    for k, v in greenlet_dict.items():
+        if not k.startswith(key_prefix):
+            continue
+        local_impl = v()
+        if local_impl is None:
+            continue
+        entry = local_impl.dicts.get(id_greenlet)
+        if entry is None:
+            # Not yet used in this greenlet.
+            continue
+        assert entry.wrgreenlet() is greenlet
+        result.append((local_impl.localtypeid, entry.localdict))
+
+    return result
+
 
 class _wrefdict(dict):
     """A dict that can be weak referenced"""
@@ -211,21 +253,24 @@ class _local_deleted(object):
 
 class _localimpl(object):
     """A class managing thread-local dicts"""
-    __slots__ = ('key', 'dicts', 'localargs', '__weakref__',)
-
-    def __init__(self, args, kwargs):
-        # The key used in the Thread objects' attribute dicts.
-        # We keep it a string for speed but make it unlikely to clash with
-        # a "real" attribute.
-        self.key = '_threading_local._localimpl.' + str(id(self))
-        # { id(Thread) -> (ref(Thread), thread-local dict) }
+    __slots__ = ('key', 'dicts',
+                 'localargs', 'localkwargs',
+                 'localtypeid',
+                 '__weakref__',)
+
+    def __init__(self, args, kwargs, local_type, id_local):
+        self.key = key_prefix + str(id(self))
+        # { id(greenlet) -> _localimpl_dict_entry(ref(greenlet), greenlet-local dict) }
         self.dicts = _wrefdict()
-        self.localargs = args, kwargs
+        self.localargs = args
+        self.localkwargs = kwargs
+        self.localtypeid = local_type, id_local
 
         # We need to create the thread dict in anticipation of
         # __init__ being called, to make sure we don't call it
         # again ourselves. MUST do this before setting any attributes.
-        _localimpl_create_dict(self)
+        greenlet = getcurrent() # pylint:disable=undefined-variable
+        _localimpl_create_dict(self, greenlet, id(greenlet))
 
 class _localimpl_dict_entry(object):
     """
@@ -247,40 +292,32 @@ class _localimpl_dict_entry(object):
 # (but not pointer chasing overhead; the vtable isn't used when we declare
 # the class final).
 
-def _localimpl_get_dict(self):
-    """Return the dict for the current thread. Raises KeyError if none
-    defined."""
-    greenlet = getcurrent() # pylint:disable=undefined-variable
-    entry = self.dicts[id(greenlet)]
-    return entry.localdict
 
-def _localimpl_create_dict(self):
+def _localimpl_create_dict(self, greenlet, id_greenlet):
     """Create a new dict for the current thread, and return it."""
     localdict = {}
     key = self.key
-    greenlet = getcurrent() # pylint:disable=undefined-variable
-    idt = id(greenlet)
 
     wrdicts = ref(self.dicts)
 
-    # When the thread is deleted, remove the local dict.
-    # Note that this is suboptimal if the thread object gets
+    # When the greenlet is deleted, remove the local dict.
+    # Note that this is suboptimal if the greenlet object gets
     # caught in a reference loop. We would like to be called
-    # as soon as the OS-level thread ends instead.
+    # as soon as the OS-level greenlet ends instead.
 
     # If we are working with a gevent.greenlet.Greenlet, we
     # can pro-actively clear out with a link, avoiding the
-    # issue described above.Use rawlink to avoid spawning any
+    # issue described above. Use rawlink to avoid spawning any
     # more greenlets.
-    greenlet_deleted = _greenlet_deleted(idt, wrdicts)
+    greenlet_deleted = _greenlet_deleted(id_greenlet, wrdicts)
 
-    try:
-        rawlink = greenlet.rawlink
-    except AttributeError:
-        wrthread = ref(greenlet, greenlet_deleted)
-    else:
+    rawlink = getattr(greenlet, 'rawlink', None)
+    if rawlink is not None:
         rawlink(greenlet_deleted)
         wrthread = ref(greenlet)
+    else:
+        wrthread = ref(greenlet, greenlet_deleted)
+
 
     # When the localimpl is deleted, remove the thread attribute.
     local_deleted = _local_deleted(key, wrthread, greenlet_deleted)
@@ -289,7 +326,7 @@ def _localimpl_create_dict(self):
     wrlocal = ref(self, local_deleted)
     greenlet.__dict__[key] = wrlocal
 
-    self.dicts[idt] = _localimpl_dict_entry(wrthread, localdict)
+    self.dicts[id_greenlet] = _localimpl_dict_entry(wrthread, localdict)
     return localdict
 
 
@@ -297,12 +334,15 @@ _marker = object()
 
 def _local_get_dict(self):
     impl = self._local__impl
+    # Cython can optimize dict[], but not dict.get()
+    greenlet = getcurrent() # pylint:disable=undefined-variable
+    idg = id(greenlet)
     try:
-        dct = _localimpl_get_dict(impl)
+        entry = impl.dicts[idg]
+        dct = entry.localdict
     except KeyError:
-        dct = _localimpl_create_dict(impl)
-        args, kw = impl.localargs
-        self.__init__(*args, **kw)
+        dct = _localimpl_create_dict(impl, greenlet, idg)
+        self.__init__(*impl.localargs, **impl.localkwargs)
     return dct
 
 def _init():
@@ -330,7 +370,7 @@ class local(object):
         if args or kw:
             if type(self).__init__ == object.__init__:
                 raise TypeError("Initialization arguments are not supported", args, kw)
-        impl = _localimpl(args, kw)
+        impl = _localimpl(args, kw, type(self), id(self))
         # pylint:disable=attribute-defined-outside-init
         self._local__impl = impl
         get, dels, sets_or_dels, sets = _local_find_descriptors(self)
@@ -467,13 +507,13 @@ class local(object):
 
     def __copy__(self):
         impl = self._local__impl
+        entry = impl.dicts[id(getcurrent())]  # pylint:disable=undefined-variable
 
-        d = _localimpl_get_dict(impl)
-        duplicate = copy(d)
+        dct = entry.localdict
+        duplicate = copy(dct)
 
         cls = type(self)
-        args, kw = impl.localargs
-        instance = cls(*args, **kw)
+        instance = cls(*impl.localargs, **impl.localkwargs)
         _local__copy_dict_from(instance, impl, duplicate)
         return instance
 

src/gevent/util.py

@@ -78,7 +78,7 @@ def format_run_info():
     .. versionchanged:: 1.3a2
        Renamed from ``dump_stacks`` to reflect the fact that this
        prints additional information about greenlets, including their
-       spawning stack, parent, and any spawn tree locals.
+       spawning stack, parent, locals, and any spawn tree locals.
     """
 
     lines = []
@@ -112,10 +112,12 @@ def _format_thread_info(lines):
     del threads
 
 def _format_greenlet_info(lines):
+    # pylint:disable=too-many-locals
     from greenlet import greenlet
     import pprint
     import traceback
     import gc
+    from gevent.local import all_local_dicts_for_greenlet
 
     def _noop():
         return None
@@ -145,6 +147,12 @@ def _format_greenlet_info(lines):
             seen_locals.add(id(spawn_tree_locals))
             lines.append("Spawn Tree Locals:\n")
             lines.append(pprint.pformat(spawn_tree_locals))
+        gr_locals = all_local_dicts_for_greenlet(ob)
+        if gr_locals:
+            lines.append("Greenlet Locals:\n")
+            for (kind, idl), vals in gr_locals:
+                lines.append("\tLocal %s at %s\n" % (kind, hex(idl)))
+                lines.append("\t" + pprint.pformat(vals))
 
 
     del lines

src/greentest/test__greenlet.py

@@ -418,34 +418,36 @@ class TestStr(greentest.TestCase):
 
     def test_function(self):
         g = gevent.Greenlet.spawn(dummy_test_func)
-        self.assertEqual(hexobj.sub('X', str(g)), '<Greenlet at X: dummy_test_func>')
+        self.assertTrue(hexobj.sub('X', str(g)).endswith('at X: dummy_test_func>'))
         assert_not_ready(g)
         g.join()
         assert_ready(g)
-        self.assertEqual(hexobj.sub('X', str(g)), '<Greenlet at X: dummy_test_func>')
+        self.assertTrue(hexobj.sub('X', str(g)).endswith(' at X: dummy_test_func>'))
 
     def test_method(self):
         g = gevent.Greenlet.spawn(A().method)
         str_g = hexobj.sub('X', str(g))
         str_g = str_g.replace(__name__, 'module')
-        self.assertEqual(str_g, '<Greenlet at X: <bound method A.method of <module.A object at X>>>')
+        self.assertTrue(str_g.startswith('<Greenlet "Greenlet-'))
+        self.assertTrue(str_g.endswith('at X: <bound method A.method of <module.A object at X>>>'))
         assert_not_ready(g)
         g.join()
         assert_ready(g)
         str_g = hexobj.sub('X', str(g))
         str_g = str_g.replace(__name__, 'module')
-        self.assertEqual(str_g, '<Greenlet at X: <bound method A.method of <module.A object at X>>>')
+        self.assertTrue(str_g.endswith('at X: <bound method A.method of <module.A object at X>>>'))
 
     def test_subclass(self):
         g = Subclass()
         str_g = hexobj.sub('X', str(g))
         str_g = str_g.replace(__name__, 'module')
-        self.assertEqual(str_g, '<Subclass at X: _run>')
+        self.assertTrue(str_g.startswith('<Subclass '))
+        self.assertTrue(str_g.endswith('at X: _run>'))
 
         g = Subclass(None, 'question', answer=42)
         str_g = hexobj.sub('X', str(g))
         str_g = str_g.replace(__name__, 'module')
-        self.assertEqual(str_g, "<Subclass at X: _run('question', answer=42)>")
+        self.assertTrue(str_g.endswith(" at X: _run('question', answer=42)>"))
 
 
 class TestJoin(AbstractGenericWaitTestCase):

src/greentest/test__hub.py

@@ -110,7 +110,8 @@ class TestWaiter(greentest.TestCase):
         waiter = Waiter()
         g = gevent.spawn(waiter.get)
         gevent.sleep(0)
-        assert str(waiter).startswith('<Waiter greenlet=<Greenlet at '), str(waiter)
+        self.assertTrue(str(waiter).startswith('<Waiter greenlet=<Greenlet "Greenlet-'))
+
         g.kill()
 
 

src/greentest/test__local.py

@@ -308,6 +308,21 @@ class GeventLocalTestCase(greentest.TestCase):
 
         self.assertEqual(count, len(deleted_sentinels))
 
+    def test_local_dicts_for_greenlet(self):
+        import gevent
+        from gevent.local import all_local_dicts_for_greenlet
+        x = MyLocal()
+        x.foo = 42
+        del x.sentinel
+
+        if greentest.PYPY:
+            import gc
+            gc.collect()
+
+        results = all_local_dicts_for_greenlet(gevent.getcurrent())
+        self.assertEqual(results,
+                         [((MyLocal, id(x)), {'foo': 42})])
+
 @greentest.skipOnPurePython("Needs C extension")
 class TestCExt(greentest.TestCase):
 

src/greentest/test__util.py

@@ -10,6 +10,11 @@ import greentest
 
 import gevent
 from gevent import util
+from gevent import local
+
+class MyLocal(local.local):
+    def __init__(self, foo):
+        self.foo = foo
 
 @greentest.skipOnPyPy("5.10.x is *very* slow formatting stacks")
 class TestFormat(greentest.TestCase):
@@ -27,19 +32,26 @@ class TestFormat(greentest.TestCase):
         self.assertNotIn("Spawn Tree Locals", value)
 
     def test_with_Greenlet(self):
+        rl = local.local()
+        rl.foo = 1
         def root():
+            l = MyLocal(42)
             gevent.getcurrent().spawn_tree_locals['a value'] = 42
             g = gevent.spawn(util.format_run_info)
             g.join()
             return g.value
 
         g = gevent.spawn(root)
+        g.name = 'Printer'
         g.join()
         value = '\n'.join(g.value)
 
         self.assertIn("Spawned at", value)
         self.assertIn("Parent greenlet", value)
         self.assertIn("Spawn Tree Locals", value)
+        self.assertIn("Greenlet Locals:", value)
+        self.assertIn('MyLocal', value)
+        self.assertIn("Printer", value) # The name is printed
 
 
 if __name__ == '__main__':