Use the PyObject APIs for libev with corecext.

CHANGES.rst

@@ -35,6 +35,12 @@ Library and Dependency Updates
      The binary wheels gevent distributes *do not* set ``EV_VERIFY``
      and don't have this issue.
 
+- Make libuv and libev use the Python memory allocators. This assists
+  with debugging. The event libraries allocate small amounts of memory
+  at startup. The allocation functions have to take the GIL, but
+  because of the limited amount of actual allocation that gets done
+  this is not expected to be a concern.
+
 1.5a3 (2020-01-01)
 ==================
 

src/gevent/_ffi/alloc.c

+#define GIL_DECLARE  PyGILState_STATE ___save
+#define GIL_ENSURE  ___save = PyGILState_Ensure();
+#define GIL_RELEASE  PyGILState_Release(___save);
+
+void* gevent_realloc(void* ptr, size_t size)
+{
+    // libev is interesting and assumes that everything can be
+    // done with realloc(), assuming that passing in a size of 0 means to
+    // free the pointer. But the C/++ standard explicitly says that
+    // this is undefined. So this wrapper function exists to do it all.
+    GIL_DECLARE;
+    void* result;
+    if(!size && !ptr) {
+	// libev for some reason often tries to free(NULL); I won't specutale
+	// why. No need to acquire the GIL or do anything in that case.
+	return NULL;
+    }
+
+    // Using PyObject_* APIs to get access to pymalloc allocator on
+    // all versions of CPython; in Python 3, PyMem_* and PyObject_* use
+    // the same allocator, but in Python 2, only PyObject_* uses pymalloc.
+    GIL_ENSURE;
+
+    if(!size) {
+	PyObject_Free(ptr);
+	result = NULL;
+    }
+    else {
+	result = PyObject_Realloc(ptr, size);
+    }
+    GIL_RELEASE;
+    return result;
+}

src/gevent/libev/_corecffi_build.py

@@ -21,6 +21,7 @@ except ImportError:
     raise
 
 thisdir = os.path.dirname(os.path.abspath(__file__))
+parentdir = os.path.abspath(os.path.join(thisdir, '..'))
 setup_dir = os.path.abspath(os.path.join(thisdir, '..', '..', '..'))
 
 
@@ -88,7 +89,10 @@ ffi.cdef(_cdef)
 ffi.set_source(
     'gevent.libev._corecffi',
     _source,
-    include_dirs=distutils_ext.include_dirs + [thisdir], # "libev.h"
+    include_dirs=distutils_ext.include_dirs + [
+        thisdir, # "libev.h"
+        parentdir, # _ffi/alloc.c
+    ],
     define_macros=macros,
     undef_macros=distutils_ext.undef_macros,
     libraries=distutils_ext.libraries,

src/gevent/libev/_corecffi_cdef.c

@@ -217,7 +217,7 @@ struct ev_loop* gevent_ev_default_loop(unsigned int flags);
 void gevent_install_sigchld_handler();
 void gevent_reset_sigchld_handler();
 
-void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents);
+extern void (*gevent_noop)(struct ev_loop *_loop, struct ev_timer *w, int revents);
 void ev_sleep (ev_tstamp delay); /* sleep for a while */
 
 /* gevent callbacks */
@@ -246,3 +246,4 @@ static void _gevent_generic_callback(struct ev_loop* loop, struct ev_watcher* wa
 static void gevent_zero_check(struct ev_check* handle);
 static void gevent_zero_timer(struct ev_timer* handle);
 static void gevent_zero_prepare(struct ev_prepare* handle);
+static void gevent_set_ev_alloc();

src/gevent/libev/_corecffi_source.c

@@ -71,3 +71,15 @@ static void gevent_zero_prepare(struct ev_prepare* handle)
 {
 	memset(handle, 0, sizeof(struct ev_prepare));
 }
+
+#include "_ffi/alloc.c"
+
+static void gevent_set_ev_alloc()
+{
+    ev_set_allocator(gevent_realloc);
+}
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunreachable-code"
+#endif

src/gevent/libev/callbacks.c

@@ -39,16 +39,11 @@ static CYTHON_INLINE void gevent_check_signals(struct PyGeventLoopObject* loop)
     ((struct PY_TYPE *)(((char *)EV_PTR) - offsetof(struct PY_TYPE, MEMBER)))
 
 
-#ifdef WITH_THREAD
-#define GIL_DECLARE  PyGILState_STATE ___save
-#define GIL_ENSURE  ___save = PyGILState_Ensure();
-#define GIL_RELEASE  PyGILState_Release(___save);
-#else
-#define GIL_DECLARE
-#define GIL_ENSURE
-#define GIL_RELEASE
-#endif
+/* define gevent_realloc with libev semantics */
+
+#include "../_ffi/alloc.c"
 
+void gevent_noop(struct ev_loop* loop, void* watcher, int revents) {}
 
 static void gevent_stop(PyObject* watcher, struct PyGeventLoopObject* loop) {
     PyObject *result, *method;
@@ -212,5 +207,4 @@ void gevent_periodic_signal_check(struct ev_loop *_loop, void *watcher, int reve
     GIL_RELEASE;
 }
 
-
 #endif  /* Py_PYTHON_H */

src/gevent/libev/callbacks.h

@@ -27,12 +27,18 @@ DEFINE_CALLBACKS
 
 void gevent_run_callbacks(struct ev_loop *, void *, int);
 
-
-
 void gevent_call(struct PyGeventLoopObject* loop, struct PyGeventCallbackObject* cb);
 
-static void gevent_noop(struct ev_loop *_loop, void *watcher, int revents) {
-}
+void* gevent_realloc(void* ptr, size_t size);
+void gevent_noop(struct ev_loop*, void* watcher, int revents);
 
 /* Only used on Win32 */
 void gevent_periodic_signal_check(struct ev_loop *, void *, int);
+
+// We're included in corecext.c. Disable a bunch of annoying warnings
+// that are in the generated code that we can't do anything about.
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdeprecated-declarations"
+#pragma clang diagnostic ignored "-Wunreachable-code"
+#endif

src/gevent/libev/corecext.pyx

@@ -74,6 +74,7 @@ cdef extern from "callbacks.h":
     void gevent_periodic_signal_check(libev.ev_loop, void*, int)
     void gevent_call(loop, callback)
     void gevent_noop(libev.ev_loop, void*, int)
+    void* gevent_realloc(void*, long size)
 
 cdef extern from "stathelper.c":
     object _pystat_fromstructstat(void*)
@@ -1303,12 +1304,12 @@ cpdef set_syserr_cb(callback):
         libev.ev_set_syserr_cb(NULL)
         __SYSERR_CALLBACK = None
     elif callable(callback):
-        libev.ev_set_syserr_cb(<void *>_syserr_cb)
+        libev.ev_set_syserr_cb(<void*>_syserr_cb)
         __SYSERR_CALLBACK = callback
     else:
         raise TypeError('Expected callable or None, got %r' % (callback, ))
 
-
+libev.ev_set_allocator(<void*>gevent_realloc)
 
 LIBEV_EMBED = bool(libev.LIBEV_EMBED)
 EV_USE_FLOOR = libev.EV_USE_FLOOR

src/gevent/libev/corecffi.py

@@ -31,6 +31,8 @@ if hasattr(libev, 'vfd_open'):
 else:
     vfd_open = vfd_free = vfd_get = lambda fd: fd
 
+libev.gevent_set_ev_alloc()
+
 #####
 ## NOTE on Windows:
 # The C implementation does several things specially for Windows;

src/gevent/libev/libev.h

 #if defined(LIBEV_EMBED) && LIBEV_EMBED
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wcomment"
+#pragma clang diagnostic ignored "-Wsign-compare"
+#pragma clang diagnostic ignored "-Wextern-initializer"
+#pragma clang diagnostic ignored "-Wbitwise-op-parentheses"
+#endif
   #include "ev.c"
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
   #undef LIBEV_EMBED
   #define LIBEV_EMBED 1
   #define gevent_ev_loop_origflags(loop) ((loop)->origflags)

src/gevent/libev/libev.pxd

@@ -151,7 +151,8 @@ cdef extern from "libev.h" nogil:
     ctypedef double ev_tstamp
 
     ev_tstamp ev_time()
-    void ev_set_syserr_cb(void *)
+    void ev_set_syserr_cb(void*)
+    void ev_set_allocator(void*)
 
     int ev_priority(void*)
     void ev_set_priority(void*, int)

src/gevent/libuv/_corecffi_build.py

@@ -31,6 +31,7 @@ LIBUV_EMBED = _setuputils.should_embed('libuv')
 ffi = FFI()
 
 thisdir = os.path.dirname(os.path.abspath(__file__))
+parentdir = os.path.abspath(os.path.join(thisdir, '..'))
 setup_py_dir = os.path.abspath(os.path.join(thisdir, '..', '..', '..'))
 libuv_dir = os.path.abspath(os.path.join(setup_py_dir, 'deps', 'libuv'))
 
@@ -255,6 +256,8 @@ if not LIBUV_EMBED:
     del LIBUV_INCLUDE_DIRS[:]
     _add_library('uv')
 
+LIBUV_INCLUDE_DIRS.append(parentdir)
+
 ffi.cdef(_cdef)
 ffi.set_source('gevent.libuv._corecffi',
                _source,
@@ -269,5 +272,5 @@ if __name__ == '__main__':
     #
     # Other than the obvious directory changes, the changes are:
     #
-    # CPPFLAGS=-Ideps/libuv/include/
+    # CPPFLAGS=-Ideps/libuv/include/ -Isrc/gevent/
     ffi.compile(verbose=True)

src/gevent/libuv/_corecffi_cdef.c

@@ -388,7 +388,10 @@ static void _gevent_fs_poll_callback3(uv_fs_poll_t* handle, int status, const uv
 
 static void gevent_uv_walk_callback_close(uv_handle_t* handle, void* arg);
 static void gevent_close_all_handles(uv_loop_t* loop);
+
+/* gevent utility functions */
 static void gevent_zero_timer(uv_timer_t* handle);
 static void gevent_zero_prepare(uv_prepare_t* handle);
 static void gevent_zero_check(uv_check_t* handle);
 static void gevent_zero_loop(uv_loop_t* handle);
+static void gevent_set_uv_alloc();

src/gevent/libuv/_corecffi_source.c

 #include <string.h>
 #include <assert.h>
 #include "uv.h"
+#include "Python.h"
 
 typedef void* GeventWatcherObject;
 #ifdef __clang__
@@ -130,8 +131,47 @@ static void gevent_zero_loop(uv_loop_t* handle)
     memset(handle, 0, sizeof(uv_loop_t));
 }
 
+#include "_ffi/alloc.c"
+
+static void* _gevent_uv_malloc(size_t size)
+{
+    return gevent_realloc(NULL, size);
+}
+
+static void* _gevent_uv_realloc(void* ptr, size_t size)
+{
+    return gevent_realloc(ptr, size);
+}
+
+static void _gevent_uv_free(void* ptr)
+{
+    gevent_realloc(ptr, 0);
+}
+
+static void* _gevent_uv_calloc(size_t count, size_t size)
+{
+    // We assume no overflows. Not using PyObject_Calloc because
+    // it's not available prior to 3.5 and isn't in PyPy.
+    void* result;
+    result = _gevent_uv_malloc(count * size);
+    if(result) {
+	memset(result, 0, count * size);
+    }
+    return result;
+}
+
+static void gevent_set_uv_alloc()
+{
+    uv_replace_allocator(_gevent_uv_malloc,
+			 _gevent_uv_realloc,
+			 _gevent_uv_calloc,
+			 _gevent_uv_free);
+}
+
 #ifdef __clang__
 #pragma clang diagnostic pop
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wunreachable-code"
 #endif
 
 /* Local Variables: */

src/gevent/libuv/loop.py

@@ -12,11 +12,11 @@ import signal
 
 from gevent._ffi import _dbg # pylint: disable=unused-import
 from gevent._ffi.loop import AbstractLoop
-from gevent.libuv import _corecffi # pylint:disable=no-name-in-module,import-error
 from gevent._ffi.loop import assign_standard_callbacks
 from gevent._ffi.loop import AbstractCallbacks
 from gevent._util import implementer
 from gevent._interfaces import ILoop
+from gevent.libuv import _corecffi # pylint:disable=no-name-in-module,import-error
 
 ffi = _corecffi.ffi
 libuv = _corecffi.lib
@@ -83,6 +83,8 @@ def get_header_version():
 def supported_backends():
     return ['default']
 
+libuv.gevent_set_uv_alloc()
+
 @implementer(ILoop)
 class loop(AbstractLoop):