Fix a lurking GC-related crash with stat watchers on PyPy, detected by CPython...

Fix a lurking GC-related crash with stat watchers on PyPy, detected by CPython re-using the memory much faster.

changelog.rst

@@ -34,6 +34,10 @@
 - The CFFI backend now call the callback of a watcher whose ``args`` attribute is
   set to ``None``, just like the Cython backend does. It also only
   allows ``args`` to be a tuple or ``None``, again matching the Cython backend.
+- PyPy/CFFI: Fix a potential crash when using stat watchers.
+- PyPy/CFFI: Encode unicode paths for stat watchers using
+  :meth:`sys.getfilesystemencoding` like the Cython backend.
+
 
 1.1rc1 (Nov 14, 2015)
 =====================

gevent/corecext.ppyx

@@ -1053,6 +1053,7 @@ cdef public class stat(watcher) [object PyGeventStatObject, type PyGeventStat_Ty
             self._paths = paths
         else:
             paths = <bytes>path
+            self._paths = paths
         libev.ev_stat_init(&self._watcher, <void *>gevent_callback_stat, <char*>paths, interval)
         self.loop = loop
         if ref:

gevent/corecffi.py

@@ -1035,23 +1035,46 @@ class child(watcher):
 class stat(watcher):
     _watcher_type = 'ev_stat'
 
+    @staticmethod
+    def _encode_path(path):
+        if isinstance(path, bytes):
+            return path
+
+        # encode for the filesystem. Not all systems (e.g., Unix)
+        # will have an encoding specified
+        encoding = sys.getfilesystemencoding() or 'utf-8'
+        try:
+            path = path.encode(encoding, 'surrogateescape')
+        except LookupError:
+            # Can't encode it, and the error handler doesn't
+            # exist. Probably on Python 2 with an astral character.
+            # Not sure how to handle this.
+            raise UnicodeEncodeError("Can't encode path to filesystem encoding")
+        return path
+
     def __init__(self, _loop, path, interval=0.0, ref=True, priority=None):
-        if not isinstance(path, bytes):
-            # XXX: Filesystem encoding? Python itself has issues here, were they fixed?
-            path = path.encode('utf-8')
+        # Store the encoded path in the same attribute that corecext does
+        self._paths = self._encode_path(path)
+
+        # Keep the original path to avoid re-encoding, especially on Python 3
+        self._path = path
+
+        # Although CFFI would automatically convert a bytes object into a char* when
+        # calling ev_stat_init(..., char*, ...), on PyPy the char* pointer is not
+        # guaranteed to live past the function call. On CPython, only with a constant/interned
+        # bytes object is the pointer guaranteed to last path the function call. (And since
+        # Python 3 is pretty much guaranteed to produce a newly-encoded bytes object above, thats
+        # rarely the case). Therefore, we must keep a reference to the produced cdata object
+        # so that the struct ev_stat_watcher's `path` pointer doesn't become invalid/deallocated
+        self._cpath = ffi.new('char[]', self._paths)
 
         watcher.__init__(self, _loop, ref=ref, priority=priority,
-                         # cffi doesn't automatically marshal byte strings to
-                         # char* in the function call; instead it passes an
-                         # empty string or garbage pointer. If the watcher's
-                         # path is incorrect, watching silently fails
-                         # (the underlying call to lstat() keeps erroring out)
-                         args=(ffi.new('char[]', path),
+                         args=(self._cpath,
                                interval))
 
     @property
     def path(self):
-        return ffi.string(self._watcher.path)
+        return self._path
 
     @property
     def attr(self):

greentest/test__core_stat.py

@@ -5,6 +5,7 @@ import os
 import sys
 import time
 
+#pylint: disable=protected-access
 
 filename = 'tmp.test__core_stat.%s' % os.getpid()
 
@@ -21,9 +22,8 @@ try:
     assert os.path.exists(filename), filename
 
     def write():
-        f = open(filename, 'wb', buffering=0)
-        f.write(b'x')
-        f.close()
+        with open(filename, 'wb', buffering=0) as f:
+            f.write(b'x')
 
     start = time.time()
     greenlet = gevent.spawn_later(DELAY, write)
@@ -32,8 +32,9 @@ try:
     # which is about 5 seconds. If we go below it's minimum check
     # threshold, it bumps it up to the minimum.
     watcher = hub.loop.stat(filename, interval=-1)
-    if hasattr(watcher, 'path'):
-        assert watcher.path == filename
+    assert watcher.path == filename, (watcher.path, filename)
+    filenames = filename if isinstance(filename, bytes) else filename.encode('ascii')
+    assert watcher._paths == filenames, (watcher._paths, filenames)
     assert watcher.interval == -1
 
     def check_attr(name, none):