Make the semaphore atomic under pypy.

Makefile

@@ -16,9 +16,15 @@ gevent/gevent.ares.c: gevent/ares.pyx gevent/*.pxd
 	$(CYTHON) -o gevent.ares.c gevent/ares.pyx
 	mv gevent.ares.* gevent/
 
-gevent/gevent._semaphore.c: gevent/_semaphore.py
+gevent/gevent._semaphore.c: gevent/_semaphore.pyx gevent/_semaphore.pxd
+# For PyPy, we need to have _semaphore named as a .pyx file so it doesn't
+# get loaded in preference to the .so. But we want to keep the definitions
+# separate in a .pxd file for ease of reading, and that only works
+# with .py files.
+	cp gevent/_semaphore.pyx gevent/_semaphore.py
 	$(CYTHON) -o gevent._semaphore.c gevent/_semaphore.py
 	mv gevent._semaphore.* gevent/
+	rm gevent/_semaphore.py
 
 gevent/gevent._util.c: gevent/_util.pyx
 	$(CYTHON) -o gevent._util.c gevent/_util.pyx

changelog.rst

@@ -11,6 +11,8 @@ Unreleased
   in :issue:`319` by Ivan Smirnov.
 - ``setup.py`` can build with newer versions of clang on OS X. They
   enforce the distinction between CFLAGS and CPPFLAGS.
+- ``gevent.lock.Semaphore`` is atomic on PyPy, just like it is on
+  CPython. This comes at a small performance cost.
 
 1.1a2 (Jul 8, 2015)
 ===================

gevent/_semaphore.pxd

+cdef class Semaphore:
+    cdef public int counter
+    cdef readonly object _links
+    cdef readonly object _notifier
+    cdef readonly int _dirty
+
+    cpdef locked(self)
+    cpdef release(self)
+    cpdef rawlink(self, object callback)
+    cpdef unlink(self, object callback)
+    cpdef wait(self, object timeout=*)
+    cpdef acquire(self, int blocking=*, object timeout=*)
+    cpdef __enter__(self)
+    cpdef __exit__(self, object t, object v, object tb)

gevent/_semaphore.py → gevent/_semaphore.pyx

@@ -6,6 +6,7 @@ from gevent.timeout import Timeout
 __all__ = ['Semaphore']
 
 
+
 class Semaphore(object):
     """A semaphore manages a counter representing the number of release() calls minus the number of acquire() calls,
     plus an initial value. The acquire() method blocks if necessary until it can return without making the counter
@@ -22,6 +23,7 @@ class Semaphore(object):
         self._links = []
         self.counter = value
         self._notifier = None
+        self._dirty = False
         # we don't want to do get_hub() here to allow module-level locks
         # without initializing the hub
 
@@ -129,5 +131,5 @@ class Semaphore(object):
     def __enter__(self):
         self.acquire()
 
-    def __exit__(self, *args):
+    def __exit__(self, t, v, tb):
         self.release()

known_failures.py

@@ -67,16 +67,6 @@ if PYPY:
 
         # BUGS:
 
-        # in CPython we compile _semaphore.py with Cython to make its operation atomic
-        # how to do atomic operations on PyPy?.
-        # Note that PyPy will compile and load the Cython version of gevent._semaphore,
-        # thus fixing this test case (making it load it is a manual process now because
-        # _semaphore.py still exists and PyPy prefers that to the .so---some things would have
-        # to be renamed to make it work automatically). However, on at least one machine, the Cython
-        # version causes the test suite to run slower: ~2:52 vs ~2:37. Is that worth the
-        # non-traceability? (Is it even repeatable? Possibly not; a lot of the test time is spent in,
-        # e.g., test__socket_dns.py doing network stuff.)
-        'test__threading_vs_settrace.py',
     ]
 
     import cffi

setup.py

@@ -209,14 +209,14 @@ else:
     ARES.libraries.append('cares')
     ARES.define_macros += [('HAVE_NETDB_H', '')]
 
-
-def make(done=[]):
-    if not done:
+_ran_make = []
+def make(targets=''):
+    if not _ran_make:
         if os.path.exists('Makefile'):
             if "PYTHON" not in os.environ:
                 os.environ["PYTHON"] = sys.executable
-            system('make')
-        done.append(1)
+            system('make ' + targets)
+        _ran_make.append(1)
 
 
 class sdist(_sdist):
@@ -317,9 +317,16 @@ elif PYPY:
     system('touch gevent/libev/__init__.py')
     system('cd gevent/libev && ./configure > configure_output.txt')
     from gevent import corecffi
-    ext_modules = [corecffi.ffi.verifier.get_extension()]
+    ext_modules = [corecffi.ffi.verifier.get_extension(),
+                   # By building the semaphore with Cython under PyPy, we get
+                   # atomic operations (specifically, exiting/releasing), at the
+                   # cost of some speed (one trivial semaphore micro-benchmark put the pure-python version
+                   # at around 1s and the compiled version at around 4s). Some clever subclassing
+                   # and having only the bare minimum be in cython might help reduce that penalty.
+                   Extension(name="gevent._semaphore",
+                             sources=["gevent/gevent._semaphore.c"])]
     include_package_data = True
-    run_make = False
+    run_make = 'gevent/gevent._semaphore.c'
 else:
     ext_modules = [CORE,
                    ARES,
@@ -333,7 +340,10 @@ else:
 
 def run_setup(ext_modules, run_make):
     if run_make:
-        make()
+        if isinstance(run_make, str):
+            make(run_make)
+        else:
+            make()
     setup(
         name='gevent',
         version=__version__,
@@ -374,5 +384,5 @@ if __name__ == '__main__':
             raise
         ext_modules.remove(ARES)
         run_setup(ext_modules, run_make=run_make)
-    if ARES not in ext_modules:
+    if not PYPY and ARES not in ext_modules:
         sys.stderr.write('\nWARNING: The gevent.ares extension has been disabled.\n')