master: add support for PyPy

neo/lib/connection.py

@@ -17,7 +17,7 @@
 from functools import wraps
 from time import time
 import msgpack
-from msgpack.exceptions import UnpackValueError
+from msgpack.exceptions import OutOfData, UnpackValueError
 
 from . import attributeTracker, logging
 from .connector import ConnectorException, ConnectorDelayedConnection
@@ -25,6 +25,17 @@ from .locking import RLock
 from .protocol import uuid_str, Errors, PacketMalformedError, Packets, \
     Unpacker
 
+try:
+    msgpack.Unpacker().read_bytes(1)
+except OutOfData: # fallback implementation
+    # monkey-patch for upstream issues 259 & 352
+    def read_bytes(self, n):
+        ret = self._read(min(n, len(self._buffer) - self._buff_i))
+        self._consume()
+        return ret
+    msgpack.Unpacker.read_bytes = read_bytes
+    del read_bytes
+
 @apply
 class dummy_read_buffer(msgpack.Unpacker):
     def feed(self, _):

neo/lib/logger.py

@@ -167,7 +167,11 @@ class NEOLogger(Logger):
                 q("PRAGMA synchronous = OFF")
             if 1: # Not only when logging everything,
                   # but also for interoperability with logrotate.
-                q("PRAGMA journal_mode = MEMORY")
+                # Close the returned cursor to work around the following
+                # OperationalError with PyPy:
+                #   cannot commit transaction - SQL statements in progress
+                # XXX: Is it enough?
+                q("PRAGMA journal_mode = MEMORY").close()
             for t, columns in (('log', (
                                   "level INTEGER NOT NULL",
                                   "pathname TEXT",

neo/scripts/runner.py

@@ -318,6 +318,9 @@ class TestRunner(BenchmarkRunner):
                  " passed.")
         parser.epilog = """
 Environment Variables:
+  NEO_PYPY                    PyPy executable to run master nodes in functional
+                              tests (and also in zodb tests depending on
+                              NEO_TEST_ZODB_FUNCTIONAL).
   NEO_TESTS_ADAPTER           Default is SQLite for threaded clusters,
                               MySQL otherwise.
 

neo/tests/functional/__init__.py

@@ -55,6 +55,26 @@ command_dict = {
 DELAY_SAFETY_MARGIN = 10
 MAX_START_TIME = 30
 
+PYPY_EXECUTABLE = os.getenv('NEO_PYPY')
+if PYPY_EXECUTABLE:
+    import neo, msgpack
+    PYPY_TEMPLATE = """\
+import os, signal, sys
+def sigusr2(*_):
+    os.close(%r)
+    os.kill(os.getpid(), signal.SIGSTOP)
+signal.signal(signal.SIGUSR2, sigusr2)
+os.close(%r)
+os.write(%r, '\\0')
+sys.path.append({!r}); import msgpack; del sys.path[-1]
+sys.path.insert(0, {!r}); import neo; del sys.path[0]
+from neo.lib import logging
+logging.default_root_handler.handle = lambda record: None
+logging.backlog(%s, %s)
+from neo.scripts.%s import main
+main()
+""".format(os.path.dirname(*msgpack.__path__), os.path.dirname(*neo.__path__))
+
 class NodeProcessError(Exception):
     pass
 
@@ -174,6 +194,13 @@ class Process(object):
                     from coverage import Coverage
                     coverage = Coverage(coverage_data_path)
                     coverage.start()
+                elif PYPY_EXECUTABLE and command == 'neomaster':
+                    os.execlp(PYPY_EXECUTABLE, PYPY_EXECUTABLE, '-c',
+                        PYPY_TEMPLATE % (
+                            w, self._coverage_fd, w,
+                            logging._max_size, logging._max_packet,
+                            command),
+                        *args)
                 # XXX: Sometimes, the handler is not called immediately.
                 #      The process is stuck at an unknown place and the test
                 #      never ends. strace unlocks:
@@ -186,7 +213,7 @@ class Process(object):
                 os.close(self._coverage_fd)
                 os.write(w, '\0')
                 sys.argv = [command] + args
-                setproctitle(self.command)
+                setproctitle(command)
                 for on_fork in self.on_fork:
                     on_fork()
                 self.run()