golang: Teach pychan to work with channels of C types, not only PyObjects

Introduce notion of data type (dtype similarly to NumPy) into pychan and
teach it to accept for send objects only matching that dtype. Likewise
teach pychan to decode raw bytes received from underlying channel into
Python object correspodningg to pychan dtype. For C dtypes, e.g.
'C.int', 'C.double' etc, contrary to chan of python objects, the
transfer can be done without depending on Python GIL. This way channels
of such C-level dtypes can be used to implement interaction in between
Python and nogil worlds.

README.rst

@@ -99,6 +99,14 @@ channels. For example::
         # default case
         ...
 
+By default `chan` creates new channel that can carry arbitrary Python objects.
+However type of channel elements can be specified via `chan(dtype=X)` - for
+example `chan(dtype='C.int')` creates new channel whose elements are C
+integers. `chan.nil(X)` creates typed nil channel. `Cython/nogil API`_
+explains how channels with non-Python dtypes, besides in-Python usage, can be
+additionally used for interaction in between Python and nogil worlds.
+
+
 Methods
 -------
 
@@ -238,6 +246,13 @@ can be used to multiplex on several channels. For example::
              # default case
              ...
 
+Channels created from Python are represented by `pychan` cdef class. Python
+channels that carry non-Python elements (`pychan.dtype != DTYPE_PYOBJECT`) can
+be converted to Cython/nogil `chan[T]` via `pychan.chan_*()`. For example
+`pychan.chan_int()` converts Python channel created via `pychan(dtype='C.int')`
+into `chan[int]`. This provides interaction mechanism in between *nogil* and
+Python worlds.
+
 `panic` stops normal execution of current goroutine by throwing a C-level
 exception. On Python/C boundaries C-level exceptions have to be converted to
 Python-level exceptions with `topyexc`. For example::

golang/_golang.pxd

@@ -110,6 +110,7 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
         _chan     *ch
         _chanop   op
         unsigned  flags
+        unsigned  user
         void      *ptxrx
         uint64_t  itxrx
         cbool     *rxok
@@ -127,9 +128,40 @@ cdef extern from "golang/libgolang.h" namespace "golang" nogil:
 cdef void topyexc() except *
 cpdef pypanic(arg)
 
-# pychan is chan<object>
+# pychan is python wrapper over chan<object> or chan<structZ|bool|int|double|...>
 from cython cimport final
 
+# DType describes type of channel elements.
+# TODO consider supporting NumPy dtypes too.
+cdef enum DType:
+    DTYPE_PYOBJECT   = 0    # chan[object]
+    DTYPE_STRUCTZ    = 1    # chan[structZ]
+    DTYPE_BOOL       = 2    # chan[bool]
+    DTYPE_INT        = 3    # chan[int]
+    DTYPE_DOUBLE     = 4    # chan[double]
+    DTYPE_NTYPES     = 5
+
+# pychan wraps a channel into python object.
+#
+# Type of channel can be either channel of python objects, or channel of
+# C-level objects. If channel elements are C-level objects, the channel - even
+# via pychan wrapper - can be used to interact with nogil world.
+#
+# There can be multiple pychan(s) wrapping a particular raw channel.
 @final
 cdef class pychan:
     cdef _chan  *_ch
+    cdef DType  dtype # type of channel elements
+
+    # pychan.nil(X) creates new nil pychan with element type X.
+    @staticmethod                  # XXX needs to be `cpdef nil()` but cython:
+    cdef pychan _nil(object dtype) #  "static cpdef methods not yet supported"
+
+    # chan_X returns ._ch wrapped into typesafe pyx/nogil-level chan[X].
+    # chan_X panics if channel type != X.
+    # X can be any C-level type, but not PyObject.
+    cdef nogil:
+        chan[structZ]   chan_structZ    (pychan pych)
+        chan[cbool]     chan_bool       (pychan pych)
+        chan[int]       chan_int        (pychan pych)
+        chan[double]    chan_double     (pychan pych)

golang/_golang_test.pyx

@@ -210,3 +210,72 @@ def test_select_win_while_queue():
 def test_select_inplace():
     with nogil:
         _test_select_inplace()
+
+
+# helpers for pychan(dtype=X)  py <-> c  tests.
+def pychan_structZ_recv(pychan pych):
+    with nogil: _pychan_structZ_recv(pych)
+    return None
+def pychan_structZ_send(pychan pych, obj):
+    if obj is not None:
+        raise TypeError("cannot convert %r to structZ" % (obj,))
+    cdef structZ _
+    with nogil:
+        _pychan_structZ_send(pych, _)
+def pychan_structZ_close(pychan pych):
+    with nogil: _pychan_structZ_close(pych)
+
+def pychan_bool_recv(pychan pych):
+    with nogil: _ = _pychan_bool_recv(pych)
+    return _
+def pychan_bool_send(pychan pych, cbool obj):
+    with nogil: _pychan_bool_send(pych, obj)
+def pychan_bool_close(pychan pych):
+    with nogil: _pychan_bool_close(pych)
+
+def pychan_int_recv(pychan pych):
+    with nogil: _ = _pychan_int_recv(pych)
+    return _
+def pychan_int_send(pychan pych, int obj):
+    with nogil: _pychan_int_send(pych, obj)
+def pychan_int_close(pychan pych):
+    with nogil: _pychan_int_close(pych)
+
+def pychan_double_recv(pychan pych):
+    with nogil: _ = _pychan_double_recv(pych)
+    return _
+def pychan_double_send(pychan pych, double obj):
+    with nogil: _pychan_double_send(pych, obj)
+def pychan_double_close(pychan pych):
+    with nogil: _pychan_double_close(pych)
+
+
+cdef nogil:
+
+    structZ _pychan_structZ_recv(pychan pych)               except +topyexc:
+        return pych.chan_structZ().recv()
+    void    _pychan_structZ_send(pychan pych, structZ obj)  except +topyexc:
+        pych.chan_structZ().send(obj)
+    void    _pychan_structZ_close(pychan pych)              except +topyexc:
+        pych.chan_structZ().close()
+
+    cbool   _pychan_bool_recv(pychan pych)                  except +topyexc:
+        return pych.chan_bool().recv()
+    void    _pychan_bool_send(pychan pych, cbool obj)       except +topyexc:
+        pych.chan_bool().send(obj)
+    void    _pychan_bool_close(pychan pych)                 except +topyexc:
+        pych.chan_bool().close()
+
+    int     _pychan_int_recv(pychan pych)                   except +topyexc:
+        return pych.chan_int().recv()
+    void    _pychan_int_send(pychan pych, int obj)          except +topyexc:
+        pych.chan_int().send(obj)
+    void    _pychan_int_close(pychan pych)                  except +topyexc:
+        pych.chan_int().close()
+
+    double  _pychan_double_recv(pychan pych)                except +topyexc:
+        return pych.chan_double().recv()
+    void    _pychan_double_send(pychan pych, double obj)    except +topyexc:
+        pych.chan_double().send(obj)
+    void    _pychan_double_close(pychan pych)               except +topyexc:
+        pych.chan_double().close()

golang/golang_test.py

@@ -29,6 +29,7 @@ from subprocess import Popen, PIPE
 from six.moves import range as xrange
 import gc, weakref
 
+from golang import _golang_test
 from golang._golang_test import pywaitBlocked as waitBlocked, pylen_recvq as len_recvq, \
         pylen_sendq as len_sendq, pypanicWhenBlocked as panicWhenBlocked
 
@@ -738,15 +739,160 @@ def _test_blockforever():
     with panics("t: blocks forever"): select((z.send, 1), z.recv)
 
 
-def test_chan_misc():
-    nilch = nilchan
+# verify chan(dtype=X) functionality.
+def test_chan_dtype_invalid():
+    with raises(TypeError) as exc:
+        chan(dtype="BadType")
+    assert exc.value.args == ("pychan: invalid dtype: 'BadType'",)
+
+chantypev = [
+    # dtype         obj     zero-obj
+    ('object',      'abc',  None),
+    ('C.structZ',   None,   None),
+    ('C.bool',      True,   False),
+    ('C.int',       4,      0),
+    ('C.double',    3.14,   0.0),
+]
+
+@mark.parametrize('dtype,obj,zobj', chantypev)
+def test_chan_dtype(dtype, obj, zobj):
+    # py -> py  (pysend/pyrecv; buffered)
+    ch = chan(1, dtype=dtype)
+    ch.send(obj)
+    obj2, ok = ch.recv_()
+    assert ok == True
+    assert type(obj2) is type(obj)
+    assert obj2 == obj
+
+    # send with different type - rejected
+    for (dtype2, obj2, _) in chantypev:
+        if dtype2 == dtype or dtype == "object":
+            continue    # X -> X; object accepts *,
+        if (dtype2, dtype) == ('C.int', 'C.double'): # int -> double  ok
+            continue
+        with raises(TypeError) as exc:
+            ch.send(obj2)
+        # XXX we can implement vvv, but it will potentially hide cause error
+        # XXX (or use raise from?)
+        #assert exc.value.args == ("type mismatch: expect %s; got %r" % (dtype, obj2),)
+        with raises(TypeError) as exc:
+            select((ch.send, obj2))
+
+    # py -> py  (pyclose/pyrecv)
+    ch.close()
+    obj2, ok = ch.recv_()
+    assert ok == False
+    assert type(obj2) is type(zobj)
+    assert obj2 == zobj
+
+    # below tests are for py <-> c interaction
+    if dtype == "object":
+        return
+    ctype = dtype[2:]  # C.int -> int
+
+    ch = chan(dtype=dtype)  # recreate after close; mode=synchronous
+
+    # recv/send/close via C
+    def crecv(ch):
+        return getattr(_golang_test, "pychan_%s_recv" % ctype)(ch)
+    def csend(ch, obj):
+        getattr(_golang_test, "pychan_%s_send" % ctype)(ch, obj)
+    def cclose(ch):
+        getattr(_golang_test, "pychan_%s_close" % ctype)(ch)
+
+    # py -> c  (pysend/crecv)
+    rx = chan()
+    def _():
+        _ = crecv(ch)
+        rx.send(_)
+    go(_)
+    ch.send(obj)
+    obj2 = rx.recv()
+    assert type(obj2) is type(obj)
+    assert obj2 == obj
+
+    # py -> c  (pyselect/crecv)
+    rx = chan()
+    def _():
+        _ = crecv(ch)
+        rx.send(_)
+    go(_)
+    _, _rx = select(
+        (ch.send, obj), # 0
+    )
+    assert (_, _rx) == (0, None)
+    obj2 = rx.recv()
+    assert type(obj2) is type(obj)
+    assert obj2 == obj
+
+    # py -> c  (pyclose/crecv)
+    rx = chan()
+    def _():
+        _ = crecv(ch)
+        rx.send(_)
+    go(_)
+    ch.close()
+    obj2 = rx.recv()
+    assert type(obj2) is type(zobj)
+    assert obj2 == zobj
+
 
-    assert nilch == nilch                   # nil == nil
+    ch = chan(dtype=dtype)  # recreate after close
+
+    # py <- c  (pyrecv/csend)
+    def _():
+        csend(ch, obj)
+    go(_)
+    obj2 = ch.recv()
+    assert type(obj2) is type(obj)
+    assert obj2 == obj
+
+    # py <- c  (pyselect/csend)
+    def _():
+        csend(ch, obj)
+    go(_)
+    _, _rx = select(
+        ch.recv,        # 0
+    )
+    assert _ == 0
+    obj2 = _rx
+    assert type(obj2) is type(obj)
+    assert obj2 == obj
+
+    # py <- c  (pyrecv/cclose)
+    def _():
+        cclose(ch)
+    go(_)
+    obj2 = ch.recv()
+    assert type(obj2) is type(zobj)
+    assert obj2 == zobj
+
+
+@mark.parametrize('dtype', [_[0] for _ in chantypev])
+def test_chan_dtype_misc(dtype):
+    nilch = chan.nil(dtype)
+
+    # nil repr
+    if dtype == "object":
+        assert repr(nilch) == "nilchan"
+    else:
+        assert repr(nilch) == ("chan.nil(%r)" % dtype)
+
+    # optimization: nil[X]() -> always same object
+    nilch_ = chan.nil(dtype)
+    assert nilch is nilch_
+    if dtype == "object":
+        assert nilch is nilchan
+
+    assert hash(nilch) == hash(nilchan)
+    assert nilch == nilch                   # nil[X] == nil[X]
+    assert nilch == nilchan                 # nil[X] == nil[*]
+    assert nilchan == nilch                 # nil[*] == nil[X]
 
     # channels can be compared, different channels differ
     assert nilch != None    # just in case
-    ch1 = chan()
-    ch2 = chan()
+    ch1 = chan(dtype=dtype)
+    ch2 = chan(dtype=dtype)
     ch3 = chan()
     assert ch1 != ch2;  assert ch1 == ch1
     assert ch1 != ch3;  assert ch2 == ch2
@@ -758,6 +904,43 @@ def test_chan_misc():
     assert nilch != ch2
     assert nilch != ch3
 
+    # .nil on chan instance     XXX doesn't work (yet ?)
+    """
+    ch = chan() # non-nil chan object instance
+    with raises(AttributeError):
+        ch.nil
+    """
+
+    # nil[X] vs nil[Y]
+    for (dtype2, _, _) in chantypev:
+        nilch2 = chan.nil(dtype2)
+        # nil[*] stands for untyped nil - it is equal to nil[X] for ∀ X
+        if dtype == "object" or dtype2 == "object":
+            if dtype != dtype2:
+                assert nilch is not nilch2
+            assert hash(nilch) == hash(nilch2)
+            assert (nilch  == nilch2)   == True
+            assert (nilch2 == nilch)    == True
+            assert (nilch  != nilch2)   == False
+            assert (nilch2 != nilch)    == False
+            continue
+
+        # nil[X] == nil[X]
+        if dtype == dtype2:
+            assert hash(nilch) == hash(nilch2)
+            assert (nilch  == nilch2)   == True
+            assert (nilch2 == nilch)    == True
+            assert (nilch  != nilch2)   == False
+            assert (nilch2 != nilch)    == False
+            continue
+
+        # nil[X] != nil[Y]
+        assert nilch is not nilch2
+        assert (nilch  == nilch2)   == False
+        assert (nilch2 == nilch)    == False
+        assert (nilch  != nilch2)   == True
+        assert (nilch2 != nilch)    == True
+
 
 def test_func():
     # test how @func(cls) works

golang/libgolang.h

@@ -164,7 +164,9 @@ typedef struct _selcase {
     _chan           *ch;        // channel
     enum _chanop    op    : 8;  // chansend/chanrecv/default
     enum _selflags  flags : 8;  // e.g. _INPLACE_DATA
-    unsigned              :16;
+    unsigned        user  : 8;  // arbitrary value that can be set by user
+                                // (e.g. pyselect stores channel type here)
+    unsigned              : 8;
     union {
         void        *ptxrx;     // chansend: ptx; chanrecv: prx
         uint64_t     itxrx;     // used instead of .ptxrx if .flags&_INPLACE_DATA != 0
@@ -191,6 +193,7 @@ _selcase _selsend(_chan *ch, const void *ptx) {
         .ch     = ch,
         .op     = _CHANSEND,
         .flags  = (enum _selflags)0,
+        .user   = 0xff,
         .ptxrx  = (void *)ptx,
         .rxok   = NULL,
     };
@@ -204,6 +207,7 @@ _selcase _selrecv(_chan *ch, void *prx) {
         .ch     = ch,
         .op     = _CHANRECV,
         .flags  = (enum _selflags)0,
+        .user   = 0xff,
         .ptxrx  = prx,
         .rxok   = NULL,
     };
@@ -217,6 +221,7 @@ _selcase _selrecv_(_chan *ch, void *prx, bool *pok) {
         .ch     = ch,
         .op     = _CHANRECV,
         .flags  = (enum _selflags)0,
+        .user   = 0xff,
         .ptxrx  = prx,
         .rxok   = pok,
     };

golang/runtime/libgolang.cpp

@@ -860,6 +860,7 @@ const _selcase _default = {
     .ch     = NULL,
     .op     = _DEFAULT,
     .flags  = (_selflags)0,
+    .user   = 0xff,
     .ptxrx  = NULL,
     .rxok   = NULL,
 };