solve most of the issues with failing async-gen tests, some asyncio tests are still failing

Cython/Compiler/ExprNodes.py

@@ -9484,12 +9484,15 @@ class YieldExprNode(ExprNode):
             if type.is_pyobject:
                 code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname))
                 code.put_xgotref(cname)
-        code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos))
+        self.generate_sent_value_handling_code(code, Naming.sent_value_cname)
         if self.result_is_used:
             self.allocate_temp_result(code)
             code.put('%s = %s; ' % (self.result(), Naming.sent_value_cname))
             code.put_incref(self.result(), py_object_type)
 
+    def generate_sent_value_handling_code(self, code, value_cname):
+        code.putln(code.error_goto_if_null(value_cname, self.pos))
+
 
 class _YieldDelegationExprNode(YieldExprNode):
     def yield_from_func(self, code):
@@ -9582,8 +9585,7 @@ class AwaitIterNextExprNode(AwaitExprNode):
     #
     # Breaks out of loop on StopAsyncIteration exception.
 
-    def fetch_iteration_result(self, code):
-        assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop"
+    def _generate_break(self, code):
         code.globalstate.use_utility_code(UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
         code.putln("PyObject* exc_type = PyErr_Occurred();")
         code.putln("if (exc_type && likely(exc_type == __Pyx_PyExc_StopAsyncIteration ||"
@@ -9591,8 +9593,20 @@ class AwaitIterNextExprNode(AwaitExprNode):
         code.putln("PyErr_Clear();")
         code.putln("break;")
         code.putln("}")
+
+    def fetch_iteration_result(self, code):
+        assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop"
+        self._generate_break(code)
         super(AwaitIterNextExprNode, self).fetch_iteration_result(code)
 
+    def generate_sent_value_handling_code(self, code, value_cname):
+        assert code.break_label, "AwaitIterNextExprNode outside of 'async for' loop"
+        code.putln("if (unlikely(!%s)) {" % value_cname)
+        self._generate_break(code)
+        # all non-break exceptions are errors, as in parent class
+        code.putln(code.error_goto(self.pos))
+        code.putln("}")
+
 
 class GlobalsExprNode(AtomicExprNode):
     type = dict_type

Cython/Compiler/Nodes.py

@@ -4025,19 +4025,19 @@ class AsyncGenNode(AsyncDefNode):
     is_asyncgen = True
 
 
-
 class GeneratorBodyDefNode(DefNode):
     # Main code body of a generator implemented as a DefNode.
     #
 
     is_generator_body = True
     is_inlined = False
+    is_async_gen = False
     inlined_comprehension_type = None  # container type for inlined comprehensions
 
-    def __init__(self, pos=None, name=None, body=None):
+    def __init__(self, pos=None, name=None, body=None, is_async_gen=False):
         super(GeneratorBodyDefNode, self).__init__(
-            pos=pos, body=body, name=name, doc=None,
-            args=[], star_arg=None, starstar_arg=None)
+            pos=pos, body=body, name=name, is_async_gen=is_async_gen,
+            doc=None, args=[], star_arg=None, starstar_arg=None)
 
     def declare_generator_body(self, env):
         prefix = env.next_id(env.scope_prefix)
@@ -4134,9 +4134,10 @@ class GeneratorBodyDefNode(DefNode):
         # on normal generator termination, we do not take the exception propagation
         # path: no traceback info is required and not creating it is much faster
         if not self.is_inlined and not self.body.is_terminator:
-            code.putln('PyErr_SetNone(PyExc_StopIteration);')
+            code.putln('PyErr_SetNone(%s);' % (
+                '__Pyx_PyExc_StopAsyncIteration' if self.is_async_gen else 'PyExc_StopIteration'))
         # ----- Error cleanup
-        if code.error_label in code.labels_used:
+        if code.label_used(code.error_label):
             if not self.body.is_terminator:
                 code.put_goto(code.return_label)
             code.put_label(code.error_label)
@@ -4145,8 +4146,7 @@ class GeneratorBodyDefNode(DefNode):
             if Future.generator_stop in env.global_scope().context.future_directives:
                 # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
                 code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
-                code.putln("if (unlikely(PyErr_ExceptionMatches(PyExc_StopIteration))) "
-                           "__Pyx_Generator_Replace_StopIteration();")
+                code.putln("__Pyx_Generator_Replace_StopIteration(%d);" % bool(self.is_async_gen))
             for cname, type in code.funcstate.all_managed_temps():
                 code.put_xdecref(cname, type)
             code.put_add_traceback(self.entry.qualified_name)
@@ -5583,10 +5583,12 @@ class ReturnStatNode(StatNode):
     #  value         ExprNode or None
     #  return_type   PyrexType
     #  in_generator  return inside of generator => raise StopIteration
+    #  in_async_gen  return inside of async generator
 
     child_attrs = ["value"]
     is_terminator = True
     in_generator = False
+    in_async_gen = False
 
     # Whether we are in a parallel section
     in_parallel = False
@@ -5598,6 +5600,8 @@ class ReturnStatNode(StatNode):
             error(self.pos, "Return not inside a function body")
             return self
         if self.value:
+            if self.in_async_gen:
+                error(self.pos, "Return with value in async generator")
             self.value = self.value.analyse_types(env)
             if return_type.is_void or return_type.is_returncode:
                 error(self.value.pos, "Return with value in void function")
@@ -5654,6 +5658,8 @@ class ReturnStatNode(StatNode):
         else:
             if self.return_type.is_pyobject:
                 if self.in_generator:
+                    if self.in_async_gen:
+                        code.put("PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); ")
                     code.putln("%s = NULL;" % Naming.retval_cname)
                 else:
                     code.put_init_to_py_none(Naming.retval_cname, self.return_type)

Cython/Compiler/ParseTreeTransforms.py

@@ -2518,7 +2518,7 @@ class MarkClosureVisitor(CythonTransform):
         if node.is_async_def:
             coroutine_type = Nodes.AsyncGenNode if collector.has_yield else Nodes.AsyncDefNode
             if collector.has_yield:
-                for yield_expr in collector.yields:
+                for yield_expr in collector.yields + collector.returns:
                     yield_expr.in_async_gen = True
         elif collector.has_await:
             found = next(y for y in collector.yields if y.is_await)
@@ -2535,7 +2535,8 @@ class MarkClosureVisitor(CythonTransform):
             retnode.in_generator = True
 
         gbody = Nodes.GeneratorBodyDefNode(
-            pos=node.pos, name=node.name, body=node.body)
+            pos=node.pos, name=node.name, body=node.body,
+            is_async_gen=node.is_async_def and collector.has_yield)
         coroutine = coroutine_type(
             pos=node.pos, name=node.name, args=node.args,
             star_arg=node.star_arg, starstar_arg=node.starstar_arg,

Cython/Utility/AsyncGen.c

 // This is copied from genobject.c in CPython 3.6.
-// Try to keep it in sync.
+// Try to keep it in sync by doing this from time to time:
+//    sed -e 's|__pyx_||ig'  Cython/Utility/AsyncGen.c | diff -udw - cpython/Objects/genobject.c | less
 
 //////////////////// AsyncGenerator.proto ////////////////////
 //@requires: Coroutine.c::Coroutine
@@ -12,14 +13,20 @@ typedef struct {
     int ag_closed;
 } __pyx_PyAsyncGenObject;
 
+typedef struct __pyx_PyAsyncGenASend_struct __pyx_PyAsyncGenASend;
 
 static PyTypeObject *__pyx__PyAsyncGenWrappedValueType = 0;
 static PyTypeObject *__pyx__PyAsyncGenASendType = 0;
 static PyTypeObject *__pyx__PyAsyncGenAThrowType = 0;
 static PyTypeObject *__pyx_AsyncGenType = 0;
+
 #define __Pyx_AsyncGen_CheckExact(obj) (Py_TYPE(obj) == __pyx_AsyncGenType)
+#define __pyx_PyAsyncGenASend_CheckExact(o) \
+                    (Py_TYPE(o) == __pyx__PyAsyncGenASendType)
 
-static PyObject *__Pyx_async_gen_anext(PyObject *o);
+static PyObject *__Pyx_async_gen_anext(__pyx_PyAsyncGenObject *o);
+static PyObject *__Pyx_async_gen_asend_iternext(__pyx_PyAsyncGenASend *o);
+static PyObject *__Pyx_async_gen_asend_send(__pyx_PyAsyncGenASend *o, PyObject *arg);
 
 static PyObject *__Pyx__PyAsyncGenValueWrapperNew(PyObject *val);
 
@@ -120,7 +127,7 @@ typedef enum {
     __PYX_AWAITABLE_STATE_CLOSED, /* closed */
 } __pyx_AwaitableState;
 
-typedef struct {
+struct __pyx_PyAsyncGenASend_struct {
     PyObject_HEAD
     __pyx_PyAsyncGenObject *ags_gen;
 
@@ -128,7 +135,7 @@ typedef struct {
     PyObject *ags_sendval;
 
     __pyx_AwaitableState ags_state;
-} __pyx_PyAsyncGenASend;
+};
 
 
 typedef struct {
@@ -167,9 +174,6 @@ static int __Pyx_ag_asend_freelist_free = 0;
 #define __pyx__PyAsyncGenWrappedValue_CheckExact(o) \
                     (Py_TYPE(o) == __pyx__PyAsyncGenWrappedValueType)
 
-#define __pyx_PyAsyncGenASend_CheckExact(o) \
-                    (Py_TYPE(o) == __pyx__PyAsyncGenASendType)
-
 
 static int
 __Pyx_async_gen_traverse(__pyx_PyAsyncGenObject *gen, visitproc visit, void *arg)
@@ -303,7 +307,7 @@ static PyMethodDef __Pyx_async_gen_methods[] = {
 };
 
 
-static PyAsyncMethods __Pyx_async_gen_as_async = {
+static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_as_async = {
     0,                                          /* am_await */
     PyObject_SelfIter,                          /* am_aiter */
     (unaryfunc)__Pyx_async_gen_anext             /* am_anext */
@@ -475,12 +479,12 @@ __Pyx_async_gen_asend_send(__pyx_PyAsyncGenASend *o, PyObject *arg)
 
     if (o->ags_state == __PYX_AWAITABLE_STATE_INIT) {
         if (arg == NULL || arg == Py_None) {
-            arg = o->ags_sendval;
+            arg = o->ags_sendval ? o->ags_sendval : Py_None;
         }
         o->ags_state = __PYX_AWAITABLE_STATE_ITER;
     }
 
-    result = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject*)o->ags_gen, arg);
+    result = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject*)o->ags_gen, arg, 0);
     result = __Pyx_async_gen_unwrap_value(o->ags_gen, result);
 
     if (result == NULL) {
@@ -494,7 +498,7 @@ __Pyx_async_gen_asend_send(__pyx_PyAsyncGenASend *o, PyObject *arg)
 static PyObject *
 __Pyx_async_gen_asend_iternext(__pyx_PyAsyncGenASend *o)
 {
-    return __Pyx_async_gen_asend_send(o, NULL);
+    return __Pyx_async_gen_asend_send(o, Py_None);
 }
 
 
@@ -535,7 +539,7 @@ static PyMethodDef __Pyx_async_gen_asend_methods[] = {
 };
 
 
-static PyAsyncMethods __Pyx_async_gen_asend_as_async = {
+static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_asend_as_async = {
     PyObject_SelfIter,                          /* am_await */
     0,                                          /* am_aiter */
     0                                           /* am_anext */
@@ -824,7 +828,7 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg)
 
     assert (o->agt_state == __PYX_AWAITABLE_STATE_ITER);
 
-    retval = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject *)gen, arg);
+    retval = __Pyx_Coroutine_SendEx((__pyx_CoroutineObject *)gen, arg, 0);
     if (o->agt_args) {
         return __Pyx_async_gen_unwrap_value(o->agt_gen, retval);
     } else {
@@ -921,7 +925,7 @@ static PyMethodDef __Pyx_async_gen_athrow_methods[] = {
 };
 
 
-static PyAsyncMethods __Pyx_async_gen_athrow_as_async = {
+static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_athrow_as_async = {
     PyObject_SelfIter,                          /* am_await */
     0,                                          /* am_aiter */
     0                                           /* am_anext */

Cython/Utility/ModuleSetupCode.c

@@ -387,15 +387,17 @@
     #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods
     #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async)
   #else
+    #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved))
+  #endif
+#else
+  #define __Pyx_PyType_AsAsync(obj) NULL
+#endif
+#ifndef __Pyx_PyAsyncMethodsStruct
     typedef struct {
         unaryfunc am_await;
         unaryfunc am_aiter;
         unaryfunc am_anext;
     } __Pyx_PyAsyncMethodsStruct;
-    #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved))
-  #endif
-#else
-  #define __Pyx_PyType_AsAsync(obj) NULL
 #endif
 
 // restrict