Commit b9e0bdcf authored by Stefan Behnel's avatar Stefan Behnel

implement "async def" statement and "await" expression (PEP 492)

parent 4ed35f23
...@@ -49,7 +49,8 @@ non_portable_builtins_map = { ...@@ -49,7 +49,8 @@ non_portable_builtins_map = {
'basestring' : ('PY_MAJOR_VERSION >= 3', 'str'), 'basestring' : ('PY_MAJOR_VERSION >= 3', 'str'),
'xrange' : ('PY_MAJOR_VERSION >= 3', 'range'), 'xrange' : ('PY_MAJOR_VERSION >= 3', 'range'),
'raw_input' : ('PY_MAJOR_VERSION >= 3', 'input'), 'raw_input' : ('PY_MAJOR_VERSION >= 3', 'input'),
} 'StopAsyncIteration': ('PY_VERSION_HEX < 0x030500B1', 'StopIteration'),
}
basicsize_builtins_map = { basicsize_builtins_map = {
# builtins whose type has a different tp_basicsize than sizeof(...) # builtins whose type has a different tp_basicsize than sizeof(...)
......
...@@ -8593,10 +8593,12 @@ class YieldExprNode(ExprNode): ...@@ -8593,10 +8593,12 @@ class YieldExprNode(ExprNode):
type = py_object_type type = py_object_type
label_num = 0 label_num = 0
is_yield_from = False is_yield_from = False
is_await = False
expr_keyword = 'yield'
def analyse_types(self, env): def analyse_types(self, env):
if not self.label_num: if not self.label_num:
error(self.pos, "'yield' not supported here") error(self.pos, "'%s' not supported here" % self.expr_keyword)
self.is_temp = 1 self.is_temp = 1
if self.arg is not None: if self.arg is not None:
self.arg = self.arg.analyse_types(env) self.arg = self.arg.analyse_types(env)
...@@ -8661,6 +8663,7 @@ class YieldExprNode(ExprNode): ...@@ -8661,6 +8663,7 @@ class YieldExprNode(ExprNode):
class YieldFromExprNode(YieldExprNode): class YieldFromExprNode(YieldExprNode):
# "yield from GEN" expression # "yield from GEN" expression
is_yield_from = True is_yield_from = True
expr_keyword = 'yield from'
def coerce_yield_argument(self, env): def coerce_yield_argument(self, env):
if not self.arg.type.is_string: if not self.arg.type.is_string:
...@@ -8668,14 +8671,17 @@ class YieldFromExprNode(YieldExprNode): ...@@ -8668,14 +8671,17 @@ class YieldFromExprNode(YieldExprNode):
error(self.pos, "yielding from non-Python object not supported") error(self.pos, "yielding from non-Python object not supported")
self.arg = self.arg.coerce_to_pyobject(env) self.arg = self.arg.coerce_to_pyobject(env)
def generate_evaluation_code(self, code): def yield_from_func(self, code):
code.globalstate.use_utility_code(UtilityCode.load_cached("YieldFrom", "Coroutine.c")) code.globalstate.use_utility_code(UtilityCode.load_cached("GeneratorYieldFrom", "Coroutine.c"))
return "__Pyx_Generator_Yield_From"
def generate_evaluation_code(self, code):
self.arg.generate_evaluation_code(code) self.arg.generate_evaluation_code(code)
code.putln("%s = __Pyx_Generator_Yield_From(%s, %s);" % ( code.putln("%s = %s(%s, %s);" % (
Naming.retval_cname, Naming.retval_cname,
self.yield_from_func(code),
Naming.generator_cname, Naming.generator_cname,
self.arg.result_as(py_object_type))) self.arg.py_result()))
self.arg.generate_disposal_code(code) self.arg.generate_disposal_code(code)
self.arg.free_temps(code) self.arg.free_temps(code)
code.put_xgotref(Naming.retval_cname) code.put_xgotref(Naming.retval_cname)
...@@ -8687,9 +8693,7 @@ class YieldFromExprNode(YieldExprNode): ...@@ -8687,9 +8693,7 @@ class YieldFromExprNode(YieldExprNode):
if self.result_is_used: if self.result_is_used:
# YieldExprNode has allocated the result temp for us # YieldExprNode has allocated the result temp for us
code.putln("%s = NULL;" % self.result()) code.putln("%s = NULL;" % self.result())
code.putln("if (unlikely(__Pyx_PyGen_FetchStopIterationValue(&%s) < 0)) %s" % ( code.put_error_if_neg(self.pos, "__Pyx_PyGen_FetchStopIterationValue(&%s)" % self.result())
self.result(),
code.error_goto(self.pos)))
code.put_gotref(self.result()) code.put_gotref(self.result())
else: else:
code.putln("PyObject* exc_type = PyErr_Occurred();") code.putln("PyObject* exc_type = PyErr_Occurred();")
...@@ -8700,6 +8704,25 @@ class YieldFromExprNode(YieldExprNode): ...@@ -8700,6 +8704,25 @@ class YieldFromExprNode(YieldExprNode):
code.putln("}") code.putln("}")
code.putln("}") code.putln("}")
class AwaitExprNode(YieldFromExprNode):
# 'await' expression node
#
# arg ExprNode the Awaitable value to await
# label_num integer yield label number
# is_yield_from boolean is a YieldFromExprNode to delegate to another generator
is_await = True
expr_keyword = 'await'
def coerce_yield_argument(self, env):
# FIXME: use same check as in YieldFromExprNode.coerce_yield_argument() ?
self.arg = self.arg.coerce_to_pyobject(env)
def yield_from_func(self, code):
code.globalstate.use_utility_code(UtilityCode.load_cached("CoroutineYieldFrom", "Coroutine.c"))
return "__Pyx_Coroutine_Yield_From"
class GlobalsExprNode(AtomicExprNode): class GlobalsExprNode(AtomicExprNode):
type = dict_type type = dict_type
is_temp = 1 is_temp = 1
......
...@@ -1574,9 +1574,11 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1574,9 +1574,11 @@ class FuncDefNode(StatNode, BlockNode):
# pymethdef_required boolean Force Python method struct generation # pymethdef_required boolean Force Python method struct generation
# directive_locals { string : ExprNode } locals defined by cython.locals(...) # directive_locals { string : ExprNode } locals defined by cython.locals(...)
# directive_returns [ExprNode] type defined by cython.returns(...) # directive_returns [ExprNode] type defined by cython.returns(...)
# star_arg PyArgDeclNode or None * argument # star_arg PyArgDeclNode or None * argument
# starstar_arg PyArgDeclNode or None ** argument # starstar_arg PyArgDeclNode or None ** argument
#
# is_async_def boolean is a Coroutine function
#
# has_fused_arguments boolean # has_fused_arguments boolean
# Whether this cdef function has fused parameters. This is needed # Whether this cdef function has fused parameters. This is needed
# by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes # by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
...@@ -1588,6 +1590,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -1588,6 +1590,7 @@ class FuncDefNode(StatNode, BlockNode):
pymethdef_required = False pymethdef_required = False
is_generator = False is_generator = False
is_generator_body = False is_generator_body = False
is_async_def = False
modifiers = [] modifiers = []
has_fused_arguments = False has_fused_arguments = False
star_arg = None star_arg = None
...@@ -3936,6 +3939,7 @@ class GeneratorDefNode(DefNode): ...@@ -3936,6 +3939,7 @@ class GeneratorDefNode(DefNode):
# #
is_generator = True is_generator = True
is_coroutine = False
needs_closure = True needs_closure = True
child_attrs = DefNode.child_attrs + ["gbody"] child_attrs = DefNode.child_attrs + ["gbody"]
...@@ -3956,8 +3960,9 @@ class GeneratorDefNode(DefNode): ...@@ -3956,8 +3960,9 @@ class GeneratorDefNode(DefNode):
qualname = code.intern_identifier(self.qualname) qualname = code.intern_identifier(self.qualname)
code.putln('{') code.putln('{')
code.putln('__pyx_CoroutineObject *gen = __Pyx_Generator_New(' code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
'(__pyx_coroutine_body_t) %s, (PyObject *) %s, %s, %s); %s' % ( '(__pyx_coroutine_body_t) %s, (PyObject *) %s, %s, %s); %s' % (
'Coroutine' if self.is_coroutine else 'Generator',
body_cname, Naming.cur_scope_cname, name, qualname, body_cname, Naming.cur_scope_cname, name, qualname,
code.error_goto_if_null('gen', self.pos))) code.error_goto_if_null('gen', self.pos)))
code.put_decref(Naming.cur_scope_cname, py_object_type) code.put_decref(Naming.cur_scope_cname, py_object_type)
...@@ -3972,13 +3977,18 @@ class GeneratorDefNode(DefNode): ...@@ -3972,13 +3977,18 @@ class GeneratorDefNode(DefNode):
code.putln('}') code.putln('}')
def generate_function_definitions(self, env, code): def generate_function_definitions(self, env, code):
env.use_utility_code(UtilityCode.load_cached("Generator", "Coroutine.c")) env.use_utility_code(UtilityCode.load_cached(
'Coroutine' if self.is_coroutine else 'Generator', "Coroutine.c"))
self.gbody.generate_function_header(code, proto=True) self.gbody.generate_function_header(code, proto=True)
super(GeneratorDefNode, self).generate_function_definitions(env, code) super(GeneratorDefNode, self).generate_function_definitions(env, code)
self.gbody.generate_function_definitions(env, code) self.gbody.generate_function_definitions(env, code)
class AsyncDefNode(GeneratorDefNode):
is_coroutine = True
class GeneratorBodyDefNode(DefNode): class GeneratorBodyDefNode(DefNode):
# Main code body of a generator implemented as a DefNode. # Main code body of a generator implemented as a DefNode.
# #
...@@ -7108,7 +7118,7 @@ class GILStatNode(NogilTryFinallyStatNode): ...@@ -7108,7 +7118,7 @@ class GILStatNode(NogilTryFinallyStatNode):
from .ParseTreeTransforms import YieldNodeCollector from .ParseTreeTransforms import YieldNodeCollector
collector = YieldNodeCollector() collector = YieldNodeCollector()
collector.visitchildren(body) collector.visitchildren(body)
if not collector.yields: if not collector.yields and not collector.awaits:
return return
if state == 'gil': if state == 'gil':
......
...@@ -200,7 +200,7 @@ class PostParse(ScopeTrackingTransform): ...@@ -200,7 +200,7 @@ class PostParse(ScopeTrackingTransform):
node.lambda_name = EncodedString(u'lambda%d' % lambda_id) node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
collector = YieldNodeCollector() collector = YieldNodeCollector()
collector.visitchildren(node.result_expr) collector.visitchildren(node.result_expr)
if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode): if collector.yields or collector.awaits or isinstance(node.result_expr, ExprNodes.YieldExprNode):
body = Nodes.ExprStatNode( body = Nodes.ExprStatNode(
node.result_expr.pos, expr=node.result_expr) node.result_expr.pos, expr=node.result_expr)
else: else:
...@@ -2205,6 +2205,7 @@ class YieldNodeCollector(TreeVisitor): ...@@ -2205,6 +2205,7 @@ class YieldNodeCollector(TreeVisitor):
def __init__(self): def __init__(self):
super(YieldNodeCollector, self).__init__() super(YieldNodeCollector, self).__init__()
self.yields = [] self.yields = []
self.awaits = []
self.returns = [] self.returns = []
self.has_return_value = False self.has_return_value = False
...@@ -2215,6 +2216,10 @@ class YieldNodeCollector(TreeVisitor): ...@@ -2215,6 +2216,10 @@ class YieldNodeCollector(TreeVisitor):
self.yields.append(node) self.yields.append(node)
self.visitchildren(node) self.visitchildren(node)
def visit_AwaitExprNode(self, node):
self.awaits.append(node)
self.visitchildren(node)
def visit_ReturnStatNode(self, node): def visit_ReturnStatNode(self, node):
self.visitchildren(node) self.visitchildren(node)
if node.value: if node.value:
...@@ -2250,27 +2255,36 @@ class MarkClosureVisitor(CythonTransform): ...@@ -2250,27 +2255,36 @@ class MarkClosureVisitor(CythonTransform):
collector = YieldNodeCollector() collector = YieldNodeCollector()
collector.visitchildren(node) collector.visitchildren(node)
if collector.yields: if node.is_async_def:
if isinstance(node, Nodes.CFuncDefNode): if collector.yields:
# Will report error later error(collector.yields[0].pos, "'yield' not allowed in async coroutines (use 'await')")
return node yields = collector.awaits
for i, yield_expr in enumerate(collector.yields, 1): elif collector.yields:
yield_expr.label_num = i if collector.awaits:
for retnode in collector.returns: error(collector.yields[0].pos, "'await' not allowed in generators (use 'yield')")
retnode.in_generator = True yields = collector.yields
else:
return node
gbody = Nodes.GeneratorBodyDefNode( for i, yield_expr in enumerate(yields, 1):
pos=node.pos, name=node.name, body=node.body) yield_expr.label_num = i
generator = Nodes.GeneratorDefNode( for retnode in collector.returns:
pos=node.pos, name=node.name, args=node.args, retnode.in_generator = True
star_arg=node.star_arg, starstar_arg=node.starstar_arg,
doc=node.doc, decorators=node.decorators, gbody = Nodes.GeneratorBodyDefNode(
gbody=gbody, lambda_name=node.lambda_name) pos=node.pos, name=node.name, body=node.body)
return generator coroutine = (Nodes.AsyncDefNode if node.is_async_def else Nodes.GeneratorDefNode)(
return node pos=node.pos, name=node.name, args=node.args,
star_arg=node.star_arg, starstar_arg=node.starstar_arg,
doc=node.doc, decorators=node.decorators,
gbody=gbody, lambda_name=node.lambda_name)
return coroutine
def visit_CFuncDefNode(self, node): def visit_CFuncDefNode(self, node):
self.visit_FuncDefNode(node) self.needs_closure = False
self.visitchildren(node)
node.needs_closure = self.needs_closure
self.needs_closure = True
if node.needs_closure and node.overridable: if node.needs_closure and node.overridable:
error(node.pos, "closures inside cpdef functions not yet supported") error(node.pos, "closures inside cpdef functions not yet supported")
return node return node
...@@ -2287,6 +2301,7 @@ class MarkClosureVisitor(CythonTransform): ...@@ -2287,6 +2301,7 @@ class MarkClosureVisitor(CythonTransform):
self.needs_closure = True self.needs_closure = True
return node return node
class CreateClosureClasses(CythonTransform): class CreateClosureClasses(CythonTransform):
# Output closure classes in module scope for all functions # Output closure classes in module scope for all functions
# that really need it. # that really need it.
......
...@@ -44,6 +44,8 @@ cdef p_typecast(PyrexScanner s) ...@@ -44,6 +44,8 @@ cdef p_typecast(PyrexScanner s)
cdef p_sizeof(PyrexScanner s) cdef p_sizeof(PyrexScanner s)
cdef p_yield_expression(PyrexScanner s) cdef p_yield_expression(PyrexScanner s)
cdef p_yield_statement(PyrexScanner s) cdef p_yield_statement(PyrexScanner s)
cdef p_await_expression(PyrexScanner s)
cdef p_async_statement(PyrexScanner s, ctx)
cdef p_power(PyrexScanner s) cdef p_power(PyrexScanner s)
cdef p_new_expr(PyrexScanner s) cdef p_new_expr(PyrexScanner s)
cdef p_trailer(PyrexScanner s, node1) cdef p_trailer(PyrexScanner s, node1)
...@@ -128,7 +130,7 @@ cdef p_IF_statement(PyrexScanner s, ctx) ...@@ -128,7 +130,7 @@ cdef p_IF_statement(PyrexScanner s, ctx)
cdef p_statement(PyrexScanner s, ctx, bint first_statement = *) cdef p_statement(PyrexScanner s, ctx, bint first_statement = *)
cdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *) cdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *)
cdef p_suite(PyrexScanner s, ctx = *) cdef p_suite(PyrexScanner s, ctx = *)
cdef tuple p_suite_with_docstring(PyrexScanner s, ctx, with_doc_only = *) cdef tuple p_suite_with_docstring(PyrexScanner s, ctx, bint with_doc_only=*)
cdef tuple _extract_docstring(node) cdef tuple _extract_docstring(node)
cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *) cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *)
...@@ -176,7 +178,7 @@ cdef p_c_modifiers(PyrexScanner s) ...@@ -176,7 +178,7 @@ cdef p_c_modifiers(PyrexScanner s)
cdef p_c_func_or_var_declaration(PyrexScanner s, pos, ctx) cdef p_c_func_or_var_declaration(PyrexScanner s, pos, ctx)
cdef p_ctypedef_statement(PyrexScanner s, ctx) cdef p_ctypedef_statement(PyrexScanner s, ctx)
cdef p_decorators(PyrexScanner s) cdef p_decorators(PyrexScanner s)
cdef p_def_statement(PyrexScanner s, list decorators = *) cdef p_def_statement(PyrexScanner s, list decorators=*, bint is_async_def=*)
cdef p_varargslist(PyrexScanner s, terminator=*, bint annotated = *) cdef p_varargslist(PyrexScanner s, terminator=*, bint annotated = *)
cdef p_py_arg_decl(PyrexScanner s, bint annotated = *) cdef p_py_arg_decl(PyrexScanner s, bint annotated = *)
cdef p_class_statement(PyrexScanner s, decorators) cdef p_class_statement(PyrexScanner s, decorators)
......
...@@ -55,6 +55,7 @@ class Ctx(object): ...@@ -55,6 +55,7 @@ class Ctx(object):
d.update(kwds) d.update(kwds)
return ctx return ctx
def p_ident(s, message="Expected an identifier"): def p_ident(s, message="Expected an identifier"):
if s.sy == 'IDENT': if s.sy == 'IDENT':
name = s.systring name = s.systring
...@@ -350,6 +351,7 @@ def p_sizeof(s): ...@@ -350,6 +351,7 @@ def p_sizeof(s):
s.expect(')') s.expect(')')
return node return node
def p_yield_expression(s): def p_yield_expression(s):
# s.sy == "yield" # s.sy == "yield"
pos = s.position() pos = s.position()
...@@ -370,19 +372,52 @@ def p_yield_expression(s): ...@@ -370,19 +372,52 @@ def p_yield_expression(s):
else: else:
return ExprNodes.YieldExprNode(pos, arg=arg) return ExprNodes.YieldExprNode(pos, arg=arg)
def p_yield_statement(s): def p_yield_statement(s):
# s.sy == "yield" # s.sy == "yield"
yield_expr = p_yield_expression(s) yield_expr = p_yield_expression(s)
return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr) return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)
#power: atom trailer* ('**' factor)*
def p_async_statement(s, ctx, decorators):
# s.sy >> 'async' ...
if s.sy == 'def':
# 'async def' statements aren't allowed in pxd files
if 'pxd' in ctx.level:
s.error('def statement not allowed here')
s.level = ctx.level
return p_def_statement(s, decorators, is_async_def=True)
elif decorators:
s.error("Decorators can only be followed by functions or classes")
elif s.sy == 'for':
#s.error("'async for' is not currently supported", fatal=False)
return p_statement(s, ctx) # TODO: implement
elif s.sy == 'with':
#s.error("'async with' is not currently supported", fatal=False)
return p_statement(s, ctx) # TODO: implement
else:
s.error("expected one of 'def', 'for', 'with' after 'async'")
def p_await_expression(s):
n1 = p_atom(s)
#power: atom_expr ('**' factor)*
#atom_expr: ['await'] atom trailer*
def p_power(s): def p_power(s):
if s.systring == 'new' and s.peek()[0] == 'IDENT': if s.systring == 'new' and s.peek()[0] == 'IDENT':
return p_new_expr(s) return p_new_expr(s)
await_pos = None
if s.sy == 'await':
await_pos = s.position()
s.next()
n1 = p_atom(s) n1 = p_atom(s)
while s.sy in ('(', '[', '.'): while s.sy in ('(', '[', '.'):
n1 = p_trailer(s, n1) n1 = p_trailer(s, n1)
if await_pos:
n1 = ExprNodes.AwaitExprNode(await_pos, arg=n1)
if s.sy == '**': if s.sy == '**':
pos = s.position() pos = s.position()
s.next() s.next()
...@@ -390,6 +425,7 @@ def p_power(s): ...@@ -390,6 +425,7 @@ def p_power(s):
n1 = ExprNodes.binop_node(pos, '**', n1, n2) n1 = ExprNodes.binop_node(pos, '**', n1, n2)
return n1 return n1
def p_new_expr(s): def p_new_expr(s):
# s.systring == 'new'. # s.systring == 'new'.
pos = s.position() pos = s.position()
...@@ -1929,12 +1965,14 @@ def p_statement(s, ctx, first_statement = 0): ...@@ -1929,12 +1965,14 @@ def p_statement(s, ctx, first_statement = 0):
s.error('decorator not allowed here') s.error('decorator not allowed here')
s.level = ctx.level s.level = ctx.level
decorators = p_decorators(s) decorators = p_decorators(s)
bad_toks = 'def', 'cdef', 'cpdef', 'class' if not ctx.allow_struct_enum_decorator and s.sy not in ('def', 'cdef', 'cpdef', 'class'):
if not ctx.allow_struct_enum_decorator and s.sy not in bad_toks: if s.sy == 'IDENT' and s.systring == 'async':
s.error("Decorators can only be followed by functions or classes") pass # handled below
else:
s.error("Decorators can only be followed by functions or classes")
elif s.sy == 'pass' and cdef_flag: elif s.sy == 'pass' and cdef_flag:
# empty cdef block # empty cdef block
return p_pass_statement(s, with_newline = 1) return p_pass_statement(s, with_newline=1)
overridable = 0 overridable = 0
if s.sy == 'cdef': if s.sy == 'cdef':
...@@ -1948,11 +1986,11 @@ def p_statement(s, ctx, first_statement = 0): ...@@ -1948,11 +1986,11 @@ def p_statement(s, ctx, first_statement = 0):
if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'): if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
s.error('cdef statement not allowed here') s.error('cdef statement not allowed here')
s.level = ctx.level s.level = ctx.level
node = p_cdef_statement(s, ctx(overridable = overridable)) node = p_cdef_statement(s, ctx(overridable=overridable))
if decorators is not None: if decorators is not None:
tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode tup = (Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode)
if ctx.allow_struct_enum_decorator: if ctx.allow_struct_enum_decorator:
tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode tup += (Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode)
if not isinstance(node, tup): if not isinstance(node, tup):
s.error("Decorators can only be followed by functions or classes") s.error("Decorators can only be followed by functions or classes")
node.decorators = decorators node.decorators = decorators
...@@ -1995,9 +2033,25 @@ def p_statement(s, ctx, first_statement = 0): ...@@ -1995,9 +2033,25 @@ def p_statement(s, ctx, first_statement = 0):
return p_try_statement(s) return p_try_statement(s)
elif s.sy == 'with': elif s.sy == 'with':
return p_with_statement(s) return p_with_statement(s)
elif s.sy == 'async':
s.next()
return p_async_statement(s, ctx, decorators)
else: else:
return p_simple_statement_list( if s.sy == 'IDENT' and s.systring == 'async':
s, ctx, first_statement = first_statement) # PEP 492 enables the async/await keywords when it spots "async def ..."
s.next()
if s.sy == 'def':
s.enable_keyword('async')
s.enable_keyword('await')
result = p_async_statement(s, ctx, decorators)
s.enable_keyword('await')
s.disable_keyword('async')
return result
elif decorators:
s.error("Decorators can only be followed by functions or classes")
s.put_back('IDENT', 'async')
return p_simple_statement_list(s, ctx, first_statement=first_statement)
def p_statement_list(s, ctx, first_statement = 0): def p_statement_list(s, ctx, first_statement = 0):
# Parse a series of statements separated by newlines. # Parse a series of statements separated by newlines.
...@@ -3002,7 +3056,8 @@ def p_decorators(s): ...@@ -3002,7 +3056,8 @@ def p_decorators(s):
s.expect_newline("Expected a newline after decorator") s.expect_newline("Expected a newline after decorator")
return decorators return decorators
def p_def_statement(s, decorators=None):
def p_def_statement(s, decorators=None, is_async_def=False):
# s.sy == 'def' # s.sy == 'def'
pos = s.position() pos = s.position()
s.next() s.next()
...@@ -3017,10 +3072,11 @@ def p_def_statement(s, decorators=None): ...@@ -3017,10 +3072,11 @@ def p_def_statement(s, decorators=None):
s.next() s.next()
return_type_annotation = p_test(s) return_type_annotation = p_test(s)
doc, body = p_suite_with_docstring(s, Ctx(level='function')) doc, body = p_suite_with_docstring(s, Ctx(level='function'))
return Nodes.DefNode(pos, name = name, args = args, return Nodes.DefNode(
star_arg = star_arg, starstar_arg = starstar_arg, pos, name=name, args=args, star_arg=star_arg, starstar_arg=starstar_arg,
doc = doc, body = body, decorators = decorators, doc=doc, body=body, decorators=decorators, is_async_def=is_async_def,
return_type_annotation = return_type_annotation) return_type_annotation=return_type_annotation)
def p_varargslist(s, terminator=')', annotated=1): def p_varargslist(s, terminator=')', annotated=1):
args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1, args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,
......
...@@ -30,6 +30,7 @@ cdef class PyrexScanner(Scanner): ...@@ -30,6 +30,7 @@ cdef class PyrexScanner(Scanner):
cdef public bint in_python_file cdef public bint in_python_file
cdef public source_encoding cdef public source_encoding
cdef set keywords cdef set keywords
cdef public dict keywords_stack
cdef public list indentation_stack cdef public list indentation_stack
cdef public indentation_char cdef public indentation_char
cdef public int bracket_nesting_level cdef public int bracket_nesting_level
...@@ -57,3 +58,5 @@ cdef class PyrexScanner(Scanner): ...@@ -57,3 +58,5 @@ cdef class PyrexScanner(Scanner):
cdef expect_indent(self) cdef expect_indent(self)
cdef expect_dedent(self) cdef expect_dedent(self)
cdef expect_newline(self, message=*, bint ignore_semicolon=*) cdef expect_newline(self, message=*, bint ignore_semicolon=*)
cdef enable_keyword(self, name)
cdef disable_keyword(self, name)
...@@ -319,6 +319,7 @@ class PyrexScanner(Scanner): ...@@ -319,6 +319,7 @@ class PyrexScanner(Scanner):
self.in_python_file = False self.in_python_file = False
self.keywords = set(pyx_reserved_words) self.keywords = set(pyx_reserved_words)
self.trace = trace_scanner self.trace = trace_scanner
self.keywords_stack = {}
self.indentation_stack = [0] self.indentation_stack = [0]
self.indentation_char = None self.indentation_char = None
self.bracket_nesting_level = 0 self.bracket_nesting_level = 0
...@@ -497,3 +498,18 @@ class PyrexScanner(Scanner): ...@@ -497,3 +498,18 @@ class PyrexScanner(Scanner):
self.expect('NEWLINE', message) self.expect('NEWLINE', message)
if useless_trailing_semicolon is not None: if useless_trailing_semicolon is not None:
warning(useless_trailing_semicolon, "useless trailing semicolon") warning(useless_trailing_semicolon, "useless trailing semicolon")
def enable_keyword(self, name):
if name in self.keywords_stack:
self.keywords_stack[name] += 1
else:
self.keywords_stack[name] = 1
self.keywords.add(name)
def disable_keyword(self, name):
count = self.keywords_stack.get(name, 1)
if count == 1:
self.keywords.discard(name)
del self.keywords_stack[name]
else:
self.keywords_stack[name] = count - 1
...@@ -13,7 +13,8 @@ eval_input: testlist NEWLINE* ENDMARKER ...@@ -13,7 +13,8 @@ eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_PY_NAME [ '(' [arglist] ')' ] NEWLINE decorator: '@' dotted_PY_NAME [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+ decorators: decorator+
decorated: decorators (classdef | funcdef | cdef_stmt) decorated: decorators (classdef | funcdef | async_funcdef | cdef_stmt)
async_funcdef: 'async' funcdef
funcdef: 'def' PY_NAME parameters ['->' test] ':' suite funcdef: 'def' PY_NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')' parameters: '(' [typedargslist] ')'
typedargslist: (tfpdef ['=' (test | '*')] (',' tfpdef ['=' (test | '*')])* [',' typedargslist: (tfpdef ['=' (test | '*')] (',' tfpdef ['=' (test | '*')])* [','
...@@ -96,7 +97,8 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)* ...@@ -96,7 +97,8 @@ shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)* arith_expr: term (('+'|'-') term)*
term: factor (('*'|'/'|'%'|'//') factor)* term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power | address | size_of | cast factor: ('+'|'-'|'~') factor | power | address | size_of | cast
power: atom trailer* ['**' factor] power: atom_expr ['**' factor]
atom_expr: ['await'] atom trailer*
atom: ('(' [yield_expr|testlist_comp] ')' | atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [testlist_comp] ']' | '[' [testlist_comp] ']' |
'{' [dictorsetmaker] '}' | '{' [dictorsetmaker] '}' |
......
//////////////////// YieldFrom.proto //////////////////// //////////////////// GeneratorYieldFrom.proto ////////////////////
static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source); static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source);
//////////////////// YieldFrom //////////////////// //////////////////// GeneratorYieldFrom ////////////////////
//@requires: Generator //@requires: Generator
static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) { static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) {
...@@ -21,6 +21,125 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject ...@@ -21,6 +21,125 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject
} }
//////////////////// CoroutineYieldFrom.proto ////////////////////
static CYTHON_INLINE PyObject* __Pyx_Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source);
//////////////////// CoroutineYieldFrom ////////////////////
//@requires: Coroutine
//@requires: GetAwaitIter
static CYTHON_INLINE PyObject* __Pyx_Coroutine_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) {
PyObject *retval;
if (__Pyx_Coroutine_CheckExact(source)) {
retval = __Pyx_Generator_Next(source);
if (retval) {
Py_INCREF(source);
gen->yieldfrom = source;
return retval;
}
} else {
PyObject *source_gen = __Pyx__Coroutine_GetAwaitableIter(source);
if (unlikely(!source_gen))
return NULL;
// source_gen is now the iterator, make the first next() call
if (__Pyx_Coroutine_CheckExact(source_gen)) {
retval = __Pyx_Generator_Next(source_gen);
} else {
retval = Py_TYPE(source_gen)->tp_iternext(source_gen);
}
if (retval) {
gen->yieldfrom = source_gen;
return retval;
}
Py_DECREF(source_gen);
}
return NULL;
}
//////////////////// GetAwaitIter.proto ////////////////////
static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o); /*proto*/
static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *o); /*proto*/
//////////////////// GetAwaitIter ////////////////////
//@requires: Coroutine
//@requires: ObjectHandling.c::PyObjectGetAttrStr
//@requires: ObjectHandling.c::PyObjectCallNoArg
//@requires: ObjectHandling.c::PyObjectCallOneArg
static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o) {
#ifdef __Pyx_Coroutine_USED
if (__Pyx_Coroutine_CheckExact(o)) {
Py_INCREF(o);
return o;
}
#endif
return __Pyx__Coroutine_GetAwaitableIter(o);
}
// copied and adapted from genobject.c in Py3.5
static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *o) {
PyObject *res;
#if PY_VERSION_HEX >= 0x030500B1
unaryfunc getter = NULL;
PyTypeObject *ot;
ot = Py_TYPE(o);
if (likely(ot->tp_as_async)) {
getter = (unaryfunc) ot->tp_as_async->am_await;
}
if (unlikely(getter)) goto slot_error;
res = (*getter)(o);
#else
PyObject *method = __Pyx_PyObject_GetAttrStr(o, PYIDENT("__await__"));
if (unlikely(!method)) goto slot_error;
#if CYTHON_COMPILING_IN_CPYTHON
if (likely(PyMethod_Check(method))) {
PyObject *self = PyMethod_GET_SELF(method);
if (likely(self)) {
PyObject *function = PyMethod_GET_FUNCTION(method);
res = __Pyx_PyObject_CallOneArg(function, self);
} else
res = __Pyx_PyObject_CallNoArg(method);
} else
#endif
res = __Pyx_PyObject_CallNoArg(method);
Py_DECREF(method);
#endif
if (unlikely(!res)) goto bad;
if (!PyIter_Check(res)) {
PyErr_Format(PyExc_TypeError,
"__await__() returned non-iterator of type '%.100s'",
Py_TYPE(res)->tp_name);
Py_CLEAR(res);
} else {
int is_coroutine = 0;
#ifdef __Pyx_Coroutine_USED
is_coroutine |= __Pyx_Coroutine_CheckExact(res);
#endif
#if PY_VERSION_HEX >= 0x030500B1
is_coroutine |= PyGen_CheckCoroutineExact(res);
#endif
if (unlikely(is_coroutine)) {
/* __await__ must return an *iterator*, not
a coroutine or another awaitable (see PEP 492) */
PyErr_SetString(PyExc_TypeError,
"__await__() returned a coroutine");
Py_CLEAR(res);
}
}
return res;
slot_error:
PyErr_Format(PyExc_TypeError,
"object %.100s can't be used in 'await' expression",
Py_TYPE(o)->tp_name);
bad:
return NULL;
}
//////////////////// pep479.proto //////////////////// //////////////////// pep479.proto ////////////////////
static void __Pyx_Generator_Replace_StopIteration(void); /*proto*/ static void __Pyx_Generator_Replace_StopIteration(void); /*proto*/
...@@ -107,6 +226,7 @@ static int __pyx_Generator_init(void); ...@@ -107,6 +226,7 @@ static int __pyx_Generator_init(void);
#include <structmember.h> #include <structmember.h>
#include <frameobject.h> #include <frameobject.h>
static PyObject *__Pyx_Generator_Next(PyObject *self);
static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value); static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value);
static PyObject *__Pyx_Coroutine_Close(PyObject *self); static PyObject *__Pyx_Coroutine_Close(PyObject *self);
static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args); static PyObject *__Pyx_Coroutine_Throw(PyObject *gen, PyObject *args);
...@@ -403,6 +523,28 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { ...@@ -403,6 +523,28 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) {
return err; return err;
} }
static PyObject *__Pyx_Generator_Next(PyObject *self) {
__pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self;
PyObject *yf = gen->yieldfrom;
if (unlikely(__Pyx_Coroutine_CheckRunning(gen)))
return NULL;
if (yf) {
PyObject *ret;
// FIXME: does this really need an INCREF() ?
//Py_INCREF(yf);
// YieldFrom code ensures that yf is an iterator
gen->is_running = 1;
ret = Py_TYPE(yf)->tp_iternext(yf);
gen->is_running = 0;
//Py_DECREF(yf);
if (likely(ret)) {
return ret;
}
return __Pyx_Coroutine_FinishDelegation(gen);
}
return __Pyx_Coroutine_SendEx(gen, Py_None);
}
static PyObject *__Pyx_Coroutine_Close(PyObject *self) { static PyObject *__Pyx_Coroutine_Close(PyObject *self) {
__pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self; __pyx_CoroutineObject *gen = (__pyx_CoroutineObject *) self;
PyObject *retval, *raised_exception; PyObject *retval, *raised_exception;
...@@ -729,6 +871,14 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_New(PyTypeObject* type, __pyx_cor ...@@ -729,6 +871,14 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_New(PyTypeObject* type, __pyx_cor
//@requires: CoroutineBase //@requires: CoroutineBase
//@requires: PatchGeneratorABC //@requires: PatchGeneratorABC
#if PY_VERSION_HEX >= 0x030500B1
static PyAsyncMethods __pyx_Coroutine_as_async {
0, /*am_await*/
0, /*am_aiter*/
0, /*am_anext*/
}
#endif
static PyTypeObject __pyx_CoroutineType_type = { static PyTypeObject __pyx_CoroutineType_type = {
PyVarObject_HEAD_INIT(0, 0) PyVarObject_HEAD_INIT(0, 0)
"coroutine", /*tp_name*/ "coroutine", /*tp_name*/
...@@ -738,10 +888,10 @@ static PyTypeObject __pyx_CoroutineType_type = { ...@@ -738,10 +888,10 @@ static PyTypeObject __pyx_CoroutineType_type = {
0, /*tp_print*/ 0, /*tp_print*/
0, /*tp_getattr*/ 0, /*tp_getattr*/
0, /*tp_setattr*/ 0, /*tp_setattr*/
#if PY_MAJOR_VERSION < 3 #if PY_VERSION_HEX >= 0x030500B1
0, /*tp_compare*/ __pyx_Coroutine_as_async, /*tp_as_async*/
#else #else
0, /*reserved*/ 0, /*tp_reserved resp. tp_compare*/
#endif #endif
0, /*tp_repr*/ 0, /*tp_repr*/
0, /*tp_as_number*/ 0, /*tp_as_number*/
...@@ -759,8 +909,9 @@ static PyTypeObject __pyx_CoroutineType_type = { ...@@ -759,8 +909,9 @@ static PyTypeObject __pyx_CoroutineType_type = {
0, /*tp_clear*/ 0, /*tp_clear*/
0, /*tp_richcompare*/ 0, /*tp_richcompare*/
offsetof(__pyx_CoroutineObject, gi_weakreflist), /*tp_weaklistoffset*/ offsetof(__pyx_CoroutineObject, gi_weakreflist), /*tp_weaklistoffset*/
// no tp_iter() as iterator is only available through __await__()
0, /*tp_iter*/ 0, /*tp_iter*/
0, /*tp_iternext*/ (iternextfunc) __Pyx_Generator_Next, /*tp_iternext*/
__pyx_Coroutine_methods, /*tp_methods*/ __pyx_Coroutine_methods, /*tp_methods*/
__pyx_Coroutine_memberlist, /*tp_members*/ __pyx_Coroutine_memberlist, /*tp_members*/
__pyx_Coroutine_getsets, /*tp_getset*/ __pyx_Coroutine_getsets, /*tp_getset*/
...@@ -790,7 +941,7 @@ static PyTypeObject __pyx_CoroutineType_type = { ...@@ -790,7 +941,7 @@ static PyTypeObject __pyx_CoroutineType_type = {
#endif #endif
}; };
static int __pyx_Generator_init(void) { static int __pyx_Coroutine_init(void) {
// on Windows, C-API functions can't be used in slots statically // on Windows, C-API functions can't be used in slots statically
__pyx_CoroutineType_type.tp_getattro = PyObject_GenericGetAttr; __pyx_CoroutineType_type.tp_getattro = PyObject_GenericGetAttr;
...@@ -801,35 +952,10 @@ static int __pyx_Generator_init(void) { ...@@ -801,35 +952,10 @@ static int __pyx_Generator_init(void) {
return 0; return 0;
} }
//////////////////// Generator //////////////////// //////////////////// Generator ////////////////////
//@requires: CoroutineBase //@requires: CoroutineBase
//@requires: PatchGeneratorABC //@requires: PatchGeneratorABC
static PyObject *__Pyx_Generator_Next(PyObject *self);
static PyObject *__Pyx_Generator_Next(PyObject *self) {
__pyx_CoroutineObject *gen = (__pyx_CoroutineObject*) self;
PyObject *yf = gen->yieldfrom;
if (unlikely(__Pyx_Coroutine_CheckRunning(gen)))
return NULL;
if (yf) {
PyObject *ret;
// FIXME: does this really need an INCREF() ?
//Py_INCREF(yf);
// YieldFrom code ensures that yf is an iterator
gen->is_running = 1;
ret = Py_TYPE(yf)->tp_iternext(yf);
gen->is_running = 0;
//Py_DECREF(yf);
if (likely(ret)) {
return ret;
}
return __Pyx_Coroutine_FinishDelegation(gen);
}
return __Pyx_Coroutine_SendEx(gen, Py_None);
}
static PyTypeObject __pyx_GeneratorType_type = { static PyTypeObject __pyx_GeneratorType_type = {
PyVarObject_HEAD_INIT(0, 0) PyVarObject_HEAD_INIT(0, 0)
"generator", /*tp_name*/ "generator", /*tp_name*/
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment