partially implement PEP 448 for function calls only

CHANGES.rst

@@ -15,6 +15,9 @@ Features added
 
 * Tracing is supported in ``nogil`` functions/sections and module init code.
 
+* PEP 448 (Additional Unpacking Generalizations) was implemented for function
+  calls.
+
 * When generators are used in a Cython module and the module imports the
   modules "inspect" and/or "asyncio", Cython enables interoperability by
   patching these modules to recognise Cython's internal generator type.

Cython/Compiler/Parsing.pxd

@@ -48,7 +48,7 @@ cdef p_power(PyrexScanner s)
 cdef p_new_expr(PyrexScanner s)
 cdef p_trailer(PyrexScanner s, node1)
 cdef p_call_parse_args(PyrexScanner s, bint allow_genexp = *)
-cdef p_call_build_packed_args(pos, positional_args, keyword_args, star_arg, starstar_arg)
+cdef p_call_build_packed_args(pos, positional_args, keyword_args)
 cdef p_call(PyrexScanner s, function)
 cdef p_index(PyrexScanner s, base)
 cdef tuple p_subscript_list(PyrexScanner s)

Cython/Compiler/Parsing.py

@@ -12,10 +12,11 @@ cython.declare(Nodes=object, ExprNodes=object, EncodedString=object,
                FileSourceDescriptor=object, lookup_unicodechar=object,
                Future=object, Options=object, error=object, warning=object,
                Builtin=object, ModuleNode=object, Utils=object,
-               re=object, _unicode=object, _bytes=object)
+               re=object, _unicode=object, _bytes=object, partial=object)
 
 import re
 from unicodedata import lookup as lookup_unicodechar
+from functools import partial
 
 from .Scanning import PyrexScanner, FileSourceDescriptor
 from . import Nodes
@@ -409,24 +410,35 @@ def p_trailer(s, node1):
         return ExprNodes.AttributeNode(pos,
             obj=node1, attribute=name)
 
+
 # arglist:  argument (',' argument)* [',']
 # argument: [test '='] test       # Really [keyword '='] test
 
-def p_call_parse_args(s, allow_genexp = True):
+# since PEP 448:
+# argument: ( test [comp_for] |
+#             test '=' test |
+#             '**' expr |
+#             star_expr )
+
+def p_call_parse_args(s, allow_genexp=True):
     # s.sy == '('
     pos = s.position()
     s.next()
     positional_args = []
     keyword_args = []
-    star_arg = None
-    starstar_arg = None
-    while s.sy not in ('**', ')'):
+    starstar_seen = False
+    last_was_tuple_unpack = False
+    while s.sy != ')':
         if s.sy == '*':
-            if star_arg:
-                s.error("only one star-arg parameter allowed",
-                        pos=s.position())
+            if starstar_seen:
+                s.error("Non-keyword arg following keyword arg", pos=s.position())
+            s.next()
+            positional_args.append(p_test(s))
+            last_was_tuple_unpack = True
+        elif s.sy == '**':
             s.next()
-            star_arg = p_test(s)
+            keyword_args.append(p_test(s))
+            starstar_seen = True
         else:
             arg = p_test(s)
             if s.sy == '=':
@@ -441,73 +453,78 @@ def p_call_parse_args(s, allow_genexp = True):
                 keyword_args.append((keyword, arg))
             else:
                 if keyword_args:
-                    s.error("Non-keyword arg following keyword arg",
-                            pos=arg.pos)
-                if star_arg:
-                    s.error("Non-keyword arg following star-arg",
-                            pos=arg.pos)
-                positional_args.append(arg)
+                    s.error("Non-keyword arg following keyword arg", pos=arg.pos)
+                if positional_args and not last_was_tuple_unpack:
+                    positional_args[-1].append(arg)
+                else:
+                    positional_args.append([arg])
+                last_was_tuple_unpack = False
         if s.sy != ',':
             break
         s.next()
 
     if s.sy == 'for':
-        if len(positional_args) == 1 and not star_arg:
-            positional_args = [ p_genexp(s, positional_args[0]) ]
-    elif s.sy == '**':
-        s.next()
-        starstar_arg = p_test(s)
-        if s.sy == ',':
-            s.next()  # FIXME: this is actually not valid Python syntax
+        if not keyword_args and not last_was_tuple_unpack:
+            if len(positional_args) == 1 and len(positional_args[0]) == 1:
+                positional_args = [p_genexp(s, positional_args[0][0])]
     s.expect(')')
-    return positional_args, keyword_args, star_arg, starstar_arg
+    return positional_args or [[]], keyword_args
 
-def p_call_build_packed_args(pos, positional_args, keyword_args,
-                             star_arg, starstar_arg):
-    arg_tuple = None
+
+def p_call_build_packed_args(pos, positional_args, keyword_args):
     keyword_dict = None
-    if positional_args or not star_arg:
-        arg_tuple = ExprNodes.TupleNode(pos,
-            args = positional_args)
-    if star_arg:
-        star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)
-        if arg_tuple:
-            arg_tuple = ExprNodes.binop_node(pos,
-                operator = '+', operand1 = arg_tuple,
-                operand2 = star_arg_tuple)
-        else:
-            arg_tuple = star_arg_tuple
-    if keyword_args or starstar_arg:
-        keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
-                          for key, value in keyword_args]
-        if starstar_arg:
-            keyword_dict = ExprNodes.KeywordArgsNode(
-                pos,
-                starstar_arg = starstar_arg,
-                keyword_args = keyword_args)
-        else:
-            keyword_dict = ExprNodes.DictNode(
-                pos, key_value_pairs = keyword_args)
+
+    subtuples = [
+        ExprNodes.TupleNode(pos, args=arg) if isinstance(arg, list) else ExprNodes.AsTupleNode(pos, arg=arg)
+        for arg in positional_args
+    ]
+    # TODO: implement a faster way to join tuples than creating each one and adding them
+    arg_tuple = reduce(partial(ExprNodes.binop_node, pos, '+'), subtuples)
+
+    if keyword_args:
+        kwargs = []
+        dict_items = []
+        for item in keyword_args:
+            if isinstance(item, tuple):
+                key, value = item
+                dict_items.append(ExprNodes.DictItemNode(pos=key.pos, key=key, value=value))
+            elif item.is_dict_literal:
+                # unpack "**{a:b}" directly
+                dict_items.extend(item.key_value_pairs)
+            else:
+                if dict_items:
+                    kwargs.append(ExprNodes.DictNode(
+                        dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
+                    dict_items = []
+                kwargs.append(item)
+
+        if dict_items:
+            kwargs.append(ExprNodes.DictNode(
+                dict_items[0].pos, key_value_pairs=dict_items, reject_duplicates=True))
+
+        if kwargs:
+            if len(kwargs) == 1 and kwargs[0].is_dict_literal:
+                # only simple keyword arguments found -> one dict
+                keyword_dict = kwargs[0]
+            else:
+                # at least one **kwargs
+                keyword_dict = ExprNodes.KeywordArgsNode(pos, keyword_args=kwargs)
+
     return arg_tuple, keyword_dict
 
+
 def p_call(s, function):
     # s.sy == '('
     pos = s.position()
+    positional_args, keyword_args = p_call_parse_args(s)
 
-    positional_args, keyword_args, star_arg, starstar_arg = \
-                     p_call_parse_args(s)
-
-    if not (keyword_args or star_arg or starstar_arg):
-        return ExprNodes.SimpleCallNode(pos,
-            function = function,
-            args = positional_args)
+    if not keyword_args and len(positional_args) == 1 and isinstance(positional_args[0], list):
+        return ExprNodes.SimpleCallNode(pos, function=function, args=positional_args[0])
     else:
-        arg_tuple, keyword_dict = p_call_build_packed_args(
-            pos, positional_args, keyword_args, star_arg, starstar_arg)
-        return ExprNodes.GeneralCallNode(pos,
-            function = function,
-            positional_args = arg_tuple,
-            keyword_args = keyword_dict)
+        arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
+        return ExprNodes.GeneralCallNode(
+            pos, function=function, positional_args=arg_tuple, keyword_args=keyword_dict)
+
 
 #lambdef: 'lambda' [varargslist] ':' test
 
@@ -2969,6 +2986,7 @@ def p_py_arg_decl(s, annotated = 1):
         annotation = p_test(s)
     return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)
 
+
 def p_class_statement(s, decorators):
     # s.sy == 'class'
     pos = s.position()
@@ -2979,10 +2997,8 @@ def p_class_statement(s, decorators):
     keyword_dict = None
     starstar_arg = None
     if s.sy == '(':
-        positional_args, keyword_args, star_arg, starstar_arg = \
-                            p_call_parse_args(s, allow_genexp = False)
-        arg_tuple, keyword_dict = p_call_build_packed_args(
-            pos, positional_args, keyword_args, star_arg, None)
+        positional_args, keyword_args = p_call_parse_args(s, allow_genexp=False)
+        arg_tuple, keyword_dict = p_call_build_packed_args(pos, positional_args, keyword_args)
     if arg_tuple is None:
         # XXX: empty arg_tuple
         arg_tuple = ExprNodes.TupleNode(pos, args=[])
@@ -2995,6 +3011,7 @@ def p_class_statement(s, decorators):
         doc=doc, body=body, decorators=decorators,
         force_py3_semantics=s.context.language_level >= 3)
 
+
 def p_c_class_definition(s, pos,  ctx):
     # s.sy == 'class'
     s.next()

Cython/Parser/Grammar

@@ -109,17 +109,29 @@ subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                  (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test | star_expr)
+                   (comp_for | (',' (test | star_expr))* [','])) )
 
 classdef: 'class' PY_NAME ['(' [arglist] ')'] ':' suite
 
-arglist: (argument ',')* (argument [',']
-                         |'*' test (',' argument)* [',' '**' test]
-                         |'**' test)
+arglist: argument (',' argument)*  [',']
+
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguements are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test '=' test |
+            '**' expr |
+            star_expr )
+
 comp_iter: comp_for | comp_if
 comp_for: 'for' exprlist ('in' or_test | for_from_clause) [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]

Cython/Utility/FunctionArguments.c

@@ -110,6 +110,17 @@ static void __Pyx_RaiseDoubleKeywordsError(
 }
 
 
+//////////////////// RaiseMappingExpected.proto ////////////////////
+
+static void __Pyx_RaiseMappingExpectedError(PyObject* arg); /*proto*/
+
+//////////////////// RaiseMappingExpected ////////////////////
+
+static void __Pyx_RaiseMappingExpectedError(PyObject* arg) {
+    PyErr_Format(PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(arg)->tp_name);
+}
+
+
 //////////////////// KeywordStringCheck.proto ////////////////////
 
 static CYTHON_INLINE int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/
@@ -290,3 +301,43 @@ invalid_keyword:
 bad:
     return -1;
 }
+
+
+//////////////////// MergeKeywords.proto ////////////////////
+
+static int __Pyx_MergeKeywords(PyObject *kwdict, PyObject *source_mapping); /*proto*/
+
+//////////////////// MergeKeywords ////////////////////
+//@requires: RaiseDoubleKeywords
+//@requires: Optimize.c::dict_iter
+
+static int __Pyx_MergeKeywords(PyObject *kwdict, PyObject *source_mapping) {
+    PyObject *iter, *key, *value;
+    int source_is_dict, result;
+    Py_ssize_t orig_length, ppos = 0;
+
+    iter = __Pyx_dict_iterator(source_mapping, 0, PYIDENT("items"), &orig_length, &source_is_dict);
+    if (unlikely(!iter)) goto bad;
+
+    while (1) {
+        result = __Pyx_dict_iter_next(iter, orig_length, &ppos, &key, &value, NULL, source_is_dict);
+        if (unlikely(result < 0)) goto bad;
+        if (!result) break;
+
+        if (unlikely(PyDict_Contains(kwdict, key))) {
+            __Pyx_RaiseDoubleKeywordsError("function", key);
+            result = -1;
+        } else {
+            result = PyDict_SetItem(kwdict, key, value);
+        }
+        Py_DECREF(key);
+        Py_DECREF(value);
+        if (unlikely(result < 0)) goto bad;
+    }
+    Py_XDECREF(iter);
+    return 0;
+
+bad:
+    Py_XDECREF(iter);
+    return -1;
+}