Nodes.py 385 KB
Newer Older
William Stein's avatar
William Stein committed
1
#
2
#   Parse tree nodes
William Stein's avatar
William Stein committed
3
#
4

5 6
from __future__ import absolute_import

7
import cython
8
cython.declare(sys=object, os=object, copy=object,
9
               Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object,
10
               py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object,
11
               StructOrUnionScope=object, PyClassScope=object,
12
               CppClassScope=object, UtilityCode=object, EncodedString=object,
13
               error_type=object, _py_int_types=object)
William Stein's avatar
William Stein committed
14

15
import sys, os, copy
Stefan Behnel's avatar
Stefan Behnel committed
16
from itertools import chain
Robert Bradshaw's avatar
Robert Bradshaw committed
17

18 19 20 21 22 23 24
from . import Builtin
from .Errors import error, warning, InternalError, CompileError
from . import Naming
from . import PyrexTypes
from . import TypeSlots
from .PyrexTypes import py_object_type, error_type
from .Symtab import (ModuleScope, LocalScope, ClosureScope,
25
                     StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope)
26
from .Code import UtilityCode
27
from .StringEncoding import EncodedString
28
from . import Future
29 30
from . import Options
from . import DebugFlags
31
from .Pythran import has_np_pythran, pythran_type, is_pythran_buffer
32
from ..Utils import add_metaclass
William Stein's avatar
William Stein committed
33

34

35 36 37 38 39
if sys.version_info[0] >= 3:
    _py_int_types = int
else:
    _py_int_types = (int, long)

40

41
def relative_position(pos):
42
    return (pos[0].get_filenametable_entry(), pos[1])
43

44 45 46 47

def embed_position(pos, docstring):
    if not Options.embed_pos_in_docstring:
        return docstring
48
    pos_line = u'File: %s (starting at line %s)' % relative_position(pos)
49 50
    if docstring is None:
        # unicode string
51
        return EncodedString(pos_line)
52 53 54 55 56 57

    # make sure we can encode the filename in the docstring encoding
    # otherwise make the docstring a unicode string
    encoding = docstring.encoding
    if encoding is not None:
        try:
Stefan Behnel's avatar
Stefan Behnel committed
58
            pos_line.encode(encoding)
59 60 61 62 63
        except UnicodeEncodeError:
            encoding = None

    if not docstring:
        # reuse the string encoding of the original docstring
64
        doc = EncodedString(pos_line)
65
    else:
66
        doc = EncodedString(pos_line + u'\n' + docstring)
67 68
    doc.encoding = encoding
    return doc
69

70

71
def analyse_type_annotation(annotation, env, assigned_value=None):
72
    base_type = None
73
    is_ambiguous = False
74 75
    explicit_pytype = explicit_ctype = False
    if annotation.is_dict_literal:
76 77
        warning(annotation.pos,
                "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.")
78 79 80 81 82 83 84 85 86 87 88 89 90
        for name, value in annotation.key_value_pairs:
            if not name.is_string_literal:
                continue
            if name.value in ('type', b'type'):
                explicit_pytype = True
                if not explicit_ctype:
                    annotation = value
            elif name.value in ('ctype', b'ctype'):
                explicit_ctype = True
                annotation = value
        if explicit_pytype and explicit_ctype:
            warning(annotation.pos, "Duplicate type declarations found in signature annotation")
    arg_type = annotation.analyse_as_type(env)
91
    if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'):
92 93 94 95 96 97 98
        # Map builtin numeric Python types to C types in safe cases.
        if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject:
            assigned_type = assigned_value.infer_type(env)
            if assigned_type and assigned_type.is_pyobject:
                # C type seems unsafe, e.g. due to 'None' default value  => ignore annotation type
                is_ambiguous = True
                arg_type = None
99 100 101 102 103 104
        # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations
        if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type):
            arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type
    elif arg_type is not None and annotation.is_string_literal:
        warning(annotation.pos,
                "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.")
105 106 107 108 109 110
    if arg_type is not None:
        if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject:
            warning(annotation.pos,
                    "Python type declaration in signature annotation does not refer to a Python type")
        base_type = CAnalysedBaseTypeNode(
            annotation.pos, type=arg_type, is_arg=True)
111 112
    elif is_ambiguous:
        warning(annotation.pos, "Ambiguous types in annotation, ignoring")
113
    else:
114
        warning(annotation.pos, "Unknown type declaration in annotation, ignoring")
115 116 117
    return base_type, arg_type


118
def write_func_call(func, codewriter_class):
119
    def f(*args, **kwds):
120
        if len(args) > 1 and isinstance(args[1], codewriter_class):
Robert Bradshaw's avatar
Robert Bradshaw committed
121 122
            # here we annotate the code with this function call
            # but only if new code is generated
123
            node, code = args[:2]
Robert Bradshaw's avatar
Robert Bradshaw committed
124
            marker = '                    /* %s -> %s.%s %s */' % (
Stefan Behnel's avatar
Stefan Behnel committed
125 126 127 128
                ' ' * code.call_level,
                node.__class__.__name__,
                func.__name__,
                node.pos[1:])
Robert Bradshaw's avatar
Robert Bradshaw committed
129 130 131 132 133 134 135
            pristine = code.buffer.stream.tell()
            code.putln(marker)
            start = code.buffer.stream.tell()
            code.call_level += 4
            res = func(*args, **kwds)
            code.call_level -= 4
            if start == code.buffer.stream.tell():
136 137
                # no code written => undo writing marker
                code.buffer.stream.truncate(pristine)
Robert Bradshaw's avatar
Robert Bradshaw committed
138
            else:
139
                marker = marker.replace('->', '<-', 1)
Robert Bradshaw's avatar
Robert Bradshaw committed
140 141 142 143
                code.putln(marker)
            return res
        else:
            return func(*args, **kwds)
144 145
    return f

146

147 148
class VerboseCodeWriter(type):
    # Set this as a metaclass to trace function calls in code.
149
    # This slows down code generation and makes much larger files.
150
    def __new__(cls, name, bases, attrs):
151
        from types import FunctionType
152
        from .Code import CCodeWriter
153 154 155
        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType):
156
                attrs[mname] = write_func_call(m, CCodeWriter)
157 158 159
        return super(VerboseCodeWriter, cls).__new__(cls, name, bases, attrs)


160 161 162 163 164 165 166 167 168 169 170 171 172
class CheckAnalysers(type):
    """Metaclass to check that type analysis functions return a node.
    """
    methods = set(['analyse_types',
                   'analyse_expressions',
                   'analyse_target_types'])

    def __new__(cls, name, bases, attrs):
        from types import FunctionType
        def check(name, func):
            def call(*args, **kwargs):
                retval = func(*args, **kwargs)
                if retval is None:
173
                    print('%s %s %s' % (name, args, kwargs))
174 175 176 177 178 179 180 181 182 183
                return retval
            return call

        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType) and mname in cls.methods:
                attrs[mname] = check(mname, m)
        return super(CheckAnalysers, cls).__new__(cls, name, bases, attrs)


184 185 186 187 188 189 190 191
def _with_metaclass(cls):
    if DebugFlags.debug_trace_code_generation:
        return add_metaclass(VerboseCodeWriter)(cls)
    #return add_metaclass(CheckAnalysers)(cls)
    return cls


@_with_metaclass
Stefan Behnel's avatar
Stefan Behnel committed
192
class Node(object):
William Stein's avatar
William Stein committed
193 194 195
    #  pos         (string, int, int)   Source file position
    #  is_name     boolean              Is a NameNode
    #  is_literal  boolean              Is a ConstNode
196

William Stein's avatar
William Stein committed
197
    is_name = 0
198
    is_none = 0
199
    is_nonecheck = 0
William Stein's avatar
William Stein committed
200
    is_literal = 0
201
    is_terminator = 0
202
    is_wrapper = False  # is a DefNode wrapper for a C function
203
    temps = None
204

Stefan Behnel's avatar
Stefan Behnel committed
205
    # All descendants should set child_attrs to a list of the attributes
206 207
    # containing nodes considered "children" in the tree. Each such attribute
    # can either contain a single node or a list of nodes. See Visitor.py.
208
    child_attrs = None
209

210 211 212
    # Subset of attributes that are evaluated in the outer scope (e.g. function default arguments).
    outer_attrs = None

213 214
    cf_state = None

215 216 217 218 219
    # This may be an additional (or 'actual') type that will be checked when
    # this node is coerced to another type. This could be useful to set when
    # the actual type to which it can coerce is known, but you want to leave
    # the type a py_object_type
    coercion_type = None
220

William Stein's avatar
William Stein committed
221 222 223
    def __init__(self, pos, **kw):
        self.pos = pos
        self.__dict__.update(kw)
224

225 226
    gil_message = "Operation"

227
    nogil_check = None
228
    in_nogil_context = False  # For use only during code generation.
229

230
    def gil_error(self, env=None):
231
        error(self.pos, "%s not allowed without gil" % self.gil_message)
232

Robert Bradshaw's avatar
Robert Bradshaw committed
233
    cpp_message = "Operation"
234

Robert Bradshaw's avatar
Robert Bradshaw committed
235 236 237 238 239 240
    def cpp_check(self, env):
        if not env.is_cpp():
            self.cpp_error()

    def cpp_error(self):
        error(self.pos, "%s only allowed in c++" % self.cpp_message)
241

242 243 244 245
    def clone_node(self):
        """Clone the node. This is defined as a shallow copy, except for member lists
           amongst the child attributes (from get_child_accessors) which are also
           copied. Lists containing child nodes are thus seen as a way for the node
Stefan Behnel's avatar
Stefan Behnel committed
246
           to hold multiple children directly; the list is not treated as a separate
247
           level in the tree."""
248 249 250
        result = copy.copy(self)
        for attrname in result.child_attrs:
            value = getattr(result, attrname)
251
            if isinstance(value, list):
252
                setattr(result, attrname, [x for x in value])
253
        return result
254 255


William Stein's avatar
William Stein committed
256
    #
257
    #  There are 3 phases of parse tree processing, applied in order to
William Stein's avatar
William Stein committed
258 259
    #  all the statements in a given scope-block:
    #
260
    #  (0) analyse_declarations
William Stein's avatar
William Stein committed
261 262 263 264
    #        Make symbol table entries for all declarations at the current
    #        level, both explicit (def, cdef, etc.) and implicit (assignment
    #        to an otherwise undeclared name).
    #
265
    #  (1) analyse_expressions
William Stein's avatar
William Stein committed
266 267
    #         Determine the result types of expressions and fill in the
    #         'type' attribute of each ExprNode. Insert coercion nodes into the
268
    #         tree where needed to convert to and from Python objects.
William Stein's avatar
William Stein committed
269 270 271 272
    #         Allocate temporary locals for intermediate results. Fill
    #         in the 'result_code' attribute of each ExprNode with a C code
    #         fragment.
    #
273
    #  (2) generate_code
William Stein's avatar
William Stein committed
274 275 276 277
    #         Emit C code for all declarations, statements and expressions.
    #         Recursively applies the 3 processing phases to the bodies of
    #         functions.
    #
278

William Stein's avatar
William Stein committed
279 280
    def analyse_declarations(self, env):
        pass
281

William Stein's avatar
William Stein committed
282 283 284
    def analyse_expressions(self, env):
        raise InternalError("analyse_expressions not implemented for %s" % \
            self.__class__.__name__)
285

William Stein's avatar
William Stein committed
286 287 288
    def generate_code(self, code):
        raise InternalError("generate_code not implemented for %s" % \
            self.__class__.__name__)
289

290 291 292 293
    def annotate(self, code):
        # mro does the wrong thing
        if isinstance(self, BlockNode):
            self.body.annotate(code)
294

295 296 297 298
    def end_pos(self):
        try:
            return self._end_pos
        except AttributeError:
Stefan Behnel's avatar
Stefan Behnel committed
299
            pos = self.pos
300 301 302
            if not self.child_attrs:
                self._end_pos = pos
                return pos
303
            for attr in self.child_attrs:
304
                child = getattr(self, attr)
305
                # Sometimes lists, sometimes nodes
306 307 308
                if child is None:
                    pass
                elif isinstance(child, list):
Stefan Behnel's avatar
Stefan Behnel committed
309 310
                    for c in child:
                        pos = max(pos, c.end_pos())
311
                else:
Stefan Behnel's avatar
Stefan Behnel committed
312 313 314
                    pos = max(pos, child.end_pos())
            self._end_pos = pos
            return pos
William Stein's avatar
William Stein committed
315

316
    def dump(self, level=0, filter_out=("pos",), cutoff=100, encountered=None):
317 318
        """Debug helper method that returns a recursive string representation of this node.
        """
319 320
        if cutoff == 0:
            return "<...nesting level cutoff...>"
321 322 323
        if encountered is None:
            encountered = set()
        if id(self) in encountered:
324
            return "<%s (0x%x) -- already output>" % (self.__class__.__name__, id(self))
325
        encountered.add(id(self))
326

327 328
        def dump_child(x, level):
            if isinstance(x, Node):
329
                return x.dump(level, filter_out, cutoff-1, encountered)
330
            elif isinstance(x, list):
Robert Bradshaw's avatar
Robert Bradshaw committed
331
                return "[%s]" % ", ".join([dump_child(item, level) for item in x])
332 333
            else:
                return repr(x)
334

335
        attrs = [(key, value) for key, value in self.__dict__.items() if key not in filter_out]
336
        if len(attrs) == 0:
337
            return "<%s (0x%x)>" % (self.__class__.__name__, id(self))
338 339
        else:
            indent = "  " * level
340
            res = "<%s (0x%x)\n" % (self.__class__.__name__, id(self))
341 342 343 344
            for key, value in attrs:
                res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
            res += "%s>" % indent
            return res
345

346 347 348 349 350 351
    def dump_pos(self, mark_column=False, marker='(#)'):
        """Debug helper method that returns the source code context of this node as a string.
        """
        if not self.pos:
            return u''
        source_desc, line, col = self.pos
352
        contents = source_desc.get_lines(encoding='ASCII', error_handling='ignore')
353
        # line numbers start at 1
354
        lines = contents[max(0, line-3):line]
355 356 357 358 359 360 361 362
        current = lines[-1]
        if mark_column:
            current = current[:col] + marker + current[col:]
        lines[-1] = current.rstrip() + u'             # <<<<<<<<<<<<<<\n'
        lines += contents[line:line+2]
        return u'"%s":%d:%d\n%s\n' % (
            source_desc.get_escaped_description(), line, col, u''.join(lines))

363 364 365 366 367 368 369 370 371 372 373 374 375 376
class CompilerDirectivesNode(Node):
    """
    Sets compiler directives for the children nodes
    """
    #  directives     {string:value}  A dictionary holding the right value for
    #                                 *all* possible directives.
    #  body           Node
    child_attrs = ["body"]

    def analyse_declarations(self, env):
        old = env.directives
        env.directives = self.directives
        self.body.analyse_declarations(env)
        env.directives = old
377

378 379 380
    def analyse_expressions(self, env):
        old = env.directives
        env.directives = self.directives
381
        self.body = self.body.analyse_expressions(env)
382
        env.directives = old
383
        return self
384 385 386 387 388 389 390 391

    def generate_function_definitions(self, env, code):
        env_old = env.directives
        code_old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_function_definitions(env, code)
        env.directives = env_old
        code.globalstate.directives = code_old
392

393 394 395 396 397
    def generate_execution_code(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_execution_code(code)
        code.globalstate.directives = old
398

399 400 401 402 403
    def annotate(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.annotate(code)
        code.globalstate.directives = old
404

Stefan Behnel's avatar
Stefan Behnel committed
405
class BlockNode(object):
William Stein's avatar
William Stein committed
406 407
    #  Mixin class for nodes representing a declaration block.

408
    def generate_cached_builtins_decls(self, env, code):
409
        entries = env.global_scope().undeclared_cached_builtins
410
        for entry in entries:
411
            code.globalstate.add_cached_builtin_decl(entry)
412
        del entries[:]
413 414 415 416

    def generate_lambda_definitions(self, env, code):
        for node in env.lambda_defs:
            node.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
417 418 419

class StatListNode(Node):
    # stats     a list of StatNode
420

421
    child_attrs = ["stats"]
422

423
    @staticmethod
424 425
    def create_analysed(pos, env, *args, **kw):
        node = StatListNode(pos, *args, **kw)
426
        return node  # No node-specific analysis needed
427

William Stein's avatar
William Stein committed
428 429
    def analyse_declarations(self, env):
        #print "StatListNode.analyse_declarations" ###
430
        for stat in self.stats:
William Stein's avatar
William Stein committed
431
            stat.analyse_declarations(env)
432

William Stein's avatar
William Stein committed
433 434
    def analyse_expressions(self, env):
        #print "StatListNode.analyse_expressions" ###
435 436
        self.stats = [stat.analyse_expressions(env)
                      for stat in self.stats]
437
        return self
438

439
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
440 441
        #print "StatListNode.generate_function_definitions" ###
        for stat in self.stats:
442
            stat.generate_function_definitions(env, code)
443

William Stein's avatar
William Stein committed
444 445 446 447 448
    def generate_execution_code(self, code):
        #print "StatListNode.generate_execution_code" ###
        for stat in self.stats:
            code.mark_pos(stat.pos)
            stat.generate_execution_code(code)
449

450 451 452
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)
453

William Stein's avatar
William Stein committed
454 455 456 457 458 459 460 461 462 463 464 465 466

class StatNode(Node):
    #
    #  Code generation for statements is split into the following subphases:
    #
    #  (1) generate_function_definitions
    #        Emit C code for the definitions of any structs,
    #        unions, enums and functions defined in the current
    #        scope-block.
    #
    #  (2) generate_execution_code
    #        Emit C code for executable statements.
    #
467

468
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
469
        pass
470

William Stein's avatar
William Stein committed
471 472 473 474 475 476
    def generate_execution_code(self, code):
        raise InternalError("generate_execution_code not implemented for %s" % \
            self.__class__.__name__)


class CDefExternNode(StatNode):
477 478 479
    #  include_file       string or None
    #  verbatim_include   string or None
    #  body               StatListNode
480

481
    child_attrs = ["body"]
482

William Stein's avatar
William Stein committed
483 484 485 486 487
    def analyse_declarations(self, env):
        old_cinclude_flag = env.in_cinclude
        env.in_cinclude = 1
        self.body.analyse_declarations(env)
        env.in_cinclude = old_cinclude_flag
488

489 490
        if self.include_file or self.verbatim_include:
            # Determine whether include should be late
Jeroen Demeyer's avatar
Jeroen Demeyer committed
491
            stats = self.body.stats
492 493 494
            if not env.directives['preliminary_late_includes_cy28']:
                late = False
            elif not stats:
495 496
                # Special case: empty 'cdef extern' blocks are early
                late = False
Jeroen Demeyer's avatar
Jeroen Demeyer committed
497
            else:
498 499
                late = all(isinstance(node, CVarDefNode) for node in stats)
            env.add_include_file(self.include_file, self.verbatim_include, late)
500

William Stein's avatar
William Stein committed
501
    def analyse_expressions(self, env):
502
        return self
503

William Stein's avatar
William Stein committed
504 505
    def generate_execution_code(self, code):
        pass
506 507 508

    def annotate(self, code):
        self.body.annotate(code)
509

William Stein's avatar
William Stein committed
510 511 512 513 514 515 516 517

class CDeclaratorNode(Node):
    # Part of a C declaration.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #      Returns (name, type) pair where name is the
518
    #      CNameDeclaratorNode of the name being declared
William Stein's avatar
William Stein committed
519 520
    #      and type is the type it is being declared as.
    #
521
    #  calling_convention  string   Calling convention of CFuncDeclaratorNode
522
    #                               for which this is a base
523

524 525
    child_attrs = []

526 527
    calling_convention = ""

528 529 530
    def analyse_templates(self):
        # Only C++ functions have templates.
        return None
William Stein's avatar
William Stein committed
531

532

William Stein's avatar
William Stein committed
533
class CNameDeclaratorNode(CDeclaratorNode):
534
    #  name    string             The Cython name being declared
Robert Bradshaw's avatar
Robert Bradshaw committed
535 536
    #  cname   string or None     C name, if specified
    #  default ExprNode or None   the value assigned on declaration
537

Robert Bradshaw's avatar
Robert Bradshaw committed
538
    child_attrs = ['default']
539

540
    default = None
541

542
    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
543
        if nonempty and self.name == '':
544
            # May have mistaken the name for the type.
545
            if base_type.is_ptr or base_type.is_array or base_type.is_buffer:
546
                error(self.pos, "Missing argument name")
547 548
            elif base_type.is_void:
                error(self.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
549 550 551
            else:
                self.name = base_type.declaration_code("", for_display=1, pyrex=1)
                base_type = py_object_type
552 553 554 555

        if base_type.is_fused and env.fused_to_specific:
            base_type = base_type.specialize(env.fused_to_specific)

556
        self.type = base_type
William Stein's avatar
William Stein committed
557
        return self, base_type
558

559

William Stein's avatar
William Stein committed
560 561
class CPtrDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode
562

563 564
    child_attrs = ["base"]

565 566 567
    def analyse_templates(self):
        return self.base.analyse_templates()

568
    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
William Stein's avatar
William Stein committed
569
        if base_type.is_pyobject:
570
            error(self.pos, "Pointer base type cannot be a Python object")
William Stein's avatar
William Stein committed
571
        ptr_type = PyrexTypes.c_ptr_type(base_type)
572
        return self.base.analyse(ptr_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
573

Danilo Freitas's avatar
Danilo Freitas committed
574 575 576 577 578 579

class CReferenceDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

580 581 582
    def analyse_templates(self):
        return self.base.analyse_templates()

583
    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
Danilo Freitas's avatar
Danilo Freitas committed
584
        if base_type.is_pyobject:
585
            error(self.pos, "Reference base type cannot be a Python object")
586
        ref_type = PyrexTypes.c_ref_type(base_type)
587
        return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
588

Danilo Freitas's avatar
Danilo Freitas committed
589

William Stein's avatar
William Stein committed
590 591 592
class CArrayDeclaratorNode(CDeclaratorNode):
    # base        CDeclaratorNode
    # dimension   ExprNode
593 594

    child_attrs = ["base", "dimension"]
595

596
    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
597
        if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
598
            from .ExprNodes import TupleNode
Robert Bradshaw's avatar
Robert Bradshaw committed
599 600 601 602 603 604 605
            if isinstance(self.dimension, TupleNode):
                args = self.dimension.args
            else:
                args = self.dimension,
            values = [v.analyse_as_type(env) for v in args]
            if None in values:
                ix = values.index(None)
606 607 608 609
                error(args[ix].pos, "Template parameter not a type")
                base_type = error_type
            else:
                base_type = base_type.specialize_here(self.pos, values)
610
            return self.base.analyse(base_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
William Stein's avatar
William Stein committed
611
        if self.dimension:
612
            self.dimension = self.dimension.analyse_const_expression(env)
William Stein's avatar
William Stein committed
613 614
            if not self.dimension.type.is_int:
                error(self.dimension.pos, "Array dimension not integer")
615 616 617 618 619 620 621
            size = self.dimension.get_constant_c_result_code()
            if size is not None:
                try:
                    size = int(size)
                except ValueError:
                    # runtime constant?
                    pass
William Stein's avatar
William Stein committed
622 623 624
        else:
            size = None
        if not base_type.is_complete():
625
            error(self.pos, "Array element type '%s' is incomplete" % base_type)
William Stein's avatar
William Stein committed
626
        if base_type.is_pyobject:
627
            error(self.pos, "Array element cannot be a Python object")
628
        if base_type.is_cfunction:
629
            error(self.pos, "Array element cannot be a function")
William Stein's avatar
William Stein committed
630
        array_type = PyrexTypes.c_array_type(base_type, size)
631
        return self.base.analyse(array_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
William Stein's avatar
William Stein committed
632 633 634 635 636


class CFuncDeclaratorNode(CDeclaratorNode):
    # base             CDeclaratorNode
    # args             [CArgDeclNode]
637
    # templates        [TemplatePlaceholderType]
William Stein's avatar
William Stein committed
638 639 640
    # has_varargs      boolean
    # exception_value  ConstNode
    # exception_check  boolean    True if PyErr_Occurred check needed
641 642
    # nogil            boolean    Can be called without gil
    # with_gil         boolean    Acquire gil around function body
643
    # is_const_method  boolean    Whether this is a const method
644

645 646
    child_attrs = ["base", "args", "exception_value"]

647
    overridable = 0
648
    optional_arg_count = 0
649
    is_const_method = 0
650 651 652 653
    templates = None

    def analyse_templates(self):
        if isinstance(self.base, CArrayDeclaratorNode):
654
            from .ExprNodes import TupleNode, NameNode
655 656 657 658 659 660 661
            template_node = self.base.dimension
            if isinstance(template_node, TupleNode):
                template_nodes = template_node.args
            elif isinstance(template_node, NameNode):
                template_nodes = [template_node]
            else:
                error(template_node.pos, "Template arguments must be a list of names")
662
                return None
663 664 665 666 667 668 669 670 671 672
            self.templates = []
            for template in template_nodes:
                if isinstance(template, NameNode):
                    self.templates.append(PyrexTypes.TemplatePlaceholderType(template.name))
                else:
                    error(template.pos, "Template arguments must be a list of names")
            self.base = self.base.base
            return self.templates
        else:
            return None
William Stein's avatar
William Stein committed
673

674
    def analyse(self, return_type, env, nonempty=0, directive_locals=None, visibility=None, in_pxd=False):
675 676
        if directive_locals is None:
            directive_locals = {}
677 678
        if nonempty:
            nonempty -= 1
William Stein's avatar
William Stein committed
679
        func_type_args = []
680
        for i, arg_node in enumerate(self.args):
681
            name_declarator, type = arg_node.analyse(
682 683
                env, nonempty=nonempty,
                is_self_arg=(i == 0 and env.is_c_class_scope and 'staticmethod' not in env.directives))
William Stein's avatar
William Stein committed
684
            name = name_declarator.name
685 686 687 688 689
            if name in directive_locals:
                type_node = directive_locals[name]
                other_type = type_node.analyse_as_type(env)
                if other_type is None:
                    error(type_node.pos, "Not a type")
Robert Bradshaw's avatar
Robert Bradshaw committed
690
                elif (type is not PyrexTypes.py_object_type
691 692 693 694 695
                      and not type.same_as(other_type)):
                    error(self.base.pos, "Signature does not agree with previous declaration")
                    error(type_node.pos, "Previous declaration here")
                else:
                    type = other_type
William Stein's avatar
William Stein committed
696
            if name_declarator.cname:
697 698
                error(self.pos, "Function argument cannot have C name specification")
            if i == 0 and env.is_c_class_scope and type.is_unspecified:
699 700
                # fix the type of self
                type = env.parent_type
William Stein's avatar
William Stein committed
701 702 703 704 705
            # Turn *[] argument into **
            if type.is_array:
                type = PyrexTypes.c_ptr_type(type.base_type)
            # Catch attempted C-style func(void) decl
            if type.is_void:
Robert Bradshaw's avatar
Robert Bradshaw committed
706
                error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
William Stein's avatar
William Stein committed
707 708 709
            func_type_args.append(
                PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
            if arg_node.default:
710
                self.optional_arg_count += 1
711 712
            elif self.optional_arg_count:
                error(self.pos, "Non-default argument follows default argument")
713

William Stein's avatar
William Stein committed
714 715
        exc_val = None
        exc_check = 0
716
        if self.exception_check == '+':
717
            env.add_include_file('ios')         # for std::ios_base::failure
718
            env.add_include_file('new')         # for std::bad_alloc
719
            env.add_include_file('stdexcept')
720
            env.add_include_file('typeinfo')    # for std::bad_cast
721 722 723
        if (return_type.is_pyobject
                and (self.exception_value or self.exception_check)
                and self.exception_check != '+'):
724
            error(self.pos, "Exception clause not allowed for function returning Python object")
William Stein's avatar
William Stein committed
725
        else:
726 727 728 729 730 731 732 733 734 735
            if self.exception_value is None and self.exception_check and self.exception_check != '+':
                # Use an explicit exception return value to speed up exception checks.
                # Even if it is not declared, we can use the default exception value of the return type,
                # unless the function is some kind of external function that we do not control.
                if return_type.exception_value is not None and (visibility != 'extern' and not in_pxd):
                    # Extension types are more difficult because the signature must match the base type signature.
                    if not env.is_c_class_scope:
                        from .ExprNodes import ConstNode
                        self.exception_value = ConstNode(
                            self.pos, value=return_type.exception_value, type=return_type)
William Stein's avatar
William Stein committed
736
            if self.exception_value:
Robert Bradshaw's avatar
Robert Bradshaw committed
737
                if self.exception_check == '+':
738
                    self.exception_value = self.exception_value.analyse_const_expression(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
739
                    exc_val_type = self.exception_value.type
740 741 742 743
                    if (not exc_val_type.is_error
                            and not exc_val_type.is_pyobject
                            and not (exc_val_type.is_cfunction
                                     and not exc_val_type.return_type.is_pyobject
744 745 746
                                     and not exc_val_type.args)
                            and not (exc_val_type == PyrexTypes.c_char_type
                                     and self.exception_value.value == '*')):
Robert Bradshaw's avatar
Robert Bradshaw committed
747
                        error(self.exception_value.pos,
748
                              "Exception value must be a Python exception or cdef function with no arguments or *.")
749
                    exc_val = self.exception_value
Robert Bradshaw's avatar
Robert Bradshaw committed
750
                else:
751
                    self.exception_value = self.exception_value.analyse_types(env).coerce_to(
752 753 754
                        return_type, env).analyse_const_expression(env)
                    exc_val = self.exception_value.get_constant_c_result_code()
                    if exc_val is None:
755
                        error(self.exception_value.pos, "Exception value must be constant")
756 757 758
                    if not return_type.assignable_from(self.exception_value.type):
                        error(self.exception_value.pos,
                              "Exception value incompatible with function return type")
759 760 761
                    if (visibility != 'extern'
                            and (return_type.is_int or return_type.is_float)
                            and self.exception_value.has_constant_result()):
762 763 764 765 766 767 768 769
                        try:
                            type_default_value = float(return_type.default_value)
                        except ValueError:
                            pass
                        else:
                            if self.exception_value.constant_result == type_default_value:
                                warning(self.pos, "Ambiguous exception value, same as default return value: %r" %
                                        self.exception_value.constant_result)
William Stein's avatar
William Stein committed
770
            exc_check = self.exception_check
771
        if return_type.is_cfunction:
772
            error(self.pos, "Function cannot return a function")
William Stein's avatar
William Stein committed
773
        func_type = PyrexTypes.CFuncType(
774
            return_type, func_type_args, self.has_varargs,
775 776 777 778 779 780
            optional_arg_count=self.optional_arg_count,
            exception_value=exc_val, exception_check=exc_check,
            calling_convention=self.base.calling_convention,
            nogil=self.nogil, with_gil=self.with_gil, is_overridable=self.overridable,
            is_const_method=self.is_const_method,
            templates=self.templates)
781

782
        if self.optional_arg_count:
783 784 785 786 787 788 789 790 791 792 793
            if func_type.is_fused:
                # This is a bit of a hack... When we need to create specialized CFuncTypes
                # on the fly because the cdef is defined in a pxd, we need to declare the specialized optional arg
                # struct
                def declare_opt_arg_struct(func_type, fused_cname):
                    self.declare_optional_arg_struct(func_type, env, fused_cname)

                func_type.declare_opt_arg_struct = declare_opt_arg_struct
            else:
                self.declare_optional_arg_struct(func_type, env)

794 795 796 797 798 799 800
        callspec = env.directives['callspec']
        if callspec:
            current = func_type.calling_convention
            if current and current != callspec:
                error(self.pos, "cannot have both '%s' and '%s' "
                      "calling conventions" % (current, callspec))
            func_type.calling_convention = callspec
801
        return self.base.analyse(func_type, env, visibility=visibility, in_pxd=in_pxd)
William Stein's avatar
William Stein committed
802

803 804 805 806 807 808 809 810 811 812 813
    def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
        """
        Declares the optional argument struct (the struct used to hold the
        values for optional arguments). For fused cdef functions, this is
        deferred as analyse_declarations is called only once (on the fused
        cdef function).
        """
        scope = StructOrUnionScope()
        arg_count_member = '%sn' % Naming.pyrex_prefix
        scope.declare_var(arg_count_member, PyrexTypes.c_int_type, self.pos)

814
        for arg in func_type.args[len(func_type.args) - self.optional_arg_count:]:
815
            scope.declare_var(arg.name, arg.type, arg.pos, allow_pyobject=True, allow_memoryview=True)
816 817 818 819 820 821 822

        struct_cname = env.mangle(Naming.opt_arg_prefix, self.base.name)

        if fused_cname is not None:
            struct_cname = PyrexTypes.get_fused_cname(fused_cname, struct_cname)

        op_args_struct = env.global_scope().declare_struct_or_union(
823 824 825 826 827 828
            name=struct_cname,
            kind='struct',
            scope=scope,
            typedef_flag=0,
            pos=self.pos,
            cname=struct_cname)
829 830 831 832 833 834

        op_args_struct.defined_in_pxd = 1
        op_args_struct.used = 1

        func_type.op_arg_struct = PyrexTypes.c_ptr_type(op_args_struct.type)

William Stein's avatar
William Stein committed
835

836 837 838 839 840
class CConstDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

841
    def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False):
842 843 844 845
        if base_type.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        const = PyrexTypes.c_const_type(base_type)
846
        return self.base.analyse(const, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd)
847 848


William Stein's avatar
William Stein committed
849 850 851 852 853 854
class CArgDeclNode(Node):
    # Item in a function declaration argument list.
    #
    # base_type      CBaseTypeNode
    # declarator     CDeclaratorNode
    # not_none       boolean            Tagged with 'not None'
855 856
    # or_none        boolean            Tagged with 'or None'
    # accept_none    boolean            Resolved boolean for not_none/or_none
William Stein's avatar
William Stein committed
857
    # default        ExprNode or None
858
    # default_value  PyObjectConst      constant for default value
859
    # annotation     ExprNode or None   Py3 function arg annotation
William Stein's avatar
William Stein committed
860
    # is_self_arg    boolean            Is the "self" arg of an extension type method
861
    # is_type_arg    boolean            Is the "class" arg of an extension type classmethod
862
    # kw_only        boolean            Is a keyword-only argument
863
    # is_dynamic     boolean            Non-literal arg stored inside CyFunction
864
    # pos_only       boolean            Is a positional-only argument
865

866
    child_attrs = ["base_type", "declarator", "default", "annotation"]
867
    outer_attrs = ["default", "annotation"]
868

William Stein's avatar
William Stein committed
869
    is_self_arg = 0
870
    is_type_arg = 0
871
    is_generic = 1
872
    kw_only = 0
873
    pos_only = 0
874 875
    not_none = 0
    or_none = 0
876 877
    type = None
    name_declarator = None
878
    default_value = None
879
    annotation = None
880
    is_dynamic = 0
881

882
    def analyse(self, env, nonempty=0, is_self_arg=False):
883 884
        if is_self_arg:
            self.base_type.is_self_arg = self.is_self_arg = True
885
        if self.type is None:
Stefan Behnel's avatar
Stefan Behnel committed
886
            # The parser may misinterpret names as types. We fix that here.
887 888
            if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '':
                if nonempty:
889 890
                    if self.base_type.is_basic_c_type:
                        # char, short, long called "int"
891
                        type = self.base_type.analyse(env, could_be_name=True)
892
                        arg_name = type.empty_declaration_code()
893 894 895
                    else:
                        arg_name = self.base_type.name
                    self.declarator.name = EncodedString(arg_name)
896 897 898 899 900
                    self.base_type.name = None
                    self.base_type.is_basic_c_type = False
                could_be_name = True
            else:
                could_be_name = False
901
            self.base_type.is_arg = True
902
            base_type = self.base_type.analyse(env, could_be_name=could_be_name)
Robert Bradshaw's avatar
Robert Bradshaw committed
903
            if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name:
904
                self.declarator.name = self.base_type.arg_name
905

906 907
            # The parser is unable to resolve the ambiguity of [] as part of the
            # type (e.g. in buffers) or empty declarator (as with arrays).
908
            # This is only arises for empty multi-dimensional arrays.
909 910
            if (base_type.is_array
                    and isinstance(self.base_type, TemplatedTypeNode)
911 912 913 914 915 916
                    and isinstance(self.declarator, CArrayDeclaratorNode)):
                declarator = self.declarator
                while isinstance(declarator.base, CArrayDeclaratorNode):
                    declarator = declarator.base
                declarator.base = self.base_type.array_declarator
                base_type = base_type.base_type
917 918

            # inject type declaration from annotations
919 920
            # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis
            if self.annotation and env and env.directives['annotation_typing'] and self.base_type.name is None:
921 922 923 924
                arg_type = self.inject_type_from_annotations(env)
                if arg_type is not None:
                    base_type = arg_type
            return self.declarator.analyse(base_type, env, nonempty=nonempty)
925
        else:
926
            return self.name_declarator, self.type
William Stein's avatar
William Stein committed
927

928 929 930
    def inject_type_from_annotations(self, env):
        annotation = self.annotation
        if not annotation:
931
            return None
932
        base_type, arg_type = analyse_type_annotation(annotation, env, assigned_value=self.default)
933 934
        if base_type is not None:
            self.base_type = base_type
935 936
        return arg_type

937 938 939 940 941 942 943 944 945
    def calculate_default_value_code(self, code):
        if self.default_value is None:
            if self.default:
                if self.default.is_literal:
                    # will not output any code, just assign the result_code
                    self.default.generate_evaluation_code(code)
                    return self.type.cast_code(self.default.result())
                self.default_value = code.get_argument_default_const(self.type)
        return self.default_value
946

947 948 949 950
    def annotate(self, code):
        if self.default:
            self.default.annotate(code)

951
    def generate_assignment_code(self, code, target=None, overloaded_assignment=False):
952 953 954 955 956 957 958
        default = self.default
        if default is None or default.is_literal:
            return
        if target is None:
            target = self.calculate_default_value_code(code)
        default.generate_evaluation_code(code)
        default.make_owned_reference(code)
959
        result = default.result() if overloaded_assignment else default.result_as(self.type)
960 961 962 963 964 965
        code.putln("%s = %s;" % (target, result))
        if self.type.is_pyobject:
            code.put_giveref(default.result())
        default.generate_post_assignment_code(code)
        default.free_temps(code)

William Stein's avatar
William Stein committed
966 967 968 969 970 971 972 973

class CBaseTypeNode(Node):
    # Abstract base class for C base type nodes.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #     Returns the type.
974

975 976
    def analyse_as_type(self, env):
        return self.analyse(env)
977

Stefan Behnel's avatar
Stefan Behnel committed
978

979 980
class CAnalysedBaseTypeNode(Node):
    # type            type
981

982
    child_attrs = []
983

984
    def analyse(self, env, could_be_name=False):
985
        return self.type
William Stein's avatar
William Stein committed
986

Stefan Behnel's avatar
Stefan Behnel committed
987

William Stein's avatar
William Stein committed
988 989 990 991 992 993
class CSimpleBaseTypeNode(CBaseTypeNode):
    # name             string
    # module_path      [string]     Qualifying name components
    # is_basic_c_type  boolean
    # signed           boolean
    # longness         integer
994
    # complex          boolean
William Stein's avatar
William Stein committed
995
    # is_self_arg      boolean      Is self argument of C method
996
    # ##is_type_arg      boolean      Is type argument of class method
William Stein's avatar
William Stein committed
997

998
    child_attrs = []
999
    arg_name = None   # in case the argument name was interpreted as a type
1000 1001 1002
    module_path = []
    is_basic_c_type = False
    complex = False
1003

1004
    def analyse(self, env, could_be_name=False):
William Stein's avatar
William Stein committed
1005
        # Return type descriptor.
1006
        #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ###
William Stein's avatar
William Stein committed
1007 1008 1009 1010 1011 1012 1013 1014 1015
        type = None
        if self.is_basic_c_type:
            type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name)
            if not type:
                error(self.pos, "Unrecognised type modifier combination")
        elif self.name == "object" and not self.module_path:
            type = py_object_type
        elif self.name is None:
            if self.is_self_arg and env.is_c_class_scope:
1016
                #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ###
William Stein's avatar
William Stein committed
1017
                type = env.parent_type
1018 1019
            ## elif self.is_type_arg and env.is_c_class_scope:
            ##     type = Builtin.type_type
William Stein's avatar
William Stein committed
1020 1021 1022
            else:
                type = py_object_type
        else:
1023
            if self.module_path:
1024 1025 1026 1027
                # Maybe it's a nested C++ class.
                scope = env
                for item in self.module_path:
                    entry = scope.lookup(item)
Robert Bradshaw's avatar
Robert Bradshaw committed
1028
                    if entry is not None and entry.is_cpp_class:
1029 1030 1031 1032
                        scope = entry.type.scope
                    else:
                        scope = None
                        break
1033

1034 1035 1036 1037 1038
                if scope is None:
                    # Maybe it's a cimport.
                    scope = env.find_imported_module(self.module_path, self.pos)
                    if scope:
                        scope.fused_to_specific = env.fused_to_specific
1039 1040
            else:
                scope = env
1041

William Stein's avatar
William Stein committed
1042
            if scope:
1043 1044
                if scope.is_c_class_scope:
                    scope = scope.global_scope()
1045 1046 1047 1048

                type = scope.lookup_type(self.name)
                if type is not None:
                    pass
1049 1050 1051
                elif could_be_name:
                    if self.is_self_arg and env.is_c_class_scope:
                        type = env.parent_type
1052 1053
                    ## elif self.is_type_arg and env.is_c_class_scope:
                    ##     type = Builtin.type_type
1054 1055
                    else:
                        type = py_object_type
1056
                    self.arg_name = EncodedString(self.name)
William Stein's avatar
William Stein committed
1057
                else:
Danilo Freitas's avatar
Danilo Freitas committed
1058 1059 1060
                    if self.templates:
                        if not self.name in self.templates:
                            error(self.pos, "'%s' is not a type identifier" % self.name)
1061
                        type = PyrexTypes.TemplatePlaceholderType(self.name)
Danilo Freitas's avatar
Danilo Freitas committed
1062 1063
                    else:
                        error(self.pos, "'%s' is not a type identifier" % self.name)
1064
        if self.complex:
1065 1066 1067
            if not type.is_numeric or type.is_complex:
                error(self.pos, "can only complexify c numeric types")
            type = PyrexTypes.CComplexType(type)
1068
            type.create_declaration_utility_code(env)
1069 1070 1071 1072 1073 1074 1075 1076
        elif type is Builtin.complex_type:
            # Special case: optimise builtin complex type into C's
            # double complex.  The parser cannot do this (as for the
            # normal scalar types) as the user may have redeclared the
            # 'complex' type.  Testing for the exact type here works.
            type = PyrexTypes.c_double_complex_type
            type.create_declaration_utility_code(env)
            self.complex = True
William Stein's avatar
William Stein committed
1077 1078 1079 1080 1081
        if type:
            return type
        else:
            return PyrexTypes.error_type

1082
class MemoryViewSliceTypeNode(CBaseTypeNode):
1083

1084
    name = 'memoryview'
1085 1086
    child_attrs = ['base_type_node', 'axes']

1087
    def analyse(self, env, could_be_name=False):
1088 1089 1090 1091

        base_type = self.base_type_node.analyse(env)
        if base_type.is_error: return base_type

1092
        from . import MemoryView
1093 1094 1095

        try:
            axes_specs = MemoryView.get_axes_specs(env, self.axes)
1096
        except CompileError as e:
1097 1098 1099 1100
            error(e.position, e.message_only)
            self.type = PyrexTypes.ErrorType()
            return self.type

1101 1102 1103 1104
        if not MemoryView.validate_axes(self.pos, axes_specs):
            self.type = error_type
        else:
            self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
1105
            self.type.validate_memslice_dtype(self.pos)
1106
            self.use_memview_utilities(env)
1107

1108
        return self.type
1109

1110
    def use_memview_utilities(self, env):
1111
        from . import MemoryView
1112 1113 1114
        env.use_utility_code(MemoryView.view_utility_code)


Robert Bradshaw's avatar
Robert Bradshaw committed
1115
class CNestedBaseTypeNode(CBaseTypeNode):
1116
    # For C++ classes that live inside other C++ classes.
Robert Bradshaw's avatar
Robert Bradshaw committed
1117 1118 1119

    # name             string
    # base_type        CBaseTypeNode
Stefan Behnel's avatar
Stefan Behnel committed
1120

Robert Bradshaw's avatar
Robert Bradshaw committed
1121
    child_attrs = ['base_type']
Stefan Behnel's avatar
Stefan Behnel committed
1122

1123
    def analyse(self, env, could_be_name=None):
Robert Bradshaw's avatar
Robert Bradshaw committed
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
        base_type = self.base_type.analyse(env)
        if base_type is PyrexTypes.error_type:
            return PyrexTypes.error_type
        if not base_type.is_cpp_class:
            error(self.pos, "'%s' is not a valid type scope" % base_type)
            return PyrexTypes.error_type
        type_entry = base_type.scope.lookup_here(self.name)
        if not type_entry or not type_entry.is_type:
            error(self.pos, "'%s.%s' is not a type identifier" % (base_type, self.name))
            return PyrexTypes.error_type
        return type_entry.type

1136

Danilo Freitas's avatar
Danilo Freitas committed
1137
class TemplatedTypeNode(CBaseTypeNode):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1138
    #  After parsing:
1139 1140
    #  positional_args  [ExprNode]        List of positional arguments
    #  keyword_args     DictNode          Keyword arguments
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1141 1142 1143
    #  base_type_node   CBaseTypeNode

    #  After analysis:
1144
    #  type             PyrexTypes.BufferType or PyrexTypes.CppClassType  ...containing the right options
1145 1146 1147

    child_attrs = ["base_type_node", "positional_args",
                   "keyword_args", "dtype_node"]
1148

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1149
    dtype_node = None
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1150 1151

    name = None
1152

1153
    def analyse(self, env, could_be_name=False, base_type=None):
1154 1155
        if base_type is None:
            base_type = self.base_type_node.analyse(env)
1156
        if base_type.is_error: return base_type
1157

1158
        if base_type.is_cpp_class and base_type.is_template_type():
1159
            # Templated class
1160
            if self.keyword_args and self.keyword_args.key_value_pairs:
Stefan Behnel's avatar
Stefan Behnel committed
1161
                error(self.pos, "c++ templates cannot take keyword arguments")
1162 1163 1164 1165
                self.type = PyrexTypes.error_type
            else:
                template_types = []
                for template_node in self.positional_args:
1166 1167 1168
                    type = template_node.analyse_as_type(env)
                    if type is None:
                        error(template_node.pos, "unknown type in template argument")
1169
                        type = error_type
1170
                    template_types.append(type)
1171
                self.type = base_type.specialize_here(self.pos, template_types)
1172

1173 1174
        elif base_type.is_pyobject:
            # Buffer
1175
            from . import Buffer
1176

1177 1178 1179 1180 1181 1182
            options = Buffer.analyse_buffer_options(
                self.pos,
                env,
                self.positional_args,
                self.keyword_args,
                base_type.buffer_defaults)
1183

Robert Bradshaw's avatar
Robert Bradshaw committed
1184 1185
            if sys.version_info[0] < 3:
                # Py 2.x enforces byte strings as keyword arguments ...
1186 1187
                options = dict([(name.encode('ASCII'), value)
                                for name, value in options.items()])
Stefan Behnel's avatar
Stefan Behnel committed
1188

1189
            self.type = PyrexTypes.BufferType(base_type, **options)
1190 1191
            if has_np_pythran(env) and is_pythran_buffer(self.type):
                self.type = PyrexTypes.PythranExpr(pythran_type(self.type), self.type)
1192

1193 1194
        else:
            # Array
1195
            empty_declarator = CNameDeclaratorNode(self.pos, name="", cname=None)
1196 1197 1198 1199
            if len(self.positional_args) > 1 or self.keyword_args.key_value_pairs:
                error(self.pos, "invalid array declaration")
                self.type = PyrexTypes.error_type
            else:
1200
                # It would be nice to merge this class with CArrayDeclaratorNode,
1201 1202 1203 1204 1205
                # but arrays are part of the declaration, not the type...
                if not self.positional_args:
                    dimension = None
                else:
                    dimension = self.positional_args[0]
1206 1207 1208 1209
                self.array_declarator = CArrayDeclaratorNode(
                    self.pos,
                    base=empty_declarator,
                    dimension=dimension)
1210
                self.type = self.array_declarator.analyse(base_type, env)[1]
1211

1212 1213 1214
        if self.type.is_fused and env.fused_to_specific:
            self.type = self.type.specialize(env.fused_to_specific)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1215
        return self.type
William Stein's avatar
William Stein committed
1216

1217

William Stein's avatar
William Stein committed
1218 1219 1220
class CComplexBaseTypeNode(CBaseTypeNode):
    # base_type   CBaseTypeNode
    # declarator  CDeclaratorNode
1221

1222 1223
    child_attrs = ["base_type", "declarator"]

1224
    def analyse(self, env, could_be_name=False):
1225
        base = self.base_type.analyse(env, could_be_name)
William Stein's avatar
William Stein committed
1226 1227 1228 1229
        _, type = self.declarator.analyse(base, env)
        return type


1230 1231 1232 1233 1234
class CTupleBaseTypeNode(CBaseTypeNode):
    # components [CBaseTypeNode]

    child_attrs = ["components"]

Robert Bradshaw's avatar
Robert Bradshaw committed
1235
    def analyse(self, env, could_be_name=False):
1236 1237 1238 1239
        component_types = []
        for c in self.components:
            type = c.analyse(env)
            if type.is_pyobject:
1240 1241
                error(c.pos, "Tuple types can't (yet) contain Python objects.")
                return error_type
1242
            component_types.append(type)
1243
        entry = env.declare_tuple_type(self.pos, component_types)
Robert Bradshaw's avatar
Robert Bradshaw committed
1244
        entry.used = True
1245
        return entry.type
1246 1247


1248 1249 1250 1251 1252 1253
class FusedTypeNode(CBaseTypeNode):
    """
    Represents a fused type in a ctypedef statement:

        ctypedef cython.fused_type(int, long, long long) integral

Mark Florisson's avatar
Mark Florisson committed
1254
    name            str                     name of this fused type
1255 1256 1257 1258 1259
    types           [CSimpleBaseTypeNode]   is the list of types to be fused
    """

    child_attrs = []

Mark Florisson's avatar
Mark Florisson committed
1260 1261 1262 1263 1264 1265 1266
    def analyse_declarations(self, env):
        type = self.analyse(env)
        entry = env.declare_typedef(self.name, type, self.pos)

        # Omit the typedef declaration that self.declarator would produce
        entry.in_cinclude = True

1267
    def analyse(self, env, could_be_name=False):
Mark Florisson's avatar
Mark Florisson committed
1268 1269 1270
        types = []
        for type_node in self.types:
            type = type_node.analyse_as_type(env)
1271

Mark Florisson's avatar
Mark Florisson committed
1272 1273 1274
            if not type:
                error(type_node.pos, "Not a type")
                continue
1275

Mark Florisson's avatar
Mark Florisson committed
1276
            if type in types:
1277 1278
                error(type_node.pos, "Type specified multiple times")
            else:
Mark Florisson's avatar
Mark Florisson committed
1279
                types.append(type)
1280

1281 1282
        # if len(self.types) == 1:
        #     return types[0]
Mark Florisson's avatar
Mark Florisson committed
1283 1284

        return PyrexTypes.FusedType(types, name=self.name)
1285

1286

1287
class CConstOrVolatileTypeNode(CBaseTypeNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
1288
    # base_type     CBaseTypeNode
1289 1290
    # is_const      boolean
    # is_volatile   boolean
Robert Bradshaw's avatar
Robert Bradshaw committed
1291 1292 1293

    child_attrs = ["base_type"]

1294
    def analyse(self, env, could_be_name=False):
Robert Bradshaw's avatar
Robert Bradshaw committed
1295 1296 1297
        base = self.base_type.analyse(env, could_be_name)
        if base.is_pyobject:
            error(self.pos,
1298 1299
                  "Const/volatile base type cannot be a Python object")
        return PyrexTypes.c_const_or_volatile_type(base, self.is_const, self.is_volatile)
Robert Bradshaw's avatar
Robert Bradshaw committed
1300 1301


William Stein's avatar
William Stein committed
1302 1303 1304 1305 1306 1307
class CVarDefNode(StatNode):
    #  C variable definition or forward/extern function declaration.
    #
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarators   [CDeclaratorNode]
1308
    #  in_pxd        boolean
Stefan Behnel's avatar
Stefan Behnel committed
1309
    #  api           boolean
1310
    #  overridable   boolean        whether it is a cpdef
1311
    #  modifiers     ['inline']
1312

1313
    #  decorators    [cython.locals(...)] or None
1314
    #  directive_locals { string : NameNode } locals defined by cython.locals(...)
1315 1316

    child_attrs = ["base_type", "declarators"]
1317

Robert Bradshaw's avatar
Robert Bradshaw committed
1318
    decorators = None
1319
    directive_locals = None
1320

1321
    def analyse_declarations(self, env, dest_scope=None):
1322 1323
        if self.directive_locals is None:
            self.directive_locals = {}
William Stein's avatar
William Stein committed
1324 1325
        if not dest_scope:
            dest_scope = env
1326
        self.dest_scope = dest_scope
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341

        if self.declarators:
            templates = self.declarators[0].analyse_templates()
        else:
            templates = None
        if templates is not None:
            if self.visibility != 'extern':
                error(self.pos, "Only extern functions allowed")
            if len(self.declarators) > 1:
                error(self.declarators[1].pos, "Can't multiply declare template types")
            env = TemplateScope('func_template', env)
            env.directives = env.outer_scope.directives
            for template_param in templates:
                env.declare_type(template_param.name, template_param, self.pos)

William Stein's avatar
William Stein committed
1342
        base_type = self.base_type.analyse(env)
1343

1344 1345
        if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or
                                                       env.is_module_scope):
1346 1347 1348
            error(self.pos, "Fused types not allowed here")
            return error_type

1349
        self.entry = None
1350
        visibility = self.visibility
1351

William Stein's avatar
William Stein committed
1352
        for declarator in self.declarators:
1353

1354
            if (len(self.declarators) > 1
1355 1356 1357 1358 1359
                    and not isinstance(declarator, CNameDeclaratorNode)
                    and env.directives['warn.multiple_declarators']):
                warning(
                    declarator.pos,
                    "Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). "
1360
                    "Each pointer declaration should be on its own line.", 1)
1361

1362
            create_extern_wrapper = (self.overridable
1363 1364
                                     and self.visibility == 'extern'
                                     and env.is_module_scope)
1365 1366
            if create_extern_wrapper:
                declarator.overridable = False
1367
            if isinstance(declarator, CFuncDeclaratorNode):
1368 1369
                name_declarator, type = declarator.analyse(
                    base_type, env, directive_locals=self.directive_locals, visibility=visibility, in_pxd=self.in_pxd)
1370
            else:
1371 1372
                name_declarator, type = declarator.analyse(
                    base_type, env, visibility=visibility, in_pxd=self.in_pxd)
William Stein's avatar
William Stein committed
1373
            if not type.is_complete():
1374
                if not (self.visibility == 'extern' and type.is_array or type.is_memoryviewslice):
1375
                    error(declarator.pos, "Variable type '%s' is incomplete" % type)
William Stein's avatar
William Stein committed
1376
            if self.visibility == 'extern' and type.is_pyobject:
1377
                error(declarator.pos, "Python object cannot be declared extern")
William Stein's avatar
William Stein committed
1378 1379
            name = name_declarator.name
            cname = name_declarator.cname
1380 1381 1382
            if name == '':
                error(declarator.pos, "Missing name in declaration.")
                return
1383 1384
            if type.is_reference and self.visibility != 'extern':
                error(declarator.pos, "C++ references cannot be declared; use a pointer instead")
William Stein's avatar
William Stein committed
1385
            if type.is_cfunction:
1386 1387
                if 'staticmethod' in env.directives:
                    type.is_static_method = True
1388 1389
                self.entry = dest_scope.declare_cfunction(
                    name, type, declarator.pos,
1390 1391
                    cname=cname, visibility=self.visibility, in_pxd=self.in_pxd,
                    api=self.api, modifiers=self.modifiers, overridable=self.overridable)
1392 1393
                if self.entry is not None:
                    self.entry.directive_locals = copy.copy(self.directive_locals)
1394
                if create_extern_wrapper:
1395
                    self.entry.type.create_to_py_utility_code(env)
1396
                    self.entry.create_wrapper = True
William Stein's avatar
William Stein committed
1397
            else:
1398
                if self.directive_locals:
1399
                    error(self.pos, "Decorators can only be followed by functions")
1400 1401 1402 1403
                self.entry = dest_scope.declare_var(
                    name, type, declarator.pos,
                    cname=cname, visibility=visibility, in_pxd=self.in_pxd,
                    api=self.api, is_cdef=1)
1404 1405
                if Options.docstrings:
                    self.entry.doc = embed_position(self.pos, self.doc)
1406

William Stein's avatar
William Stein committed
1407 1408 1409 1410 1411 1412

class CStructOrUnionDefNode(StatNode):
    #  name          string
    #  cname         string or None
    #  kind          "struct" or "union"
    #  typedef_flag  boolean
1413
    #  visibility    "public" or "private"
1414
    #  api           boolean
Stefan Behnel's avatar
Stefan Behnel committed
1415
    #  in_pxd        boolean
William Stein's avatar
William Stein committed
1416 1417
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1418
    #  packed        boolean
1419

1420
    child_attrs = ["attributes"]
Vitja Makarov's avatar
Vitja Makarov committed
1421

1422
    def declare(self, env, scope=None):
William Stein's avatar
William Stein committed
1423 1424
        self.entry = env.declare_struct_or_union(
            self.name, self.kind, scope, self.typedef_flag, self.pos,
1425 1426
            self.cname, visibility=self.visibility, api=self.api,
            packed=self.packed)
1427 1428 1429 1430 1431 1432

    def analyse_declarations(self, env):
        scope = None
        if self.attributes is not None:
            scope = StructOrUnionScope(self.name)
        self.declare(env, scope)
William Stein's avatar
William Stein committed
1433
        if self.attributes is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1434 1435
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
William Stein's avatar
William Stein committed
1436 1437
            for attr in self.attributes:
                attr.analyse_declarations(env, scope)
1438 1439 1440 1441
            if self.visibility != 'extern':
                for attr in scope.var_entries:
                    type = attr.type
                    while type.is_array:
1442 1443
                        type = type.base_type
                    if type == self.entry.type:
1444
                        error(attr.pos, "Struct cannot contain itself as a member.")
1445

William Stein's avatar
William Stein committed
1446
    def analyse_expressions(self, env):
1447
        return self
1448

William Stein's avatar
William Stein committed
1449 1450 1451 1452
    def generate_execution_code(self, code):
        pass


1453
class CppClassNode(CStructOrUnionDefNode, BlockNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
1454 1455 1456

    #  name          string
    #  cname         string or None
1457
    #  visibility    "extern"
Robert Bradshaw's avatar
Robert Bradshaw committed
1458 1459 1460
    #  in_pxd        boolean
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1461
    #  base_classes  [CBaseTypeNode]
1462
    #  templates     [(string, bool)] or None
1463 1464 1465
    #  decorators    [DecoratorNode] or None

    decorators = None
Robert Bradshaw's avatar
Robert Bradshaw committed
1466

1467 1468 1469 1470
    def declare(self, env):
        if self.templates is None:
            template_types = None
        else:
1471 1472 1473 1474 1475
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
            num_optional_templates = sum(not required for _, required in self.templates)
            if num_optional_templates and not all(required for _, required in self.templates[:-num_optional_templates]):
                error(self.pos, "Required template parameters must precede optional template parameters.")
1476
        self.entry = env.declare_cpp_class(
1477 1478
            self.name, None, self.pos, self.cname,
            base_classes=[], visibility=self.visibility, templates=template_types)
1479

Robert Bradshaw's avatar
Robert Bradshaw committed
1480
    def analyse_declarations(self, env):
1481 1482 1483 1484 1485 1486
        if self.templates is None:
            template_types = template_names = None
        else:
            template_names = [template_name for template_name, _ in self.templates]
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
Robert Bradshaw's avatar
Robert Bradshaw committed
1487
        scope = None
1488
        if self.attributes is not None:
1489
            scope = CppClassScope(self.name, env, templates=template_names)
1490 1491 1492 1493 1494 1495
        def base_ok(base_class):
            if base_class.is_cpp_class or base_class.is_struct:
                return True
            else:
                error(self.pos, "Base class '%s' not a struct or class." % base_class)
        base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
1496
        self.entry = env.declare_cpp_class(
1497
            self.name, scope, self.pos,
1498
            self.cname, base_class_types, visibility=self.visibility, templates=template_types)
1499 1500
        if self.entry is None:
            return
Danilo Freitas's avatar
Danilo Freitas committed
1501
        self.entry.is_cpp_class = 1
Stefan Behnel's avatar
Stefan Behnel committed
1502 1503
        if scope is not None:
            scope.type = self.entry.type
1504
        defined_funcs = []
1505 1506 1507 1508 1509 1510 1511
        def func_attributes(attributes):
            for attr in attributes:
                if isinstance(attr, CFuncDefNode):
                    yield attr
                elif isinstance(attr, CompilerDirectivesNode):
                    for sub_attr in func_attributes(attr.body.stats):
                        yield sub_attr
Robert Bradshaw's avatar
Robert Bradshaw committed
1512 1513 1514 1515
        if self.attributes is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for attr in self.attributes:
1516 1517 1518
                declare = getattr(attr, 'declare', None)
                if declare:
                    attr.declare(scope)
Robert Bradshaw's avatar
Robert Bradshaw committed
1519
                attr.analyse_declarations(scope)
1520 1521 1522
            for func in func_attributes(self.attributes):
                defined_funcs.append(func)
                if self.templates is not None:
1523
                    func.template_declaration = "template <typename %s>" % ", typename ".join(template_names)
1524 1525 1526 1527
        self.body = StatListNode(self.pos, stats=defined_funcs)
        self.scope = scope

    def analyse_expressions(self, env):
1528 1529
        self.body = self.body.analyse_expressions(self.entry.type.scope)
        return self
1530 1531 1532 1533 1534 1535 1536 1537 1538 1539

    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(self.entry.type.scope, code)

    def generate_execution_code(self, code):
        self.body.generate_execution_code(code)

    def annotate(self, code):
        self.body.annotate(code)

Robert Bradshaw's avatar
Robert Bradshaw committed
1540

William Stein's avatar
William Stein committed
1541 1542 1543 1544 1545
class CEnumDefNode(StatNode):
    #  name           string or None
    #  cname          string or None
    #  items          [CEnumDefItemNode]
    #  typedef_flag   boolean
1546
    #  visibility     "public" or "private" or "extern"
1547
    #  api            boolean
Stefan Behnel's avatar
Stefan Behnel committed
1548
    #  in_pxd         boolean
1549
    #  create_wrapper boolean
William Stein's avatar
William Stein committed
1550
    #  entry          Entry
Robert Bradshaw's avatar
Robert Bradshaw committed
1551

1552
    child_attrs = ["items"]
1553

1554
    def declare(self, env):
1555 1556 1557 1558
         self.entry = env.declare_enum(
             self.name, self.pos,
             cname=self.cname, typedef_flag=self.typedef_flag,
             visibility=self.visibility, api=self.api,
1559
             create_wrapper=self.create_wrapper)
1560

William Stein's avatar
William Stein committed
1561
    def analyse_declarations(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
1562 1563 1564 1565 1566
        if self.items is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for item in self.items:
                item.analyse_declarations(env, self.entry)
William Stein's avatar
William Stein committed
1567

1568
    def analyse_expressions(self, env):
1569
        return self
1570

William Stein's avatar
William Stein committed
1571
    def generate_execution_code(self, code):
1572
        if self.visibility == 'public' or self.api:
1573
            code.mark_pos(self.pos)
1574
            temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
1575 1576
            for item in self.entry.enum_values:
                code.putln("%s = PyInt_FromLong(%s); %s" % (
1577 1578 1579
                    temp,
                    item.cname,
                    code.error_goto_if_null(temp, item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1580
                code.put_gotref(temp)
1581
                code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % (
1582 1583 1584 1585
                    Naming.moddict_cname,
                    item.name,
                    temp,
                    code.error_goto(item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1586
                code.put_decref_clear(temp, PyrexTypes.py_object_type)
1587
            code.funcstate.release_temp(temp)
William Stein's avatar
William Stein committed
1588 1589 1590 1591 1592 1593


class CEnumDefItemNode(StatNode):
    #  name     string
    #  cname    string or None
    #  value    ExprNode or None
1594

1595 1596
    child_attrs = ["value"]

William Stein's avatar
William Stein committed
1597 1598
    def analyse_declarations(self, env, enum_entry):
        if self.value:
1599
            self.value = self.value.analyse_const_expression(env)
1600 1601
            if not self.value.type.is_int:
                self.value = self.value.coerce_to(PyrexTypes.c_int_type, env)
1602
                self.value = self.value.analyse_const_expression(env)
1603 1604 1605 1606 1607
        entry = env.declare_const(
            self.name, enum_entry.type,
            self.value, self.pos, cname=self.cname,
            visibility=enum_entry.visibility, api=enum_entry.api,
            create_wrapper=enum_entry.create_wrapper and enum_entry.name is None)
William Stein's avatar
William Stein committed
1608
        enum_entry.enum_values.append(entry)
1609
        if enum_entry.name:
1610
            enum_entry.type.values.append(entry.name)
William Stein's avatar
William Stein committed
1611 1612 1613


class CTypeDefNode(StatNode):
Stefan Behnel's avatar
Stefan Behnel committed
1614 1615 1616
    #  base_type    CBaseTypeNode
    #  declarator   CDeclaratorNode
    #  visibility   "public" or "private"
1617
    #  api          boolean
Stefan Behnel's avatar
Stefan Behnel committed
1618
    #  in_pxd       boolean
1619 1620

    child_attrs = ["base_type", "declarator"]
1621

William Stein's avatar
William Stein committed
1622 1623
    def analyse_declarations(self, env):
        base = self.base_type.analyse(env)
1624 1625
        name_declarator, type = self.declarator.analyse(
            base, env, visibility=self.visibility, in_pxd=self.in_pxd)
William Stein's avatar
William Stein committed
1626 1627
        name = name_declarator.name
        cname = name_declarator.cname
1628

1629 1630 1631
        entry = env.declare_typedef(
            name, type, self.pos,
            cname=cname, visibility=self.visibility, api=self.api)
1632

Mark Florisson's avatar
Mark Florisson committed
1633
        if type.is_fused:
1634 1635
            entry.in_cinclude = True

Mark Florisson's avatar
Mark Florisson committed
1636 1637
        if self.in_pxd and not env.in_cinclude:
            entry.defined_in_pxd = 1
1638

William Stein's avatar
William Stein committed
1639
    def analyse_expressions(self, env):
1640 1641
        return self

William Stein's avatar
William Stein committed
1642 1643 1644 1645 1646 1647 1648 1649 1650 1651
    def generate_execution_code(self, code):
        pass


class FuncDefNode(StatNode, BlockNode):
    #  Base class for function definition nodes.
    #
    #  return_type     PyrexType
    #  #filename        string        C name of filename string const
    #  entry           Symtab.Entry
1652
    #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
Vitja Makarov's avatar
Vitja Makarov committed
1653
    #  needs_outer_scope boolean      Whether or not this function requires outer scope
1654
    #  pymethdef_required boolean     Force Python method struct generation
1655 1656
    #  directive_locals { string : ExprNode } locals defined by cython.locals(...)
    #  directive_returns [ExprNode] type defined by cython.returns(...)
1657 1658 1659 1660 1661
    #  star_arg      PyArgDeclNode or None  * argument
    #  starstar_arg  PyArgDeclNode or None  ** argument
    #
    #  is_async_def  boolean          is a Coroutine function
    #
1662 1663 1664 1665 1666
    #  has_fused_arguments  boolean
    #       Whether this cdef function has fused parameters. This is needed
    #       by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
    #       with fused argument types with a FusedCFuncDefNode

1667
    py_func = None
1668
    needs_closure = False
Vitja Makarov's avatar
Vitja Makarov committed
1669
    needs_outer_scope = False
1670
    pymethdef_required = False
1671
    is_generator = False
1672
    is_generator_body = False
1673
    is_async_def = False
Robert Bradshaw's avatar
Robert Bradshaw committed
1674
    modifiers = []
1675
    has_fused_arguments = False
1676 1677
    star_arg = None
    starstar_arg = None
1678
    is_cyfunction = False
1679
    code_object = None
1680

1681
    def analyse_default_values(self, env):
1682
        default_seen = 0
1683 1684
        for arg in self.args:
            if arg.default:
1685
                default_seen = 1
1686
                if arg.is_generic:
1687
                    arg.default = arg.default.analyse_types(env)
1688
                    arg.default = arg.default.coerce_to(arg.type, env)
1689
                else:
1690
                    error(arg.pos, "This argument cannot have a default value")
1691
                    arg.default = None
1692 1693 1694 1695
            elif arg.kw_only:
                default_seen = 1
            elif default_seen:
                error(arg.pos, "Non-default argument following default argument")
1696

1697
    def analyse_annotation(self, env, annotation):
Stefan Behnel's avatar
Stefan Behnel committed
1698
        # Annotations can not only contain valid Python expressions but arbitrary type references.
1699 1700
        if annotation is None:
            return None
1701
        if not env.directives['annotation_typing'] or annotation.analyse_as_type(env) is None:
1702 1703 1704
            annotation = annotation.analyse_types(env)
        return annotation

1705 1706 1707
    def analyse_annotations(self, env):
        for arg in self.args:
            if arg.annotation:
1708
                arg.annotation = self.analyse_annotation(env, arg.annotation)
1709

1710
    def align_argument_type(self, env, arg):
1711
        # @cython.locals()
1712
        directive_locals = self.directive_locals
1713
        orig_type = arg.type
1714 1715 1716
        if arg.name in directive_locals:
            type_node = directive_locals[arg.name]
            other_type = type_node.analyse_as_type(env)
1717
        elif isinstance(arg, CArgDeclNode) and arg.annotation and env.directives['annotation_typing']:
1718 1719
            type_node = arg.annotation
            other_type = arg.inject_type_from_annotations(env)
1720 1721
            if other_type is None:
                return arg
1722 1723 1724 1725
        else:
            return arg
        if other_type is None:
            error(type_node.pos, "Not a type")
1726
        elif orig_type is not py_object_type and not orig_type.same_as(other_type):
1727 1728 1729 1730
            error(arg.base_type.pos, "Signature does not agree with previous declaration")
            error(type_node.pos, "Previous declaration here")
        else:
            arg.type = other_type
1731 1732
        return arg

1733 1734
    def need_gil_acquisition(self, lenv):
        return 0
1735

1736 1737
    def create_local_scope(self, env):
        genv = env
1738
        while genv.is_py_class_scope or genv.is_c_class_scope:
1739
            genv = genv.outer_scope
1740
        if self.needs_closure:
1741
            lenv = ClosureScope(name=self.entry.name,
1742 1743
                                outer_scope=genv,
                                parent_scope=env,
1744
                                scope_name=self.entry.cname)
1745
        else:
1746 1747 1748
            lenv = LocalScope(name=self.entry.name,
                              outer_scope=genv,
                              parent_scope=env)
William Stein's avatar
William Stein committed
1749
        lenv.return_type = self.return_type
1750 1751 1752
        type = self.entry.type
        if type.is_cfunction:
            lenv.nogil = type.nogil and not type.with_gil
1753
        self.local_scope = lenv
1754
        lenv.directives = env.directives
1755
        return lenv
1756

1757 1758 1759
    def generate_function_body(self, env, code):
        self.body.generate_execution_code(code)

1760
    def generate_function_definitions(self, env, code):
1761
        from . import Buffer
1762
        if self.return_type.is_memoryviewslice:
1763
            from . import MemoryView
1764 1765

        lenv = self.local_scope
Vitja Makarov's avatar
Vitja Makarov committed
1766
        if lenv.is_closure_scope and not lenv.is_passthrough:
1767 1768 1769 1770 1771
            outer_scope_cname = "%s->%s" % (Naming.cur_scope_cname,
                                            Naming.outer_scope_cname)
        else:
            outer_scope_cname = Naming.outer_scope_cname
        lenv.mangle_closure_cnames(outer_scope_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
1772 1773
        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)
Stefan Behnel's avatar
Stefan Behnel committed
1774
        # generate lambda function definitions
1775
        self.generate_lambda_definitions(lenv, code)
1776

1777 1778
        is_getbuffer_slot = (self.entry.name == "__getbuffer__" and
                             self.entry.scope.is_c_class_scope)
1779 1780 1781
        is_releasebuffer_slot = (self.entry.name == "__releasebuffer__" and
                                 self.entry.scope.is_c_class_scope)
        is_buffer_slot = is_getbuffer_slot or is_releasebuffer_slot
1782
        if is_buffer_slot:
1783 1784
            if 'cython_unused' not in self.modifiers:
                self.modifiers = self.modifiers + ['cython_unused']
1785

1786
        preprocessor_guard = self.get_preprocessor_guard()
1787

1788
        profile = code.globalstate.directives['profile']
1789 1790
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
1791 1792
            if linetrace:
                code.use_fast_gil_utility_code()
1793 1794
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("Profile", "Profile.c"))
1795

1796
        # Generate C code for header and body of function
1797
        code.enter_cfunc_scope(lenv)
1798
        code.return_from_error_cleanup_label = code.new_label()
1799
        code.funcstate.gil_owned = not lenv.nogil
1800

William Stein's avatar
William Stein committed
1801
        # ----- Top-level constants used by this function
1802
        code.mark_pos(self.pos)
1803
        self.generate_cached_builtins_decls(lenv, code)
William Stein's avatar
William Stein committed
1804 1805
        # ----- Function header
        code.putln("")
1806 1807 1808 1809

        if preprocessor_guard:
            code.putln(preprocessor_guard)

1810 1811
        with_pymethdef = (self.needs_assignment_synthesis(env, code) or
                          self.pymethdef_required)
1812
        if self.py_func:
1813 1814 1815
            self.py_func.generate_function_header(
                code, with_pymethdef=with_pymethdef, proto_only=True)
        self.generate_function_header(code, with_pymethdef=with_pymethdef)
William Stein's avatar
William Stein committed
1816
        # ----- Local variable declarations
1817 1818 1819 1820
        # Find function scope
        cenv = env
        while cenv.is_py_class_scope or cenv.is_c_class_scope:
            cenv = cenv.outer_scope
Vitja Makarov's avatar
Vitja Makarov committed
1821
        if self.needs_closure:
1822
            code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
Robert Bradshaw's avatar
Robert Bradshaw committed
1823
            code.putln(";")
Vitja Makarov's avatar
Vitja Makarov committed
1824 1825 1826 1827
        elif self.needs_outer_scope:
            if lenv.is_passthrough:
                code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
                code.putln(";")
1828
            code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
1829
            code.putln(";")
William Stein's avatar
William Stein committed
1830
        self.generate_argument_declarations(lenv, code)
1831

1832
        for entry in lenv.var_entries:
1833
            if not (entry.in_closure or entry.is_arg):
1834
                code.put_var_declaration(entry)
1835

1836
        # Initialize the return variable __pyx_r
William Stein's avatar
William Stein committed
1837 1838
        init = ""
        if not self.return_type.is_void:
1839 1840
            if self.return_type.is_pyobject:
                init = " = NULL"
1841
            elif self.return_type.is_memoryviewslice:
1842
                init = ' = ' + MemoryView.memslice_entry_init
1843

1844 1845 1846
            code.putln("%s%s;" % (
                self.return_type.declaration_code(Naming.retval_cname),
                init))
1847

1848
        tempvardecl_code = code.insertion_point()
William Stein's avatar
William Stein committed
1849
        self.generate_keyword_list(code)
1850

1851
        # ----- GIL acquisition
1852
        acquire_gil = self.acquire_gil
1853

Mark Florisson's avatar
Mark Florisson committed
1854 1855
        # See if we need to acquire the GIL for variable declarations, or for
        # refnanny only
1856

1857 1858 1859
        # Closures are not currently possible for cdef nogil functions,
        # but check them anyway
        have_object_args = self.needs_closure or self.needs_outer_scope
Mark Florisson's avatar
Mark Florisson committed
1860 1861 1862 1863 1864
        for arg in lenv.arg_entries:
            if arg.type.is_pyobject:
                have_object_args = True
                break

1865 1866
        used_buffer_entries = [entry for entry in lenv.buffer_entries if entry.used]

Mark Florisson's avatar
Mark Florisson committed
1867
        acquire_gil_for_var_decls_only = (
1868 1869
            lenv.nogil and lenv.has_with_gil_block and
            (have_object_args or used_buffer_entries))
Mark Florisson's avatar
Mark Florisson committed
1870 1871

        acquire_gil_for_refnanny_only = (
1872 1873
            lenv.nogil and lenv.has_with_gil_block and not
            acquire_gil_for_var_decls_only)
Mark Florisson's avatar
Mark Florisson committed
1874 1875

        use_refnanny = not lenv.nogil or lenv.has_with_gil_block
1876 1877 1878

        if acquire_gil or acquire_gil_for_var_decls_only:
            code.put_ensure_gil()
1879
            code.funcstate.gil_owned = True
1880 1881
        elif lenv.nogil and lenv.has_with_gil_block:
            code.declare_gilstate()
1882

1883
        if profile or linetrace:
1884 1885 1886 1887 1888
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                tempvardecl_code.put_trace_declarations()
                code_object = self.code_object.calculate_result_code(code) if self.code_object else None
                code.put_trace_frame_init(code_object)
1889

1890 1891 1892 1893
        # ----- Special check for getbuffer
        if is_getbuffer_slot:
            self.getbuffer_check(code)

1894
        # ----- set up refnanny
1895
        if use_refnanny:
1896
            tempvardecl_code.put_declare_refcount_context()
1897 1898
            code.put_setup_refcount_context(
                self.entry.name, acquire_gil=acquire_gil_for_refnanny_only)
Mark Florisson's avatar
Mark Florisson committed
1899

1900
        # ----- Automatic lead-ins for certain special functions
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1901 1902
        if is_getbuffer_slot:
            self.getbuffer_init(code)
1903
        # ----- Create closure scope object
Robert Bradshaw's avatar
Robert Bradshaw committed
1904
        if self.needs_closure:
1905
            tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
1906 1907 1908 1909
            slot_func_cname = TypeSlots.get_slot_function(lenv.scope_class.type.scope, tp_slot)
            if not slot_func_cname:
                slot_func_cname = '%s->tp_new' % lenv.scope_class.type.typeptr_cname
            code.putln("%s = (%s)%s(%s, %s, NULL);" % (
1910
                Naming.cur_scope_cname,
1911
                lenv.scope_class.type.empty_declaration_code(),
1912
                slot_func_cname,
1913
                lenv.scope_class.type.typeptr_cname,
1914 1915
                Naming.empty_tuple))
            code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
1916 1917 1918
            # Scope unconditionally DECREFed on return.
            code.putln("%s = %s;" % (
                Naming.cur_scope_cname,
1919 1920
                lenv.scope_class.type.cast_code("Py_None")))
            code.put_incref("Py_None", py_object_type)
1921 1922
            code.putln(code.error_goto(self.pos))
            code.putln("} else {")
Robert Bradshaw's avatar
Robert Bradshaw committed
1923
            code.put_gotref(Naming.cur_scope_cname)
1924
            code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
1925
            # Note that it is unsafe to decref the scope at this point.
Vitja Makarov's avatar
Vitja Makarov committed
1926
        if self.needs_outer_scope:
1927 1928 1929
            if self.is_cyfunction:
                code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
                    outer_scope_cname,
1930
                    cenv.scope_class.type.empty_declaration_code(),
1931 1932 1933 1934
                    Naming.self_cname))
            else:
                code.putln("%s = (%s) %s;" % (
                    outer_scope_cname,
1935
                    cenv.scope_class.type.empty_declaration_code(),
1936
                    Naming.self_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1937
            if lenv.is_passthrough:
Stefan Behnel's avatar
Stefan Behnel committed
1938
                code.putln("%s = %s;" % (Naming.cur_scope_cname, outer_scope_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1939
            elif self.needs_closure:
1940
                # inner closures own a reference to their outer parent
1941
                code.put_incref(outer_scope_cname, cenv.scope_class.type)
1942
                code.put_giveref(outer_scope_cname)
1943
        # ----- Trace function call
1944
        if profile or linetrace:
1945 1946
            # this looks a bit late, but if we don't get here due to a
            # fatal error before hand, it's not really worth tracing
1947 1948 1949 1950 1951 1952 1953 1954
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.is_wrapper:
                    trace_name = self.entry.name + " (wrapper)"
                else:
                    trace_name = self.entry.name
                code.put_trace_call(
                    trace_name, self.pos, nogil=not code.funcstate.gil_owned)
1955
            code.funcstate.can_trace = True
William Stein's avatar
William Stein committed
1956
        # ----- Fetch arguments
1957
        self.generate_argument_parsing_code(env, code)
1958
        # If an argument is assigned to in the body, we must
Robert Bradshaw's avatar
Robert Bradshaw committed
1959
        # incref it to properly keep track of refcounts.
1960
        is_cdef = isinstance(self, CFuncDefNode)
Robert Bradshaw's avatar
Robert Bradshaw committed
1961
        for entry in lenv.arg_entries:
1962
            if entry.type.is_pyobject:
1963
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
1964
                    code.put_var_incref(entry)
1965

1966
            # Note: defaults are always incref-ed. For def functions, we
luz.paz's avatar
luz.paz committed
1967
            #       we acquire arguments from object conversion, so we have
1968 1969
            #       new references. If we are a cdef function, we need to
            #       incref our arguments
1970
            elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1:
1971
                code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
1972
        for entry in lenv.var_entries:
1973
            if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure:
1974 1975 1976 1977
                if entry.xdecref_cleanup:
                    code.put_var_xincref(entry)
                else:
                    code.put_var_incref(entry)
1978

1979 1980
        # ----- Initialise local buffer auxiliary variables
        for entry in lenv.var_entries + lenv.arg_entries:
1981 1982
            if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                Buffer.put_init_vars(entry, code)
1983

1984
        # ----- Check and convert arguments
William Stein's avatar
William Stein committed
1985
        self.generate_argument_type_tests(code)
1986 1987 1988
        # ----- Acquire buffer arguments
        for entry in lenv.arg_entries:
            if entry.type.is_buffer:
1989 1990
                Buffer.put_acquire_arg_buffer(entry, code, self.pos)

1991 1992
        if acquire_gil_for_var_decls_only:
            code.put_release_ensured_gil()
1993
            code.funcstate.gil_owned = False
1994

1995 1996 1997
        # -------------------------
        # ----- Function body -----
        # -------------------------
1998
        self.generate_function_body(env, code)
1999

2000
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
2001
        code.putln("")
Robert Bradshaw's avatar
Robert Bradshaw committed
2002
        code.putln("/* function exit code */")
2003 2004

        # ----- Default return value
2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015
        if not self.body.is_terminator:
            if self.return_type.is_pyobject:
                #if self.return_type.is_extension_type:
                #    lhs = "(PyObject *)%s" % Naming.retval_cname
                #else:
                lhs = Naming.retval_cname
                code.put_init_to_py_none(lhs, self.return_type)
            else:
                val = self.return_type.default_value
                if val:
                    code.putln("%s = %s;" % (Naming.retval_cname, val))
2016 2017
                elif not self.return_type.is_void:
                    code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
William Stein's avatar
William Stein committed
2018
        # ----- Error cleanup
2019
        if code.error_label in code.labels_used:
2020 2021
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
2022
            code.put_label(code.error_label)
2023
            for cname, type in code.funcstate.all_managed_temps():
2024
                code.put_xdecref(cname, type, have_gil=not lenv.nogil)
2025 2026 2027

            # Clean up buffers -- this calls a Python function
            # so need to save and restore error state
2028
            buffers_present = len(used_buffer_entries) > 0
2029
            #memslice_entries = [e for e in lenv.entries.values() if e.type.is_memoryviewslice]
2030
            if buffers_present:
2031
                code.globalstate.use_utility_code(restore_exception_utility_code)
2032
                code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
2033 2034
                code.putln("__Pyx_PyThreadState_declare")
                code.putln("__Pyx_PyThreadState_assign")
2035
                code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
2036 2037
                for entry in used_buffer_entries:
                    Buffer.put_release_buffer_code(code, entry)
2038
                    #code.putln("%s = 0;" % entry.cname)
2039
                code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
2040

2041 2042 2043 2044 2045 2046
            if self.return_type.is_memoryviewslice:
                MemoryView.put_init_entry(Naming.retval_cname, code)
                err_val = Naming.retval_cname
            else:
                err_val = self.error_value()

2047 2048
            exc_check = self.caller_will_check_exceptions()
            if err_val is not None or exc_check:
2049
                # TODO: Fix exception tracing (though currently unused by cProfile).
Robert Bradshaw's avatar
Robert Bradshaw committed
2050 2051
                # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                # code.put_trace_exception()
2052

Mark Florisson's avatar
Mark Florisson committed
2053
                if lenv.nogil and not lenv.has_with_gil_block:
2054 2055 2056
                    code.putln("{")
                    code.put_ensure_gil()

2057
                code.put_add_traceback(self.entry.qualified_name)
2058

Mark Florisson's avatar
Mark Florisson committed
2059
                if lenv.nogil and not lenv.has_with_gil_block:
2060 2061
                    code.put_release_ensured_gil()
                    code.putln("}")
2062
            else:
2063 2064 2065
                warning(self.entry.pos,
                        "Unraisable exception in function '%s'." %
                        self.entry.qualified_name, 0)
2066
                code.put_unraisable(self.entry.qualified_name, lenv.nogil)
2067 2068 2069 2070
            default_retval = self.return_type.default_value
            if err_val is None and default_retval:
                err_val = default_retval
            if err_val is not None:
2071 2072
                if err_val != Naming.retval_cname:
                    code.putln("%s = %s;" % (Naming.retval_cname, err_val))
2073 2074
            elif not self.return_type.is_void:
                code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
2075 2076 2077 2078 2079 2080 2081

            if is_getbuffer_slot:
                self.getbuffer_error_cleanup(code)

            # If we are using the non-error cleanup section we should
            # jump past it if we have an error. The if-test below determine
            # whether this section is used.
2082
            if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice:
2083 2084 2085
                code.put_goto(code.return_from_error_cleanup_label)

        # ----- Non-error return cleanup
William Stein's avatar
William Stein committed
2086
        code.put_label(code.return_label)
2087 2088
        for entry in used_buffer_entries:
            Buffer.put_release_buffer_code(code, entry)
2089 2090
        if is_getbuffer_slot:
            self.getbuffer_normal_cleanup(code)
2091 2092 2093

        if self.return_type.is_memoryviewslice:
            # See if our return value is uninitialized on non-error return
2094
            # from . import MemoryView
2095
            # MemoryView.err_if_nogil_initialized_check(self.pos, env)
2096
            cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname))
2097 2098 2099 2100 2101
            code.putln(
                'if (%s) {' % cond)
            if env.nogil:
                code.put_ensure_gil()
            code.putln(
2102
                'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");')
2103 2104 2105 2106 2107
            if env.nogil:
                code.put_release_ensured_gil()
            code.putln(
                '}')

2108 2109
        # ----- Return cleanup for both error and no-error return
        code.put_label(code.return_from_error_cleanup_label)
2110

2111
        for entry in lenv.var_entries:
2112 2113
            if not entry.used or entry.in_closure:
                continue
2114

2115
            if entry.type.is_memoryviewslice:
2116
                code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil)
2117
            elif entry.type.is_pyobject:
2118
                if not entry.is_arg or len(entry.cf_assignments) > 1:
2119 2120 2121 2122
                    if entry.xdecref_cleanup:
                        code.put_var_xdecref(entry)
                    else:
                        code.put_var_decref(entry)
2123

Robert Bradshaw's avatar
Robert Bradshaw committed
2124
        # Decref any increfed args
2125
        for entry in lenv.arg_entries:
2126
            if entry.type.is_pyobject:
2127
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
2128
                    code.put_var_decref(entry)
2129 2130 2131 2132
            elif (entry.type.is_memoryviewslice and
                  (not is_cdef or len(entry.cf_assignments) > 1)):
                # decref slices of def functions and acquired slices from cdef
                # functions, but not borrowed slices from cdef functions.
2133 2134
                code.put_xdecref_memoryviewslice(entry.cname,
                                                 have_gil=not lenv.nogil)
Robert Bradshaw's avatar
Robert Bradshaw committed
2135 2136
        if self.needs_closure:
            code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
2137

2138
        # ----- Return
2139
        # This code is duplicated in ModuleNode.generate_module_init_func
2140 2141 2142 2143
        if not lenv.nogil:
            default_retval = self.return_type.default_value
            err_val = self.error_value()
            if err_val is None and default_retval:
Stefan Behnel's avatar
Stefan Behnel committed
2144
                err_val = default_retval  # FIXME: why is err_val not used?
2145 2146
            if self.return_type.is_pyobject:
                code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
2147

2148 2149
        if self.entry.is_special and self.entry.name == "__hash__":
            # Returning -1 for __hash__ is supposed to signal an error
2150
            # We do as Python instances and coerce -1 into -2.
2151
            code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
2152
                Naming.retval_cname, Naming.retval_cname))
2153

2154 2155
        if profile or linetrace:
            code.funcstate.can_trace = False
2156 2157 2158 2159 2160 2161 2162 2163
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.return_type.is_pyobject:
                    code.put_trace_return(
                        Naming.retval_cname, nogil=not code.funcstate.gil_owned)
                else:
                    code.put_trace_return(
                        "Py_None", nogil=not code.funcstate.gil_owned)
2164

2165
        if not lenv.nogil:
Stefan Behnel's avatar
Stefan Behnel committed
2166
            # GIL holding function
2167
            code.put_finish_refcount_context()
2168

2169 2170
        if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
            # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
2171
            code.put_release_ensured_gil()
2172
            code.funcstate.gil_owned = False
2173

William Stein's avatar
William Stein committed
2174
        if not self.return_type.is_void:
2175
            code.putln("return %s;" % Naming.retval_cname)
2176

William Stein's avatar
William Stein committed
2177
        code.putln("}")
2178 2179 2180 2181

        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

2182
        # ----- Go back and insert temp variable declarations
2183
        tempvardecl_code.put_temp_declarations(code.funcstate)
2184

2185
        # ----- Python version
2186
        code.exit_cfunc_scope()
2187
        if self.py_func:
2188
            self.py_func.generate_function_definitions(env, code)
2189
        self.generate_wrapper_functions(code)
William Stein's avatar
William Stein committed
2190 2191 2192 2193

    def declare_argument(self, env, arg):
        if arg.type.is_void:
            error(arg.pos, "Invalid use of 'void'")
2194
        elif not arg.type.is_complete() and not (arg.type.is_array or arg.type.is_memoryviewslice):
2195
            error(arg.pos, "Argument type '%s' is incomplete" % arg.type)
2196 2197 2198 2199
        entry = env.declare_arg(arg.name, arg.type, arg.pos)
        if arg.annotation:
            entry.annotation = arg.annotation
        return entry
2200

2201 2202 2203
    def generate_arg_type_test(self, arg, code):
        # Generate type test for one argument.
        if arg.type.typeobj_is_available():
2204 2205
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ArgTypeTest", "FunctionArguments.c"))
2206 2207 2208 2209
            typeptr_cname = arg.type.typeptr_cname
            arg_code = "((PyObject *)%s)" % arg.entry.cname
            code.putln(
                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % (
2210
                    arg_code,
2211 2212 2213
                    typeptr_cname,
                    arg.accept_none,
                    arg.name,
2214
                    arg.type.is_builtin_type and arg.type.require_exact,
2215 2216
                    code.error_goto(arg.pos)))
        else:
2217
            error(arg.pos, "Cannot test type of extern C class without type object name specification")
2218 2219 2220

    def generate_arg_none_check(self, arg, code):
        # Generate None check for one argument.
2221 2222 2223 2224 2225 2226
        if arg.type.is_memoryviewslice:
            cname = "%s.memview" % arg.entry.cname
        else:
            cname = arg.entry.cname

        code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname)
2227 2228
        code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % (
            max(200, len(arg.name)), arg.name,
2229 2230
            code.error_goto(arg.pos)))
        code.putln('}')
2231

2232
    def generate_wrapper_functions(self, code):
William Stein's avatar
William Stein committed
2233 2234 2235
        pass

    def generate_execution_code(self, code):
2236
        code.mark_pos(self.pos)
2237 2238
        # Evaluate and store argument default values
        for arg in self.args:
2239 2240
            if not arg.is_dynamic:
                arg.generate_assignment_code(code)
William Stein's avatar
William Stein committed
2241

2242
    #
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2243
    # Special code for the __getbuffer__ function
2244
    #
2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
    def _get_py_buffer_info(self):
        py_buffer = self.local_scope.arg_entries[1]
        try:
            # Check builtin definition of struct Py_buffer
            obj_type = py_buffer.type.base_type.scope.entries['obj'].type
        except (AttributeError, KeyError):
            # User code redeclared struct Py_buffer
            obj_type = None
        return py_buffer, obj_type

    # Old Python 3 used to support write-locks on buffer-like objects by
    # calling PyObject_GetBuffer() with a view==NULL parameter. This obscure
    # feature is obsolete, it was almost never used (only one instance in
    # `Modules/posixmodule.c` in Python 3.1) and it is now officially removed
    # (see bpo-14203). We add an extra check here to prevent legacy code from
    # from trying to use the feature and prevent segmentation faults.
    def getbuffer_check(self, code):
        py_buffer, _ = self._get_py_buffer_info()
        view = py_buffer.cname
        code.putln("if (%s == NULL) {" % view)
        code.putln("PyErr_SetString(PyExc_BufferError, "
                   "\"PyObject_GetBuffer: view==NULL argument is obsolete\");")
        code.putln("return -1;")
2268
        code.putln("}")
2269

2270 2271 2272 2273 2274 2275 2276 2277 2278
    def getbuffer_init(self, code):
        py_buffer, obj_type = self._get_py_buffer_info()
        view = py_buffer.cname
        if obj_type and obj_type.is_pyobject:
            code.put_init_to_py_none("%s->obj" % view, obj_type)
            code.put_giveref("%s->obj" % view) # Do not refnanny object within structs
        else:
            code.putln("%s->obj = NULL;" % view)

2279
    def getbuffer_error_cleanup(self, code):
2280 2281 2282 2283 2284 2285 2286 2287 2288
        py_buffer, obj_type = self._get_py_buffer_info()
        view = py_buffer.cname
        if obj_type and obj_type.is_pyobject:
            code.putln("if (%s->obj != NULL) {" % view)
            code.put_gotref("%s->obj" % view)
            code.put_decref_clear("%s->obj" % view, obj_type)
            code.putln("}")
        else:
            code.putln("Py_CLEAR(%s->obj);" % view)
2289 2290

    def getbuffer_normal_cleanup(self, code):
2291 2292 2293 2294 2295 2296 2297
        py_buffer, obj_type = self._get_py_buffer_info()
        view = py_buffer.cname
        if obj_type and obj_type.is_pyobject:
            code.putln("if (%s->obj == Py_None) {" % view)
            code.put_gotref("%s->obj" % view)
            code.put_decref_clear("%s->obj" % view, obj_type)
            code.putln("}")
William Stein's avatar
William Stein committed
2298

2299
    def get_preprocessor_guard(self):
2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
        if not self.entry.is_special:
            return None
        name = self.entry.name
        slot = TypeSlots.method_name_to_slot.get(name)
        if not slot:
            return None
        if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
            return None
        if name in ("__getbuffer__", "__releasebuffer__") and self.entry.scope.is_c_class_scope:
            return None
        return slot.preprocessor_guard_code()
2311 2312


William Stein's avatar
William Stein committed
2313 2314 2315
class CFuncDefNode(FuncDefNode):
    #  C function definition.
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
2316
    #  modifiers     ['inline']
William Stein's avatar
William Stein committed
2317 2318 2319
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarator    CDeclaratorNode
2320 2321 2322
    #  cfunc_declarator  the CFuncDeclarator of this function
    #                    (this is also available through declarator or a
    #                     base thereof)
William Stein's avatar
William Stein committed
2323
    #  body          StatListNode
2324
    #  api           boolean
2325
    #  decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2326
    #
2327
    #  with_gil      boolean    Acquire GIL around body
William Stein's avatar
William Stein committed
2328
    #  type          CFuncType
2329
    #  py_func       wrapper for calling from Python
2330
    #  overridable   whether or not this is a cpdef function
2331
    #  inline_in_pxd whether this is an inline function in a pxd file
2332
    #  template_declaration  String or None   Used for c++ class methods
Robert Bradshaw's avatar
Robert Bradshaw committed
2333
    #  is_const_method whether this is a const method
Robert Bradshaw's avatar
Robert Bradshaw committed
2334
    #  is_static_method whether this is a static method
2335
    #  is_c_class_method whether this is a cclass method
2336

2337 2338
    child_attrs = ["base_type", "declarator", "body", "py_func_stat", "decorators"]
    outer_attrs = ["decorators"]
2339 2340

    inline_in_pxd = False
2341
    decorators = None
2342
    directive_locals = None
2343
    directive_returns = None
2344
    override = None
2345
    template_declaration = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2346
    is_const_method = False
2347
    py_func_stat = None
2348

William Stein's avatar
William Stein committed
2349 2350
    def unqualified_name(self):
        return self.entry.name
2351

2352 2353 2354 2355 2356
    @property
    def code_object(self):
        # share the CodeObject with the cpdef wrapper (if available)
        return self.py_func.code_object if self.py_func else None

William Stein's avatar
William Stein committed
2357
    def analyse_declarations(self, env):
2358
        is_property = 0
2359 2360 2361 2362
        if self.decorators:
            for decorator in self.decorators:
                func = decorator.decorator
                if func.is_name:
mattip's avatar
mattip committed
2363
                    if func.name == 'property':
2364
                        is_property = 1
mattip's avatar
mattip committed
2365
                    elif func.name == 'staticmethod':
2366
                        pass
2367 2368
                    else:
                        error(self.pos, "Cannot handle %s decorators yet" % func.name)
mattip's avatar
mattip committed
2369 2370 2371
                else:
                    error(self.pos,
                          "Cannot handle %s decorators yet" % type(func).__name__)
2372

2373
        self.is_c_class_method = env.is_c_class_scope
2374 2375
        if self.directive_locals is None:
            self.directive_locals = {}
2376
        self.directive_locals.update(env.directives.get('locals', {}))
2377 2378 2379 2380 2381 2382 2383
        if self.directive_returns is not None:
            base_type = self.directive_returns.analyse_as_type(env)
            if base_type is None:
                error(self.directive_returns.pos, "Not a type")
                base_type = PyrexTypes.error_type
        else:
            base_type = self.base_type.analyse(env)
2384
        self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod')
2385
        # The 2 here is because we need both function and argument names.
2386
        if isinstance(self.declarator, CFuncDeclaratorNode):
mattip's avatar
mattip committed
2387
            name_declarator, typ = self.declarator.analyse(
2388
                base_type, env, nonempty=2 * (self.body is not None),
2389
                directive_locals=self.directive_locals, visibility=self.visibility)
2390
        else:
mattip's avatar
mattip committed
2391
            name_declarator, typ = self.declarator.analyse(
2392
                base_type, env, nonempty=2 * (self.body is not None), visibility=self.visibility)
mattip's avatar
mattip committed
2393
        if not typ.is_cfunction:
2394
            error(self.pos, "Suite attached to non-function declaration")
William Stein's avatar
William Stein committed
2395 2396 2397 2398
        # Remember the actual type according to the function header
        # written here, because the type in the symbol table entry
        # may be different if we're overriding a C method inherited
        # from the base type of an extension type.
mattip's avatar
mattip committed
2399 2400
        self.type = typ
        typ.is_overridable = self.overridable
2401 2402 2403
        declarator = self.declarator
        while not hasattr(declarator, 'args'):
            declarator = declarator.base
2404 2405

        self.cfunc_declarator = declarator
2406
        self.args = declarator.args
2407

2408 2409 2410
        opt_arg_count = self.cfunc_declarator.optional_arg_count
        if (self.visibility == 'public' or self.api) and opt_arg_count:
            error(self.cfunc_declarator.pos,
2411
                  "Function with optional arguments may not be declared public or api")
2412

mattip's avatar
mattip committed
2413
        if typ.exception_check == '+' and self.visibility != 'extern':
2414 2415 2416
            warning(self.cfunc_declarator.pos,
                    "Only extern functions can throw C++ exceptions.")

mattip's avatar
mattip committed
2417
        for formal_arg, type_arg in zip(self.args, typ.args):
2418
            self.align_argument_type(env, type_arg)
2419
            formal_arg.type = type_arg.type
2420
            formal_arg.name = type_arg.name
2421
            formal_arg.cname = type_arg.cname
2422

2423 2424
            self._validate_type_visibility(type_arg.type, type_arg.pos, env)

2425 2426 2427
            if type_arg.type.is_fused:
                self.has_fused_arguments = True

2428 2429
            if type_arg.type.is_buffer and 'inline' in self.modifiers:
                warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
2430

2431
            if type_arg.type.is_buffer or type_arg.type.is_pythran_expr:
2432 2433
                if self.type.nogil:
                    error(formal_arg.pos,
2434
                          "Buffer may not be acquired without the GIL. Consider using memoryview slices instead.")
2435 2436 2437
                elif 'inline' in self.modifiers:
                    warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)

mattip's avatar
mattip committed
2438
        self._validate_type_visibility(typ.return_type, self.pos, env)
2439

William Stein's avatar
William Stein committed
2440 2441
        name = name_declarator.name
        cname = name_declarator.cname
2442

mattip's avatar
mattip committed
2443 2444
        typ.is_const_method = self.is_const_method
        typ.is_static_method = self.is_static_method
mattip's avatar
mattip committed
2445

William Stein's avatar
William Stein committed
2446
        self.entry = env.declare_cfunction(
mattip's avatar
mattip committed
2447
            name, typ, self.pos,
2448 2449 2450
            cname=cname, visibility=self.visibility, api=self.api,
            defining=self.body is not None, modifiers=self.modifiers,
            overridable=self.overridable)
2451 2452 2453 2454
        if is_property:
            self.entry.is_property = 1
            env.property_entries.append(self.entry)
            env.cfunc_entries.remove(self.entry)
2455
        self.entry.inline_func_in_pxd = self.inline_in_pxd
mattip's avatar
mattip committed
2456
        self.return_type = typ.return_type
2457
        if self.return_type.is_array and self.visibility != 'extern':
2458
            error(self.pos, "Function cannot return an array")
2459 2460
        if self.return_type.is_cpp_class:
            self.return_type.check_nullary_constructor(self.pos, "used as a return value")
2461

Robert Bradshaw's avatar
Robert Bradshaw committed
2462
        if self.overridable and not env.is_module_scope and not self.is_static_method:
2463 2464 2465
            if len(self.args) < 1 or not self.args[0].type.is_pyobject:
                # An error will be produced in the cdef function
                self.overridable = False
2466

2467 2468 2469 2470
        self.declare_cpdef_wrapper(env)
        self.create_local_scope(env)

    def declare_cpdef_wrapper(self, env):
2471
        if self.overridable:
2472 2473 2474
            if self.is_static_method:
                # TODO(robertwb): Finish this up, perhaps via more function refactoring.
                error(self.pos, "static cpdef methods not yet supported")
2475
            name = self.entry.name
2476
            py_func_body = self.call_self_node(is_module_scope=env.is_module_scope)
2477 2478 2479 2480 2481 2482
            if self.is_static_method:
                from .ExprNodes import NameNode
                decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))]
                decorators[0].decorator.analyse_types(env)
            else:
                decorators = []
2483 2484 2485 2486 2487 2488 2489 2490 2491
            self.py_func = DefNode(pos=self.pos,
                                   name=self.entry.name,
                                   args=self.args,
                                   star_arg=None,
                                   starstar_arg=None,
                                   doc=self.doc,
                                   body=py_func_body,
                                   decorators=decorators,
                                   is_wrapper=1)
2492
            self.py_func.is_module_scope = env.is_module_scope
2493
            self.py_func.analyse_declarations(env)
2494
            self.py_func.entry.is_overridable = True
2495
            self.py_func_stat = StatListNode(self.pos, stats=[self.py_func])
2496
            self.py_func.type = PyrexTypes.py_object_type
2497
            self.entry.as_variable = self.py_func.entry
2498
            self.entry.used = self.entry.as_variable.used = True
2499 2500
            # Reset scope entry the above cfunction
            env.entries[name] = self.entry
2501
            if (not self.entry.is_final_cmethod and
2502 2503
                    (not env.is_module_scope or Options.lookup_module_cpdef)):
                self.override = OverrideCheckNode(self.pos, py_func=self.py_func)
2504
                self.body = StatListNode(self.pos, stats=[self.override, self.body])
2505

2506 2507 2508 2509 2510 2511 2512 2513 2514 2515
    def _validate_type_visibility(self, type, pos, env):
        """
        Ensure that types used in cdef functions are public or api, or
        defined in a C header.
        """
        public_or_api = (self.visibility == 'public' or self.api)
        entry = getattr(type, 'entry', None)
        if public_or_api and entry and env.is_module_scope:
            if not (entry.visibility in ('public', 'extern') or
                    entry.api or entry.in_cinclude):
2516
                error(pos, "Function declared public or api may not have private types")
2517

2518
    def call_self_node(self, omit_optional_args=0, is_module_scope=0):
2519
        from . import ExprNodes
2520 2521 2522 2523
        args = self.type.args
        if omit_optional_args:
            args = args[:len(args) - self.type.optional_arg_count]
        arg_names = [arg.name for arg in args]
2524
        if is_module_scope:
2525
            cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
2526 2527 2528 2529
            call_arg_names = arg_names
            skip_dispatch = Options.lookup_module_cpdef
        elif self.type.is_static_method:
            class_entry = self.entry.scope.parent_type.entry
2530 2531 2532
            class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
            class_node.entry = class_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
2533 2534 2535
            # Calling static c(p)def methods on an instance disallowed.
            # TODO(robertwb): Support by passing self to check for override?
            skip_dispatch = True
2536
        else:
2537 2538 2539 2540
            type_entry = self.type.args[0].type.entry
            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
            type_arg.entry = type_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
2541
        skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
2542 2543 2544 2545 2546
        c_call = ExprNodes.SimpleCallNode(
            self.pos,
            function=cfunc,
            args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
            wrapper_call=skip_dispatch)
2547 2548
        return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)

William Stein's avatar
William Stein committed
2549 2550 2551 2552 2553
    def declare_arguments(self, env):
        for arg in self.type.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            self.declare_argument(env, arg)
2554

2555
    def need_gil_acquisition(self, lenv):
2556 2557
        return self.type.with_gil

2558
    def nogil_check(self, env):
2559
        type = self.type
2560
        with_gil = type.with_gil
2561 2562 2563 2564
        if type.nogil and not with_gil:
            if type.return_type.is_pyobject:
                error(self.pos,
                      "Function with Python return type cannot be declared nogil")
2565
            for entry in self.local_scope.var_entries:
2566
                if entry.type.is_pyobject and not entry.in_with_gil_block:
2567 2568
                    error(self.pos, "Function declared nogil has Python locals or temporaries")

2569
    def analyse_expressions(self, env):
2570
        self.local_scope.directives = env.directives
2571 2572 2573 2574
        if self.py_func_stat is not None:
            # this will also analyse the default values and the function name assignment
            self.py_func_stat = self.py_func_stat.analyse_expressions(env)
        elif self.py_func is not None:
2575
            # this will also analyse the default values
2576
            self.py_func = self.py_func.analyse_expressions(env)
2577 2578
        else:
            self.analyse_default_values(env)
2579
            self.analyse_annotations(env)
2580
        self.acquire_gil = self.need_gil_acquisition(self.local_scope)
2581
        return self
2582

Robert Bradshaw's avatar
Robert Bradshaw committed
2583 2584 2585
    def needs_assignment_synthesis(self, env, code=None):
        return False

2586
    def generate_function_header(self, code, with_pymethdef, with_opt_args=1, with_dispatch=1, cname=None):
2587
        scope = self.local_scope
William Stein's avatar
William Stein committed
2588 2589
        arg_decls = []
        type = self.type
2590
        for arg in type.args[:len(type.args)-type.optional_arg_count]:
2591 2592 2593 2594 2595
            arg_decl = arg.declaration_code()
            entry = scope.lookup(arg.name)
            if not entry.cf_used:
                arg_decl = 'CYTHON_UNUSED %s' % arg_decl
            arg_decls.append(arg_decl)
2596
        if with_dispatch and self.overridable:
2597 2598 2599 2600 2601 2602
            dispatch_arg = PyrexTypes.c_int_type.declaration_code(
                Naming.skip_dispatch_cname)
            if self.override:
                arg_decls.append(dispatch_arg)
            else:
                arg_decls.append('CYTHON_UNUSED %s' % dispatch_arg)
2603 2604
        if type.optional_arg_count and with_opt_args:
            arg_decls.append(type.op_arg_struct.declaration_code(Naming.optional_args_cname))
William Stein's avatar
William Stein committed
2605 2606 2607 2608
        if type.has_varargs:
            arg_decls.append("...")
        if not arg_decls:
            arg_decls = ["void"]
2609 2610
        if cname is None:
            cname = self.entry.func_cname
2611
        entity = type.function_header_code(cname, ', '.join(arg_decls))
2612
        if self.entry.visibility == 'private' and '::' not in cname:
2613
            storage_class = "static "
William Stein's avatar
William Stein committed
2614
        else:
2615
            storage_class = ""
2616
        dll_linkage = None
2617
        modifiers = code.build_function_modifiers(self.entry.func_modifiers)
Robert Bradshaw's avatar
Robert Bradshaw committed
2618

2619 2620
        header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage)
        #print (storage_class, modifiers, header)
2621
        needs_proto = self.is_c_class_method
2622
        if self.template_declaration:
2623 2624
            if needs_proto:
                code.globalstate.parts['module_declarations'].putln(self.template_declaration)
2625
            code.putln(self.template_declaration)
2626
        if needs_proto:
2627 2628
            code.globalstate.parts['module_declarations'].putln(
                "%s%s%s; /* proto*/" % (storage_class, modifiers, header))
2629
        code.putln("%s%s%s {" % (storage_class, modifiers, header))
William Stein's avatar
William Stein committed
2630 2631

    def generate_argument_declarations(self, env, code):
2632
        scope = self.local_scope
2633
        for arg in self.args:
2634
            if arg.default:
2635
                entry = scope.lookup(arg.name)
2636
                if self.override or entry.cf_used:
2637 2638 2639
                    result = arg.calculate_default_value_code(code)
                    code.putln('%s = %s;' % (
                        arg.type.declaration_code(arg.cname), result))
2640

William Stein's avatar
William Stein committed
2641 2642
    def generate_keyword_list(self, code):
        pass
2643

2644
    def generate_argument_parsing_code(self, env, code):
2645
        i = 0
2646
        used = 0
2647
        scope = self.local_scope
2648 2649
        if self.type.optional_arg_count:
            code.putln('if (%s) {' % Naming.optional_args_cname)
2650
            for arg in self.args:
2651
                if arg.default:
2652
                    entry = scope.lookup(arg.name)
2653
                    if self.override or entry.cf_used:
2654 2655 2656 2657 2658 2659 2660 2661 2662 2663
                        code.putln('if (%s->%sn > %s) {' %
                                   (Naming.optional_args_cname,
                                    Naming.pyrex_prefix, i))
                        declarator = arg.declarator
                        while not hasattr(declarator, 'name'):
                            declarator = declarator.base
                        code.putln('%s = %s->%s;' %
                                   (arg.cname, Naming.optional_args_cname,
                                    self.type.opt_arg_cname(declarator.name)))
                        used += 1
2664
                    i += 1
2665
            for _ in range(used):
2666
                code.putln('}')
2667
            code.putln('}')
2668

2669 2670 2671 2672 2673 2674 2675 2676 2677 2678
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure and not arg.default:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(scope.lookup_here(arg.name))


William Stein's avatar
William Stein committed
2679 2680
    def generate_argument_conversion_code(self, code):
        pass
2681

William Stein's avatar
William Stein committed
2682
    def generate_argument_type_tests(self, code):
2683 2684 2685 2686 2687
        # Generate type tests for args whose type in a parent
        # class is a supertype of the declared type.
        for arg in self.type.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
2688 2689
            elif arg.type.is_pyobject and not arg.accept_none:
                self.generate_arg_none_check(arg, code)
2690

2691
    def generate_execution_code(self, code):
2692 2693 2694
        if code.globalstate.directives['linetrace']:
            code.mark_pos(self.pos)
            code.putln("")  # generate line tracing code
2695 2696 2697 2698
        super(CFuncDefNode, self).generate_execution_code(code)
        if self.py_func_stat:
            self.py_func_stat.generate_execution_code(code)

William Stein's avatar
William Stein committed
2699 2700 2701 2702
    def error_value(self):
        if self.return_type.is_pyobject:
            return "0"
        else:
2703 2704
            #return None
            return self.entry.type.exception_value
2705

William Stein's avatar
William Stein committed
2706
    def caller_will_check_exceptions(self):
2707
        return self.entry.type.exception_check
2708

2709 2710
    def generate_wrapper_functions(self, code):
        # If the C signature of a function has changed, we need to generate
2711
        # wrappers to put in the slots here.
2712 2713 2714 2715 2716 2717 2718
        k = 0
        entry = self.entry
        func_type = entry.type
        while entry.prev_entry is not None:
            k += 1
            entry = entry.prev_entry
            entry.func_cname = "%s%swrap_%s" % (self.entry.func_cname, Naming.pyrex_prefix, k)
2719
            code.putln()
2720 2721 2722 2723 2724
            self.generate_function_header(
                code, 0,
                with_dispatch=entry.type.is_overridable,
                with_opt_args=entry.type.optional_arg_count,
                cname=entry.func_cname)
2725 2726 2727 2728
            if not self.return_type.is_void:
                code.put('return ')
            args = self.type.args
            arglist = [arg.cname for arg in args[:len(args)-self.type.optional_arg_count]]
2729 2730 2731 2732 2733 2734 2735 2736
            if entry.type.is_overridable:
                arglist.append(Naming.skip_dispatch_cname)
            elif func_type.is_overridable:
                arglist.append('0')
            if entry.type.optional_arg_count:
                arglist.append(Naming.optional_args_cname)
            elif func_type.optional_arg_count:
                arglist.append('NULL')
2737 2738
            code.putln('%s(%s);' % (self.entry.func_cname, ', '.join(arglist)))
            code.putln('}')
2739

William Stein's avatar
William Stein committed
2740 2741 2742 2743 2744

class PyArgDeclNode(Node):
    # Argument which must be a Python object (used
    # for * and ** arguments).
    #
2745 2746 2747
    # name        string
    # entry       Symtab.Entry
    # annotation  ExprNode or None   Py3 argument annotation
2748
    child_attrs = []
2749 2750
    is_self_arg = False
    is_type_arg = False
2751 2752 2753

    def generate_function_definitions(self, env, code):
        self.entry.generate_function_definitions(env, code)
2754

2755

2756 2757 2758
class DecoratorNode(Node):
    # A decorator
    #
2759
    # decorator    NameNode or CallNode or AttributeNode
2760 2761
    child_attrs = ['decorator']

William Stein's avatar
William Stein committed
2762 2763 2764 2765 2766

class DefNode(FuncDefNode):
    # A Python function definition.
    #
    # name          string                 the Python name of the function
Stefan Behnel's avatar
Stefan Behnel committed
2767
    # lambda_name   string                 the internal name of a lambda 'function'
2768
    # decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2769
    # args          [CArgDeclNode]         formal arguments
2770
    # doc           EncodedString or None
William Stein's avatar
William Stein committed
2771
    # body          StatListNode
2772 2773
    # return_type_annotation
    #               ExprNode or None       the Py3 return type annotation
William Stein's avatar
William Stein committed
2774 2775 2776 2777
    #
    #  The following subnode is constructed internally
    #  when the def statement is inside a Python class definition.
    #
2778 2779 2780
    #  fused_py_func        DefNode     The original fused cpdef DefNode
    #                                   (in case this is a specialization)
    #  specialized_cpdefs   [DefNode]   list of specialized cpdef DefNodes
2781
    #  py_cfunc_node  PyCFunctionNode/InnerFunctionNode   The PyCFunction to create and assign
2782 2783
    #
    # decorator_indirection IndirectionNode Used to remove __Pyx_Method_ClassMethod for fused functions
2784

2785
    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "return_type_annotation"]
2786
    outer_attrs = ["decorators", "return_type_annotation"]
2787

2788 2789 2790
    is_staticmethod = False
    is_classmethod = False

Stefan Behnel's avatar
Stefan Behnel committed
2791
    lambda_name = None
2792
    reqd_kw_flags_cname = "0"
2793
    is_wrapper = 0
2794
    no_assignment_synthesis = 0
2795
    decorators = None
2796
    return_type_annotation = None
2797
    entry = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2798
    acquire_gil = 0
2799
    self_in_stararg = 0
2800
    py_cfunc_node = None
2801
    requires_classobj = False
2802
    defaults_struct = None # Dynamic kwrds structure name
2803
    doc = None
2804

2805 2806
    fused_py_func = False
    specialized_cpdefs = None
2807 2808 2809
    py_wrapper = None
    py_wrapper_required = True
    func_cname = None
2810

2811 2812
    defaults_getter = None

2813 2814
    def __init__(self, pos, **kwds):
        FuncDefNode.__init__(self, pos, **kwds)
2815
        k = rk = r = 0
2816 2817
        for arg in self.args:
            if arg.kw_only:
2818
                k += 1
2819
                if not arg.default:
2820 2821 2822 2823 2824 2825
                    rk += 1
            if not arg.default:
                r += 1
        self.num_kwonly_args = k
        self.num_required_kw_args = rk
        self.num_required_args = r
2826

2827 2828
    def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, except_val=None, modifiers=None,
                     nogil=False, with_gil=False):
2829 2830 2831 2832
        if self.star_arg:
            error(self.star_arg.pos, "cdef function cannot have star argument")
        if self.starstar_arg:
            error(self.starstar_arg.pos, "cdef function cannot have starstar argument")
2833 2834
        exception_value, exception_check = except_val or (None, False)

2835 2836 2837 2838
        if cfunc is None:
            cfunc_args = []
            for formal_arg in self.args:
                name_declarator, type = formal_arg.analyse(scope, nonempty=1)
2839 2840
                cfunc_args.append(PyrexTypes.CFuncTypeArg(name=name_declarator.name,
                                                          cname=None,
2841
                                                          annotation=formal_arg.annotation,
2842 2843 2844 2845 2846 2847
                                                          type=py_object_type,
                                                          pos=formal_arg.pos))
            cfunc_type = PyrexTypes.CFuncType(return_type=py_object_type,
                                              args=cfunc_args,
                                              has_varargs=False,
                                              exception_value=None,
2848
                                              exception_check=exception_check,
2849 2850
                                              nogil=nogil,
                                              with_gil=with_gil,
2851
                                              is_overridable=overridable)
2852
            cfunc = CVarDefNode(self.pos, type=cfunc_type)
2853
        else:
2854 2855
            if scope is None:
                scope = cfunc.scope
2856 2857 2858
            cfunc_type = cfunc.type
            if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
                error(self.pos, "wrong number of arguments")
Stefan Behnel's avatar
Stefan Behnel committed
2859
                error(cfunc.pos, "previous declaration here")
2860 2861
            for i, (formal_arg, type_arg) in enumerate(zip(self.args, cfunc_type.args)):
                name_declarator, type = formal_arg.analyse(scope, nonempty=1,
2862
                                                           is_self_arg=(i == 0 and scope.is_c_class_scope))
Stefan Behnel's avatar
Stefan Behnel committed
2863
                if type is None or type is PyrexTypes.py_object_type:
2864 2865
                    formal_arg.type = type_arg.type
                    formal_arg.name_declarator = name_declarator
2866 2867 2868 2869

        if exception_value is None and cfunc_type.exception_value is not None:
            from .ExprNodes import ConstNode
            exception_value = ConstNode(
2870
                self.pos, value=cfunc_type.exception_value, type=cfunc_type.return_type)
2871
        declarator = CFuncDeclaratorNode(self.pos,
2872 2873 2874 2875 2876 2877 2878
                                         base=CNameDeclaratorNode(self.pos, name=self.name, cname=None),
                                         args=self.args,
                                         has_varargs=False,
                                         exception_check=cfunc_type.exception_check,
                                         exception_value=exception_value,
                                         with_gil=cfunc_type.with_gil,
                                         nogil=cfunc_type.nogil)
2879
        return CFuncDefNode(self.pos,
2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892
                            modifiers=modifiers or [],
                            base_type=CAnalysedBaseTypeNode(self.pos, type=cfunc_type.return_type),
                            declarator=declarator,
                            body=self.body,
                            doc=self.doc,
                            overridable=cfunc_type.is_overridable,
                            type=cfunc_type,
                            with_gil=cfunc_type.with_gil,
                            nogil=cfunc_type.nogil,
                            visibility='private',
                            api=False,
                            directive_locals=getattr(cfunc, 'directive_locals', {}),
                            directive_returns=returns)
2893

2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904
    def is_cdef_func_compatible(self):
        """Determines if the function's signature is compatible with a
        cdef function.  This can be used before calling
        .as_cfunction() to see if that will be successful.
        """
        if self.needs_closure:
            return False
        if self.star_arg or self.starstar_arg:
            return False
        return True

William Stein's avatar
William Stein committed
2905
    def analyse_declarations(self, env):
2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919
        if self.decorators:
            for decorator in self.decorators:
                func = decorator.decorator
                if func.is_name:
                    self.is_classmethod |= func.name == 'classmethod'
                    self.is_staticmethod |= func.name == 'staticmethod'

        if self.is_classmethod and env.lookup_here('classmethod'):
            # classmethod() was overridden - not much we can do here ...
            self.is_classmethod = False
        if self.is_staticmethod and env.lookup_here('staticmethod'):
            # staticmethod() was overridden - not much we can do here ...
            self.is_staticmethod = False

2920
        if self.name == '__new__' and env.is_py_class_scope:
Vitja Makarov's avatar
Vitja Makarov committed
2921 2922
            self.is_staticmethod = 1

2923
        self.analyse_argument_types(env)
2924 2925 2926 2927
        if self.name == '<lambda>':
            self.declare_lambda_function(env)
        else:
            self.declare_pyfunction(env)
2928

2929 2930
        self.analyse_signature(env)
        self.return_type = self.entry.signature.return_type()
2931 2932 2933
        # if a signature annotation provides a more specific return object type, use it
        if self.return_type is py_object_type and self.return_type_annotation:
            if env.directives['annotation_typing'] and not self.entry.is_special:
2934
                _, return_type = analyse_type_annotation(self.return_type_annotation, env)
2935 2936 2937
                if return_type and return_type.is_pyobject:
                    self.return_type = return_type

2938 2939
        self.create_local_scope(env)

2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
        self.py_wrapper = DefNodeWrapper(
            self.pos,
            target=self,
            name=self.entry.name,
            args=self.args,
            star_arg=self.star_arg,
            starstar_arg=self.starstar_arg,
            return_type=self.return_type)
        self.py_wrapper.analyse_declarations(env)

2950
    def analyse_argument_types(self, env):
2951
        self.directive_locals = env.directives.get('locals', {})
2952
        allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
2953 2954 2955 2956

        f2s = env.fused_to_specific
        env.fused_to_specific = None

William Stein's avatar
William Stein committed
2957
        for arg in self.args:
2958 2959 2960 2961
            if hasattr(arg, 'name'):
                name_declarator = None
            else:
                base_type = arg.base_type.analyse(env)
2962 2963 2964 2965 2966 2967 2968
                # If we hare in pythran mode and we got a buffer supported by
                # Pythran, we change this node to a fused type
                if has_np_pythran(env) and base_type.is_pythran_expr:
                    base_type = PyrexTypes.FusedType([
                        base_type,
                        #PyrexTypes.PythranExpr(pythran_type(self.type, "numpy_texpr")),
                        base_type.org_buffer])
2969 2970 2971
                name_declarator, type = \
                    arg.declarator.analyse(base_type, env)
                arg.name = name_declarator.name
2972
                arg.type = type
2973 2974 2975 2976

                if type.is_fused:
                    self.has_fused_arguments = True

2977
            self.align_argument_type(env, arg)
2978
            if name_declarator and name_declarator.cname:
2979
                error(self.pos, "Python function argument cannot have C name specification")
2980
            arg.type = arg.type.as_argument_type()
William Stein's avatar
William Stein committed
2981 2982 2983 2984
            arg.hdr_type = None
            arg.needs_conversion = 0
            arg.needs_type_test = 0
            arg.is_generic = 1
2985
            if arg.type.is_pyobject or arg.type.is_buffer or arg.type.is_memoryviewslice:
2986 2987 2988 2989
                if arg.or_none:
                    arg.accept_none = True
                elif arg.not_none:
                    arg.accept_none = False
2990
                elif (arg.type.is_extension_type or arg.type.is_builtin_type
2991
                        or arg.type.is_buffer or arg.type.is_memoryviewslice):
2992 2993 2994 2995 2996 2997
                    if arg.default and arg.default.constant_result is None:
                        # special case: def func(MyType obj = None)
                        arg.accept_none = True
                    else:
                        # default depends on compiler directive
                        arg.accept_none = allow_none_for_extension_args
2998 2999 3000
                else:
                    # probably just a plain 'object'
                    arg.accept_none = True
3001
            else:
3002
                arg.accept_none = True # won't be used, but must be there
3003
                if arg.not_none:
3004
                    error(arg.pos, "Only Python type arguments can have 'not None'")
3005
                if arg.or_none:
3006
                    error(arg.pos, "Only Python type arguments can have 'or None'")
3007 3008
        env.fused_to_specific = f2s

3009 3010 3011 3012 3013
        if has_np_pythran(env):
            self.np_args_idx = [i for i,a in enumerate(self.args) if a.type.is_numpy_buffer]
        else:
            self.np_args_idx = []

William Stein's avatar
William Stein committed
3014
    def analyse_signature(self, env):
3015
        if self.entry.is_special:
3016
            if self.decorators:
3017
                error(self.pos, "special functions of cdef classes cannot have decorators")
3018 3019 3020 3021
            self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg)
        elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg):
            # Use the simpler calling signature for zero- and one-argument functions.
            if self.entry.signature is TypeSlots.pyfunction_signature:
3022 3023
                if len(self.args) == 0:
                    self.entry.signature = TypeSlots.pyfunction_noargs
3024 3025 3026
                elif len(self.args) == 1:
                    if self.args[0].default is None and not self.args[0].kw_only:
                        self.entry.signature = TypeSlots.pyfunction_onearg
3027 3028 3029
            elif self.entry.signature is TypeSlots.pymethod_signature:
                if len(self.args) == 1:
                    self.entry.signature = TypeSlots.unaryfunc
3030 3031 3032
                elif len(self.args) == 2:
                    if self.args[1].default is None and not self.args[1].kw_only:
                        self.entry.signature = TypeSlots.ibinaryfunc
3033

William Stein's avatar
William Stein committed
3034 3035
        sig = self.entry.signature
        nfixed = sig.num_fixed_args()
3036 3037
        if (sig is TypeSlots.pymethod_signature and nfixed == 1
               and len(self.args) == 0 and self.star_arg):
3038 3039 3040 3041 3042 3043 3044
            # this is the only case where a diverging number of
            # arguments is not an error - when we have no explicit
            # 'self' parameter as in method(*args)
            sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used
            self.self_in_stararg = 1
            nfixed = 0

3045 3046
        if self.is_staticmethod and env.is_c_class_scope:
            nfixed = 0
3047
            self.self_in_stararg = True  # FIXME: why for staticmethods?
3048 3049 3050 3051 3052 3053

            self.entry.signature = sig = copy.copy(sig)
            sig.fixed_arg_format = "*"
            sig.is_staticmethod = True
            sig.has_generic_args = True

3054
        if ((self.is_classmethod or self.is_staticmethod) and
3055
                self.has_fused_arguments and env.is_c_class_scope):
3056 3057
            del self.decorator_indirection.stats[:]

3058 3059 3060 3061 3062 3063 3064
        for i in range(min(nfixed, len(self.args))):
            arg = self.args[i]
            arg.is_generic = 0
            if sig.is_self_arg(i) and not self.is_staticmethod:
                if self.is_classmethod:
                    arg.is_type_arg = 1
                    arg.hdr_type = arg.type = Builtin.type_type
William Stein's avatar
William Stein committed
3065
                else:
3066 3067 3068
                    arg.is_self_arg = 1
                    arg.hdr_type = arg.type = env.parent_type
                arg.needs_conversion = 0
William Stein's avatar
William Stein committed
3069
            else:
3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084
                arg.hdr_type = sig.fixed_arg_type(i)
                if not arg.type.same_as(arg.hdr_type):
                    if arg.hdr_type.is_pyobject and arg.type.is_pyobject:
                        arg.needs_type_test = 1
                    else:
                        arg.needs_conversion = 1
            if arg.needs_conversion:
                arg.hdr_cname = Naming.arg_prefix + arg.name
            else:
                arg.hdr_cname = Naming.var_prefix + arg.name

        if nfixed > len(self.args):
            self.bad_signature()
            return
        elif nfixed < len(self.args):
William Stein's avatar
William Stein committed
3085 3086 3087
            if not sig.has_generic_args:
                self.bad_signature()
            for arg in self.args:
3088
                if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type):
William Stein's avatar
William Stein committed
3089
                    arg.needs_type_test = 1
3090

William Stein's avatar
William Stein committed
3091 3092 3093 3094
    def bad_signature(self):
        sig = self.entry.signature
        expected_str = "%d" % sig.num_fixed_args()
        if sig.has_generic_args:
Stefan Behnel's avatar
Stefan Behnel committed
3095
            expected_str += " or more"
William Stein's avatar
William Stein committed
3096 3097 3098 3099 3100
        name = self.name
        if name.startswith("__") and name.endswith("__"):
            desc = "Special method"
        else:
            desc = "Method"
3101 3102
        error(self.pos, "%s %s has wrong number of arguments (%d declared, %s expected)" % (
            desc, self.name, len(self.args), expected_str))
3103

William Stein's avatar
William Stein committed
3104
    def declare_pyfunction(self, env):
3105 3106
        #print "DefNode.declare_pyfunction:", self.name, "in", env ###
        name = self.name
Stefan Behnel's avatar
Stefan Behnel committed
3107
        entry = env.lookup_here(name)
3108 3109
        if entry:
            if entry.is_final_cmethod and not env.parent_type.is_final_type:
Stefan Behnel's avatar
Stefan Behnel committed
3110
                error(self.pos, "Only final types can have final Python (def/cpdef) methods")
3111
            if entry.type.is_cfunction and not entry.is_builtin_cmethod and not self.is_wrapper:
3112
                warning(self.pos, "Overriding cdef method with def method.", 5)
3113
        entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper)
3114
        self.entry = entry
3115
        prefix = env.next_id(env.scope_prefix)
3116
        self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name
3117 3118
        if Options.docstrings:
            entry.doc = embed_position(self.pos, self.doc)
3119
            entry.doc_cname = Naming.funcdoc_prefix + prefix + name
3120
            if entry.is_special:
3121 3122
                if entry.name in TypeSlots.invisible or not entry.doc or (
                        entry.name in '__getattr__' and env.directives['fast_getattr']):
3123 3124 3125
                    entry.wrapperbase_cname = None
                else:
                    entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name
3126 3127
        else:
            entry.doc = None
3128

Stefan Behnel's avatar
Stefan Behnel committed
3129
    def declare_lambda_function(self, env):
3130
        entry = env.declare_lambda_function(self.lambda_name, self.pos)
Stefan Behnel's avatar
Stefan Behnel committed
3131 3132
        entry.doc = None
        self.entry = entry
3133
        self.entry.pyfunc_cname = entry.cname
Stefan Behnel's avatar
Stefan Behnel committed
3134

William Stein's avatar
William Stein committed
3135 3136 3137 3138 3139 3140 3141 3142 3143 3144
    def declare_arguments(self, env):
        for arg in self.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            if arg.needs_conversion:
                arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
                if arg.type.is_pyobject:
                    arg.entry.init = "0"
            else:
                arg.entry = self.declare_argument(env, arg)
3145
            arg.entry.is_arg = 1
3146
            arg.entry.used = 1
William Stein's avatar
William Stein committed
3147 3148 3149 3150 3151 3152
            arg.entry.is_self_arg = arg.is_self_arg
        self.declare_python_arg(env, self.star_arg)
        self.declare_python_arg(env, self.starstar_arg)

    def declare_python_arg(self, env, arg):
        if arg:
3153
            if env.directives['infer_types'] != False:
3154 3155 3156 3157
                type = PyrexTypes.unspecified_type
            else:
                type = py_object_type
            entry = env.declare_var(arg.name, type, arg.pos)
3158
            entry.is_arg = 1
3159 3160 3161 3162
            entry.used = 1
            entry.init = "0"
            entry.xdecref_cleanup = 1
            arg.entry = entry
3163

William Stein's avatar
William Stein committed
3164
    def analyse_expressions(self, env):
3165
        self.local_scope.directives = env.directives
William Stein's avatar
William Stein committed
3166
        self.analyse_default_values(env)
3167 3168
        self.analyse_annotations(env)
        if self.return_type_annotation:
3169
            self.return_type_annotation = self.analyse_annotation(env, self.return_type_annotation)
3170

3171
        if not self.needs_assignment_synthesis(env) and self.decorators:
3172
            for decorator in self.decorators[::-1]:
3173
                decorator.decorator = decorator.decorator.analyse_expressions(env)
3174

3175
        self.py_wrapper.prepare_argument_coercion(env)
3176
        return self
3177

3178
    def needs_assignment_synthesis(self, env, code=None):
3179
        if self.is_staticmethod:
3180
            return True
3181
        if self.specialized_cpdefs or self.entry.is_fused_specialized:
3182
            return False
3183 3184
        if self.no_assignment_synthesis:
            return False
3185 3186
        if self.entry.is_special:
            return False
3187
        if self.entry.is_anonymous:
3188
            return True
3189
        if env.is_module_scope or env.is_c_class_scope:
3190
            if code is None:
3191
                return self.local_scope.directives['binding']
3192 3193 3194 3195
            else:
                return code.globalstate.directives['binding']
        return env.is_py_class_scope or env.is_closure_scope

3196 3197 3198 3199
    def error_value(self):
        return self.entry.signature.error_value

    def caller_will_check_exceptions(self):
3200
        return self.entry.signature.exception_check
3201 3202

    def generate_function_definitions(self, env, code):
3203
        if self.defaults_getter:
3204 3205
            # defaults getter must never live in class scopes, it's always a module function
            self.defaults_getter.generate_function_definitions(env.global_scope(), code)
3206

3207 3208 3209 3210 3211 3212 3213
        # Before closure cnames are mangled
        if self.py_wrapper_required:
            # func_cname might be modified by @cname
            self.py_wrapper.func_cname = self.entry.func_cname
            self.py_wrapper.generate_function_definitions(env, code)
        FuncDefNode.generate_function_definitions(self, env, code)

3214
    def generate_function_header(self, code, with_pymethdef, proto_only=0):
3215 3216 3217 3218 3219
        if proto_only:
            if self.py_wrapper_required:
                self.py_wrapper.generate_function_header(
                    code, with_pymethdef, True)
            return
William Stein's avatar
William Stein committed
3220
        arg_code_list = []
3221
        if self.entry.signature.has_dummy_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3222 3223 3224
            self_arg = 'PyObject *%s' % Naming.self_cname
            if not self.needs_outer_scope:
                self_arg = 'CYTHON_UNUSED ' + self_arg
3225 3226 3227 3228 3229 3230 3231 3232
            arg_code_list.append(self_arg)

        def arg_decl_code(arg):
            entry = arg.entry
            if entry.in_closure:
                cname = entry.original_cname
            else:
                cname = entry.cname
3233
            decl = entry.type.declaration_code(cname)
Stefan Behnel's avatar
Stefan Behnel committed
3234 3235 3236
            if not entry.cf_used:
                decl = 'CYTHON_UNUSED ' + decl
            return decl
3237 3238 3239 3240 3241 3242 3243

        for arg in self.args:
            arg_code_list.append(arg_decl_code(arg))
        if self.star_arg:
            arg_code_list.append(arg_decl_code(self.star_arg))
        if self.starstar_arg:
            arg_code_list.append(arg_decl_code(self.starstar_arg))
3244 3245 3246 3247
        if arg_code_list:
            arg_code = ', '.join(arg_code_list)
        else:
            arg_code = 'void'  # No arguments
3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305
        dc = self.return_type.declaration_code(self.entry.pyfunc_cname)

        decls_code = code.globalstate['decls']
        preprocessor_guard = self.get_preprocessor_guard()
        if preprocessor_guard:
            decls_code.putln(preprocessor_guard)
        decls_code.putln(
            "static %s(%s); /* proto */" % (dc, arg_code))
        if preprocessor_guard:
            decls_code.putln("#endif")
        code.putln("static %s(%s) {" % (dc, arg_code))

    def generate_argument_declarations(self, env, code):
        pass

    def generate_keyword_list(self, code):
        pass

    def generate_argument_parsing_code(self, env, code):
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(arg.entry)
        for arg in self.star_arg, self.starstar_arg:
            if arg:
                put_into_closure(arg.entry)

    def generate_argument_type_tests(self, code):
        pass


class DefNodeWrapper(FuncDefNode):
    # DefNode python wrapper code generator

    defnode = None
    target = None # Target DefNode

    def __init__(self, *args, **kwargs):
        FuncDefNode.__init__(self, *args, **kwargs)
        self.num_kwonly_args = self.target.num_kwonly_args
        self.num_required_kw_args = self.target.num_required_kw_args
        self.num_required_args = self.target.num_required_args
        self.self_in_stararg = self.target.self_in_stararg
        self.signature = None

    def analyse_declarations(self, env):
        target_entry = self.target.entry
        name = self.name
        prefix = env.next_id(env.scope_prefix)
        target_entry.func_cname = Naming.pywrap_prefix + prefix + name
        target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name

        self.signature = target_entry.signature

3306 3307
        self.np_args_idx = self.target.np_args_idx

3308
    def prepare_argument_coercion(self, env):
3309 3310 3311 3312 3313 3314 3315 3316
        # This is only really required for Cython utility code at this time,
        # everything else can be done during code generation.  But we expand
        # all utility code here, simply because we cannot easily distinguish
        # different code types.
        for arg in self.args:
            if not arg.type.is_pyobject:
                if not arg.type.create_from_py_utility_code(env):
                    pass # will fail later
Robert Bradshaw's avatar
Robert Bradshaw committed
3317
            elif arg.hdr_type and not arg.hdr_type.is_pyobject:
3318 3319
                if not arg.hdr_type.create_to_py_utility_code(env):
                    pass # will fail later
3320

3321 3322 3323 3324 3325 3326 3327 3328 3329
        if self.starstar_arg and not self.starstar_arg.entry.cf_used:
            # we will set the kwargs argument to NULL instead of a new dict
            # and must therefore correct the control flow state
            entry = self.starstar_arg.entry
            entry.xdecref_cleanup = 1
            for ass in entry.cf_assignments:
                if not ass.is_arg and ass.lhs.is_name:
                    ass.lhs.cf_maybe_null = True

3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
    def signature_has_nongeneric_args(self):
        argcount = len(self.args)
        if argcount == 0 or (
                argcount == 1 and (self.args[0].is_self_arg or
                                   self.args[0].is_type_arg)):
            return 0
        return 1

    def signature_has_generic_args(self):
        return self.signature.has_generic_args

    def generate_function_body(self, code):
        args = []
        if self.signature.has_dummy_arg:
            args.append(Naming.self_cname)
        for arg in self.args:
3346 3347 3348
            if arg.hdr_type and not (arg.type.is_memoryviewslice or
                                     arg.type.is_struct or
                                     arg.type.is_complex):
3349 3350 3351
                args.append(arg.type.cast_code(arg.entry.cname))
            else:
                args.append(arg.entry.cname)
3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364
        if self.star_arg:
            args.append(self.star_arg.entry.cname)
        if self.starstar_arg:
            args.append(self.starstar_arg.entry.cname)
        args = ', '.join(args)
        if not self.return_type.is_void:
            code.put('%s = ' % Naming.retval_cname)
        code.putln('%s(%s);' % (
            self.target.entry.pyfunc_cname, args))

    def generate_function_definitions(self, env, code):
        lenv = self.target.local_scope
        # Generate C code for header and body of function
3365
        code.mark_pos(self.pos)
3366 3367 3368 3369 3370 3371
        code.putln("")
        code.putln("/* Python wrapper */")
        preprocessor_guard = self.target.get_preprocessor_guard()
        if preprocessor_guard:
            code.putln(preprocessor_guard)

3372
        code.enter_cfunc_scope(lenv)
3373 3374
        code.return_from_error_cleanup_label = code.new_label()

Vitja Makarov's avatar
Vitja Makarov committed
3375
        with_pymethdef = (self.target.needs_assignment_synthesis(env, code) or
3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398
                          self.target.pymethdef_required)
        self.generate_function_header(code, with_pymethdef)
        self.generate_argument_declarations(lenv, code)
        tempvardecl_code = code.insertion_point()

        if self.return_type.is_pyobject:
            retval_init = ' = 0'
        else:
            retval_init = ''
        if not self.return_type.is_void:
            code.putln('%s%s;' % (
                self.return_type.declaration_code(Naming.retval_cname),
                retval_init))
        code.put_declare_refcount_context()
        code.put_setup_refcount_context('%s (wrapper)' % self.name)

        self.generate_argument_parsing_code(lenv, code)
        self.generate_argument_type_tests(code)
        self.generate_function_body(code)

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)

3399 3400 3401 3402
        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

3403 3404 3405 3406 3407 3408
        # ----- Error cleanup
        if code.error_label in code.labels_used:
            code.put_goto(code.return_label)
            code.put_label(code.error_label)
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
3409 3410 3411
            err_val = self.error_value()
            if err_val is not None:
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
        for entry in lenv.var_entries:
            if entry.is_arg and entry.type.is_pyobject:
                code.put_var_decref(entry)

        code.put_finish_refcount_context()
        if not self.return_type.is_void:
            code.putln("return %s;" % Naming.retval_cname)
        code.putln('}')
        code.exit_cfunc_scope()
        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

    def generate_function_header(self, code, with_pymethdef, proto_only=0):
        arg_code_list = []
        sig = self.signature

3431
        if sig.has_dummy_arg or self.self_in_stararg:
3432 3433 3434 3435
            arg_code = "PyObject *%s" % Naming.self_cname
            if not sig.has_dummy_arg:
                arg_code = 'CYTHON_UNUSED ' + arg_code
            arg_code_list.append(arg_code)
3436

William Stein's avatar
William Stein committed
3437 3438
        for arg in self.args:
            if not arg.is_generic:
3439
                if arg.is_self_arg or arg.is_type_arg:
William Stein's avatar
William Stein committed
3440 3441
                    arg_code_list.append("PyObject *%s" % arg.hdr_cname)
                else:
3442 3443 3444 3445
                    arg_code_list.append(
                        arg.hdr_type.declaration_code(arg.hdr_cname))
        entry = self.target.entry
        if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]:
3446
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
3447
        if entry.scope.is_c_class_scope and entry.name == "__ipow__":
Lisandro Dalcin's avatar
Lisandro Dalcin committed
3448
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
William Stein's avatar
William Stein committed
3449 3450
        if sig.has_generic_args:
            arg_code_list.append(
3451 3452
                "PyObject *%s, PyObject *%s" % (
                    Naming.args_cname, Naming.kwds_cname))
William Stein's avatar
William Stein committed
3453
        arg_code = ", ".join(arg_code_list)
3454 3455 3456 3457

        # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__'
        mf = ""
        if (entry.name in ("__getbuffer__", "__releasebuffer__")
3458
                and entry.scope.is_c_class_scope):
3459
            mf = "CYTHON_UNUSED "
3460
            with_pymethdef = False
3461

3462
        dc = self.return_type.declaration_code(entry.func_cname)
3463
        header = "static %s%s(%s)" % (mf, dc, arg_code)
William Stein's avatar
William Stein committed
3464
        code.putln("%s; /*proto*/" % header)
3465

3466
        if proto_only:
3467
            if self.target.fused_py_func:
3468 3469 3470
                # If we are the specialized version of the cpdef, we still
                # want the prototype for the "fused cpdef", in case we're
                # checking to see if our method was overridden in Python
3471
                self.target.fused_py_func.generate_function_header(
3472
                    code, with_pymethdef, proto_only=True)
3473
            return
3474

3475 3476 3477 3478
        if (Options.docstrings and entry.doc and
                not self.target.fused_py_func and
                not entry.scope.is_property_scope and
                (not entry.is_special or entry.wrapperbase_cname)):
3479
            # h_code = code.globalstate['h_code']
3480
            docstr = entry.doc
3481

Stefan Behnel's avatar
Stefan Behnel committed
3482
            if docstr.is_unicode:
3483
                docstr = docstr.as_utf8_string()
3484

3485
            if not (entry.is_special and entry.name in ('__getbuffer__', '__releasebuffer__')):
3486
                code.putln('PyDoc_STRVAR(%s, %s);' % (
3487
                    entry.doc_cname,
3488
                    docstr.as_c_string_literal()))
3489

3490
            if entry.is_special:
3491
                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
3492
                code.putln(
3493
                    "struct wrapperbase %s;" % entry.wrapperbase_cname)
3494
                code.putln('#endif')
3495

3496
        if with_pymethdef or self.target.fused_py_func:
William Stein's avatar
William Stein committed
3497
            code.put(
3498
                "static PyMethodDef %s = " % entry.pymethdef_cname)
3499
            code.put_pymethoddef(self.target.entry, ";", allow_skip=False)
William Stein's avatar
William Stein committed
3500 3501 3502 3503
        code.putln("%s {" % header)

    def generate_argument_declarations(self, env, code):
        for arg in self.args:
3504
            if arg.is_generic:
3505 3506
                if arg.needs_conversion:
                    code.putln("PyObject *%s = 0;" % arg.hdr_cname)
3507
                else:
3508
                    code.put_var_declaration(arg.entry)
3509 3510 3511
        for entry in env.var_entries:
            if entry.is_arg:
                code.put_var_declaration(entry)
3512

3513
    def generate_argument_parsing_code(self, env, code):
Stefan Behnel's avatar
Stefan Behnel committed
3514 3515
        # Generate fast equivalent of PyArg_ParseTuple call for
        # generic arguments, if any, including args/kwargs
3516 3517
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
Stefan Behnel's avatar
Stefan Behnel committed
3518
        end_label = code.new_label("argument_unpacking_done")
3519

3520 3521 3522
        has_kwonly_args = self.num_kwonly_args > 0
        has_star_or_kw_args = self.star_arg is not None \
            or self.starstar_arg is not None or has_kwonly_args
3523

3524
        for arg in self.args:
3525
            if not arg.type.is_pyobject:
3526
                if not arg.type.create_from_py_utility_code(env):
3527
                    pass  # will fail later
3528

3529
        if not self.signature_has_generic_args():
3530 3531
            if has_star_or_kw_args:
                error(self.pos, "This method cannot have * or keyword arguments")
3532
            self.generate_argument_conversion_code(code)
3533

3534 3535
        elif not self.signature_has_nongeneric_args():
            # func(*args) or func(**kw) or func(*args, **kw)
3536
            self.generate_stararg_copy_code(code)
3537

3538
        else:
3539
            self.generate_tuple_and_keyword_parsing_code(self.args, end_label, code)
3540

3541 3542
        code.error_label = old_error_label
        if code.label_used(our_error_label):
3543 3544
            if not code.label_used(end_label):
                code.put_goto(end_label)
3545 3546 3547 3548 3549
            code.put_label(our_error_label)
            if has_star_or_kw_args:
                self.generate_arg_decref(self.star_arg, code)
                if self.starstar_arg:
                    if self.starstar_arg.entry.xdecref_cleanup:
3550
                        code.put_var_xdecref_clear(self.starstar_arg.entry)
3551
                    else:
3552
                        code.put_var_decref_clear(self.starstar_arg.entry)
3553
            code.put_add_traceback(self.target.entry.qualified_name)
3554
            code.put_finish_refcount_context()
3555
            code.putln("return %s;" % self.error_value())
3556
        if code.label_used(end_label):
3557 3558
            code.put_label(end_label)

William Stein's avatar
William Stein committed
3559 3560
    def generate_arg_xdecref(self, arg, code):
        if arg:
3561
            code.put_var_xdecref_clear(arg.entry)
3562

3563 3564
    def generate_arg_decref(self, arg, code):
        if arg:
3565
            code.put_var_decref_clear(arg.entry)
William Stein's avatar
William Stein committed
3566

3567 3568
    def generate_stararg_copy_code(self, code):
        if not self.star_arg:
3569 3570
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3571 3572 3573
            code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" %
                       Naming.args_cname)
            code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % (
3574
                self.name, Naming.args_cname, self.error_value()))
3575
            code.putln("}")
3576

3577
        if self.starstar_arg:
3578
            if self.star_arg or not self.starstar_arg.entry.cf_used:
3579 3580 3581 3582 3583 3584
                kwarg_check = "unlikely(%s)" % Naming.kwds_cname
            else:
                kwarg_check = "%s" % Naming.kwds_cname
        else:
            kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
                Naming.kwds_cname, Naming.kwds_cname)
3585 3586
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c"))
3587
        code.putln(
3588 3589
            "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
                kwarg_check, Naming.kwds_cname, self.name,
3590
                bool(self.starstar_arg), self.error_value()))
3591

3592 3593
        if self.starstar_arg and self.starstar_arg.entry.cf_used:
            if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references):
3594 3595
                code.putln("if (%s) {" % kwarg_check)
                code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % (
3596 3597 3598 3599
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    self.starstar_arg.entry.cname,
                    self.error_value()))
3600 3601
                code.put_gotref(self.starstar_arg.entry.cname)
                code.putln("} else {")
3602
                code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,))
3603 3604 3605
                code.putln("}")
                self.starstar_arg.entry.xdecref_cleanup = 1
            else:
3606
                code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % (
3607 3608 3609
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    Naming.kwds_cname))
3610
                code.putln("if (unlikely(!%s)) return %s;" % (
3611
                    self.starstar_arg.entry.cname, self.error_value()))
3612 3613
                self.starstar_arg.entry.xdecref_cleanup = 0
                code.put_gotref(self.starstar_arg.entry.cname)
3614

3615
        if self.self_in_stararg and not self.target.is_staticmethod:
3616 3617
            # need to create a new tuple with 'self' inserted as first item
            code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % (
3618 3619 3620
                self.star_arg.entry.cname,
                Naming.args_cname,
                self.star_arg.entry.cname))
3621
            if self.starstar_arg and self.starstar_arg.entry.cf_used:
3622
                code.putln("{")
3623
                code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type)
3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645
                code.putln("return %s;" % self.error_value())
                code.putln("}")
            else:
                code.putln("return %s;" % self.error_value())
            code.put_gotref(self.star_arg.entry.cname)
            code.put_incref(Naming.self_cname, py_object_type)
            code.put_giveref(Naming.self_cname)
            code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % (
                self.star_arg.entry.cname, Naming.self_cname))
            temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % (
                temp, temp, Naming.args_cname, temp))
            code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % (
                Naming.args_cname, temp))
            code.put_incref("item", py_object_type)
            code.put_giveref("item")
            code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % (
                self.star_arg.entry.cname, temp))
            code.putln("}")
            code.funcstate.release_temp(temp)
            self.star_arg.entry.xdecref_cleanup = 0
        elif self.star_arg:
3646 3647
            code.put_incref(Naming.args_cname, py_object_type)
            code.putln("%s = %s;" % (
3648 3649
                self.star_arg.entry.cname,
                Naming.args_cname))
3650 3651
            self.star_arg.entry.xdecref_cleanup = 0

3652
    def generate_tuple_and_keyword_parsing_code(self, args, success_label, code):
3653
        argtuple_error_label = code.new_label("argtuple_error")
3654

3655 3656 3657
        positional_args = []
        required_kw_only_args = []
        optional_kw_only_args = []
3658
        num_pos_only_args = 0
3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671
        for arg in args:
            if arg.is_generic:
                if arg.default:
                    if not arg.is_self_arg and not arg.is_type_arg:
                        if arg.kw_only:
                            optional_kw_only_args.append(arg)
                        else:
                            positional_args.append(arg)
                elif arg.kw_only:
                    required_kw_only_args.append(arg)
                elif not arg.is_self_arg and not arg.is_type_arg:
                    positional_args.append(arg)

3672 3673 3674
                if arg.pos_only:
                    num_pos_only_args += 1

3675 3676 3677 3678
        # sort required kw-only args before optional ones to avoid special
        # cases in the unpacking code
        kw_only_args = required_kw_only_args + optional_kw_only_args

3679
        min_positional_args = self.num_required_args - self.num_required_kw_args
3680
        if len(args) > 0 and (args[0].is_self_arg or args[0].is_type_arg):
3681 3682
            min_positional_args -= 1
        max_positional_args = len(positional_args)
3683 3684
        has_fixed_positional_count = not self.star_arg and \
            min_positional_args == max_positional_args
3685
        has_kw_only_args = bool(kw_only_args)
3686

3687 3688 3689
        if self.starstar_arg or self.star_arg:
            self.generate_stararg_init_code(max_positional_args, code)

3690 3691
        code.putln('{')
        all_args = tuple(positional_args) + tuple(kw_only_args)
3692
        code.putln("static PyObject **%s[] = {%s};" % (
3693
            Naming.pykwdlist_cname,
3694
            ','.join(['&%s' % code.intern_identifier(arg.name)
3695
                      for arg in all_args if not arg.pos_only] + ['0'])))
3696 3697 3698 3699 3700 3701 3702 3703 3704 3705

        # Before being converted and assigned to the target variables,
        # borrowed references to all unpacked argument values are
        # collected into a local PyObject* array called "values",
        # regardless if they were taken from default arguments,
        # positional arguments or keyword arguments.  Note that
        # C-typed default arguments are handled at conversion time,
        # so their array value is NULL in the end if no argument
        # was passed for them.
        self.generate_argument_values_setup_code(all_args, code)
3706

3707
        # --- optimised code when we receive keyword arguments
3708 3709 3710
        code.putln("if (%s(%s)) {" % (
            (self.num_required_kw_args > 0) and "likely" or "unlikely",
            Naming.kwds_cname))
3711 3712
        self.generate_keyword_unpacking_code(
            min_positional_args, max_positional_args,
3713 3714
            num_pos_only_args, has_fixed_positional_count,
            has_kw_only_args, all_args, argtuple_error_label, code)
3715 3716

        # --- optimised code when we do not receive any keyword arguments
3717
        if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
3718 3719 3720 3721 3722 3723 3724
            # Python raises arg tuple related errors first, so we must
            # check the length here
            if min_positional_args == max_positional_args and not self.star_arg:
                compare = '!='
            else:
                compare = '<'
            code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % (
3725
                Naming.args_cname, compare, min_positional_args))
3726
            code.put_goto(argtuple_error_label)
3727

3728 3729 3730 3731
        if self.num_required_kw_args:
            # pure error case: keywords required but not passed
            if max_positional_args > min_positional_args and not self.star_arg:
                code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % (
3732
                    Naming.args_cname, max_positional_args))
3733 3734 3735 3736
                code.put_goto(argtuple_error_label)
            code.putln('} else {')
            for i, arg in enumerate(kw_only_args):
                if not arg.default:
3737
                    pystring_cname = code.intern_identifier(arg.name)
3738
                    # required keyword-only argument missing
3739 3740
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
3741
                    code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
3742 3743
                        self.name,
                        pystring_cname))
3744 3745
                    code.putln(code.error_goto(self.pos))
                    break
3746

3747
        else:
3748
            # optimised tuple unpacking code
3749
            code.putln('} else {')
3750 3751 3752 3753 3754
            if min_positional_args == max_positional_args:
                # parse the exact number of positional arguments from
                # the args tuple
                for i, arg in enumerate(positional_args):
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
3755
            else:
3756 3757 3758 3759 3760 3761 3762 3763
                # parse the positional arguments from the variable length
                # args tuple and reject illegal argument tuple sizes
                code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname)
                if self.star_arg:
                    code.putln('default:')
                reversed_args = list(enumerate(positional_args))[::-1]
                for i, arg in reversed_args:
                    if i >= min_positional_args-1:
Robert Bradshaw's avatar
Robert Bradshaw committed
3764 3765
                        if i != reversed_args[0][0]:
                            code.putln('CYTHON_FALLTHROUGH;')
3766
                        code.put('case %2d: ' % (i+1))
3767 3768
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
                if min_positional_args == 0:
Robert Bradshaw's avatar
Robert Bradshaw committed
3769
                    code.putln('CYTHON_FALLTHROUGH;')
3770 3771 3772 3773 3774 3775 3776 3777 3778
                    code.put('case  0: ')
                code.putln('break;')
                if self.star_arg:
                    if min_positional_args:
                        for i in range(min_positional_args-1, -1, -1):
                            code.putln('case %2d:' % i)
                        code.put_goto(argtuple_error_label)
                else:
                    code.put('default: ')
3779
                    code.put_goto(argtuple_error_label)
3780 3781
                code.putln('}')

3782
        code.putln('}') # end of the conditional unpacking blocks
3783

3784 3785 3786
        # Convert arg values to their final type and assign them.
        # Also inject non-Python default arguments, which do cannot
        # live in the values[] array.
3787 3788
        for i, arg in enumerate(all_args):
            self.generate_arg_assignment(arg, "values[%d]" % i, code)
3789

3790
        code.putln('}') # end of the whole argument unpacking block
3791 3792 3793 3794

        if code.label_used(argtuple_error_label):
            code.put_goto(success_label)
            code.put_label(argtuple_error_label)
3795 3796
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3797
            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
3798 3799 3800
                self.name, has_fixed_positional_count,
                min_positional_args, max_positional_args,
                Naming.args_cname))
3801 3802
            code.putln(code.error_goto(self.pos))

3803 3804 3805 3806 3807 3808 3809 3810
    def generate_arg_assignment(self, arg, item, code):
        if arg.type.is_pyobject:
            # Python default arguments were already stored in 'item' at the very beginning
            if arg.is_generic:
                item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
            entry = arg.entry
            code.putln("%s = %s;" % (entry.cname, item))
        else:
3811
            if arg.type.from_py_function:
3812 3813 3814
                if arg.default:
                    # C-typed default arguments must be handled here
                    code.putln('if (%s) {' % item)
3815 3816
                code.putln(arg.type.from_py_call_code(
                    item, arg.entry.cname, arg.pos, code))
3817 3818
                if arg.default:
                    code.putln('} else {')
3819 3820 3821
                    code.putln("%s = %s;" % (
                        arg.entry.cname,
                        arg.calculate_default_value_code(code)))
3822 3823 3824 3825 3826 3827
                    if arg.type.is_memoryviewslice:
                        code.put_incref_memoryviewslice(arg.entry.cname,
                                                        have_gil=True)
                    code.putln('}')
            else:
                error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type)
3828

3829
    def generate_stararg_init_code(self, max_positional_args, code):
3830
        if self.starstar_arg:
3831
            self.starstar_arg.entry.xdecref_cleanup = 0
Stefan Behnel's avatar
Stefan Behnel committed
3832
            code.putln('%s = PyDict_New(); if (unlikely(!%s)) return %s;' % (
3833 3834 3835
                self.starstar_arg.entry.cname,
                self.starstar_arg.entry.cname,
                self.error_value()))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3836
            code.put_gotref(self.starstar_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3837 3838 3839
        if self.star_arg:
            self.star_arg.entry.xdecref_cleanup = 0
            code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
3840 3841
                Naming.args_cname,
                max_positional_args))
3842
            code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
3843 3844
                self.star_arg.entry.cname, Naming.args_cname,
                max_positional_args, Naming.args_cname))
3845
            code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3846
            if self.starstar_arg:
3847
                code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
3848 3849 3850 3851
            code.put_finish_refcount_context()
            code.putln('return %s;' % self.error_value())
            code.putln('}')
            code.put_gotref(self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3852 3853 3854 3855
            code.putln('} else {')
            code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
            code.put_incref(Naming.empty_tuple, py_object_type)
            code.putln('}')
3856

3857
    def generate_argument_values_setup_code(self, args, code):
3858
        max_args = len(args)
Stefan Behnel's avatar
Stefan Behnel committed
3859 3860
        # the 'values' array collects borrowed references to arguments
        # before doing any type coercion etc.
3861
        code.putln("PyObject* values[%d] = {%s};" % (
Stefan Behnel's avatar
Stefan Behnel committed
3862
            max_args, ','.join('0'*max_args)))
3863

3864
        if self.target.defaults_struct:
3865
            code.putln('%s *%s = __Pyx_CyFunction_Defaults(%s, %s);' % (
3866 3867
                self.target.defaults_struct, Naming.dynamic_args_cname,
                self.target.defaults_struct, Naming.self_cname))
3868

3869 3870
        # assign borrowed Python default values to the values array,
        # so that they can be overwritten by received arguments below
3871
        for i, arg in enumerate(args):
3872 3873 3874 3875
            if arg.default and arg.type.is_pyobject:
                default_value = arg.calculate_default_value_code(code)
                code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))

3876
    def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
3877 3878
                                        num_pos_only_args, has_fixed_positional_count,
                                        has_kw_only_args, all_args, argtuple_error_label, code):
3879
        code.putln('Py_ssize_t kw_args;')
3880
        code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname)
3881
        # copy the values from the args tuple and check that it's not too long
3882
        code.putln('switch (pos_args) {')
Stefan Behnel's avatar
Stefan Behnel committed
3883 3884
        if self.star_arg:
            code.putln('default:')
3885
        for i in range(max_positional_args-1, -1, -1):
3886
            code.put('case %2d: ' % (i+1))
3887
            code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (
3888
                i, Naming.args_cname, i))
3889
            code.putln('CYTHON_FALLTHROUGH;')
3890
        code.putln('case  0: break;')
3891
        if not self.star_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3892
            code.put('default: ') # more arguments than allowed
3893
            code.put_goto(argtuple_error_label)
3894 3895
        code.putln('}')

3896 3897 3898 3899 3900 3901 3902 3903 3904
        # The code above is very often (but not always) the same as
        # the optimised non-kwargs tuple unpacking code, so we keep
        # the code block above at the very top, before the following
        # 'external' PyDict_Size() call, to make it easy for the C
        # compiler to merge the two separate tuple unpacking
        # implementations into one when they turn out to be identical.

        # If we received kwargs, fill up the positional/required
        # arguments with values from the kw dict
3905
        code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname)
3906
        if self.num_required_args or max_positional_args > 0:
Stefan Behnel's avatar
Stefan Behnel committed
3907
            last_required_arg = -1
3908
            last_required_posonly_arg = -1
Stefan Behnel's avatar
Stefan Behnel committed
3909 3910 3911
            for i, arg in enumerate(all_args):
                if not arg.default:
                    last_required_arg = i
3912 3913
                if arg.pos_only and not arg.default:
                    last_required_posonly_arg = i
3914 3915
            if last_required_arg < max_positional_args:
                last_required_arg = max_positional_args-1
Stefan Behnel's avatar
Stefan Behnel committed
3916
            if max_positional_args > 0:
3917
                code.putln('switch (pos_args) {')
3918
            for i, arg in enumerate(all_args[:last_required_arg+1]):
Stefan Behnel's avatar
Stefan Behnel committed
3919
                if max_positional_args > 0 and i <= max_positional_args:
3920 3921
                    if i != 0:
                        code.putln('CYTHON_FALLTHROUGH;')
3922 3923 3924 3925
                    if self.star_arg and i == max_positional_args:
                        code.putln('default:')
                    else:
                        code.putln('case %2d:' % i)
3926
                pystring_cname = code.intern_identifier(arg.name)
3927 3928 3929
                if arg.pos_only:
                    if i == last_required_posonly_arg:
                        code.put_goto(argtuple_error_label)
3930
                    elif i == last_required_arg:
3931 3932
                        code.putln('break;')
                    continue
3933
                if arg.default:
3934
                    if arg.kw_only:
3935
                        # optional kw-only args are handled separately below
3936
                        continue
3937
                    code.putln('if (kw_args > 0) {')
3938
                    # don't overwrite default argument
3939
                    code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, %s);' % (
3940
                        Naming.kwds_cname, pystring_cname))
3941
                    code.putln('if (value) { values[%d] = value; kw_args--; }' % i)
3942 3943
                    code.putln('}')
                else:
3944
                    code.putln('if (likely((values[%d] = __Pyx_PyDict_GetItemStr(%s, %s)) != 0)) kw_args--;' % (
3945
                        i, Naming.kwds_cname, pystring_cname))
3946 3947 3948 3949 3950 3951 3952 3953 3954 3955
                    if i < min_positional_args:
                        if i == 0:
                            # special case: we know arg 0 is missing
                            code.put('else ')
                            code.put_goto(argtuple_error_label)
                        else:
                            # print the correct number of values (args or
                            # kwargs) that were passed into positional
                            # arguments up to this point
                            code.putln('else {')
3956 3957
                            code.globalstate.use_utility_code(
                                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3958
                            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
3959 3960
                                self.name, has_fixed_positional_count,
                                min_positional_args, max_positional_args, i))
3961 3962 3963
                            code.putln(code.error_goto(self.pos))
                            code.putln('}')
                    elif arg.kw_only:
3964
                        code.putln('else {')
3965 3966
                        code.globalstate.use_utility_code(
                            UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
3967 3968
                        code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
                            self.name, pystring_cname))
3969 3970
                        code.putln(code.error_goto(self.pos))
                        code.putln('}')
Stefan Behnel's avatar
Stefan Behnel committed
3971 3972
            if max_positional_args > 0:
                code.putln('}')
3973

3974
        if has_kw_only_args:
3975
            # unpack optional keyword-only arguments separately because
3976
            # checking for interned strings in a dict is faster than iterating
3977
            self.generate_optional_kwonly_args_unpacking_code(all_args, code)
3978

3979
        code.putln('if (unlikely(kw_args > 0)) {')
3980 3981
        # non-positional/-required kw args left in dict: default args,
        # kw-only args, **kwargs or error
Stefan Behnel's avatar
Stefan Behnel committed
3982 3983 3984 3985 3986
        #
        # This is sort of a catch-all: except for checking required
        # arguments, this will always do the right thing for unpacking
        # keyword arguments, so that we can concentrate on optimising
        # common cases above.
3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001
        #
        # ParseOptionalKeywords() needs to know how many of the arguments
        # that could be passed as keywords have in fact been passed as
        # positional args.
        if num_pos_only_args > 0:
            # There are positional-only arguments which we don't want to count,
            # since they cannot be keyword arguments.  Subtract the number of
            # pos-only arguments from the number of positional arguments we got.
            # If we get a negative number then none of the keyword arguments were
            # passed as positional args.
            code.putln('const Py_ssize_t kwd_pos_args = (pos_args < %d) ? 0 : (pos_args - %d);' % (
                num_pos_only_args, num_pos_only_args))
        elif max_positional_args > 0:
            code.putln('const Py_ssize_t kwd_pos_args = pos_args;')

4002 4003 4004
        if max_positional_args == 0:
            pos_arg_count = "0"
        elif self.star_arg:
4005 4006 4007 4008 4009 4010 4011
            # If there is a *arg, the number of used positional args could be larger than
            # the number of possible keyword arguments.  But ParseOptionalKeywords() uses the
            # number of positional args as an index into the keyword argument name array,
            # if this is larger than the number of kwd args we get a segfault.  So round
            # this down to max_positional_args - num_pos_only_args (= num possible kwd args).
            code.putln("const Py_ssize_t used_pos_args = (kwd_pos_args < %d) ? kwd_pos_args : %d;" % (
                max_positional_args - num_pos_only_args, max_positional_args - num_pos_only_args))
Stefan Behnel's avatar
Stefan Behnel committed
4012 4013
            pos_arg_count = "used_pos_args"
        else:
4014
            pos_arg_count = "kwd_pos_args"
4015 4016 4017 4018
        if num_pos_only_args > 0 and num_pos_only_args < len(all_args):
            values_array = 'values + %d' % (num_pos_only_args)
        else:
            values_array = 'values'
4019 4020
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
4021
        code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, "%s") < 0)) %s' % (
4022 4023 4024
            Naming.kwds_cname,
            Naming.pykwdlist_cname,
            self.starstar_arg and self.starstar_arg.entry.cname or '0',
4025
            values_array,
4026 4027 4028
            pos_arg_count,
            self.name,
            code.error_goto(self.pos)))
4029
        code.putln('}')
4030

4031 4032 4033
    def generate_optional_kwonly_args_unpacking_code(self, all_args, code):
        optional_args = []
        first_optional_arg = -1
4034
        num_posonly_args = 0
4035
        for i, arg in enumerate(all_args):
4036 4037
            if arg.pos_only:
                num_posonly_args += 1
4038 4039 4040 4041 4042
            if not arg.kw_only or not arg.default:
                continue
            if not optional_args:
                first_optional_arg = i
            optional_args.append(arg.name)
4043 4044 4045 4046
        if num_posonly_args > 0:
            posonly_correction = '-%d' % num_posonly_args
        else:
            posonly_correction = ''
4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061
        if optional_args:
            if len(optional_args) > 1:
                # if we receive more than the named kwargs, we either have **kwargs
                # (in which case we must iterate anyway) or it's an error (which we
                # also handle during iteration) => skip this part if there are more
                code.putln('if (kw_args > 0 && %s(kw_args <= %d)) {' % (
                    not self.starstar_arg and 'likely' or '',
                    len(optional_args)))
                code.putln('Py_ssize_t index;')
                # not unrolling the loop here reduces the C code overhead
                code.putln('for (index = %d; index < %d && kw_args > 0; index++) {' % (
                    first_optional_arg, first_optional_arg + len(optional_args)))
            else:
                code.putln('if (kw_args == 1) {')
                code.putln('const Py_ssize_t index = %d;' % first_optional_arg)
4062 4063
            code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, *%s[index%s]);' % (
                Naming.kwds_cname, Naming.pykwdlist_cname, posonly_correction))
4064 4065 4066 4067 4068
            code.putln('if (value) { values[index] = value; kw_args--; }')
            if len(optional_args) > 1:
                code.putln('}')
            code.putln('}')

William Stein's avatar
William Stein committed
4069
    def generate_argument_conversion_code(self, code):
4070 4071 4072
        # Generate code to convert arguments from signature type to
        # declared type, if needed.  Also copies signature arguments
        # into closure fields.
William Stein's avatar
William Stein committed
4073 4074 4075 4076 4077 4078 4079 4080 4081
        for arg in self.args:
            if arg.needs_conversion:
                self.generate_arg_conversion(arg, code)

    def generate_arg_conversion(self, arg, code):
        # Generate conversion code for one argument.
        old_type = arg.hdr_type
        new_type = arg.type
        if old_type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
4082 4083 4084 4085
            if arg.default:
                code.putln("if (%s) {" % arg.hdr_cname)
            else:
                code.putln("assert(%s); {" % arg.hdr_cname)
William Stein's avatar
William Stein committed
4086
            self.generate_arg_conversion_from_pyobject(arg, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
4087
            code.putln("}")
William Stein's avatar
William Stein committed
4088 4089 4090 4091
        elif new_type.is_pyobject:
            self.generate_arg_conversion_to_pyobject(arg, code)
        else:
            if new_type.assignable_from(old_type):
4092
                code.putln("%s = %s;" % (arg.entry.cname, arg.hdr_cname))
William Stein's avatar
William Stein committed
4093
            else:
4094
                error(arg.pos, "Cannot convert 1 argument from '%s' to '%s'" % (old_type, new_type))
4095

William Stein's avatar
William Stein committed
4096 4097
    def generate_arg_conversion_from_pyobject(self, arg, code):
        new_type = arg.type
4098
        # copied from CoerceFromPyTypeNode
4099 4100 4101 4102 4103 4104 4105
        if new_type.from_py_function:
            code.putln(new_type.from_py_call_code(
                arg.hdr_cname,
                arg.entry.cname,
                arg.pos,
                code,
            ))
William Stein's avatar
William Stein committed
4106
        else:
4107
            error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
4108

William Stein's avatar
William Stein committed
4109 4110 4111 4112
    def generate_arg_conversion_to_pyobject(self, arg, code):
        old_type = arg.hdr_type
        func = old_type.to_py_function
        if func:
Robert Bradshaw's avatar
Robert Bradshaw committed
4113
            code.putln("%s = %s(%s); %s" % (
William Stein's avatar
William Stein committed
4114 4115 4116
                arg.entry.cname,
                func,
                arg.hdr_cname,
Robert Bradshaw's avatar
Robert Bradshaw committed
4117
                code.error_goto_if_null(arg.entry.cname, arg.pos)))
4118
            code.put_var_gotref(arg.entry)
William Stein's avatar
William Stein committed
4119
        else:
4120
            error(arg.pos, "Cannot convert argument of type '%s' to Python object" % old_type)
William Stein's avatar
William Stein committed
4121 4122 4123 4124 4125 4126 4127 4128

    def generate_argument_type_tests(self, code):
        # Generate type tests for args whose signature
        # type is PyObject * and whose declared type is
        # a subtype thereof.
        for arg in self.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
4129 4130 4131
            elif not arg.accept_none and (arg.type.is_pyobject or
                                          arg.type.is_buffer or
                                          arg.type.is_memoryviewslice):
4132 4133
                self.generate_arg_none_check(arg, code)

William Stein's avatar
William Stein committed
4134
    def error_value(self):
4135
        return self.signature.error_value
4136

4137 4138

class GeneratorDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
4139
    # Generator function node that creates a new generator instance when called.
4140
    #
Stefan Behnel's avatar
Stefan Behnel committed
4141
    # gbody          GeneratorBodyDefNode   the function implementing the generator
4142 4143 4144
    #

    is_generator = True
4145
    is_coroutine = False
4146
    is_iterable_coroutine = False
4147 4148
    is_asyncgen = False
    gen_type_name = 'Generator'
4149 4150
    needs_closure = True

Stefan Behnel's avatar
Stefan Behnel committed
4151
    child_attrs = DefNode.child_attrs + ["gbody"]
4152

4153
    def __init__(self, pos, **kwargs):
4154
        # XXX: don't actually needs a body
4155 4156
        kwargs['body'] = StatListNode(pos, stats=[], is_terminator=True)
        super(GeneratorDefNode, self).__init__(pos, **kwargs)
4157 4158 4159 4160 4161 4162 4163 4164

    def analyse_declarations(self, env):
        super(GeneratorDefNode, self).analyse_declarations(env)
        self.gbody.local_scope = self.local_scope
        self.gbody.analyse_declarations(env)

    def generate_function_body(self, env, code):
        body_cname = self.gbody.entry.func_cname
4165 4166
        name = code.intern_identifier(self.name)
        qualname = code.intern_identifier(self.qualname)
4167
        module_name = code.intern_identifier(self.module_name)
4168

4169
        code.putln('{')
4170
        code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
4171
                   '(__pyx_coroutine_body_t) %s, %s, (PyObject *) %s, %s, %s, %s); %s' % (
4172
                       self.gen_type_name,
4173 4174
                       body_cname, self.code_object.calculate_result_code(code) if self.code_object else 'NULL',
                       Naming.cur_scope_cname, name, qualname, module_name,
4175 4176
                       code.error_goto_if_null('gen', self.pos)))
        code.put_decref(Naming.cur_scope_cname, py_object_type)
4177
        if self.requires_classobj:
4178
            classobj_cname = 'gen->classobj'
4179 4180 4181 4182
            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                classobj_cname, Naming.self_cname))
            code.put_incref(classobj_cname, py_object_type)
            code.put_giveref(classobj_cname)
4183
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4184
        code.putln('return (PyObject *) gen;')
4185
        code.putln('}')
4186 4187

    def generate_function_definitions(self, env, code):
4188
        env.use_utility_code(UtilityCode.load_cached(self.gen_type_name, "Coroutine.c"))
4189 4190 4191 4192 4193
        self.gbody.generate_function_header(code, proto=True)
        super(GeneratorDefNode, self).generate_function_definitions(env, code)
        self.gbody.generate_function_definitions(env, code)


4194
class AsyncDefNode(GeneratorDefNode):
4195
    gen_type_name = 'Coroutine'
4196 4197 4198
    is_coroutine = True


4199 4200 4201 4202 4203
class IterableAsyncDefNode(AsyncDefNode):
    gen_type_name = 'IterableCoroutine'
    is_iterable_coroutine = True


4204 4205 4206 4207 4208
class AsyncGenNode(AsyncDefNode):
    gen_type_name = 'AsyncGen'
    is_asyncgen = True


4209
class GeneratorBodyDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
4210
    # Main code body of a generator implemented as a DefNode.
4211 4212 4213
    #

    is_generator_body = True
4214
    is_inlined = False
4215
    is_async_gen_body = False
4216
    inlined_comprehension_type = None  # container type for inlined comprehensions
4217

4218
    def __init__(self, pos=None, name=None, body=None, is_async_gen_body=False):
4219
        super(GeneratorBodyDefNode, self).__init__(
4220
            pos=pos, body=body, name=name, is_async_gen_body=is_async_gen_body,
4221
            doc=None, args=[], star_arg=None, starstar_arg=None)
4222

4223 4224
    def declare_generator_body(self, env):
        prefix = env.next_id(env.scope_prefix)
Vitja Makarov's avatar
Vitja Makarov committed
4225
        name = env.next_id('generator')
4226 4227 4228 4229
        cname = Naming.genbody_prefix + prefix + name
        entry = env.declare_var(None, py_object_type, self.pos,
                                cname=cname, visibility='private')
        entry.func_cname = cname
4230 4231 4232 4233 4234 4235
        entry.qualified_name = EncodedString(self.name)
        self.entry = entry

    def analyse_declarations(self, env):
        self.analyse_argument_types(env)
        self.declare_generator_body(env)
4236 4237

    def generate_function_header(self, code, proto=False):
4238
        header = "static PyObject *%s(__pyx_CoroutineObject *%s, CYTHON_UNUSED PyThreadState *%s, PyObject *%s)" % (
4239
            self.entry.func_cname,
4240
            Naming.generator_cname,
4241
            Naming.local_tstate_cname,
4242 4243 4244 4245
            Naming.sent_value_cname)
        if proto:
            code.putln('%s; /* proto */' % header)
        else:
Stefan Behnel's avatar
Stefan Behnel committed
4246
            code.putln('%s /* generator body */\n{' % header)
4247 4248 4249 4250 4251 4252 4253 4254

    def generate_function_definitions(self, env, code):
        lenv = self.local_scope

        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)

        # Generate C code for header and body of function
4255
        code.enter_cfunc_scope(lenv)
4256 4257 4258 4259 4260 4261 4262 4263
        code.return_from_error_cleanup_label = code.new_label()

        # ----- Top-level constants used by this function
        code.mark_pos(self.pos)
        self.generate_cached_builtins_decls(lenv, code)
        # ----- Function header
        code.putln("")
        self.generate_function_header(code)
4264
        closure_init_code = code.insertion_point()
4265 4266 4267 4268
        # ----- Local variables
        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
        tempvardecl_code = code.insertion_point()
        code.put_declare_refcount_context()
4269
        code.put_setup_refcount_context(self.entry.name or self.entry.qualified_name)
4270 4271 4272
        profile = code.globalstate.directives['profile']
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
4273
            tempvardecl_code.put_trace_declarations()
4274 4275 4276
            code.funcstate.can_trace = True
            code_object = self.code_object.calculate_result_code(code) if self.code_object else None
            code.put_trace_frame_init(code_object)
4277 4278 4279 4280 4281 4282 4283 4284 4285 4286

        # ----- Resume switch point.
        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
        resume_code = code.insertion_point()
        first_run_label = code.new_label('first_run')
        code.use_label(first_run_label)
        code.put_label(first_run_label)
        code.putln('%s' %
                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))

4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303
        # ----- prepare target container for inlined comprehension
        if self.is_inlined and self.inlined_comprehension_type is not None:
            target_type = self.inlined_comprehension_type
            if target_type is Builtin.list_type:
                comp_init = 'PyList_New(0)'
            elif target_type is Builtin.set_type:
                comp_init = 'PySet_New(NULL)'
            elif target_type is Builtin.dict_type:
                comp_init = 'PyDict_New()'
            else:
                raise InternalError(
                    "invalid type of inlined comprehension: %s" % target_type)
            code.putln("%s = %s; %s" % (
                Naming.retval_cname, comp_init,
                code.error_goto_if_null(Naming.retval_cname, self.pos)))
            code.put_gotref(Naming.retval_cname)

4304 4305
        # ----- Function body
        self.generate_function_body(env, code)
4306
        # ----- Closure initialization
4307
        if lenv.scope_class.type.scope.var_entries:
4308 4309 4310 4311
            closure_init_code.putln('%s = %s;' % (
                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
                lenv.scope_class.type.cast_code('%s->closure' %
                                                Naming.generator_cname)))
4312
            # FIXME: this silences a potential "unused" warning => try to avoid unused closures in more cases
4313
            code.putln("CYTHON_MAYBE_UNUSED_VAR(%s);" % Naming.cur_scope_cname)
4314

4315 4316 4317
        if profile or linetrace:
            code.funcstate.can_trace = False

4318 4319 4320 4321
        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

Stefan Behnel's avatar
Stefan Behnel committed
4322 4323
        # on normal generator termination, we do not take the exception propagation
        # path: no traceback info is required and not creating it is much faster
4324
        if not self.is_inlined and not self.body.is_terminator:
4325 4326 4327
            if self.is_async_gen_body:
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
4328
            code.putln('PyErr_SetNone(%s);' % (
4329
                '__Pyx_PyExc_StopAsyncIteration' if self.is_async_gen_body else 'PyExc_StopIteration'))
4330
        # ----- Error cleanup
4331
        if code.label_used(code.error_label):
4332 4333
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
4334
            code.put_label(code.error_label)
4335 4336
            if self.is_inlined and self.inlined_comprehension_type is not None:
                code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4337 4338
            if Future.generator_stop in env.global_scope().context.future_directives:
                # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
4339
                code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
4340
                code.putln("__Pyx_Generator_Replace_StopIteration(%d);" % bool(self.is_async_gen_body))
4341 4342
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
4343
            code.put_add_traceback(self.entry.qualified_name)
4344 4345 4346

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
4347 4348 4349 4350
        if self.is_inlined:
            code.put_xgiveref(Naming.retval_cname)
        else:
            code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4351 4352
        # For Py3.7, clearing is already done below.
        code.putln("#if !CYTHON_USE_EXC_INFO_STACK")
4353
        code.putln("__Pyx_Coroutine_ResetAndClearException(%s);" % Naming.generator_cname)
4354
        code.putln("#endif")
4355
        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
4356
        # clean up as early as possible to help breaking any reference cycles
4357
        code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)
4358 4359 4360
        if profile or linetrace:
            code.put_trace_return(Naming.retval_cname,
                                  nogil=not code.funcstate.gil_owned)
4361
        code.put_finish_refcount_context()
4362
        code.putln("return %s;" % Naming.retval_cname)
4363 4364 4365 4366 4367
        code.putln("}")

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)
        # ----- Generator resume code
4368 4369 4370
        if profile or linetrace:
            resume_code.put_trace_call(self.entry.qualified_name, self.pos,
                                       nogil=not code.funcstate.gil_owned)
4371
        resume_code.putln("switch (%s->resume_label) {" % (
Stefan Behnel's avatar
Stefan Behnel committed
4372
                       Naming.generator_cname))
4373

4374
        resume_code.putln("case 0: goto %s;" % first_run_label)
4375

4376 4377
        for i, label in code.yield_labels:
            resume_code.putln("case %d: goto %s;" % (i, label))
Stefan Behnel's avatar
Stefan Behnel committed
4378
        resume_code.putln("default: /* CPython raises the right error here */")
4379 4380 4381
        if profile or linetrace:
            resume_code.put_trace_return("Py_None",
                                         nogil=not code.funcstate.gil_owned)
4382
        resume_code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4383 4384
        resume_code.putln("return NULL;")
        resume_code.putln("}")
4385 4386 4387 4388

        code.exit_cfunc_scope()


4389 4390
class OverrideCheckNode(StatNode):
    # A Node for dispatching to the def method if it
Unknown's avatar
Unknown committed
4391
    # is overridden.
4392 4393 4394 4395 4396 4397
    #
    #  py_func
    #
    #  args
    #  func_temp
    #  body
4398

Robert Bradshaw's avatar
Robert Bradshaw committed
4399
    child_attrs = ['body']
4400

4401
    body = None
Robert Bradshaw's avatar
Robert Bradshaw committed
4402

4403 4404
    def analyse_expressions(self, env):
        self.args = env.arg_entries
4405 4406 4407 4408
        if self.py_func.is_module_scope:
            first_arg = 0
        else:
            first_arg = 1
4409
        from . import ExprNodes
4410
        self.func_node = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
Stefan Behnel's avatar
Stefan Behnel committed
4411 4412
        call_node = ExprNodes.SimpleCallNode(
            self.pos, function=self.func_node,
4413 4414
            args=[ExprNodes.NameNode(self.pos, name=arg.name)
                  for arg in self.args[first_arg:]])
4415 4416
        if env.return_type.is_void or env.return_type.is_returncode:
            self.body = StatListNode(self.pos, stats=[
4417 4418
                ExprStatNode(self.pos, expr=call_node),
                ReturnStatNode(self.pos, value=None)])
4419 4420
        else:
            self.body = ReturnStatNode(self.pos, value=call_node)
4421 4422
        self.body = self.body.analyse_expressions(env)
        return self
4423

4424
    def generate_execution_code(self, code):
4425
        interned_attr_cname = code.intern_identifier(self.py_func.entry.name)
4426
        # Check to see if we are an extension type
4427 4428 4429 4430
        if self.py_func.is_module_scope:
            self_arg = "((PyObject *)%s)" % Naming.module_cname
        else:
            self_arg = "((PyObject *)%s)" % self.args[0].cname
4431
        code.putln("/* Check if called by wrapper */")
4432
        code.putln("if (unlikely(%s)) ;" % Naming.skip_dispatch_cname)
Stefan Behnel's avatar
Stefan Behnel committed
4433
        code.putln("/* Check if overridden in Python */")
4434 4435 4436
        if self.py_func.is_module_scope:
            code.putln("else {")
        else:
4437 4438 4439
            code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0)"
                       " || (Py_TYPE(%s)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % (
                self_arg, self_arg))
4440

4441
        code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
4442 4443
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("PyDictVersioning", "ObjectHandling.c"))
4444 4445 4446 4447
        # TODO: remove the object dict version check by 'inlining' the getattr implementation for methods.
        # This would allow checking the dict versions around _PyType_Lookup() if it returns a descriptor,
        # and would (tada!) make this check a pure type based thing instead of supporting only a single
        # instance at a time.
4448 4449 4450 4451 4452 4453
        code.putln("static PY_UINT64_T %s = __PYX_DICT_VERSION_INIT, %s = __PYX_DICT_VERSION_INIT;" % (
            Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
        code.putln("if (unlikely(!__Pyx_object_dict_version_matches(%s, %s, %s))) {" % (
            self_arg, Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
        code.putln("PY_UINT64_T %s = __Pyx_get_tp_dict_version(%s);" % (
            Naming.type_dict_guard_temp, self_arg))
4454 4455
        code.putln("#endif")

4456 4457
        func_node_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.func_node.set_cname(func_node_temp)
4458
        # need to get attribute manually--scope would return cdef method
4459 4460
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("PyObjectGetAttrStr", "ObjectHandling.c"))
4461
        err = code.error_goto_if_null(func_node_temp, self.pos)
4462
        code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % (
4463 4464
            func_node_temp, self_arg, interned_attr_cname, err))
        code.put_gotref(func_node_temp)
4465

4466
        is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp
4467
        is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)(void*)%s)" % (
4468
            func_node_temp, self.py_func.entry.func_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
4469
        code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden))
4470 4471
        self.body.generate_execution_code(code)
        code.putln("}")
4472 4473 4474 4475

        # NOTE: it's not 100% sure that we catch the exact versions here that were used for the lookup,
        # but it is very unlikely that the versions change during lookup, and the type dict safe guard
        # should increase the chance of detecting such a case.
4476
        code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
4477 4478 4479 4480
        code.putln("%s = __Pyx_get_tp_dict_version(%s);" % (
            Naming.tp_dict_version_temp, self_arg))
        code.putln("%s = __Pyx_get_object_dict_version(%s);" % (
            Naming.obj_dict_version_temp, self_arg))
4481 4482 4483 4484 4485
        # Safety check that the type dict didn't change during the lookup.  Since CPython looks up the
        # attribute (descriptor) first in the type dict and then in the instance dict or through the
        # descriptor, the only really far-away lookup when we get here is one in the type dict. So we
        # double check the type dict version before and afterwards to guard against later changes of
        # the type dict during the lookup process.
4486 4487 4488 4489
        code.putln("if (unlikely(%s != %s)) {" % (
            Naming.type_dict_guard_temp, Naming.tp_dict_version_temp))
        code.putln("%s = %s = __PYX_DICT_VERSION_INIT;" % (
            Naming.tp_dict_version_temp, Naming.obj_dict_version_temp))
4490 4491 4492
        code.putln("}")
        code.putln("#endif")

4493 4494
        code.put_decref_clear(func_node_temp, PyrexTypes.py_object_type)
        code.funcstate.release_temp(func_node_temp)
4495

4496
        code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS")
4497 4498 4499
        code.putln("}")
        code.putln("#endif")

Robert Bradshaw's avatar
Robert Bradshaw committed
4500
        code.putln("}")
4501

4502

Robert Bradshaw's avatar
Robert Bradshaw committed
4503 4504
class ClassDefNode(StatNode, BlockNode):
    pass
4505

4506

Robert Bradshaw's avatar
Robert Bradshaw committed
4507
class PyClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4508 4509
    #  A Python class definition.
    #
Stefan Behnel's avatar
Stefan Behnel committed
4510
    #  name     EncodedString   Name of the class
4511
    #  doc      string or None  The class docstring
William Stein's avatar
William Stein committed
4512 4513 4514
    #  body     StatNode        Attribute definition code
    #  entry    Symtab.Entry
    #  scope    PyClassScope
4515
    #  decorators    [DecoratorNode]        list of decorators or None
William Stein's avatar
William Stein committed
4516 4517 4518
    #
    #  The following subnodes are constructed internally:
    #
4519
    #  doc_node NameNode   '__doc__' name that is made available to the class body
4520
    #  dict     DictNode   Class dictionary or Py3 namespace
William Stein's avatar
William Stein committed
4521 4522
    #  classobj ClassNode  Class object
    #  target   NameNode   Variable to assign class object to
4523

4524
    child_attrs = ["doc_node", "body", "dict", "metaclass", "mkw", "bases", "class_result",
4525
                   "target", "class_cell", "decorators"]
4526
    decorators = None
4527
    class_result = None
4528 4529 4530
    is_py3_style_class = False  # Python3 style class (kwargs)
    metaclass = None
    mkw = None
4531
    doc_node = None
4532

4533
    def __init__(self, pos, name, bases, doc, body, decorators=None,
4534
                 keyword_args=None, force_py3_semantics=False):
William Stein's avatar
William Stein committed
4535 4536 4537 4538
        StatNode.__init__(self, pos)
        self.name = name
        self.doc = doc
        self.body = body
4539
        self.decorators = decorators
4540
        self.bases = bases
4541
        from . import ExprNodes
4542
        if self.doc and Options.docstrings:
4543
            doc = embed_position(self.pos, self.doc)
4544
            doc_node = ExprNodes.StringNode(pos, value=doc)
4545
            self.doc_node = ExprNodes.NameNode(name=EncodedString('__doc__'), type=py_object_type, pos=pos)
William Stein's avatar
William Stein committed
4546 4547
        else:
            doc_node = None
4548 4549

        allow_py2_metaclass = not force_py3_semantics
4550
        if keyword_args:
4551 4552
            allow_py2_metaclass = False
            self.is_py3_style_class = True
4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566
            if keyword_args.is_dict_literal:
                if keyword_args.key_value_pairs:
                    for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
                        if item.key.value == 'metaclass':
                            if self.metaclass is not None:
                                error(item.pos, "keyword argument 'metaclass' passed multiple times")
                            # special case: we already know the metaclass,
                            # so we don't need to do the "build kwargs,
                            # find metaclass" dance at runtime
                            self.metaclass = item.value
                            del keyword_args.key_value_pairs[i]
                    self.mkw = keyword_args
                else:
                    assert self.metaclass is not None
4567
            else:
4568
                # MergedDictNode
4569
                self.mkw = ExprNodes.ProxyNode(keyword_args)
4570 4571

        if force_py3_semantics or self.bases or self.mkw or self.metaclass:
4572
            if self.metaclass is None:
4573
                if keyword_args and not keyword_args.is_dict_literal:
4574 4575 4576 4577
                    # **kwargs may contain 'metaclass' arg
                    mkdict = self.mkw
                else:
                    mkdict = None
4578 4579 4580 4581 4582 4583 4584
                if (not mkdict and
                        self.bases.is_sequence_constructor and
                        not self.bases.args):
                    pass  # no base classes => no inherited metaclass
                else:
                    self.metaclass = ExprNodes.PyClassMetaclassNode(
                        pos, mkw=mkdict, bases=self.bases)
4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597
                needs_metaclass_calculation = False
            else:
                needs_metaclass_calculation = True

            self.dict = ExprNodes.PyClassNamespaceNode(
                pos, name=name, doc=doc_node,
                metaclass=self.metaclass, bases=self.bases, mkw=self.mkw)
            self.classobj = ExprNodes.Py3ClassNode(
                pos, name=name,
                bases=self.bases, dict=self.dict, doc=doc_node,
                metaclass=self.metaclass, mkw=self.mkw,
                calculate_metaclass=needs_metaclass_calculation,
                allow_py2_metaclass=allow_py2_metaclass)
4598
        else:
4599 4600 4601 4602 4603 4604 4605
            # no bases, no metaclass => old style class creation
            self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
            self.classobj = ExprNodes.ClassNode(
                pos, name=name,
                bases=bases, dict=self.dict, doc=doc_node)

        self.target = ExprNodes.NameNode(pos, name=name)
4606
        self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
4607

4608 4609
    def as_cclass(self):
        """
4610
        Return this node as if it were declared as an extension class
4611
        """
4612
        if self.is_py3_style_class:
4613 4614
            error(self.classobj.pos, "Python3 style class could not be represented as C class")
            return
4615

4616
        from . import ExprNodes
4617
        return CClassDefNode(self.pos,
4618 4619 4620
                             visibility='private',
                             module_name=None,
                             class_name=self.name,
4621
                             bases=self.classobj.bases or ExprNodes.TupleNode(self.pos, args=[]),
4622 4623 4624 4625
                             decorators=self.decorators,
                             body=self.body,
                             in_pxd=False,
                             doc=self.doc)
4626

4627 4628
    def create_scope(self, env):
        genv = env
4629 4630
        while genv.is_py_class_scope or genv.is_c_class_scope:
            genv = genv.outer_scope
4631
        cenv = self.scope = PyClassScope(name=self.name, outer_scope=genv)
4632
        return cenv
4633

William Stein's avatar
William Stein committed
4634
    def analyse_declarations(self, env):
4635 4636
        class_result = self.classobj
        if self.decorators:
4637
            from .ExprNodes import SimpleCallNode
4638 4639 4640
            for decorator in self.decorators[::-1]:
                class_result = SimpleCallNode(
                    decorator.pos,
4641 4642
                    function=decorator.decorator,
                    args=[class_result])
4643
            self.decorators = None
4644 4645
        self.class_result = class_result
        self.class_result.analyse_declarations(env)
William Stein's avatar
William Stein committed
4646
        self.target.analyse_target_declaration(env)
4647
        cenv = self.create_scope(env)
4648
        cenv.directives = env.directives
4649
        cenv.class_obj_cname = self.target.entry.cname
4650 4651
        if self.doc_node:
            self.doc_node.analyse_target_declaration(cenv)
4652
        self.body.analyse_declarations(cenv)
4653

William Stein's avatar
William Stein committed
4654
    def analyse_expressions(self, env):
4655
        if self.bases:
4656
            self.bases = self.bases.analyse_expressions(env)
4657
        if self.metaclass:
4658
            self.metaclass = self.metaclass.analyse_expressions(env)
4659
        if self.mkw:
4660 4661 4662
            self.mkw = self.mkw.analyse_expressions(env)
        self.dict = self.dict.analyse_expressions(env)
        self.class_result = self.class_result.analyse_expressions(env)
4663
        cenv = self.scope
4664
        self.body = self.body.analyse_expressions(cenv)
4665
        self.target.analyse_target_expression(env, self.classobj)
4666 4667
        self.class_cell = self.class_cell.analyse_expressions(cenv)
        return self
4668

4669
    def generate_function_definitions(self, env, code):
4670
        self.generate_lambda_definitions(self.scope, code)
4671
        self.body.generate_function_definitions(self.scope, code)
4672

William Stein's avatar
William Stein committed
4673
    def generate_execution_code(self, code):
4674
        code.mark_pos(self.pos)
4675 4676
        code.pyclass_stack.append(self)
        cenv = self.scope
4677
        if self.bases:
4678
            self.bases.generate_evaluation_code(code)
4679
        if self.mkw:
4680
            self.mkw.generate_evaluation_code(code)
4681
        if self.metaclass:
4682
            self.metaclass.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
4683
        self.dict.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4684
        cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
4685
        self.class_cell.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4686
        self.body.generate_execution_code(code)
4687
        self.class_result.generate_evaluation_code(code)
4688 4689 4690
        self.class_cell.generate_injection_code(
            code, self.class_result.result())
        self.class_cell.generate_disposal_code(code)
4691
        cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
4692
        self.target.generate_assignment_code(self.class_result, code)
William Stein's avatar
William Stein committed
4693
        self.dict.generate_disposal_code(code)
4694
        self.dict.free_temps(code)
4695
        if self.metaclass:
4696 4697
            self.metaclass.generate_disposal_code(code)
            self.metaclass.free_temps(code)
4698 4699 4700 4701
        if self.mkw:
            self.mkw.generate_disposal_code(code)
            self.mkw.free_temps(code)
        if self.bases:
4702 4703
            self.bases.generate_disposal_code(code)
            self.bases.free_temps(code)
4704
        code.pyclass_stack.pop()
William Stein's avatar
William Stein committed
4705

4706

Robert Bradshaw's avatar
Robert Bradshaw committed
4707
class CClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4708 4709 4710 4711
    #  An extension type definition.
    #
    #  visibility         'private' or 'public' or 'extern'
    #  typedef_flag       boolean
Stefan Behnel's avatar
Stefan Behnel committed
4712
    #  api                boolean
William Stein's avatar
William Stein committed
4713 4714 4715
    #  module_name        string or None    For import of extern type objects
    #  class_name         string            Unqualified name of class
    #  as_name            string or None    Name to declare as in this scope
4716
    #  bases              TupleNode         Base class(es)
William Stein's avatar
William Stein committed
4717 4718
    #  objstruct_name     string or None    Specified C name of object struct
    #  typeobj_name       string or None    Specified C name of type object
4719
    #  check_size         'warn', 'error', 'ignore'     What to do if tp_basicsize does not match
William Stein's avatar
William Stein committed
4720
    #  in_pxd             boolean           Is in a .pxd file
4721
    #  decorators         [DecoratorNode]   list of decorators or None
William Stein's avatar
William Stein committed
4722 4723 4724 4725
    #  doc                string or None
    #  body               StatNode or None
    #  entry              Symtab.Entry
    #  base_type          PyExtensionType or None
4726 4727
    #  buffer_defaults_node DictNode or None Declares defaults for a buffer
    #  buffer_defaults_pos
4728

4729
    child_attrs = ["body"]
4730 4731
    buffer_defaults_node = None
    buffer_defaults_pos = None
4732 4733 4734 4735
    typedef_flag = False
    api = False
    objstruct_name = None
    typeobj_name = None
4736
    check_size = None
4737
    decorators = None
4738
    shadow = False
4739

Robert Bradshaw's avatar
Robert Bradshaw committed
4740 4741
    def buffer_defaults(self, env):
        if not hasattr(self, '_buffer_defaults'):
4742
            from . import Buffer
Robert Bradshaw's avatar
Robert Bradshaw committed
4743 4744 4745 4746 4747 4748 4749 4750 4751
            if self.buffer_defaults_node:
                self._buffer_defaults = Buffer.analyse_buffer_options(
                    self.buffer_defaults_pos,
                    env, [], self.buffer_defaults_node,
                    need_complete=False)
            else:
                self._buffer_defaults = None
        return self._buffer_defaults

4752 4753 4754 4755 4756 4757 4758 4759
    def declare(self, env):
        if self.module_name and self.visibility != 'extern':
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return None
        else:
            home_scope = env
4760

4761
        self.entry = home_scope.declare_c_class(
4762 4763 4764 4765 4766 4767 4768 4769 4770 4771
            name=self.class_name,
            pos=self.pos,
            defining=0,
            implementing=0,
            module_name=self.module_name,
            base_type=None,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
4772
            check_size = self.check_size,
4773 4774 4775
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4776

William Stein's avatar
William Stein committed
4777 4778 4779 4780
    def analyse_declarations(self, env):
        #print "CClassDefNode.analyse_declarations:", self.class_name
        #print "...visibility =", self.visibility
        #print "...module_name =", self.module_name
4781

William Stein's avatar
William Stein committed
4782
        if env.in_cinclude and not self.objstruct_name:
4783
            error(self.pos, "Object struct name specification required for C class defined in 'extern from' block")
4784
        if self.decorators:
4785
            error(self.pos, "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
William Stein's avatar
William Stein committed
4786
        self.base_type = None
4787 4788
        # Now that module imports are cached, we need to
        # import the modules for extern classes.
4789 4790 4791 4792 4793 4794 4795
        if self.module_name:
            self.module = None
            for module in env.cimported_modules:
                if module.name == self.module_name:
                    self.module = module
            if self.module is None:
                self.module = ModuleScope(self.module_name, None, env.context)
4796
                self.module.has_extern_class = 1
Robert Bradshaw's avatar
Robert Bradshaw committed
4797
                env.add_imported_module(self.module)
4798

4799
        if self.bases.args:
Robert Bradshaw's avatar
Robert Bradshaw committed
4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822
            base = self.bases.args[0]
            base_type = base.analyse_as_type(env)
            if base_type in (PyrexTypes.c_int_type, PyrexTypes.c_long_type, PyrexTypes.c_float_type):
                # Use the Python rather than C variant of these types.
                base_type = env.lookup(base_type.sign_and_name()).type
            if base_type is None:
                error(base.pos, "First base of '%s' is not an extension type" % self.class_name)
            elif base_type == PyrexTypes.py_object_type:
                base_class_scope = None
            elif not base_type.is_extension_type and \
                     not (base_type.is_builtin_type and base_type.objstruct_cname):
                error(base.pos, "'%s' is not an extension type" % base_type)
            elif not base_type.is_complete():
                error(base.pos, "Base class '%s' of type '%s' is incomplete" % (
                    base_type.name, self.class_name))
            elif base_type.scope and base_type.scope.directives and \
                     base_type.is_final_type:
                error(base.pos, "Base class '%s' of type '%s' is final" % (
                    base_type, self.class_name))
            elif base_type.is_builtin_type and \
                     base_type.name in ('tuple', 'str', 'bytes'):
                error(base.pos, "inheritance from PyVarObject types like '%s' is not currently supported"
                      % base_type.name)
William Stein's avatar
William Stein committed
4823
            else:
Robert Bradshaw's avatar
Robert Bradshaw committed
4824 4825 4826
                self.base_type = base_type
            if env.directives.get('freelist', 0) > 0 and base_type != PyrexTypes.py_object_type:
                warning(self.pos, "freelists cannot be used on subtypes, only the base class can manage them", 1)
4827

William Stein's avatar
William Stein committed
4828
        has_body = self.body is not None
4829 4830 4831 4832 4833 4834
        if has_body and self.base_type and not self.base_type.scope:
            # To properly initialize inherited attributes, the base type must
            # be analysed before this type.
            self.base_type.defered_declarations.append(lambda : self.analyse_declarations(env))
            return

4835
        if self.module_name and self.visibility != 'extern':
4836 4837 4838 4839 4840 4841
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return
        else:
            home_scope = env
4842 4843

        if self.visibility == 'extern':
4844
            if (self.module_name == '__builtin__' and
4845 4846
                    self.class_name in Builtin.builtin_types and
                    env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython
4847
                warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
4848

4849
        self.entry = home_scope.declare_c_class(
4850 4851 4852 4853 4854 4855 4856 4857
            name=self.class_name,
            pos=self.pos,
            defining=has_body and self.in_pxd,
            implementing=has_body and not self.in_pxd,
            module_name=self.module_name,
            base_type=self.base_type,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
4858
            check_size=self.check_size,
4859 4860 4861 4862 4863
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4864

4865 4866
        if self.shadow:
            home_scope.lookup(self.class_name).as_variable = self.entry
4867
        if home_scope is not env and self.visibility == 'extern':
4868
            env.add_imported_entry(self.class_name, self.entry, self.pos)
4869
        self.scope = scope = self.entry.type.scope
4870 4871
        if scope is not None:
            scope.directives = env.directives
4872

4873
        if self.doc and Options.docstrings:
4874
            scope.doc = embed_position(self.pos, self.doc)
4875

William Stein's avatar
William Stein committed
4876 4877
        if has_body:
            self.body.analyse_declarations(scope)
4878
            dict_entry = self.scope.lookup_here("__dict__")
4879
            if dict_entry and dict_entry.is_variable and (not scope.defined and not scope.implemented):
4880 4881
                dict_entry.getter_cname = self.scope.mangle_internal("__dict__getter")
                self.scope.declare_property("__dict__", dict_entry.doc, dict_entry.pos)
William Stein's avatar
William Stein committed
4882 4883 4884 4885
            if self.in_pxd:
                scope.defined = 1
            else:
                scope.implemented = 1
4886

4887
        if len(self.bases.args) > 1:
Robert Bradshaw's avatar
Robert Bradshaw committed
4888 4889
            if not has_body or self.in_pxd:
                error(self.bases.args[1].pos, "Only declare first base in declaration.")
4890 4891
            # At runtime, we check that the other bases are heap types
            # and that a __dict__ is added if required.
Robert Bradshaw's avatar
Robert Bradshaw committed
4892 4893 4894 4895 4896 4897 4898 4899 4900 4901
            for other_base in self.bases.args[1:]:
                if other_base.analyse_as_type(env):
                    error(other_base.pos, "Only one extension type base class allowed.")
            self.entry.type.early_init = 0
            from . import ExprNodes
            self.type_init_args = ExprNodes.TupleNode(
                self.pos,
                args=[ExprNodes.IdentifierStringNode(self.pos, value=self.class_name),
                      self.bases,
                      ExprNodes.DictNode(self.pos, key_value_pairs=[])])
4902
        elif self.base_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
4903 4904
            self.entry.type.early_init = self.base_type.is_external or self.base_type.early_init
            self.type_init_args = None
4905
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
4906 4907
            self.entry.type.early_init = 1
            self.type_init_args = None
4908

William Stein's avatar
William Stein committed
4909
        env.allocate_vtable_names(self.entry)
4910

4911 4912 4913
        for thunk in self.entry.type.defered_declarations:
            thunk()

William Stein's avatar
William Stein committed
4914 4915
    def analyse_expressions(self, env):
        if self.body:
Robert Bradshaw's avatar
Robert Bradshaw committed
4916
            scope = self.entry.type.scope
4917
            self.body = self.body.analyse_expressions(scope)
4918 4919
        if self.type_init_args:
            self.type_init_args.analyse_expressions(env)
4920
        return self
4921

4922
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
4923
        if self.body:
4924 4925
            self.generate_lambda_definitions(self.scope, code)
            self.body.generate_function_definitions(self.scope, code)
4926

William Stein's avatar
William Stein committed
4927 4928 4929
    def generate_execution_code(self, code):
        # This is needed to generate evaluation code for
        # default values of method arguments.
4930
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4931 4932
        if self.body:
            self.body.generate_execution_code(code)
4933 4934
        if not self.entry.type.early_init:
            if self.type_init_args:
Robert Bradshaw's avatar
Robert Bradshaw committed
4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957
                self.type_init_args.generate_evaluation_code(code)
                bases = "PyTuple_GET_ITEM(%s, 1)" % self.type_init_args.result()
                first_base = "((PyTypeObject*)PyTuple_GET_ITEM(%s, 0))" % bases
                # Let Python do the base types compatibility checking.
                trial_type = code.funcstate.allocate_temp(PyrexTypes.py_object_type, True)
                code.putln("%s = PyType_Type.tp_new(&PyType_Type, %s, NULL);" % (
                    trial_type, self.type_init_args.result()))
                code.putln(code.error_goto_if_null(trial_type, self.pos))
                code.put_gotref(trial_type)
                code.putln("if (((PyTypeObject*) %s)->tp_base != %s) {" % (
                    trial_type, first_base))
                code.putln("PyErr_Format(PyExc_TypeError, \"best base '%s' must be equal to first base '%s'\",")
                code.putln("             ((PyTypeObject*) %s)->tp_base->tp_name, %s->tp_name);" % (
                           trial_type, first_base))
                code.putln(code.error_goto(self.pos))
                code.putln("}")
                code.funcstate.release_temp(trial_type)
                code.put_incref(bases, PyrexTypes.py_object_type)
                code.put_giveref(bases)
                code.putln("%s.tp_bases = %s;" % (self.entry.type.typeobj_cname, bases))
                code.put_decref_clear(trial_type, PyrexTypes.py_object_type)
                self.type_init_args.generate_disposal_code(code)
                self.type_init_args.free_temps(code)
4958

Robert Bradshaw's avatar
Robert Bradshaw committed
4959
            self.generate_type_ready_code(self.entry, code, True)
4960 4961 4962

    # Also called from ModuleNode for early init types.
    @staticmethod
Robert Bradshaw's avatar
Robert Bradshaw committed
4963
    def generate_type_ready_code(entry, code, heap_type_bases=False):
4964 4965 4966 4967 4968
        # Generate a call to PyType_Ready for an extension
        # type defined in this module.
        type = entry.type
        typeobj_cname = type.typeobj_cname
        scope = type.scope
Robert Bradshaw's avatar
Robert Bradshaw committed
4969 4970 4971 4972 4973 4974
        if not scope:  # could be None if there was an error
            return
        if entry.visibility != 'extern':
            for slot in TypeSlots.slot_table:
                slot.generate_dynamic_init_code(scope, code)
            if heap_type_bases:
4975 4976 4977 4978 4979
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached('PyType_Ready', 'ExtensionTypes.c'))
                readyfunc = "__Pyx_PyType_Ready"
            else:
                readyfunc = "PyType_Ready"
Robert Bradshaw's avatar
Robert Bradshaw committed
4980
            code.putln(
4981 4982
                "if (%s(&%s) < 0) %s" % (
                    readyfunc,
Robert Bradshaw's avatar
Robert Bradshaw committed
4983 4984 4985 4986 4987
                    typeobj_cname,
                    code.error_goto(entry.pos)))
            # Don't inherit tp_print from builtin types, restoring the
            # behavior of using tp_repr or tp_str instead.
            code.putln("%s.tp_print = 0;" % typeobj_cname)
4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007

            # Use specialised attribute lookup for types with generic lookup but no instance dict.
            getattr_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_getattro')
            dictoffset_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_dictoffset')
            if getattr_slot_func == '0' and dictoffset_slot_func == '0':
                if type.is_final_type:
                    py_cfunc = "__Pyx_PyObject_GenericGetAttrNoDict"  # grepable
                    utility_func = "PyObject_GenericGetAttrNoDict"
                else:
                    py_cfunc = "__Pyx_PyObject_GenericGetAttr"
                    utility_func = "PyObject_GenericGetAttr"
                code.globalstate.use_utility_code(UtilityCode.load_cached(utility_func, "ObjectHandling.c"))

                code.putln("if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) &&"
                           " likely(!%s.tp_dictoffset && %s.tp_getattro == PyObject_GenericGetAttr)) {" % (
                    typeobj_cname, typeobj_cname))
                code.putln("%s.tp_getattro = %s;" % (
                    typeobj_cname, py_cfunc))
                code.putln("}")

Robert Bradshaw's avatar
Robert Bradshaw committed
5008 5009 5010 5011 5012 5013 5014
            # Fix special method docstrings. This is a bit of a hack, but
            # unless we let PyType_Ready create the slot wrappers we have
            # a significant performance hit. (See trac #561.)
            for func in entry.type.scope.pyfunc_entries:
                is_buffer = func.name in ('__getbuffer__', '__releasebuffer__')
                if (func.is_special and Options.docstrings and
                        func.wrapperbase_cname and not is_buffer):
Stefan Behnel's avatar
Stefan Behnel committed
5015 5016
                    slot = TypeSlots.method_name_to_slot.get(func.name)
                    preprocessor_guard = slot.preprocessor_guard_code() if slot else None
Robert Bradshaw's avatar
Robert Bradshaw committed
5017 5018 5019 5020
                    if preprocessor_guard:
                        code.putln(preprocessor_guard)
                    code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
                    code.putln("{")
5021
                    code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
5022
                        'PyObject *wrapper = PyObject_GetAttrString((PyObject *)&%s, "%s"); %s' % (
5023
                            typeobj_cname,
Robert Bradshaw's avatar
Robert Bradshaw committed
5024 5025
                            func.name,
                            code.error_goto_if_null('wrapper', entry.pos)))
5026
                    code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041
                        "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {")
                    code.putln(
                        "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % (
                            func.wrapperbase_cname))
                    code.putln(
                        "%s.doc = %s;" % (func.wrapperbase_cname, func.doc_cname))
                    code.putln(
                        "((PyWrapperDescrObject *)wrapper)->d_base = &%s;" % (
                            func.wrapperbase_cname))
                    code.putln("}")
                    code.putln("}")
                    code.putln('#endif')
                    if preprocessor_guard:
                        code.putln('#endif')
            if type.vtable_cname:
5042 5043
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached('SetVTable', 'ImportExport.c'))
Robert Bradshaw's avatar
Robert Bradshaw committed
5044 5045 5046 5047 5048
                code.putln(
                    "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % (
                        typeobj_cname,
                        type.vtabptr_cname,
                        code.error_goto(entry.pos)))
5049 5050 5051 5052 5053 5054
                if heap_type_bases:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached('MergeVTables', 'ImportExport.c'))
                    code.putln("if (__Pyx_MergeVtables(&%s) < 0) %s" % (
                        typeobj_cname,
                        code.error_goto(entry.pos)))
5055
            if not type.scope.is_internal and not type.scope.directives.get('internal'):
Robert Bradshaw's avatar
Robert Bradshaw committed
5056 5057 5058 5059
                # scope.is_internal is set for types defined by
                # Cython (such as closures), the 'internal'
                # directive is set by users
                code.putln(
5060
                    'if (PyObject_SetAttr(%s, %s, (PyObject *)&%s) < 0) %s' % (
Robert Bradshaw's avatar
Robert Bradshaw committed
5061
                        Naming.module_cname,
5062
                        code.intern_identifier(scope.class_name),
Robert Bradshaw's avatar
Robert Bradshaw committed
5063 5064 5065 5066 5067 5068 5069 5070
                        typeobj_cname,
                        code.error_goto(entry.pos)))
            weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
            if weakref_entry:
                if weakref_entry.type is py_object_type:
                    tp_weaklistoffset = "%s.tp_weaklistoffset" % typeobj_cname
                    if type.typedef_flag:
                        objstruct = type.objstruct_cname
5071
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088
                        objstruct = "struct %s" % type.objstruct_cname
                    code.putln("if (%s == 0) %s = offsetof(%s, %s);" % (
                        tp_weaklistoffset,
                        tp_weaklistoffset,
                        objstruct,
                        weakref_entry.cname))
                else:
                    error(weakref_entry.pos, "__weakref__ slot must be of type 'object'")
            if scope.lookup_here("__reduce_cython__") if not scope.is_closure_class_scope else None:
                # Unfortunately, we cannot reliably detect whether a
                # superclass defined __reduce__ at compile time, so we must
                # do so at runtime.
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c'))
                code.putln('if (__Pyx_setup_reduce((PyObject*)&%s) < 0) %s' % (
                              typeobj_cname,
                              code.error_goto(entry.pos)))
5089 5090 5091 5092 5093 5094
        # Generate code to initialise the typeptr of an extension
        # type defined in this module to point to its type object.
        if type.typeobj_cname:
            code.putln(
                "%s = &%s;" % (
                    type.typeptr_cname, type.typeobj_cname))
5095

5096
    def annotate(self, code):
5097 5098
        if self.type_init_args:
            self.type_init_args.annotate(code)
5099 5100
        if self.body:
            self.body.annotate(code)
William Stein's avatar
William Stein committed
5101 5102 5103 5104 5105 5106


class PropertyNode(StatNode):
    #  Definition of a property in an extension type.
    #
    #  name   string
5107
    #  doc    EncodedString or None    Doc string
5108
    #  entry  Symtab.Entry
William Stein's avatar
William Stein committed
5109
    #  body   StatListNode
5110

5111 5112
    child_attrs = ["body"]

William Stein's avatar
William Stein committed
5113
    def analyse_declarations(self, env):
5114 5115 5116
        self.entry = env.declare_property(self.name, self.doc, self.pos)
        self.entry.scope.directives = env.directives
        self.body.analyse_declarations(self.entry.scope)
5117

William Stein's avatar
William Stein committed
5118
    def analyse_expressions(self, env):
5119 5120
        self.body = self.body.analyse_expressions(env)
        return self
5121

5122 5123
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
5124 5125 5126 5127

    def generate_execution_code(self, code):
        pass

5128 5129 5130
    def annotate(self, code):
        self.body.annotate(code)

William Stein's avatar
William Stein committed
5131 5132 5133 5134 5135

class GlobalNode(StatNode):
    # Global variable declaration.
    #
    # names    [string]
5136

5137 5138
    child_attrs = []

William Stein's avatar
William Stein committed
5139 5140 5141 5142 5143
    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_global(name, self.pos)

    def analyse_expressions(self, env):
5144
        return self
5145

William Stein's avatar
William Stein committed
5146 5147 5148 5149
    def generate_execution_code(self, code):
        pass


5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161
class NonlocalNode(StatNode):
    # Nonlocal variable declaration via the 'nonlocal' keyword.
    #
    # names    [string]

    child_attrs = []

    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_nonlocal(name, self.pos)

    def analyse_expressions(self, env):
5162
        return self
5163 5164 5165 5166 5167

    def generate_execution_code(self, code):
        pass


William Stein's avatar
William Stein committed
5168 5169 5170 5171
class ExprStatNode(StatNode):
    #  Expression used as a statement.
    #
    #  expr   ExprNode
5172 5173

    child_attrs = ["expr"]
5174

Robert Bradshaw's avatar
Robert Bradshaw committed
5175
    def analyse_declarations(self, env):
5176
        from . import ExprNodes
5177 5178 5179
        expr = self.expr
        if isinstance(expr, ExprNodes.GeneralCallNode):
            func = expr.function.as_cython_attribute()
Robert Bradshaw's avatar
Robert Bradshaw committed
5180
            if func == u'declare':
5181
                args, kwds = expr.explicit_args_kwds()
Robert Bradshaw's avatar
Robert Bradshaw committed
5182
                if len(args):
5183
                    error(expr.pos, "Variable names must be specified.")
Robert Bradshaw's avatar
Robert Bradshaw committed
5184 5185 5186 5187 5188
                for var, type_node in kwds.key_value_pairs:
                    type = type_node.analyse_as_type(env)
                    if type is None:
                        error(type_node.pos, "Unknown type")
                    else:
5189
                        env.declare_var(var.value, type, var.pos, is_cdef=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
5190
                self.__class__ = PassStatNode
5191
        elif getattr(expr, 'annotation', None) is not None:
5192 5193 5194 5195 5196 5197 5198
            if expr.is_name:
                # non-code variable annotation, e.g. "name: type"
                expr.declare_from_annotation(env)
                self.__class__ = PassStatNode
            elif expr.is_attribute or expr.is_subscript:
                # unused expression with annotation, e.g. "a[0]: type" or "a.xyz : type"
                self.__class__ = PassStatNode
5199

William Stein's avatar
William Stein committed
5200
    def analyse_expressions(self, env):
5201
        self.expr.result_is_used = False  # hint that .result() may safely be left empty
5202
        self.expr = self.expr.analyse_expressions(env)
5203 5204
        # Repeat in case of node replacement.
        self.expr.result_is_used = False  # hint that .result() may safely be left empty
5205
        return self
5206

5207
    def nogil_check(self, env):
5208
        if self.expr.type.is_pyobject and self.expr.is_temp:
5209 5210 5211 5212
            self.gil_error()

    gil_message = "Discarding owned Python object"

William Stein's avatar
William Stein committed
5213
    def generate_execution_code(self, code):
5214
        code.mark_pos(self.pos)
5215
        self.expr.result_is_used = False  # hint that .result() may safely be left empty
William Stein's avatar
William Stein committed
5216
        self.expr.generate_evaluation_code(code)
5217
        if not self.expr.is_temp and self.expr.result():
5218 5219 5220 5221
            result = self.expr.result()
            if not self.expr.type.is_void:
                result = "(void)(%s)" % result
            code.putln("%s;" % result)
William Stein's avatar
William Stein committed
5222
        self.expr.generate_disposal_code(code)
5223
        self.expr.free_temps(code)
William Stein's avatar
William Stein committed
5224

5225 5226 5227
    def generate_function_definitions(self, env, code):
        self.expr.generate_function_definitions(env, code)

5228 5229 5230
    def annotate(self, code):
        self.expr.annotate(code)

William Stein's avatar
William Stein committed
5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241

class AssignmentNode(StatNode):
    #  Abstract base class for assignment nodes.
    #
    #  The analyse_expressions and generate_execution_code
    #  phases of assignments are split into two sub-phases
    #  each, to enable all the right hand sides of a
    #  parallel assignment to be evaluated before assigning
    #  to any of the left hand sides.

    def analyse_expressions(self, env):
5242
        node = self.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5243
        if isinstance(node, AssignmentNode) and not isinstance(node, ParallelAssignmentNode):
5244
            if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
5245 5246
                error(self.pos, "Storing unsafe C derivative of temporary Python reference")
        return node
5247

5248 5249 5250
#       def analyse_expressions(self, env):
#           self.analyse_expressions_1(env)
#           self.analyse_expressions_2(env)
William Stein's avatar
William Stein committed
5251 5252

    def generate_execution_code(self, code):
5253
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5254 5255
        self.generate_rhs_evaluation_code(code)
        self.generate_assignment_code(code)
5256

William Stein's avatar
William Stein committed
5257 5258 5259 5260 5261 5262

class SingleAssignmentNode(AssignmentNode):
    #  The simplest case:
    #
    #    a = b
    #
5263 5264 5265 5266
    #  lhs                      ExprNode      Left hand side
    #  rhs                      ExprNode      Right hand side
    #  first                    bool          Is this guaranteed the first assignment to lhs?
    #  is_overloaded_assignment bool          Is this assignment done via an overloaded operator=
5267 5268
    #  exception_check
    #  exception_value
5269

5270
    child_attrs = ["lhs", "rhs"]
5271
    first = False
5272
    is_overloaded_assignment = False
5273
    declaration_only = False
William Stein's avatar
William Stein committed
5274 5275

    def analyse_declarations(self, env):
5276
        from . import ExprNodes
5277

5278 5279
        # handle declarations of the form x = cython.foo()
        if isinstance(self.rhs, ExprNodes.CallNode):
5280
            func_name = self.rhs.function.as_cython_attribute()
5281 5282 5283
            if func_name:
                args, kwds = self.rhs.explicit_args_kwds()
                if func_name in ['declare', 'typedef']:
5284 5285
                    if len(args) > 2:
                        error(args[2].pos, "Invalid positional argument.")
5286
                        return
5287 5288 5289 5290 5291 5292 5293 5294
                    if kwds is not None:
                        kwdict = kwds.compile_time_value(None)
                        if func_name == 'typedef' or 'visibility' not in kwdict:
                            error(kwds.pos, "Invalid keyword argument.")
                            return
                        visibility = kwdict['visibility']
                    else:
                        visibility = 'private'
5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308
                    type = args[0].analyse_as_type(env)
                    if type is None:
                        error(args[0].pos, "Unknown type")
                        return
                    lhs = self.lhs
                    if func_name == 'declare':
                        if isinstance(lhs, ExprNodes.NameNode):
                            vars = [(lhs.name, lhs.pos)]
                        elif isinstance(lhs, ExprNodes.TupleNode):
                            vars = [(var.name, var.pos) for var in lhs.args]
                        else:
                            error(lhs.pos, "Invalid declaration")
                            return
                        for var, pos in vars:
5309
                            env.declare_var(var, type, pos, is_cdef=True, visibility=visibility)
Robert Bradshaw's avatar
Robert Bradshaw committed
5310 5311 5312 5313 5314
                        if len(args) == 2:
                            # we have a value
                            self.rhs = args[1]
                        else:
                            self.declaration_only = True
5315
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
5316
                        self.declaration_only = True
5317 5318
                        if not isinstance(lhs, ExprNodes.NameNode):
                            error(lhs.pos, "Invalid declaration.")
5319
                        env.declare_typedef(lhs.name, type, self.pos, visibility='private')
5320

5321 5322 5323
                elif func_name in ['struct', 'union']:
                    self.declaration_only = True
                    if len(args) > 0 or kwds is None:
5324
                        error(self.rhs.pos, "Struct or union members must be given by name.")
5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341
                        return
                    members = []
                    for member, type_node in kwds.key_value_pairs:
                        type = type_node.analyse_as_type(env)
                        if type is None:
                            error(type_node.pos, "Unknown type")
                        else:
                            members.append((member.value, type, member.pos))
                    if len(members) < len(kwds.key_value_pairs):
                        return
                    if not isinstance(self.lhs, ExprNodes.NameNode):
                        error(self.lhs.pos, "Invalid declaration.")
                    name = self.lhs.name
                    scope = StructOrUnionScope(name)
                    env.declare_struct_or_union(name, func_name, scope, False, self.rhs.pos)
                    for member, type, pos in members:
                        scope.declare_var(member, type, pos)
5342

Mark Florisson's avatar
Mark Florisson committed
5343 5344 5345 5346 5347 5348 5349 5350
                elif func_name == 'fused_type':
                    # dtype = cython.fused_type(...)
                    self.declaration_only = True
                    if kwds:
                        error(self.rhs.function.pos,
                              "fused_type does not take keyword arguments")

                    fusednode = FusedTypeNode(self.rhs.pos,
5351
                                              name=self.lhs.name, types=args)
Mark Florisson's avatar
Mark Florisson committed
5352 5353
                    fusednode.analyse_declarations(env)

5354 5355 5356 5357
        if self.declaration_only:
            return
        else:
            self.lhs.analyse_target_declaration(env)
5358

5359 5360
    def analyse_types(self, env, use_temp=0):
        from . import ExprNodes
5361

5362
        self.rhs = self.rhs.analyse_types(env)
5363

5364 5365 5366
        unrolled_assignment = self.unroll_rhs(env)
        if unrolled_assignment:
            return unrolled_assignment
Robert Bradshaw's avatar
Robert Bradshaw committed
5367

5368
        self.lhs = self.lhs.analyse_target_types(env)
5369
        self.lhs.gil_assignment_check(env)
5370 5371 5372
        unrolled_assignment = self.unroll_lhs(env)
        if unrolled_assignment:
            return unrolled_assignment
5373

5374 5375 5376
        if isinstance(self.lhs, ExprNodes.MemoryViewIndexNode):
            self.lhs.analyse_broadcast_operation(self.rhs)
            self.lhs = self.lhs.analyse_as_memview_scalar_assignment(self.rhs)
5377
        elif self.lhs.type.is_array:
5378
            if not isinstance(self.lhs, ExprNodes.SliceIndexNode):
5379
                # cannot assign to C array, only to its full slice
5380
                self.lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None)
5381
                self.lhs = self.lhs.analyse_target_types(env)
5382

5383 5384 5385 5386
        if self.lhs.type.is_cpp_class:
            op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type])
            if op:
                rhs = self.rhs
5387
                self.is_overloaded_assignment = True
5388 5389 5390 5391
                self.exception_check = op.type.exception_check
                self.exception_value = op.type.exception_value
                if self.exception_check == '+' and self.exception_value is None:
                    env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
5392
            else:
5393
                rhs = self.rhs.coerce_to(self.lhs.type, env)
5394
        else:
5395 5396
            rhs = self.rhs.coerce_to(self.lhs.type, env)

5397 5398 5399
        if use_temp or rhs.is_attribute or (
                not rhs.is_name and not rhs.is_literal and
                rhs.type.is_pyobject):
5400
            # things like (cdef) attribute access are not safe (traverses pointers)
5401 5402 5403 5404
            rhs = rhs.coerce_to_temp(env)
        elif rhs.type.is_pyobject:
            rhs = rhs.coerce_to_simple(env)
        self.rhs = rhs
5405
        return self
5406

5407 5408
    def unroll(self, node, target_size, env):
        from . import ExprNodes, UtilNodes
5409 5410 5411 5412

        base = node
        start_node = stop_node = step_node = check_node = None

5413
        if node.type.is_ctuple:
5414
            slice_size = node.type.size
5415

5416 5417 5418
        elif node.type.is_ptr or node.type.is_array:
            while isinstance(node, ExprNodes.SliceIndexNode) and not (node.start or node.stop):
                base = node = node.base
5419 5420 5421
            if isinstance(node, ExprNodes.SliceIndexNode):
                base = node.base
                start_node = node.start
5422 5423
                if start_node:
                    start_node = start_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5424
                stop_node = node.stop
5425 5426 5427
                if stop_node:
                    stop_node = stop_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
                else:
5428
                    if node.type.is_array and node.type.size:
5429 5430
                        stop_node = ExprNodes.IntNode(
                            self.pos, value=str(node.type.size),
5431
                            constant_result=(node.type.size if isinstance(node.type.size, _py_int_types)
5432
                                             else ExprNodes.constant_value_not_set))
5433 5434 5435
                    else:
                        error(self.pos, "C array iteration requires known end index")
                        return
5436
                step_node = None  #node.step
5437 5438
                if step_node:
                    step_node = step_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5439

5440 5441 5442 5443
                # TODO: Factor out SliceIndexNode.generate_slice_guard_code() for use here.
                def get_const(node, none_value):
                    if node is None:
                        return none_value
5444
                    elif node.has_constant_result():
5445 5446
                        return node.constant_result
                    else:
5447 5448
                        raise ValueError("Not a constant.")

5449 5450 5451 5452 5453
                try:
                    slice_size = (get_const(stop_node, None) - get_const(start_node, 0)) / get_const(step_node, 1)
                except ValueError:
                    error(self.pos, "C array assignment currently requires known endpoints")
                    return
5454 5455 5456

            elif node.type.is_array:
                slice_size = node.type.size
5457
                if not isinstance(slice_size, _py_int_types):
5458
                    return  # might still work when coercing to Python
5459 5460 5461 5462 5463 5464
            else:
                return

        else:
            return

5465 5466 5467 5468 5469
        if slice_size != target_size:
            error(self.pos, "Assignment to/from slice of wrong length, expected %s, got %s" % (
                slice_size, target_size))
            return

5470
        items = []
5471 5472 5473
        base = UtilNodes.LetRefNode(base)
        refs = [base]
        if start_node and not start_node.is_literal:
5474 5475
            start_node = UtilNodes.LetRefNode(start_node)
            refs.append(start_node)
5476
        if stop_node and not stop_node.is_literal:
5477 5478
            stop_node = UtilNodes.LetRefNode(stop_node)
            refs.append(stop_node)
5479
        if step_node and not step_node.is_literal:
5480 5481
            step_node = UtilNodes.LetRefNode(step_node)
            refs.append(step_node)
5482

5483
        for ix in range(target_size):
5484
            ix_node = ExprNodes.IntNode(self.pos, value=str(ix), constant_result=ix, type=PyrexTypes.c_py_ssize_t_type)
5485
            if step_node is not None:
5486 5487 5488 5489 5490
                if step_node.has_constant_result():
                    step_value = ix_node.constant_result * step_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(step_value), constant_result=step_value)
                else:
                    ix_node = ExprNodes.MulNode(self.pos, operator='*', operand1=step_node, operand2=ix_node)
5491
            if start_node is not None:
5492 5493 5494 5495 5496 5497 5498
                if start_node.has_constant_result() and ix_node.has_constant_result():
                    index_value = ix_node.constant_result + start_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(index_value), constant_result=index_value)
                else:
                    ix_node = ExprNodes.AddNode(
                        self.pos, operator='+', operand1=start_node, operand2=ix_node)
            items.append(ExprNodes.IndexNode(self.pos, base=base, index=ix_node.analyse_types(env)))
5499 5500
        return check_node, refs, items

5501
    def unroll_assignments(self, refs, check_node, lhs_list, rhs_list, env):
5502
        from . import UtilNodes
5503
        assignments = []
5504
        for lhs, rhs in zip(lhs_list, rhs_list):
5505
            assignments.append(SingleAssignmentNode(self.pos, lhs=lhs, rhs=rhs, first=self.first))
5506
        node = ParallelAssignmentNode(pos=self.pos, stats=assignments).analyse_expressions(env)
5507
        if check_node:
5508
            node = StatListNode(pos=self.pos, stats=[check_node, node])
5509
        for ref in refs[::-1]:
5510 5511
            node = UtilNodes.LetNode(ref, node)
        return node
5512

5513
    def unroll_rhs(self, env):
5514
        from . import ExprNodes
5515 5516
        if not isinstance(self.lhs, ExprNodes.TupleNode):
            return
5517 5518
        if any(arg.is_starred for arg in self.lhs.args):
            return
5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529

        unrolled = self.unroll(self.rhs, len(self.lhs.args), env)
        if not unrolled:
            return
        check_node, refs, rhs = unrolled
        return self.unroll_assignments(refs, check_node, self.lhs.args, rhs, env)

    def unroll_lhs(self, env):
        if self.lhs.type.is_ctuple:
            # Handled directly.
            return
5530
        from . import ExprNodes
5531 5532 5533 5534 5535 5536 5537 5538 5539
        if not isinstance(self.rhs, ExprNodes.TupleNode):
            return

        unrolled = self.unroll(self.lhs, len(self.rhs.args), env)
        if not unrolled:
            return
        check_node, refs, lhs = unrolled
        return self.unroll_assignments(refs, check_node, lhs, self.rhs.args, env)

William Stein's avatar
William Stein committed
5540 5541
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5542

5543
    def generate_assignment_code(self, code, overloaded_assignment=False):
5544 5545 5546 5547 5548 5549 5550 5551 5552
        if self.is_overloaded_assignment:
            self.lhs.generate_assignment_code(
                self.rhs,
                code,
                overloaded_assignment=self.is_overloaded_assignment,
                exception_check=self.exception_check,
                exception_value=self.exception_value)
        else:
            self.lhs.generate_assignment_code(self.rhs, code)
William Stein's avatar
William Stein committed
5553

5554 5555 5556
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5557 5558 5559 5560
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

William Stein's avatar
William Stein committed
5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571

class CascadedAssignmentNode(AssignmentNode):
    #  An assignment with multiple left hand sides:
    #
    #    a = b = c
    #
    #  lhs_list   [ExprNode]   Left hand sides
    #  rhs        ExprNode     Right hand sides
    #
    #  Used internally:
    #
5572 5573 5574
    #  coerced_values       [ExprNode]   RHS coerced to all distinct LHS types
    #  cloned_values        [ExprNode]   cloned RHS value for each LHS
    #  assignment_overloads [Bool]       If each assignment uses a C++ operator=
5575

5576 5577
    child_attrs = ["lhs_list", "rhs", "coerced_values", "cloned_values"]
    cloned_values = None
5578
    coerced_values = None
5579
    assignment_overloads = None
5580

William Stein's avatar
William Stein committed
5581 5582 5583
    def analyse_declarations(self, env):
        for lhs in self.lhs_list:
            lhs.analyse_target_declaration(env)
5584

5585
    def analyse_types(self, env, use_temp=0):
5586
        from .ExprNodes import CloneNode, ProxyNode
5587

5588
        # collect distinct types used on the LHS
5589
        lhs_types = set()
5590 5591
        for i, lhs in enumerate(self.lhs_list):
            lhs = self.lhs_list[i] = lhs.analyse_target_types(env)
5592 5593 5594
            lhs.gil_assignment_check(env)
            lhs_types.add(lhs.type)

5595
        rhs = self.rhs.analyse_types(env)
5596
        # common special case: only one type needed on the LHS => coerce only once
5597
        if len(lhs_types) == 1:
5598 5599 5600 5601 5602 5603 5604
            # Avoid coercion for overloaded assignment operators.
            if next(iter(lhs_types)).is_cpp_class:
                op = env.lookup_operator('=', [lhs, self.rhs])
                if not op:
                    rhs = rhs.coerce_to(lhs_types.pop(), env)
            else:
                rhs = rhs.coerce_to(lhs_types.pop(), env)
5605 5606 5607

        if not rhs.is_name and not rhs.is_literal and (
                use_temp or rhs.is_attribute or rhs.type.is_pyobject):
5608
            rhs = rhs.coerce_to_temp(env)
5609
        else:
5610
            rhs = rhs.coerce_to_simple(env)
5611 5612
        self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs

5613
        # clone RHS and coerce it to all distinct LHS types
5614 5615
        self.coerced_values = []
        coerced_values = {}
5616
        self.assignment_overloads = []
5617
        for lhs in self.lhs_list:
5618 5619
            overloaded = lhs.type.is_cpp_class and env.lookup_operator('=', [lhs, self.rhs])
            self.assignment_overloads.append(overloaded)
5620
            if lhs.type not in coerced_values and lhs.type != rhs.type:
5621
                rhs = CloneNode(self.rhs)
5622
                if not overloaded:
5623
                    rhs = rhs.coerce_to(lhs.type, env)
5624 5625
                self.coerced_values.append(rhs)
                coerced_values[lhs.type] = rhs
5626

5627 5628
        # clone coerced values for all LHS assignments
        self.cloned_values = []
William Stein's avatar
William Stein committed
5629
        for lhs in self.lhs_list:
5630
            rhs = coerced_values.get(lhs.type, self.rhs)
5631
            self.cloned_values.append(CloneNode(rhs))
5632
        return self
5633

William Stein's avatar
William Stein committed
5634 5635
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5636

5637
    def generate_assignment_code(self, code, overloaded_assignment=False):
5638
        # prepare all coercions
5639 5640
        for rhs in self.coerced_values:
            rhs.generate_evaluation_code(code)
5641
        # assign clones to LHS
5642
        for lhs, rhs, overload in zip(self.lhs_list, self.cloned_values, self.assignment_overloads):
William Stein's avatar
William Stein committed
5643
            rhs.generate_evaluation_code(code)
5644
            lhs.generate_assignment_code(rhs, code, overloaded_assignment=overload)
5645 5646 5647 5648
        # dispose of coerced values and original RHS
        for rhs_value in self.coerced_values:
            rhs_value.generate_disposal_code(code)
            rhs_value.free_temps(code)
William Stein's avatar
William Stein committed
5649
        self.rhs.generate_disposal_code(code)
5650
        self.rhs.free_temps(code)
William Stein's avatar
William Stein committed
5651

5652 5653 5654
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5655
    def annotate(self, code):
5656 5657
        for rhs in self.coerced_values:
            rhs.annotate(code)
5658
        for lhs, rhs in zip(self.lhs_list, self.cloned_values):
5659 5660
            lhs.annotate(code)
            rhs.annotate(code)
5661
        self.rhs.annotate(code)
5662

5663

William Stein's avatar
William Stein committed
5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676
class ParallelAssignmentNode(AssignmentNode):
    #  A combined packing/unpacking assignment:
    #
    #    a, b, c =  d, e, f
    #
    #  This has been rearranged by the parser into
    #
    #    a = d ; b = e ; c = f
    #
    #  but we must evaluate all the right hand sides
    #  before assigning to any of the left hand sides.
    #
    #  stats     [AssignmentNode]   The constituent assignments
5677

5678 5679
    child_attrs = ["stats"]

William Stein's avatar
William Stein committed
5680 5681 5682
    def analyse_declarations(self, env):
        for stat in self.stats:
            stat.analyse_declarations(env)
5683

William Stein's avatar
William Stein committed
5684
    def analyse_expressions(self, env):
5685 5686
        self.stats = [stat.analyse_types(env, use_temp=1)
                      for stat in self.stats]
5687
        return self
5688

Robert Bradshaw's avatar
Robert Bradshaw committed
5689 5690
#    def analyse_expressions(self, env):
#        for stat in self.stats:
5691
#            stat.analyse_expressions_1(env, use_temp=1)
Robert Bradshaw's avatar
Robert Bradshaw committed
5692 5693
#        for stat in self.stats:
#            stat.analyse_expressions_2(env)
5694

William Stein's avatar
William Stein committed
5695
    def generate_execution_code(self, code):
5696
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5697 5698 5699 5700 5701
        for stat in self.stats:
            stat.generate_rhs_evaluation_code(code)
        for stat in self.stats:
            stat.generate_assignment_code(code)

5702 5703 5704 5705
    def generate_function_definitions(self, env, code):
        for stat in self.stats:
            stat.generate_function_definitions(env, code)

5706 5707 5708 5709 5710
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)


5711
class InPlaceAssignmentNode(AssignmentNode):
Craig Citro's avatar
Craig Citro committed
5712
    #  An in place arithmetic operand:
5713 5714 5715 5716 5717 5718 5719
    #
    #    a += b
    #    a -= b
    #    ...
    #
    #  lhs      ExprNode      Left hand side
    #  rhs      ExprNode      Right hand side
Stefan Behnel's avatar
Stefan Behnel committed
5720
    #  operator char          one of "+-*/%^&|"
5721
    #
5722 5723 5724 5725 5726 5727 5728
    #  This code is a bit tricky because in order to obey Python
    #  semantics the sub-expressions (e.g. indices) of the lhs must
    #  not be evaluated twice. So we must re-use the values calculated
    #  in evaluation phase for the assignment phase as well.
    #  Fortunately, the type of the lhs node is fairly constrained
    #  (it must be a NameNode, AttributeNode, or IndexNode).

5729
    child_attrs = ["lhs", "rhs"]
5730

5731 5732
    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)
5733

5734
    def analyse_types(self, env):
5735 5736
        self.rhs = self.rhs.analyse_types(env)
        self.lhs = self.lhs.analyse_target_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5737

5738
        # When assigning to a fully indexed buffer or memoryview, coerce the rhs
5739
        if self.lhs.is_memview_index or self.lhs.is_buffer_access:
5740
            self.rhs = self.rhs.coerce_to(self.lhs.type, env)
5741 5742 5743
        elif self.lhs.type.is_string and self.operator in '+-':
            # use pointer arithmetic for char* LHS instead of string concat
            self.rhs = self.rhs.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5744
        return self
5745

5746
    def generate_execution_code(self, code):
5747
        code.mark_pos(self.pos)
5748 5749 5750
        lhs, rhs = self.lhs, self.rhs
        rhs.generate_evaluation_code(code)
        lhs.generate_subexpr_evaluation_code(code)
5751 5752 5753 5754 5755
        c_op = self.operator
        if c_op == "//":
            c_op = "/"
        elif c_op == "**":
            error(self.pos, "No C inplace power operator")
5756 5757
        if lhs.is_buffer_access or lhs.is_memview_index:
            if lhs.type.is_pyobject:
5758
                error(self.pos, "In-place operators not allowed on object buffers in this release.")
5759
            if c_op in ('/', '%') and lhs.type.is_int and not code.globalstate.directives['cdivision']:
Robert Bradshaw's avatar
Robert Bradshaw committed
5760
                error(self.pos, "In-place non-c divide operators not allowed on int buffers.")
5761 5762 5763
            lhs.generate_buffer_setitem_code(rhs, code, c_op)
        elif lhs.is_memview_slice:
            error(self.pos, "Inplace operators not supported on memoryview slices")
5764 5765 5766
        else:
            # C++
            # TODO: make sure overload is declared
5767 5768 5769 5770 5771
            code.putln("%s %s= %s;" % (lhs.result(), c_op, rhs.result()))
        lhs.generate_subexpr_disposal_code(code)
        lhs.free_subexpr_temps(code)
        rhs.generate_disposal_code(code)
        rhs.free_temps(code)
5772

5773 5774 5775
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)
5776

5777
    def create_binop_node(self):
5778
        from . import ExprNodes
5779
        return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
5780

William Stein's avatar
William Stein committed
5781 5782 5783 5784

class PrintStatNode(StatNode):
    #  print statement
    #
5785
    #  arg_tuple         TupleNode
5786
    #  stream            ExprNode or None (stdout)
5787
    #  append_newline    boolean
5788

5789
    child_attrs = ["arg_tuple", "stream"]
5790

William Stein's avatar
William Stein committed
5791
    def analyse_expressions(self, env):
5792
        if self.stream:
5793 5794 5795 5796
            stream = self.stream.analyse_expressions(env)
            self.stream = stream.coerce_to_pyobject(env)
        arg_tuple = self.arg_tuple.analyse_expressions(env)
        self.arg_tuple = arg_tuple.coerce_to_pyobject(env)
5797
        env.use_utility_code(printing_utility_code)
5798 5799
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            env.use_utility_code(printing_one_utility_code)
5800
        return self
5801

5802
    nogil_check = Node.gil_error
5803
    gil_message = "Python print statement"
5804

William Stein's avatar
William Stein committed
5805
    def generate_execution_code(self, code):
5806
        code.mark_pos(self.pos)
5807 5808 5809 5810 5811
        if self.stream:
            self.stream.generate_evaluation_code(code)
            stream_result = self.stream.py_result()
        else:
            stream_result = '0'
5812 5813 5814
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            arg = self.arg_tuple.args[0]
            arg.generate_evaluation_code(code)
5815

5816
            code.putln(
5817 5818
                "if (__Pyx_PrintOne(%s, %s) < 0) %s" % (
                    stream_result,
5819 5820 5821 5822 5823 5824 5825
                    arg.py_result(),
                    code.error_goto(self.pos)))
            arg.generate_disposal_code(code)
            arg.free_temps(code)
        else:
            self.arg_tuple.generate_evaluation_code(code)
            code.putln(
5826 5827
                "if (__Pyx_Print(%s, %s, %d) < 0) %s" % (
                    stream_result,
5828 5829 5830 5831 5832
                    self.arg_tuple.py_result(),
                    self.append_newline,
                    code.error_goto(self.pos)))
            self.arg_tuple.generate_disposal_code(code)
            self.arg_tuple.free_temps(code)
5833

5834 5835 5836 5837
        if self.stream:
            self.stream.generate_disposal_code(code)
            self.stream.free_temps(code)

5838
    def generate_function_definitions(self, env, code):
5839 5840
        if self.stream:
            self.stream.generate_function_definitions(env, code)
5841
        self.arg_tuple.generate_function_definitions(env, code)
5842

5843
    def annotate(self, code):
5844 5845
        if self.stream:
            self.stream.annotate(code)
5846
        self.arg_tuple.annotate(code)
William Stein's avatar
William Stein committed
5847 5848


5849 5850 5851 5852 5853 5854 5855 5856 5857
class ExecStatNode(StatNode):
    #  exec statement
    #
    #  args     [ExprNode]

    child_attrs = ["args"]

    def analyse_expressions(self, env):
        for i, arg in enumerate(self.args):
5858
            arg = arg.analyse_expressions(env)
5859 5860 5861
            arg = arg.coerce_to_pyobject(env)
            self.args[i] = arg
        env.use_utility_code(Builtin.pyexec_utility_code)
5862
        return self
5863

5864
    nogil_check = Node.gil_error
5865 5866 5867
    gil_message = "Python exec statement"

    def generate_execution_code(self, code):
5868
        code.mark_pos(self.pos)
5869 5870 5871
        args = []
        for arg in self.args:
            arg.generate_evaluation_code(code)
5872
            args.append(arg.py_result())
5873
        args = tuple(args + ['0', '0'][:3-len(args)])
5874
        temp_result = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
5875
        code.putln("%s = __Pyx_PyExec3(%s, %s, %s);" % ((temp_result,) + args))
5876 5877
        for arg in self.args:
            arg.generate_disposal_code(code)
5878
            arg.free_temps(code)
5879
        code.putln(
5880 5881 5882 5883
            code.error_goto_if_null(temp_result, self.pos))
        code.put_gotref(temp_result)
        code.put_decref_clear(temp_result, py_object_type)
        code.funcstate.release_temp(temp_result)
5884 5885 5886 5887 5888 5889

    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)


William Stein's avatar
William Stein committed
5890 5891 5892 5893
class DelStatNode(StatNode):
    #  del statement
    #
    #  args     [ExprNode]
5894

5895
    child_attrs = ["args"]
5896
    ignore_nonexisting = False
5897

William Stein's avatar
William Stein committed
5898 5899 5900
    def analyse_declarations(self, env):
        for arg in self.args:
            arg.analyse_target_declaration(env)
5901

William Stein's avatar
William Stein committed
5902
    def analyse_expressions(self, env):
5903 5904
        for i, arg in enumerate(self.args):
            arg = self.args[i] = arg.analyse_target_expression(env, None)
5905
            if arg.type.is_pyobject or (arg.is_name and arg.type.is_memoryviewslice):
5906 5907
                if arg.is_name and arg.entry.is_cglobal:
                    error(arg.pos, "Deletion of global C variable")
Robert Bradshaw's avatar
Robert Bradshaw committed
5908
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
Robert Bradshaw's avatar
Robert Bradshaw committed
5909
                self.cpp_check(env)
5910
            elif arg.type.is_cpp_class:
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5911
                error(arg.pos, "Deletion of non-heap C++ object")
5912 5913
            elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
                pass  # del ba[i]
5914
            else:
Robert Bradshaw's avatar
Robert Bradshaw committed
5915
                error(arg.pos, "Deletion of non-Python, non-C++ object")
5916
            #arg.release_target_temp(env)
5917
        return self
5918

5919
    def nogil_check(self, env):
5920 5921
        for arg in self.args:
            if arg.type.is_pyobject:
5922
                self.gil_error()
5923

5924 5925
    gil_message = "Deleting Python object"

William Stein's avatar
William Stein committed
5926
    def generate_execution_code(self, code):
5927
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5928
        for arg in self.args:
5929 5930 5931
            if (arg.type.is_pyobject or
                    arg.type.is_memoryviewslice or
                    arg.is_subscript and arg.base.type is Builtin.bytearray_type):
5932 5933
                arg.generate_deletion_code(
                    code, ignore_nonexisting=self.ignore_nonexisting)
5934
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
5935
                arg.generate_evaluation_code(code)
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5936
                code.putln("delete %s;" % arg.result())
5937
                arg.generate_disposal_code(code)
William Stein's avatar
William Stein committed
5938 5939
            # else error reported earlier

5940 5941 5942 5943
    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)

William Stein's avatar
William Stein committed
5944 5945 5946

class PassStatNode(StatNode):
    #  pass statement
5947 5948

    child_attrs = []
5949

William Stein's avatar
William Stein committed
5950
    def analyse_expressions(self, env):
5951
        return self
5952

William Stein's avatar
William Stein committed
5953 5954 5955 5956
    def generate_execution_code(self, code):
        pass


5957 5958 5959 5960 5961 5962 5963 5964 5965
class IndirectionNode(StatListNode):
    """
    This adds an indirection so that the node can be shared and a subtree can
    be removed at any time by clearing self.stats.
    """

    def __init__(self, stats):
        super(IndirectionNode, self).__init__(stats[0].pos, stats=stats)

5966

William Stein's avatar
William Stein committed
5967 5968
class BreakStatNode(StatNode):

5969
    child_attrs = []
5970
    is_terminator = True
5971

William Stein's avatar
William Stein committed
5972
    def analyse_expressions(self, env):
5973
        return self
5974

William Stein's avatar
William Stein committed
5975
    def generate_execution_code(self, code):
5976
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5977 5978 5979
        if not code.break_label:
            error(self.pos, "break statement not inside loop")
        else:
5980
            code.put_goto(code.break_label)
William Stein's avatar
William Stein committed
5981 5982 5983 5984


class ContinueStatNode(StatNode):

5985
    child_attrs = []
5986
    is_terminator = True
5987

William Stein's avatar
William Stein committed
5988
    def analyse_expressions(self, env):
5989
        return self
5990

William Stein's avatar
William Stein committed
5991
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
5992
        if not code.continue_label:
William Stein's avatar
William Stein committed
5993
            error(self.pos, "continue statement not inside loop")
Stefan Behnel's avatar
Stefan Behnel committed
5994 5995 5996
            return
        code.mark_pos(self.pos)
        code.put_goto(code.continue_label)
William Stein's avatar
William Stein committed
5997 5998 5999 6000 6001 6002 6003


class ReturnStatNode(StatNode):
    #  return statement
    #
    #  value         ExprNode or None
    #  return_type   PyrexType
6004
    #  in_generator  return inside of generator => raise StopIteration
6005
    #  in_async_gen  return inside of async generator
6006

6007
    child_attrs = ["value"]
6008
    is_terminator = True
6009
    in_generator = False
6010
    in_async_gen = False
6011

6012 6013 6014
    # Whether we are in a parallel section
    in_parallel = False

William Stein's avatar
William Stein committed
6015 6016 6017 6018 6019
    def analyse_expressions(self, env):
        return_type = env.return_type
        self.return_type = return_type
        if not return_type:
            error(self.pos, "Return not inside a function body")
6020
            return self
William Stein's avatar
William Stein committed
6021
        if self.value:
6022 6023
            if self.in_async_gen:
                error(self.pos, "Return with value in async generator")
6024
            self.value = self.value.analyse_types(env)
William Stein's avatar
William Stein committed
6025
            if return_type.is_void or return_type.is_returncode:
6026
                error(self.value.pos, "Return with value in void function")
William Stein's avatar
William Stein committed
6027 6028 6029 6030
            else:
                self.value = self.value.coerce_to(env.return_type, env)
        else:
            if (not return_type.is_void
6031 6032 6033
                    and not return_type.is_pyobject
                    and not return_type.is_returncode):
                error(self.pos, "Return value required")
6034
        return self
6035

6036
    def nogil_check(self, env):
6037
        if self.return_type.is_pyobject:
6038
            self.gil_error()
6039 6040 6041

    gil_message = "Returning Python object"

William Stein's avatar
William Stein committed
6042
    def generate_execution_code(self, code):
6043
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6044 6045 6046
        if not self.return_type:
            # error reported earlier
            return
6047 6048

        value = self.value
6049
        if self.return_type.is_pyobject:
6050 6051 6052 6053
            code.put_xdecref(Naming.retval_cname, self.return_type)
            if value and value.is_none:
                # Use specialised default handling for "return None".
                value = None
6054

6055 6056
        if value:
            value.generate_evaluation_code(code)
6057
            if self.return_type.is_memoryviewslice:
6058
                from . import MemoryView
6059
                MemoryView.put_acquire_memoryviewslice(
6060 6061
                    lhs_cname=Naming.retval_cname,
                    lhs_type=self.return_type,
6062 6063
                    lhs_pos=value.pos,
                    rhs=value,
6064 6065
                    code=code,
                    have_gil=self.in_nogil_context)
6066 6067
            elif self.in_generator:
                # return value == raise StopIteration(value), but uncatchable
6068
                code.globalstate.use_utility_code(
6069
                    UtilityCode.load_cached("ReturnWithStopIteration", "Coroutine.c"))
6070 6071
                code.putln("%s = NULL; __Pyx_ReturnWithStopIteration(%s);" % (
                    Naming.retval_cname,
6072 6073
                    value.py_result()))
                value.generate_disposal_code(code)
6074
            else:
6075
                value.make_owned_reference(code)
6076 6077
                code.putln("%s = %s;" % (
                    Naming.retval_cname,
6078 6079 6080
                    value.result_as(self.return_type)))
            value.generate_post_assignment_code(code)
            value.free_temps(code)
William Stein's avatar
William Stein committed
6081 6082
        else:
            if self.return_type.is_pyobject:
6083
                if self.in_generator:
6084
                    if self.in_async_gen:
6085 6086
                        code.globalstate.use_utility_code(
                            UtilityCode.load_cached("StopAsyncIteration", "Coroutine.c"))
6087
                        code.put("PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); ")
6088 6089 6090
                    code.putln("%s = NULL;" % Naming.retval_cname)
                else:
                    code.put_init_to_py_none(Naming.retval_cname, self.return_type)
William Stein's avatar
William Stein committed
6091
            elif self.return_type.is_returncode:
6092 6093
                self.put_return(code, self.return_type.default_value)

6094
        for cname, type in code.funcstate.temps_holding_reference():
6095
            code.put_decref_clear(cname, type)
6096

6097
        code.put_goto(code.return_label)
6098

6099 6100 6101 6102 6103
    def put_return(self, code, value):
        if self.in_parallel:
            code.putln_openmp("#pragma omp critical(__pyx_returning)")
        code.putln("%s = %s;" % (Naming.retval_cname, value))

6104 6105 6106
    def generate_function_definitions(self, env, code):
        if self.value is not None:
            self.value.generate_function_definitions(env, code)
6107

6108 6109 6110
    def annotate(self, code):
        if self.value:
            self.value.annotate(code)
William Stein's avatar
William Stein committed
6111 6112 6113 6114 6115 6116 6117 6118


class RaiseStatNode(StatNode):
    #  raise statement
    #
    #  exc_type    ExprNode or None
    #  exc_value   ExprNode or None
    #  exc_tb      ExprNode or None
Haoyu Bai's avatar
Haoyu Bai committed
6119
    #  cause       ExprNode or None
6120

Haoyu Bai's avatar
Haoyu Bai committed
6121
    child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"]
6122
    is_terminator = True
6123

William Stein's avatar
William Stein committed
6124 6125
    def analyse_expressions(self, env):
        if self.exc_type:
6126 6127
            exc_type = self.exc_type.analyse_types(env)
            self.exc_type = exc_type.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
6128
        if self.exc_value:
6129 6130
            exc_value = self.exc_value.analyse_types(env)
            self.exc_value = exc_value.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
6131
        if self.exc_tb:
6132 6133
            exc_tb = self.exc_tb.analyse_types(env)
            self.exc_tb = exc_tb.coerce_to_pyobject(env)
Haoyu Bai's avatar
Haoyu Bai committed
6134
        if self.cause:
6135 6136
            cause = self.cause.analyse_types(env)
            self.cause = cause.coerce_to_pyobject(env)
6137 6138 6139 6140
        # special cases for builtin exceptions
        self.builtin_exc_name = None
        if self.exc_type and not self.exc_value and not self.exc_tb:
            exc = self.exc_type
6141
            from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
6142
            if (isinstance(exc, ExprNodes.SimpleCallNode) and
6143 6144
                    not (exc.args or (exc.arg_tuple is not None and exc.arg_tuple.args))):
                exc = exc.function  # extract the exception type
6145 6146 6147 6148
            if exc.is_name and exc.entry.is_builtin:
                self.builtin_exc_name = exc.name
                if self.builtin_exc_name == 'MemoryError':
                    self.exc_type = None # has a separate implementation
6149
        return self
6150

6151
    nogil_check = Node.gil_error
6152 6153
    gil_message = "Raising exception"

William Stein's avatar
William Stein committed
6154
    def generate_execution_code(self, code):
6155
        code.mark_pos(self.pos)
6156 6157 6158 6159
        if self.builtin_exc_name == 'MemoryError':
            code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos))
            return

William Stein's avatar
William Stein committed
6160 6161 6162
        if self.exc_type:
            self.exc_type.generate_evaluation_code(code)
            type_code = self.exc_type.py_result()
6163 6164
            if self.exc_type.is_name:
                code.globalstate.use_entry_utility_code(self.exc_type.entry)
William Stein's avatar
William Stein committed
6165
        else:
Stefan Behnel's avatar
Stefan Behnel committed
6166
            type_code = "0"
William Stein's avatar
William Stein committed
6167 6168 6169 6170 6171 6172 6173 6174 6175 6176
        if self.exc_value:
            self.exc_value.generate_evaluation_code(code)
            value_code = self.exc_value.py_result()
        else:
            value_code = "0"
        if self.exc_tb:
            self.exc_tb.generate_evaluation_code(code)
            tb_code = self.exc_tb.py_result()
        else:
            tb_code = "0"
Haoyu Bai's avatar
Haoyu Bai committed
6177 6178 6179 6180 6181
        if self.cause:
            self.cause.generate_evaluation_code(code)
            cause_code = self.cause.py_result()
        else:
            cause_code = "0"
6182
        code.globalstate.use_utility_code(raise_utility_code)
6183
        code.putln(
Haoyu Bai's avatar
Haoyu Bai committed
6184
            "__Pyx_Raise(%s, %s, %s, %s);" % (
6185 6186
                type_code,
                value_code,
Haoyu Bai's avatar
Haoyu Bai committed
6187 6188 6189
                tb_code,
                cause_code))
        for obj in (self.exc_type, self.exc_value, self.exc_tb, self.cause):
6190 6191 6192
            if obj:
                obj.generate_disposal_code(code)
                obj.free_temps(code)
William Stein's avatar
William Stein committed
6193 6194 6195
        code.putln(
            code.error_goto(self.pos))

6196 6197 6198 6199 6200 6201 6202
    def generate_function_definitions(self, env, code):
        if self.exc_type is not None:
            self.exc_type.generate_function_definitions(env, code)
        if self.exc_value is not None:
            self.exc_value.generate_function_definitions(env, code)
        if self.exc_tb is not None:
            self.exc_tb.generate_function_definitions(env, code)
Haoyu Bai's avatar
Haoyu Bai committed
6203 6204
        if self.cause is not None:
            self.cause.generate_function_definitions(env, code)
6205

6206 6207 6208 6209 6210 6211 6212
    def annotate(self, code):
        if self.exc_type:
            self.exc_type.annotate(code)
        if self.exc_value:
            self.exc_value.annotate(code)
        if self.exc_tb:
            self.exc_tb.annotate(code)
Haoyu Bai's avatar
Haoyu Bai committed
6213 6214
        if self.cause:
            self.cause.annotate(code)
6215

William Stein's avatar
William Stein committed
6216

6217 6218
class ReraiseStatNode(StatNode):

6219
    child_attrs = []
6220
    is_terminator = True
6221

6222
    def analyse_expressions(self, env):
6223
        return self
6224

6225
    nogil_check = Node.gil_error
6226 6227
    gil_message = "Raising exception"

6228
    def generate_execution_code(self, code):
6229
        code.mark_pos(self.pos)
6230
        vars = code.funcstate.exc_vars
6231
        if vars:
6232
            code.globalstate.use_utility_code(restore_exception_utility_code)
6233 6234 6235 6236
            code.put_giveref(vars[0])
            code.put_giveref(vars[1])
            # fresh exceptions may not have a traceback yet (-> finally!)
            code.put_xgiveref(vars[2])
6237
            code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars))
6238 6239 6240
            for varname in vars:
                code.put("%s = 0; " % varname)
            code.putln()
6241 6242
            code.putln(code.error_goto(self.pos))
        else:
6243 6244 6245
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ReRaiseException", "Exceptions.c"))
            code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos))
6246

William Stein's avatar
William Stein committed
6247 6248 6249 6250 6251
class AssertStatNode(StatNode):
    #  assert statement
    #
    #  cond    ExprNode
    #  value   ExprNode or None
6252

6253 6254
    child_attrs = ["cond", "value"]

William Stein's avatar
William Stein committed
6255 6256 6257
    def analyse_expressions(self, env):
        self.cond = self.cond.analyse_boolean_expression(env)
        if self.value:
6258
            value = self.value.analyse_types(env)
6259 6260
            if value.type is Builtin.tuple_type or not value.type.is_builtin_type:
                # prevent tuple values from being interpreted as argument value tuples
6261
                from .ExprNodes import TupleNode
6262
                value = TupleNode(value.pos, args=[value], slow=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
6263
                self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env)
6264 6265
            else:
                self.value = value.coerce_to_pyobject(env)
6266
        return self
6267

6268
    nogil_check = Node.gil_error
6269
    gil_message = "Raising exception"
6270

William Stein's avatar
William Stein committed
6271
    def generate_execution_code(self, code):
6272
        code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS")
6273
        code.putln("if (unlikely(!Py_OptimizeFlag)) {")
6274
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6275 6276
        self.cond.generate_evaluation_code(code)
        code.putln(
6277
            "if (unlikely(!%s)) {" % self.cond.result())
William Stein's avatar
William Stein committed
6278
        if self.value:
6279
            self.value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
6280
            code.putln(
6281
                "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result())
6282
            self.value.generate_disposal_code(code)
6283
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
6284 6285 6286 6287
        else:
            code.putln(
                "PyErr_SetNone(PyExc_AssertionError);")
        code.putln(
6288
            code.error_goto(self.pos))
William Stein's avatar
William Stein committed
6289 6290 6291
        code.putln(
            "}")
        self.cond.generate_disposal_code(code)
6292
        self.cond.free_temps(code)
6293 6294
        code.putln(
            "}")
6295
        code.putln("#endif")
William Stein's avatar
William Stein committed
6296

6297 6298 6299 6300 6301
    def generate_function_definitions(self, env, code):
        self.cond.generate_function_definitions(env, code)
        if self.value is not None:
            self.value.generate_function_definitions(env, code)

6302 6303 6304 6305 6306 6307
    def annotate(self, code):
        self.cond.annotate(code)
        if self.value:
            self.value.annotate(code)


William Stein's avatar
William Stein committed
6308 6309 6310 6311 6312
class IfStatNode(StatNode):
    #  if statement
    #
    #  if_clauses   [IfClauseNode]
    #  else_clause  StatNode or None
6313 6314

    child_attrs = ["if_clauses", "else_clause"]
6315

William Stein's avatar
William Stein committed
6316 6317 6318 6319 6320
    def analyse_declarations(self, env):
        for if_clause in self.if_clauses:
            if_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6321

William Stein's avatar
William Stein committed
6322
    def analyse_expressions(self, env):
6323
        self.if_clauses = [if_clause.analyse_expressions(env) for if_clause in self.if_clauses]
William Stein's avatar
William Stein committed
6324
        if self.else_clause:
6325 6326
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6327

William Stein's avatar
William Stein committed
6328
    def generate_execution_code(self, code):
6329
        code.mark_pos(self.pos)
6330
        end_label = code.new_label()
6331
        last = len(self.if_clauses)
6332 6333 6334 6335
        if self.else_clause:
            # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that.
            self._set_branch_hint(self.if_clauses[-1], self.else_clause, inverse=True)
        else:
6336
            last -= 1  # avoid redundant goto at end of last if-clause
6337
        for i, if_clause in enumerate(self.if_clauses):
6338
            self._set_branch_hint(if_clause, if_clause.body)
6339
            if_clause.generate_execution_code(code, end_label, is_last=i == last)
6340
        if self.else_clause:
6341
            code.mark_pos(self.else_clause.pos)
6342
            code.putln("/*else*/ {")
William Stein's avatar
William Stein committed
6343
            self.else_clause.generate_execution_code(code)
6344 6345
            code.putln("}")
        code.put_label(end_label)
6346

6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361
    def _set_branch_hint(self, clause, statements_node, inverse=False):
        if not statements_node.is_terminator:
            return
        if not isinstance(statements_node, StatListNode) or not statements_node.stats:
            return
        # Anything that unconditionally raises exceptions should be considered unlikely.
        if isinstance(statements_node.stats[-1], (RaiseStatNode, ReraiseStatNode)):
            if len(statements_node.stats) > 1:
                # Allow simple statements before the 'raise', but no conditions, loops, etc.
                non_branch_nodes = (ExprStatNode, AssignmentNode, DelStatNode, GlobalNode, NonlocalNode)
                for node in statements_node.stats[:-1]:
                    if not isinstance(node, non_branch_nodes):
                        return
            clause.branch_hint = 'likely' if inverse else 'unlikely'

6362 6363 6364 6365 6366
    def generate_function_definitions(self, env, code):
        for clause in self.if_clauses:
            clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
6367

6368 6369 6370 6371 6372
    def annotate(self, code):
        for if_clause in self.if_clauses:
            if_clause.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
6373 6374 6375 6376 6377 6378 6379


class IfClauseNode(Node):
    #  if or elif clause in an if statement
    #
    #  condition   ExprNode
    #  body        StatNode
6380

6381
    child_attrs = ["condition", "body"]
6382
    branch_hint = None
6383

William Stein's avatar
William Stein committed
6384 6385
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
6386

William Stein's avatar
William Stein committed
6387
    def analyse_expressions(self, env):
6388
        self.condition = self.condition.analyse_temp_boolean_expression(env)
6389 6390
        self.body = self.body.analyse_expressions(env)
        return self
6391

6392
    def generate_execution_code(self, code, end_label, is_last):
William Stein's avatar
William Stein committed
6393
        self.condition.generate_evaluation_code(code)
6394
        code.mark_pos(self.pos)
6395 6396 6397 6398
        condition = self.condition.result()
        if self.branch_hint:
            condition = '%s(%s)' % (self.branch_hint, condition)
        code.putln("if (%s) {" % condition)
6399 6400
        self.condition.generate_disposal_code(code)
        self.condition.free_temps(code)
William Stein's avatar
William Stein committed
6401
        self.body.generate_execution_code(code)
6402 6403
        code.mark_pos(self.pos, trace=False)
        if not (is_last or self.body.is_terminator):
6404
            code.put_goto(end_label)
William Stein's avatar
William Stein committed
6405
        code.putln("}")
6406

6407 6408 6409 6410
    def generate_function_definitions(self, env, code):
        self.condition.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)

6411 6412 6413
    def annotate(self, code):
        self.condition.annotate(code)
        self.body.annotate(code)
6414

6415 6416 6417 6418 6419 6420

class SwitchCaseNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # conditions    [ExprNode]
    # body          StatNode
6421

6422
    child_attrs = ['conditions', 'body']
6423

6424
    def generate_condition_evaluation_code(self, code):
6425
        for cond in self.conditions:
6426
            cond.generate_evaluation_code(code)
6427 6428 6429 6430

    def generate_execution_code(self, code):
        num_conditions = len(self.conditions)
        line_tracing_enabled = code.globalstate.directives['linetrace']
Stefan Behnel's avatar
Stefan Behnel committed
6431
        for i, cond in enumerate(self.conditions, 1):
6432
            code.putln("case %s:" % cond.result())
6433
            code.mark_pos(cond.pos)  # Tracing code must appear *after* the 'case' statement.
Stefan Behnel's avatar
Stefan Behnel committed
6434
            if line_tracing_enabled and i < num_conditions:
6435 6436
                # Allow fall-through after the line tracing code.
                code.putln('CYTHON_FALLTHROUGH;')
6437
        self.body.generate_execution_code(code)
6438
        code.mark_pos(self.pos, trace=False)
6439
        code.putln("break;")
6440 6441 6442 6443 6444

    def generate_function_definitions(self, env, code):
        for cond in self.conditions:
            cond.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
6445

6446 6447 6448
    def annotate(self, code):
        for cond in self.conditions:
            cond.annotate(code)
6449
        self.body.annotate(code)
6450

6451

6452 6453 6454 6455 6456 6457
class SwitchStatNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # test          ExprNode
    # cases         [SwitchCaseNode]
    # else_clause   StatNode or None
6458

6459
    child_attrs = ['test', 'cases', 'else_clause']
6460

6461
    def generate_execution_code(self, code):
6462
        self.test.generate_evaluation_code(code)
6463 6464 6465 6466
        # Make sure all conditions are evaluated before going into the switch() statement.
        # This is required in order to prevent any execution code from leaking into the space between the cases.
        for case in self.cases:
            case.generate_condition_evaluation_code(code)
6467
        code.mark_pos(self.pos)
6468
        code.putln("switch (%s) {" % self.test.result())
6469 6470 6471 6472 6473
        for case in self.cases:
            case.generate_execution_code(code)
        if self.else_clause is not None:
            code.putln("default:")
            self.else_clause.generate_execution_code(code)
6474
            code.putln("break;")
6475 6476 6477 6478 6479
        else:
            # Always generate a default clause to prevent C compiler warnings
            # about unmatched enum values (it was not the user who decided to
            # generate the switch statement, so shouldn't be bothered).
            code.putln("default: break;")
6480 6481
        code.putln("}")

6482 6483 6484 6485 6486 6487 6488
    def generate_function_definitions(self, env, code):
        self.test.generate_function_definitions(env, code)
        for case in self.cases:
            case.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6489 6490 6491 6492
    def annotate(self, code):
        self.test.annotate(code)
        for case in self.cases:
            case.annotate(code)
6493 6494
        if self.else_clause is not None:
            self.else_clause.annotate(code)
6495

6496

6497
class LoopNode(object):
6498
    pass
6499

6500

6501
class WhileStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6502 6503 6504 6505 6506
    #  while statement
    #
    #  condition    ExprNode
    #  body         StatNode
    #  else_clause  StatNode
6507 6508

    child_attrs = ["condition", "body", "else_clause"]
6509

William Stein's avatar
William Stein committed
6510 6511 6512 6513
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6514

William Stein's avatar
William Stein committed
6515
    def analyse_expressions(self, env):
6516 6517
        if self.condition:
            self.condition = self.condition.analyse_temp_boolean_expression(env)
6518
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6519
        if self.else_clause:
6520 6521
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6522

William Stein's avatar
William Stein committed
6523
    def generate_execution_code(self, code):
6524
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6525 6526 6527
        old_loop_labels = code.new_loop_labels()
        code.putln(
            "while (1) {")
6528 6529 6530 6531
        if self.condition:
            self.condition.generate_evaluation_code(code)
            self.condition.generate_disposal_code(code)
            code.putln(
6532
                "if (!%s) break;" % self.condition.result())
6533
            self.condition.free_temps(code)
William Stein's avatar
William Stein committed
6534
        self.body.generate_execution_code(code)
6535
        code.put_label(code.continue_label)
William Stein's avatar
William Stein committed
6536 6537 6538 6539
        code.putln("}")
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
6540
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
6541 6542 6543 6544 6545
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)

6546
    def generate_function_definitions(self, env, code):
6547 6548
        if self.condition:
            self.condition.generate_function_definitions(env, code)
6549 6550 6551 6552
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6553
    def annotate(self, code):
6554 6555
        if self.condition:
            self.condition.annotate(code)
6556 6557 6558 6559
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6560

6561 6562 6563 6564
class DictIterationNextNode(Node):
    # Helper node for calling PyDict_Next() inside of a WhileStatNode
    # and checking the dictionary size for changes.  Created in
    # Optimize.py.
6565 6566 6567
    child_attrs = ['dict_obj', 'expected_size', 'pos_index_var',
                   'coerced_key_var', 'coerced_value_var', 'coerced_tuple_var',
                   'key_target', 'value_target', 'tuple_target', 'is_dict_flag']
6568

6569 6570 6571 6572 6573 6574
    coerced_key_var = key_ref = None
    coerced_value_var = value_ref = None
    coerced_tuple_var = tuple_ref = None

    def __init__(self, dict_obj, expected_size, pos_index_var,
                 key_target, value_target, tuple_target, is_dict_flag):
6575 6576
        Node.__init__(
            self, dict_obj.pos,
6577 6578 6579 6580 6581 6582 6583 6584 6585
            dict_obj=dict_obj,
            expected_size=expected_size,
            pos_index_var=pos_index_var,
            key_target=key_target,
            value_target=value_target,
            tuple_target=tuple_target,
            is_dict_flag=is_dict_flag,
            is_temp=True,
            type=PyrexTypes.c_bint_type)
6586 6587

    def analyse_expressions(self, env):
6588
        from . import ExprNodes
6589 6590 6591 6592
        self.dict_obj = self.dict_obj.analyse_types(env)
        self.expected_size = self.expected_size.analyse_types(env)
        if self.pos_index_var:
            self.pos_index_var = self.pos_index_var.analyse_types(env)
6593
        if self.key_target:
6594
            self.key_target = self.key_target.analyse_target_types(env)
6595
            self.key_ref = ExprNodes.TempNode(self.key_target.pos, PyrexTypes.py_object_type)
6596 6597
            self.coerced_key_var = self.key_ref.coerce_to(self.key_target.type, env)
        if self.value_target:
6598
            self.value_target = self.value_target.analyse_target_types(env)
6599
            self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
6600 6601
            self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
        if self.tuple_target:
6602
            self.tuple_target = self.tuple_target.analyse_target_types(env)
6603
            self.tuple_ref = ExprNodes.TempNode(self.tuple_target.pos, PyrexTypes.py_object_type)
6604
            self.coerced_tuple_var = self.tuple_ref.coerce_to(self.tuple_target.type, env)
6605 6606
        self.is_dict_flag = self.is_dict_flag.analyse_types(env)
        return self
6607 6608 6609 6610 6611

    def generate_function_definitions(self, env, code):
        self.dict_obj.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
6612
        code.globalstate.use_utility_code(UtilityCode.load_cached("dict_iter", "Optimize.c"))
6613 6614
        self.dict_obj.generate_evaluation_code(code)

6615 6616 6617 6618 6619 6620 6621 6622
        assignments = []
        temp_addresses = []
        for var, result, target in [(self.key_ref, self.coerced_key_var, self.key_target),
                                    (self.value_ref, self.coerced_value_var, self.value_target),
                                    (self.tuple_ref, self.coerced_tuple_var, self.tuple_target)]:
            if target is None:
                addr = 'NULL'
            else:
6623 6624 6625
                assignments.append((var, result, target))
                var.allocate(code)
                addr = '&%s' % var.result()
6626 6627 6628 6629 6630
            temp_addresses.append(addr)

        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
        code.putln("%s = __Pyx_dict_iter_next(%s, %s, &%s, %s, %s, %s, %s);" % (
            result_temp,
6631
            self.dict_obj.py_result(),
6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643
            self.expected_size.result(),
            self.pos_index_var.result(),
            temp_addresses[0],
            temp_addresses[1],
            temp_addresses[2],
            self.is_dict_flag.result()
        ))
        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
        code.funcstate.release_temp(result_temp)

        # evaluate all coercions before the assignments
6644 6645
        for var, result, target in assignments:
            code.put_gotref(var.result())
6646
        for var, result, target in assignments:
6647
            result.generate_evaluation_code(code)
6648
        for var, result, target in assignments:
6649
            target.generate_assignment_code(result, code)
6650
            var.release(code)
6651

6652

6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712
class SetIterationNextNode(Node):
    # Helper node for calling _PySet_NextEntry() inside of a WhileStatNode
    # and checking the set size for changes.  Created in Optimize.py.
    child_attrs = ['set_obj', 'expected_size', 'pos_index_var',
                   'coerced_value_var', 'value_target', 'is_set_flag']

    coerced_value_var = value_ref = None

    def __init__(self, set_obj, expected_size, pos_index_var, value_target, is_set_flag):
        Node.__init__(
            self, set_obj.pos,
            set_obj=set_obj,
            expected_size=expected_size,
            pos_index_var=pos_index_var,
            value_target=value_target,
            is_set_flag=is_set_flag,
            is_temp=True,
            type=PyrexTypes.c_bint_type)

    def analyse_expressions(self, env):
        from . import ExprNodes
        self.set_obj = self.set_obj.analyse_types(env)
        self.expected_size = self.expected_size.analyse_types(env)
        self.pos_index_var = self.pos_index_var.analyse_types(env)
        self.value_target = self.value_target.analyse_target_types(env)
        self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
        self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
        self.is_set_flag = self.is_set_flag.analyse_types(env)
        return self

    def generate_function_definitions(self, env, code):
        self.set_obj.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
        code.globalstate.use_utility_code(UtilityCode.load_cached("set_iter", "Optimize.c"))
        self.set_obj.generate_evaluation_code(code)

        value_ref = self.value_ref
        value_ref.allocate(code)

        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
        code.putln("%s = __Pyx_set_iter_next(%s, %s, &%s, &%s, %s);" % (
            result_temp,
            self.set_obj.py_result(),
            self.expected_size.result(),
            self.pos_index_var.result(),
            value_ref.result(),
            self.is_set_flag.result()
        ))
        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
        code.funcstate.release_temp(result_temp)

        # evaluate all coercions before the assignments
        code.put_gotref(value_ref.result())
        self.coerced_value_var.generate_evaluation_code(code)
        self.value_target.generate_assignment_code(self.coerced_value_var, code)
        value_ref.release(code)


Robert Bradshaw's avatar
Robert Bradshaw committed
6713
def ForStatNode(pos, **kw):
6714
    if 'iterator' in kw:
6715 6716 6717 6718
        if kw['iterator'].is_async:
            return AsyncForStatNode(pos, **kw)
        else:
            return ForInStatNode(pos, **kw)
Robert Bradshaw's avatar
Robert Bradshaw committed
6719 6720 6721
    else:
        return ForFromStatNode(pos, **kw)

6722 6723 6724

class _ForInStatNode(LoopNode, StatNode):
    #  Base class of 'for-in' statements.
William Stein's avatar
William Stein committed
6725 6726
    #
    #  target        ExprNode
6727
    #  iterator      IteratorNode | AIterAwaitExprNode(AsyncIteratorNode)
William Stein's avatar
William Stein committed
6728 6729
    #  body          StatNode
    #  else_clause   StatNode
6730 6731
    #  item          NextNode | AwaitExprNode(AsyncNextNode)
    #  is_async      boolean        true for 'async for' statements
6732

6733
    child_attrs = ["target", "item", "iterator", "body", "else_clause"]
6734
    item = None
6735 6736 6737 6738
    is_async = False

    def _create_item_node(self):
        raise NotImplementedError("must be implemented by subclasses")
6739

William Stein's avatar
William Stein committed
6740 6741 6742 6743 6744
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6745
        self._create_item_node()
6746

William Stein's avatar
William Stein committed
6747
    def analyse_expressions(self, env):
6748
        self.target = self.target.analyse_target_types(env)
6749
        self.iterator = self.iterator.analyse_expressions(env)
6750
        self._create_item_node()  # must rewrap self.item after analysis
6751
        self.item = self.item.analyse_expressions(env)
6752 6753 6754
        if (not self.is_async and
                (self.iterator.type.is_ptr or self.iterator.type.is_array) and
                self.target.type.assignable_from(self.iterator.type)):
6755 6756 6757 6758
            # C array slice optimization.
            pass
        else:
            self.item = self.item.coerce_to(self.target.type, env)
6759
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6760
        if self.else_clause:
6761 6762
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
William Stein's avatar
William Stein committed
6763 6764

    def generate_execution_code(self, code):
6765
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6766 6767
        old_loop_labels = code.new_loop_labels()
        self.iterator.generate_evaluation_code(code)
Mark Florisson's avatar
Mark Florisson committed
6768
        code.putln("for (;;) {")
William Stein's avatar
William Stein committed
6769 6770 6771
        self.item.generate_evaluation_code(code)
        self.target.generate_assignment_code(self.item, code)
        self.body.generate_execution_code(code)
6772
        code.mark_pos(self.pos)
6773
        code.put_label(code.continue_label)
Mark Florisson's avatar
Mark Florisson committed
6774
        code.putln("}")
William Stein's avatar
William Stein committed
6775 6776
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
6777

William Stein's avatar
William Stein committed
6778
        if self.else_clause:
6779 6780 6781 6782 6783 6784 6785
            # in nested loops, the 'else' block can contain a
            # 'continue' statement for the outer loop, but we may need
            # to generate cleanup code before taking that path, so we
            # intercept it here
            orig_continue_label = code.continue_label
            code.continue_label = code.new_label('outer_continue')

William Stein's avatar
William Stein committed
6786 6787 6788
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
6789 6790 6791

            if code.label_used(code.continue_label):
                code.put_goto(break_label)
6792
                code.mark_pos(self.pos)
6793 6794 6795 6796 6797
                code.put_label(code.continue_label)
                self.iterator.generate_disposal_code(code)
                code.put_goto(orig_continue_label)
            code.set_loop_labels(old_loop_labels)

6798
        code.mark_pos(self.pos)
6799 6800
        if code.label_used(break_label):
            code.put_label(break_label)
William Stein's avatar
William Stein committed
6801
        self.iterator.generate_disposal_code(code)
6802
        self.iterator.free_temps(code)
William Stein's avatar
William Stein committed
6803

6804 6805 6806 6807 6808 6809 6810
    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.iterator.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6811 6812 6813 6814 6815 6816 6817 6818
    def annotate(self, code):
        self.target.annotate(code)
        self.iterator.annotate(code)
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
        self.item.annotate(code)

William Stein's avatar
William Stein committed
6819

6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832
class ForInStatNode(_ForInStatNode):
    #  'for' statement

    is_async = False

    def _create_item_node(self):
        from .ExprNodes import NextNode
        self.item = NextNode(self.iterator)


class AsyncForStatNode(_ForInStatNode):
    #  'async for' statement
    #
6833
    #  iterator      AIterAwaitExprNode(AsyncIteratorNode)
6834 6835 6836 6837
    #  item          AwaitIterNextExprNode(AsyncIteratorNode)

    is_async = True

6838
    def __init__(self, pos, **kw):
6839 6840 6841
        assert 'item' not in kw
        from . import ExprNodes
        # AwaitExprNodes must appear before running MarkClosureVisitor
6842
        kw['item'] = ExprNodes.AwaitIterNextExprNode(kw['iterator'].pos, arg=None)
6843 6844 6845 6846 6847 6848 6849
        _ForInStatNode.__init__(self, pos, **kw)

    def _create_item_node(self):
        from . import ExprNodes
        self.item.arg = ExprNodes.AsyncNextNode(self.iterator)


6850
class ForFromStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6851 6852 6853 6854 6855 6856 6857
    #  for name from expr rel name rel expr
    #
    #  target        NameNode
    #  bound1        ExprNode
    #  relation1     string
    #  relation2     string
    #  bound2        ExprNode
6858
    #  step          ExprNode or None
William Stein's avatar
William Stein committed
6859 6860 6861 6862 6863
    #  body          StatNode
    #  else_clause   StatNode or None
    #
    #  Used internally:
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
6864
    #  from_range         bool
6865
    #  is_py_target       bool
6866
    #  loopvar_node       ExprNode (usually a NameNode or temp node)
William Stein's avatar
William Stein committed
6867
    #  py_loopvar_node    PyTempNode or None
6868
    child_attrs = ["target", "bound1", "bound2", "step", "body", "else_clause"]
6869 6870

    is_py_target = False
6871
    loopvar_node = None
6872
    py_loopvar_node = None
Robert Bradshaw's avatar
Robert Bradshaw committed
6873
    from_range = False
6874

6875 6876 6877 6878 6879 6880 6881
    gil_message = "For-loop using object bounds or target"

    def nogil_check(self, env):
        for x in (self.target, self.bound1, self.bound2):
            if x.type.is_pyobject:
                self.gil_error()

Robert Bradshaw's avatar
Robert Bradshaw committed
6882 6883 6884 6885 6886
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6887

William Stein's avatar
William Stein committed
6888
    def analyse_expressions(self, env):
6889
        from . import ExprNodes
6890 6891 6892
        self.target = self.target.analyse_target_types(env)
        self.bound1 = self.bound1.analyse_types(env)
        self.bound2 = self.bound2.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6893 6894
        if self.step is not None:
            if isinstance(self.step, ExprNodes.UnaryMinusNode):
6895 6896
                warning(self.step.pos, "Probable infinite loop in for-from-by statement. "
                        "Consider switching the directions of the relations.", 2)
6897
            self.step = self.step.analyse_types(env)
6898

6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914
        self.set_up_loop(env)
        target_type = self.target.type
        if not (target_type.is_pyobject or target_type.is_numeric):
            error(self.target.pos, "for-from loop variable must be c numeric type or Python object")

        self.body = self.body.analyse_expressions(env)
        if self.else_clause:
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self

    def set_up_loop(self, env):
        from . import ExprNodes

        target_type = self.target.type
        if target_type.is_numeric:
            loop_type = target_type
6915
        else:
6916 6917 6918
            if target_type.is_enum:
                warning(self.target.pos,
                        "Integer loops over enum values are fragile. Please cast to a safe integer type instead.")
6919
            loop_type = PyrexTypes.c_long_type if target_type.is_pyobject else PyrexTypes.c_int_type
Robert Bradshaw's avatar
Robert Bradshaw committed
6920 6921 6922 6923 6924 6925 6926 6927
            if not self.bound1.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound1.type)
            if not self.bound2.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound2.type)
            if self.step is not None and not self.step.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.step.type)
        self.bound1 = self.bound1.coerce_to(loop_type, env)
        self.bound2 = self.bound2.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6928 6929
        if not self.bound2.is_literal:
            self.bound2 = self.bound2.coerce_to_temp(env)
6930
        if self.step is not None:
6931
            self.step = self.step.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6932 6933
            if not self.step.is_literal:
                self.step = self.step.coerce_to_temp(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6934

6935
        if target_type.is_numeric or target_type.is_enum:
Robert Bradshaw's avatar
Robert Bradshaw committed
6936
            self.is_py_target = False
6937 6938
            if isinstance(self.target, ExprNodes.BufferIndexNode):
                raise error(self.pos, "Buffer or memoryview slicing/indexing not allowed as for-loop target.")
6939
            self.loopvar_node = self.target
William Stein's avatar
William Stein committed
6940 6941
            self.py_loopvar_node = None
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6942 6943
            self.is_py_target = True
            c_loopvar_node = ExprNodes.TempNode(self.pos, loop_type, env)
6944
            self.loopvar_node = c_loopvar_node
6945
            self.py_loopvar_node = ExprNodes.CloneNode(c_loopvar_node).coerce_to_pyobject(env)
6946

William Stein's avatar
William Stein committed
6947
    def generate_execution_code(self, code):
6948
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6949
        old_loop_labels = code.new_loop_labels()
Robert Bradshaw's avatar
Robert Bradshaw committed
6950
        from_range = self.from_range
William Stein's avatar
William Stein committed
6951 6952 6953
        self.bound1.generate_evaluation_code(code)
        self.bound2.generate_evaluation_code(code)
        offset, incop = self.relation_table[self.relation1]
6954 6955
        if self.step is not None:
            self.step.generate_evaluation_code(code)
Magnus Lie Hetland's avatar
Magnus Lie Hetland committed
6956
            step = self.step.result()
Stefan Behnel's avatar
Stefan Behnel committed
6957 6958 6959 6960
            incop = "%s=%s" % (incop[0], step)  # e.g. '++' => '+= STEP'
        else:
            step = '1'

6961
        from . import ExprNodes
6962 6963 6964 6965
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.allocate(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.allocate(code)
6966 6967 6968

        loopvar_type = PyrexTypes.c_long_type if self.target.type.is_enum else self.target.type

6969
        if from_range and not self.is_py_target:
6970
            loopvar_name = code.funcstate.allocate_temp(loopvar_type, False)
6971
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6972
            loopvar_name = self.loopvar_node.result()
6973
        if loopvar_type.is_int and not loopvar_type.signed and self.relation2[0] == '>':
6974 6975
            # Handle the case where the endpoint of an unsigned int iteration
            # is within step of 0.
6976 6977 6978 6979 6980
            code.putln("for (%s = %s%s + %s; %s %s %s + %s; ) { %s%s;" % (
                loopvar_name,
                self.bound1.result(), offset, step,
                loopvar_name, self.relation2, self.bound2.result(), step,
                loopvar_name, incop))
6981
        else:
6982 6983 6984 6985 6986
            code.putln("for (%s = %s%s; %s %s %s; %s%s) {" % (
                loopvar_name,
                self.bound1.result(), offset,
                loopvar_name, self.relation2, self.bound2.result(),
                loopvar_name, incop))
6987 6988 6989

        coerced_loopvar_node = self.py_loopvar_node
        if coerced_loopvar_node is None and from_range:
6990
            coerced_loopvar_node = ExprNodes.RawCNameExprNode(self.target.pos, loopvar_type, loopvar_name)
6991 6992 6993 6994
        if coerced_loopvar_node is not None:
            coerced_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(coerced_loopvar_node, code)

William Stein's avatar
William Stein committed
6995 6996
        self.body.generate_execution_code(code)
        code.put_label(code.continue_label)
6997

6998
        if not from_range and self.py_loopvar_node:
6999 7000 7001
            # This mess is to make for..from loops with python targets behave
            # exactly like those with C targets with regards to re-assignment
            # of the loop variable.
7002
            if self.target.entry.is_pyglobal:
7003
                # We know target is a NameNode, this is the only ugly case.
7004
                target_node = ExprNodes.PyTempNode(self.target.pos, None)
7005 7006
                target_node.allocate(code)
                interned_cname = code.intern_identifier(self.target.entry.name)
7007 7008 7009
                if self.target.entry.scope.is_module_scope:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c"))
7010
                    lookup_func = '__Pyx_GetModuleGlobalName(%s, %s); %s'
7011 7012 7013
                else:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c"))
7014
                    lookup_func = '__Pyx_GetNameInClass(%s, {}, %s); %s'.format(
7015
                        self.target.entry.scope.namespace_cname)
7016
                code.putln(lookup_func % (
7017
                    target_node.result(),
7018
                    interned_cname,
7019
                    code.error_goto_if_null(target_node.result(), self.target.pos)))
7020
                code.put_gotref(target_node.result())
7021 7022
            else:
                target_node = self.target
7023 7024
            from_py_node = ExprNodes.CoerceFromPyTypeNode(
                self.loopvar_node.type, target_node, self.target.entry.scope)
7025 7026
            from_py_node.temp_code = loopvar_name
            from_py_node.generate_result_code(code)
7027
            if self.target.entry.is_pyglobal:
7028 7029
                code.put_decref(target_node.result(), target_node.type)
                target_node.release(code)
7030

Robert Bradshaw's avatar
Robert Bradshaw committed
7031
        code.putln("}")
7032

7033
        if not from_range and self.py_loopvar_node:
7034 7035
            # This is potentially wasteful, but we don't want the semantics to
            # depend on whether or not the loop is a python type.
7036 7037
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
7038
        if from_range and not self.is_py_target:
7039
            code.funcstate.release_temp(loopvar_name)
7040

William Stein's avatar
William Stein committed
7041 7042 7043 7044 7045 7046 7047 7048
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)
        self.bound1.generate_disposal_code(code)
7049
        self.bound1.free_temps(code)
William Stein's avatar
William Stein committed
7050
        self.bound2.generate_disposal_code(code)
7051
        self.bound2.free_temps(code)
7052 7053 7054 7055
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.release(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.release(code)
7056 7057
        if self.step is not None:
            self.step.generate_disposal_code(code)
7058
            self.step.free_temps(code)
7059

William Stein's avatar
William Stein committed
7060 7061 7062 7063 7064
    relation_table = {
        # {relop : (initial offset, increment op)}
        '<=': ("",   "++"),
        '<' : ("+1", "++"),
        '>=': ("",   "--"),
7065
        '>' : ("-1", "--"),
William Stein's avatar
William Stein committed
7066
    }
7067 7068 7069 7070 7071 7072 7073 7074 7075 7076

    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.bound1.generate_function_definitions(env, code)
        self.bound2.generate_function_definitions(env, code)
        if self.step is not None:
            self.step.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
7077

7078 7079 7080 7081 7082
    def annotate(self, code):
        self.target.annotate(code)
        self.bound1.annotate(code)
        self.bound2.annotate(code)
        if self.step:
7083
            self.step.annotate(code)
7084 7085 7086
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
7087 7088


7089 7090 7091
class WithStatNode(StatNode):
    """
    Represents a Python with statement.
7092

7093
    Implemented by the WithTransform as follows:
7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110

        MGR = EXPR
        EXIT = MGR.__exit__
        VALUE = MGR.__enter__()
        EXC = True
        try:
            try:
                TARGET = VALUE  # optional
                BODY
            except:
                EXC = False
                if not EXIT(*EXCINFO):
                    raise
        finally:
            if EXC:
                EXIT(None, None, None)
            MGR = EXIT = VALUE = None
7111 7112
    """
    #  manager          The with statement manager object
7113
    #  target           ExprNode  the target lhs of the __enter__() call
7114
    #  body             StatNode
7115
    #  enter_call       ExprNode  the call to the __enter__() method
7116
    #  exit_var         String    the cname of the __exit__() method reference
7117

7118
    child_attrs = ["manager", "enter_call", "target", "body"]
7119

7120
    enter_call = None
7121
    target_temp = None
7122 7123 7124

    def analyse_declarations(self, env):
        self.manager.analyse_declarations(env)
7125
        self.enter_call.analyse_declarations(env)
7126 7127 7128
        self.body.analyse_declarations(env)

    def analyse_expressions(self, env):
7129 7130
        self.manager = self.manager.analyse_types(env)
        self.enter_call = self.enter_call.analyse_types(env)
7131 7132 7133 7134
        if self.target:
            # set up target_temp before descending into body (which uses it)
            from .ExprNodes import TempNode
            self.target_temp = TempNode(self.enter_call.pos, self.enter_call.type)
7135 7136
        self.body = self.body.analyse_expressions(env)
        return self
7137

7138 7139
    def generate_function_definitions(self, env, code):
        self.manager.generate_function_definitions(env, code)
7140
        self.enter_call.generate_function_definitions(env, code)
7141 7142
        self.body.generate_function_definitions(env, code)

7143
    def generate_execution_code(self, code):
7144
        code.mark_pos(self.pos)
7145 7146 7147
        code.putln("/*with:*/ {")
        self.manager.generate_evaluation_code(code)
        self.exit_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
7148
        code.globalstate.use_utility_code(
7149 7150
            UtilityCode.load_cached("PyObjectLookupSpecial", "ObjectHandling.c"))
        code.putln("%s = __Pyx_PyObject_LookupSpecial(%s, %s); %s" % (
7151 7152
            self.exit_var,
            self.manager.py_result(),
7153
            code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')),
7154 7155 7156 7157 7158 7159 7160 7161
            code.error_goto_if_null(self.exit_var, self.pos),
            ))
        code.put_gotref(self.exit_var)

        # need to free exit_var in the face of exceptions during setup
        old_error_label = code.new_error_label()
        intermediate_error_label = code.error_label

7162
        self.enter_call.generate_evaluation_code(code)
7163 7164 7165 7166 7167 7168 7169
        if self.target:
            # The temp result will be cleaned up by the WithTargetAssignmentStatNode
            # after assigning its result to the target of the 'with' statement.
            self.target_temp.allocate(code)
            self.enter_call.make_owned_reference(code)
            code.putln("%s = %s;" % (self.target_temp.result(), self.enter_call.result()))
            self.enter_call.generate_post_assignment_code(code)
7170
        else:
7171 7172 7173
            self.enter_call.generate_disposal_code(code)
        self.enter_call.free_temps(code)

7174 7175 7176 7177 7178 7179
        self.manager.generate_disposal_code(code)
        self.manager.free_temps(code)

        code.error_label = old_error_label
        self.body.generate_execution_code(code)

7180 7181 7182 7183 7184 7185 7186
        if code.label_used(intermediate_error_label):
            step_over_label = code.new_label()
            code.put_goto(step_over_label)
            code.put_label(intermediate_error_label)
            code.put_decref_clear(self.exit_var, py_object_type)
            code.put_goto(old_error_label)
            code.put_label(step_over_label)
7187 7188 7189 7190

        code.funcstate.release_temp(self.exit_var)
        code.putln('}')

7191

7192 7193 7194 7195
class WithTargetAssignmentStatNode(AssignmentNode):
    # The target assignment of the 'with' statement value (return
    # value of the __enter__() call).
    #
7196
    # This is a special cased assignment that properly cleans up the RHS.
7197
    #
7198 7199 7200
    # lhs       ExprNode      the assignment target
    # rhs       ExprNode      a (coerced) TempNode for the rhs (from WithStatNode)
    # with_node WithStatNode  the surrounding with-statement
7201

7202 7203 7204
    child_attrs = ["rhs", "lhs"]
    with_node = None
    rhs = None
7205 7206 7207 7208

    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)

7209
    def analyse_expressions(self, env):
7210
        self.lhs = self.lhs.analyse_target_types(env)
7211
        self.lhs.gil_assignment_check(env)
7212
        self.rhs = self.with_node.target_temp.coerce_to(self.lhs.type, env)
7213
        return self
7214 7215 7216 7217

    def generate_execution_code(self, code):
        self.rhs.generate_evaluation_code(code)
        self.lhs.generate_assignment_code(self.rhs, code)
7218
        self.with_node.target_temp.release(code)
7219 7220 7221 7222 7223

    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

7224

William Stein's avatar
William Stein committed
7225 7226 7227 7228 7229 7230
class TryExceptStatNode(StatNode):
    #  try .. except statement
    #
    #  body             StatNode
    #  except_clauses   [ExceptClauseNode]
    #  else_clause      StatNode or None
7231

7232
    child_attrs = ["body", "except_clauses", "else_clause"]
7233
    in_generator = False
7234

William Stein's avatar
William Stein committed
7235 7236 7237 7238 7239 7240
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        for except_clause in self.except_clauses:
            except_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
7241

William Stein's avatar
William Stein committed
7242
    def analyse_expressions(self, env):
7243
        self.body = self.body.analyse_expressions(env)
7244
        default_clause_seen = 0
7245 7246
        for i, except_clause in enumerate(self.except_clauses):
            except_clause = self.except_clauses[i] = except_clause.analyse_expressions(env)
7247 7248 7249 7250 7251
            if default_clause_seen:
                error(except_clause.pos, "default 'except:' must be last")
            if not except_clause.pattern:
                default_clause_seen = 1
        self.has_default_clause = default_clause_seen
William Stein's avatar
William Stein committed
7252
        if self.else_clause:
7253 7254
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
7255

7256
    nogil_check = Node.gil_error
7257 7258
    gil_message = "Try-except statement"

William Stein's avatar
William Stein committed
7259
    def generate_execution_code(self, code):
7260 7261 7262
        code.mark_pos(self.pos)  # before changing the error label, in case of tracing errors
        code.putln("{")

7263
        old_return_label = code.return_label
7264
        old_break_label = code.break_label
7265
        old_continue_label = code.continue_label
William Stein's avatar
William Stein committed
7266 7267
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
7268 7269 7270
        except_end_label = code.new_label('exception_handled')
        except_error_label = code.new_label('except_error')
        except_return_label = code.new_label('except_return')
7271
        try_return_label = code.new_label('try_return')
7272 7273
        try_break_label = code.new_label('try_break') if old_break_label else None
        try_continue_label = code.new_label('try_continue') if old_continue_label else None
7274
        try_end_label = code.new_label('try_end')
7275

7276
        exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
7277
                         for _ in range(3)]
7278
        save_exc = code.insertion_point()
William Stein's avatar
William Stein committed
7279 7280
        code.putln(
            "/*try:*/ {")
7281
        code.return_label = try_return_label
7282
        code.break_label = try_break_label
7283
        code.continue_label = try_continue_label
William Stein's avatar
William Stein committed
7284
        self.body.generate_execution_code(code)
7285
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
7286 7287
        code.putln(
            "}")
7288
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
7289 7290 7291 7292
        can_raise = code.label_used(our_error_label)

        if can_raise:
            # inject code before the try block to save away the exception state
7293
            code.globalstate.use_utility_code(reset_exception_utility_code)
7294 7295 7296
            if not self.in_generator:
                save_exc.putln("__Pyx_PyThreadState_declare")
                save_exc.putln("__Pyx_PyThreadState_assign")
7297 7298
            save_exc.putln("__Pyx_ExceptionSave(%s);" % (
                ', '.join(['&%s' % var for var in exc_save_vars])))
7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309
            for var in exc_save_vars:
                save_exc.put_xgotref(var)

            def restore_saved_exception():
                for name in exc_save_vars:
                    code.put_xgiveref(name)
                code.putln("__Pyx_ExceptionReset(%s);" %
                           ', '.join(exc_save_vars))
        else:
            # try block cannot raise exceptions, but we had to allocate the temps above,
            # so just keep the C compiler from complaining about them being unused
7310 7311
            mark_vars_used =  ["(void)%s;" % var for var in exc_save_vars]
            save_exc.putln("%s /* mark used */" % ' '.join(mark_vars_used))
7312 7313 7314 7315

            def restore_saved_exception():
                pass

7316 7317
        code.error_label = except_error_label
        code.return_label = except_return_label
7318
        normal_case_terminates = self.body.is_terminator
William Stein's avatar
William Stein committed
7319
        if self.else_clause:
7320
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
7321 7322 7323 7324 7325
            code.putln(
                "/*else:*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln(
                "}")
7326 7327
            if not normal_case_terminates:
                normal_case_terminates = self.else_clause.is_terminator
7328

7329
        if can_raise:
7330 7331 7332 7333
            if not normal_case_terminates:
                for var in exc_save_vars:
                    code.put_xdecref_clear(var, py_object_type)
                code.put_goto(try_end_label)
7334 7335 7336
            code.put_label(our_error_label)
            for temp_name, temp_type in temps_to_clean_up:
                code.put_xdecref_clear(temp_name, temp_type)
7337 7338 7339 7340

            outer_except = code.funcstate.current_except
            # Currently points to self, but the ExceptClauseNode would also be ok. Change if needed.
            code.funcstate.current_except = self
7341 7342
            for except_clause in self.except_clauses:
                except_clause.generate_handling_code(code, except_end_label)
7343 7344
            code.funcstate.current_except = outer_except

7345 7346 7347 7348 7349 7350 7351 7352
            if not self.has_default_clause:
                code.put_goto(except_error_label)

        for exit_label, old_label in [(except_error_label, old_error_label),
                                      (try_break_label, old_break_label),
                                      (try_continue_label, old_continue_label),
                                      (try_return_label, old_return_label),
                                      (except_return_label, old_return_label)]:
7353
            if code.label_used(exit_label):
7354
                if not normal_case_terminates and not code.label_used(try_end_label):
7355
                    code.put_goto(try_end_label)
7356
                code.put_label(exit_label)
7357
                code.mark_pos(self.pos, trace=False)
7358 7359
                if can_raise:
                    restore_saved_exception()
7360
                code.put_goto(old_label)
7361 7362

        if code.label_used(except_end_label):
7363
            if not normal_case_terminates and not code.label_used(try_end_label):
7364
                code.put_goto(try_end_label)
7365
            code.put_label(except_end_label)
7366 7367
            if can_raise:
                restore_saved_exception()
7368 7369
        if code.label_used(try_end_label):
            code.put_label(try_end_label)
7370 7371
        code.putln("}")

7372 7373 7374
        for cname in exc_save_vars:
            code.funcstate.release_temp(cname)

7375
        code.return_label = old_return_label
7376
        code.break_label = old_break_label
7377
        code.continue_label = old_continue_label
7378
        code.error_label = old_error_label
William Stein's avatar
William Stein committed
7379

7380 7381 7382 7383 7384 7385 7386
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        for except_clause in self.except_clauses:
            except_clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

7387 7388 7389 7390 7391 7392 7393
    def annotate(self, code):
        self.body.annotate(code)
        for except_node in self.except_clauses:
            except_node.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
7394 7395 7396 7397

class ExceptClauseNode(Node):
    #  Part of try ... except statement.
    #
7398
    #  pattern        [ExprNode]
William Stein's avatar
William Stein committed
7399 7400
    #  target         ExprNode or None
    #  body           StatNode
7401
    #  excinfo_target TupleNode(3*ResultRefNode) or None   optional target for exception info (not owned here!)
William Stein's avatar
William Stein committed
7402 7403 7404
    #  match_flag     string             result of exception match
    #  exc_value      ExcValueNode       used internally
    #  function_name  string             qualified name of enclosing function
7405
    #  exc_vars       (string * 3)       local exception variables
7406
    #  is_except_as   bool               Py3-style "except ... as xyz"
7407 7408 7409 7410

    # excinfo_target is never set by the parser, but can be set by a transform
    # in order to extract more extensive information about the exception as a
    # sys.exc_info()-style tuple into a target variable
7411

7412
    child_attrs = ["pattern", "target", "body", "exc_value"]
7413

7414
    exc_value = None
7415
    excinfo_target = None
7416
    is_except_as = False
7417

William Stein's avatar
William Stein committed
7418 7419 7420 7421
    def analyse_declarations(self, env):
        if self.target:
            self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
7422

William Stein's avatar
William Stein committed
7423 7424 7425
    def analyse_expressions(self, env):
        self.function_name = env.qualified_name
        if self.pattern:
7426 7427
            # normalise/unpack self.pattern into a list
            for i, pattern in enumerate(self.pattern):
7428
                pattern = pattern.analyse_expressions(env)
7429
                self.pattern[i] = pattern.coerce_to_pyobject(env)
7430

William Stein's avatar
William Stein committed
7431
        if self.target:
7432
            from . import ExprNodes
7433
            self.exc_value = ExprNodes.ExcValueNode(self.pos)
7434
            self.target = self.target.analyse_target_expression(env, self.exc_value)
7435

7436 7437
        self.body = self.body.analyse_expressions(env)
        return self
7438

William Stein's avatar
William Stein committed
7439 7440
    def generate_handling_code(self, code, end_label):
        code.mark_pos(self.pos)
7441

William Stein's avatar
William Stein committed
7442
        if self.pattern:
7443
            has_non_literals = not all(
7444
                pattern.is_literal or pattern.is_simple() and not pattern.is_temp
7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459
                for pattern in self.pattern)

            if has_non_literals:
                # For non-trivial exception check expressions, hide the live exception from C-API calls.
                exc_vars = [code.funcstate.allocate_temp(py_object_type, manage_ref=True)
                            for _ in range(3)]
                code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
                code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % tuple(exc_vars))
                code.globalstate.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c"))
                exc_test_func = "__Pyx_PyErr_GivenExceptionMatches(%s, %%s)" % exc_vars[0]
            else:
                exc_vars = ()
                code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c"))
                exc_test_func = "__Pyx_PyErr_ExceptionMatches(%s)"

7460 7461 7462
            exc_tests = []
            for pattern in self.pattern:
                pattern.generate_evaluation_code(code)
7463
                exc_tests.append(exc_test_func % pattern.py_result())
7464

7465 7466
            match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
            code.putln("%s = %s;" % (match_flag, ' || '.join(exc_tests)))
7467 7468 7469
            for pattern in self.pattern:
                pattern.generate_disposal_code(code)
                pattern.free_temps(code)
7470 7471 7472 7473 7474 7475 7476

            if has_non_literals:
                code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(exc_vars))
                code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
                for temp in exc_vars:
                    code.funcstate.release_temp(temp)

William Stein's avatar
William Stein committed
7477 7478
            code.putln(
                "if (%s) {" %
7479 7480
                    match_flag)
            code.funcstate.release_temp(match_flag)
William Stein's avatar
William Stein committed
7481
        else:
7482
            code.putln("/*except:*/ {")
7483

7484 7485
        if (not getattr(self.body, 'stats', True)
                and self.excinfo_target is None
7486
                and self.target is None):
Stefan Behnel's avatar
Stefan Behnel committed
7487 7488
            # most simple case: no exception variable, empty body (pass)
            # => reset the exception state, done
7489 7490
            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
            code.putln("__Pyx_ErrRestore(0,0,0);")
7491 7492 7493
            code.put_goto(end_label)
            code.putln("}")
            return
7494

7495
        exc_vars = [code.funcstate.allocate_temp(py_object_type, manage_ref=True)
7496
                    for _ in range(3)]
7497
        code.put_add_traceback(self.function_name)
William Stein's avatar
William Stein committed
7498
        # We always have to fetch the exception value even if
7499
        # there is no target, because this also normalises the
William Stein's avatar
William Stein committed
7500
        # exception and stores it in the thread state.
7501 7502
        code.globalstate.use_utility_code(get_exception_utility_code)
        exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
7503 7504
        code.putln("if (__Pyx_GetException(%s) < 0) %s" % (
            exc_args, code.error_goto(self.pos)))
Stefan Behnel's avatar
Stefan Behnel committed
7505 7506
        for var in exc_vars:
            code.put_gotref(var)
William Stein's avatar
William Stein committed
7507
        if self.target:
7508
            self.exc_value.set_var(exc_vars[1])
7509
            self.exc_value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
7510
            self.target.generate_assignment_code(self.exc_value, code)
7511
        if self.excinfo_target is not None:
7512
            for tempvar, node in zip(exc_vars, self.excinfo_target.args):
7513
                node.set_var(tempvar)
7514

7515 7516 7517 7518
        old_break_label, old_continue_label = code.break_label, code.continue_label
        code.break_label = code.new_label('except_break')
        code.continue_label = code.new_label('except_continue')

7519
        old_exc_vars = code.funcstate.exc_vars
7520
        code.funcstate.exc_vars = exc_vars
William Stein's avatar
William Stein committed
7521
        self.body.generate_execution_code(code)
7522
        code.funcstate.exc_vars = old_exc_vars
Stefan Behnel's avatar
Stefan Behnel committed
7523

7524 7525
        if not self.body.is_terminator:
            for var in exc_vars:
7526 7527 7528
                # FIXME: XDECREF() is needed to allow re-raising (which clears the exc_vars),
                # but I don't think it's the right solution.
                code.put_xdecref_clear(var, py_object_type)
7529
            code.put_goto(end_label)
7530

Stefan Behnel's avatar
Stefan Behnel committed
7531 7532 7533 7534 7535 7536 7537
        for new_label, old_label in [(code.break_label, old_break_label),
                                     (code.continue_label, old_continue_label)]:
            if code.label_used(new_label):
                code.put_label(new_label)
                for var in exc_vars:
                    code.put_decref_clear(var, py_object_type)
                code.put_goto(old_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
7538
        code.break_label = old_break_label
7539
        code.continue_label = old_continue_label
7540

7541 7542
        for temp in exc_vars:
            code.funcstate.release_temp(temp)
7543

William Stein's avatar
William Stein committed
7544 7545 7546
        code.putln(
            "}")

7547 7548 7549 7550
    def generate_function_definitions(self, env, code):
        if self.target is not None:
            self.target.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
7551

7552
    def annotate(self, code):
7553
        if self.pattern:
7554 7555
            for pattern in self.pattern:
                pattern.annotate(code)
7556 7557 7558 7559
        if self.target:
            self.target.annotate(code)
        self.body.annotate(code)

William Stein's avatar
William Stein committed
7560 7561 7562 7563 7564 7565

class TryFinallyStatNode(StatNode):
    #  try ... finally statement
    #
    #  body             StatNode
    #  finally_clause   StatNode
7566
    #  finally_except_clause  deep-copy of finally_clause for exception case
7567
    #  in_generator     inside of generator => must store away current exception also in return case
7568
    #
Stefan Behnel's avatar
Stefan Behnel committed
7569 7570
    #  Each of the continue, break, return and error gotos runs
    #  into its own deep-copy of the finally block code.
William Stein's avatar
William Stein committed
7571 7572 7573
    #  In addition, if we're doing an error, we save the
    #  exception on entry to the finally block and restore
    #  it on exit.
7574

7575
    child_attrs = ["body", "finally_clause", "finally_except_clause"]
7576

7577
    preserve_exception = 1
7578

7579 7580
    # handle exception case, in addition to return/break/continue
    handle_error_case = True
7581
    func_return_type = None
7582
    finally_except_clause = None
7583

7584
    is_try_finally_in_nogil = False
7585
    in_generator = False
7586

7587
    @staticmethod
7588 7589 7590
    def create_analysed(pos, env, body, finally_clause):
        node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
        return node
7591

William Stein's avatar
William Stein committed
7592 7593
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
7594 7595
        self.finally_except_clause = copy.deepcopy(self.finally_clause)
        self.finally_except_clause.analyse_declarations(env)
William Stein's avatar
William Stein committed
7596
        self.finally_clause.analyse_declarations(env)
7597

William Stein's avatar
William Stein committed
7598
    def analyse_expressions(self, env):
7599 7600
        self.body = self.body.analyse_expressions(env)
        self.finally_clause = self.finally_clause.analyse_expressions(env)
7601
        self.finally_except_clause = self.finally_except_clause.analyse_expressions(env)
Stefan Behnel's avatar
Stefan Behnel committed
7602 7603
        if env.return_type and not env.return_type.is_void:
            self.func_return_type = env.return_type
7604
        return self
7605

7606
    nogil_check = Node.gil_error
7607 7608
    gil_message = "Try-finally statement"

William Stein's avatar
William Stein committed
7609
    def generate_execution_code(self, code):
7610 7611 7612
        code.mark_pos(self.pos)  # before changing the error label, in case of tracing errors
        code.putln("/*try:*/ {")

William Stein's avatar
William Stein committed
7613 7614 7615 7616
        old_error_label = code.error_label
        old_labels = code.all_new_labels()
        new_labels = code.get_all_labels()
        new_error_label = code.error_label
7617 7618
        if not self.handle_error_case:
            code.error_label = old_error_label
William Stein's avatar
William Stein committed
7619
        catch_label = code.new_label()
7620

Stefan Behnel's avatar
Stefan Behnel committed
7621 7622
        was_in_try_finally = code.funcstate.in_try_finally
        code.funcstate.in_try_finally = 1
7623

William Stein's avatar
William Stein committed
7624
        self.body.generate_execution_code(code)
7625

Stefan Behnel's avatar
Stefan Behnel committed
7626
        code.funcstate.in_try_finally = was_in_try_finally
7627 7628
        code.putln("}")

7629
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
7630
        code.mark_pos(self.finally_clause.pos)
7631 7632
        code.putln("/*finally:*/ {")

7633 7634 7635
        # Reset labels only after writing out a potential line trace call for correct nogil error handling.
        code.set_all_labels(old_labels)

7636 7637 7638 7639 7640 7641 7642 7643 7644
        def fresh_finally_clause(_next=[self.finally_clause]):
            # generate the original subtree once and always keep a fresh copy
            node = _next[0]
            node_copy = copy.deepcopy(node)
            if node is self.finally_clause:
                _next[0] = node_copy
            else:
                node = node_copy
            return node
7645 7646

        preserve_error = self.preserve_exception and code.label_used(new_error_label)
7647
        needs_success_cleanup = not self.finally_clause.is_terminator
7648 7649 7650

        if not self.body.is_terminator:
            code.putln('/*normal exit:*/{')
7651 7652
            fresh_finally_clause().generate_execution_code(code)
            if not self.finally_clause.is_terminator:
7653
                code.put_goto(catch_label)
7654
            code.putln('}')
7655

7656
        if preserve_error:
7657
            code.put_label(new_error_label)
7658
            code.putln('/*exception exit:*/{')
7659 7660
            if not self.in_generator:
                code.putln("__Pyx_PyThreadState_declare")
7661 7662
            if self.is_try_finally_in_nogil:
                code.declare_gilstate()
7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673
            if needs_success_cleanup:
                exc_lineno_cnames = tuple([
                    code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
                    for _ in range(2)])
                exc_filename_cname = code.funcstate.allocate_temp(
                    PyrexTypes.CPtrType(PyrexTypes.c_const_type(PyrexTypes.c_char_type)),
                    manage_ref=False)
            else:
                exc_lineno_cnames = exc_filename_cname = None
            exc_vars = tuple([
                code.funcstate.allocate_temp(py_object_type, manage_ref=False)
7674
                for _ in range(6)])
7675
            self.put_error_catcher(
7676
                code, temps_to_clean_up, exc_vars, exc_lineno_cnames, exc_filename_cname)
7677
            finally_old_labels = code.all_new_labels()
7678

7679
            code.putln('{')
7680 7681
            old_exc_vars = code.funcstate.exc_vars
            code.funcstate.exc_vars = exc_vars[:3]
7682
            self.finally_except_clause.generate_execution_code(code)
7683
            code.funcstate.exc_vars = old_exc_vars
7684 7685
            code.putln('}')

7686
            if needs_success_cleanup:
7687 7688 7689 7690 7691 7692
                self.put_error_uncatcher(code, exc_vars, exc_lineno_cnames, exc_filename_cname)
                if exc_lineno_cnames:
                    for cname in exc_lineno_cnames:
                        code.funcstate.release_temp(cname)
                if exc_filename_cname:
                    code.funcstate.release_temp(exc_filename_cname)
7693
                code.put_goto(old_error_label)
7694

7695 7696 7697 7698
            for new_label, old_label in zip(code.get_all_labels(), finally_old_labels):
                if not code.label_used(new_label):
                    continue
                code.put_label(new_label)
7699
                self.put_error_cleaner(code, exc_vars)
7700
                code.put_goto(old_label)
7701 7702 7703

            for cname in exc_vars:
                code.funcstate.release_temp(cname)
7704
            code.putln('}')
7705

7706
        code.set_all_labels(old_labels)
7707
        return_label = code.return_label
7708 7709
        exc_vars = ()

7710 7711 7712 7713 7714
        for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)):
            if not code.label_used(new_label):
                continue
            if new_label == new_error_label and preserve_error:
                continue  # handled above
7715

7716
            code.putln('%s: {' % new_label)
7717
            ret_temp = None
7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735
            if old_label == return_label:
                # return actually raises an (uncatchable) exception in generators that we must preserve
                if self.in_generator:
                    exc_vars = tuple([
                        code.funcstate.allocate_temp(py_object_type, manage_ref=False)
                        for _ in range(6)])
                    self.put_error_catcher(code, [], exc_vars)
                if not self.finally_clause.is_terminator:
                    # store away return value for later reuse
                    if (self.func_return_type and
                            not self.is_try_finally_in_nogil and
                            not isinstance(self.finally_clause, GILExitNode)):
                        ret_temp = code.funcstate.allocate_temp(
                            self.func_return_type, manage_ref=False)
                        code.putln("%s = %s;" % (ret_temp, Naming.retval_cname))
                        if self.func_return_type.is_pyobject:
                            code.putln("%s = 0;" % Naming.retval_cname)

7736
            fresh_finally_clause().generate_execution_code(code)
7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747

            if old_label == return_label:
                if ret_temp:
                    code.putln("%s = %s;" % (Naming.retval_cname, ret_temp))
                    if self.func_return_type.is_pyobject:
                        code.putln("%s = 0;" % ret_temp)
                    code.funcstate.release_temp(ret_temp)
                    ret_temp = None
                if self.in_generator:
                    self.put_error_uncatcher(code, exc_vars)

7748
            if not self.finally_clause.is_terminator:
7749 7750
                code.put_goto(old_label)
            code.putln('}')
7751 7752

        # End finally
7753
        code.put_label(catch_label)
William Stein's avatar
William Stein committed
7754 7755 7756
        code.putln(
            "}")

7757 7758 7759 7760
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        self.finally_clause.generate_function_definitions(env, code)

7761
    def put_error_catcher(self, code, temps_to_clean_up, exc_vars,
7762
                          exc_lineno_cnames=None, exc_filename_cname=None):
7763
        code.globalstate.use_utility_code(restore_exception_utility_code)
7764 7765
        code.globalstate.use_utility_code(get_exception_utility_code)
        code.globalstate.use_utility_code(swap_exception_utility_code)
7766 7767 7768

        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7769
        code.putln("__Pyx_PyThreadState_assign")
7770

7771
        code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
7772 7773
        for temp_name, type in temps_to_clean_up:
            code.put_xdecref_clear(temp_name, type)
7774

7775 7776 7777 7778 7779 7780 7781 7782 7783
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3)"
                   " __Pyx_ExceptionSwap(&%s, &%s, &%s);" % exc_vars[3:])
        code.putln("if ((PY_MAJOR_VERSION < 3) ||"
                   # if __Pyx_GetException() fails in Py3,
                   # store the newly raised exception instead
                   " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) "
                   "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2))
7784 7785 7786 7787 7788 7789 7790
        for var in exc_vars:
            code.put_xgotref(var)
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                exc_lineno_cnames[0], Naming.lineno_cname,
                exc_lineno_cnames[1], Naming.clineno_cname,
                exc_filename_cname, Naming.filename_cname))
7791 7792 7793 7794

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7795
    def put_error_uncatcher(self, code, exc_vars, exc_lineno_cnames=None, exc_filename_cname=None):
7796
        code.globalstate.use_utility_code(restore_exception_utility_code)
7797
        code.globalstate.use_utility_code(reset_exception_utility_code)
7798 7799 7800 7801

        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)

7802 7803 7804
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7805 7806 7807
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7808
        code.putln("}")
7809
        for var in exc_vars[:3]:
7810
            code.put_xgiveref(var)
7811
        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3])
7812 7813 7814 7815

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7816
        code.putln(' '.join(["%s = 0;" % var for var in exc_vars]))
7817 7818 7819 7820 7821
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                Naming.lineno_cname, exc_lineno_cnames[0],
                Naming.clineno_cname, exc_lineno_cnames[1],
                Naming.filename_cname, exc_filename_cname))
7822

7823
    def put_error_cleaner(self, code, exc_vars):
7824
        code.globalstate.use_utility_code(reset_exception_utility_code)
7825 7826
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7827

7828 7829 7830
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7831 7832 7833
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7834
        code.putln("}")
7835
        for var in exc_vars[:3]:
7836
            code.put_xdecref_clear(var, py_object_type)
7837 7838
        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()
7839
        code.putln(' '.join(["%s = 0;"]*3) % exc_vars[3:])
William Stein's avatar
William Stein committed
7840

7841 7842 7843 7844
    def annotate(self, code):
        self.body.annotate(code)
        self.finally_clause.annotate(code)

William Stein's avatar
William Stein committed
7845

7846 7847 7848 7849 7850 7851 7852 7853 7854 7855
class NogilTryFinallyStatNode(TryFinallyStatNode):
    """
    A try/finally statement that may be used in nogil code sections.
    """

    preserve_exception = False
    nogil_check = None


class GILStatNode(NogilTryFinallyStatNode):
7856 7857 7858
    #  'with gil' or 'with nogil' statement
    #
    #   state   string   'gil' or 'nogil'
7859

7860
    child_attrs = ["condition"] + NogilTryFinallyStatNode.child_attrs
7861 7862
    state_temp = None

7863
    def __init__(self, pos, state, body, condition=None):
7864
        self.state = state
7865
        self.condition = condition
7866
        self.create_state_temp_if_needed(pos, state, body)
7867 7868
        TryFinallyStatNode.__init__(
            self, pos,
7869 7870 7871 7872 7873
            body=body,
            finally_clause=GILExitNode(
                pos, state=state, state_temp=self.state_temp))

    def create_state_temp_if_needed(self, pos, state, body):
7874
        from .ParseTreeTransforms import YieldNodeCollector
7875 7876
        collector = YieldNodeCollector()
        collector.visitchildren(body)
7877
        if not collector.yields:
7878 7879
            return

7880 7881 7882 7883
        if state == 'gil':
            temp_type = PyrexTypes.c_gilstate_type
        else:
            temp_type = PyrexTypes.c_threadstate_ptr_type
7884
        from . import ExprNodes
7885
        self.state_temp = ExprNodes.TempNode(pos, temp_type)
7886

7887 7888 7889 7890
    def analyse_declarations(self, env):
        env._in_with_gil_block = (self.state == 'gil')
        if self.state == 'gil':
            env.has_with_gil_block = True
7891

7892 7893 7894
        if self.condition is not None:
            self.condition.analyse_declarations(env)

7895 7896
        return super(GILStatNode, self).analyse_declarations(env)

7897
    def analyse_expressions(self, env):
7898 7899
        env.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
7900 7901 7902 7903

        if self.condition is not None:
            self.condition = self.condition.analyse_expressions(env)

7904
        was_nogil = env.nogil
7905
        env.nogil = self.state == 'nogil'
7906
        node = TryFinallyStatNode.analyse_expressions(self, env)
7907
        env.nogil = was_nogil
7908
        return node
7909

7910
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
7911
        code.mark_pos(self.pos)
7912
        code.begin_block()
7913 7914 7915 7916 7917
        if self.state_temp:
            self.state_temp.allocate(code)
            variable = self.state_temp.result()
        else:
            variable = None
7918

7919
        old_gil_config = code.funcstate.gil_owned
7920
        if self.state == 'gil':
7921
            code.put_ensure_gil(variable=variable)
7922
            code.funcstate.gil_owned = True
7923
        else:
7924
            code.put_release_gil(variable=variable)
7925
            code.funcstate.gil_owned = False
7926

7927
        TryFinallyStatNode.generate_execution_code(self, code)
7928

7929 7930
        if self.state_temp:
            self.state_temp.release(code)
7931

7932
        code.funcstate.gil_owned = old_gil_config
7933
        code.end_block()
7934 7935 7936


class GILExitNode(StatNode):
7937 7938 7939 7940 7941
    """
    Used as the 'finally' block in a GILStatNode

    state   string   'gil' or 'nogil'
    """
7942

7943
    child_attrs = []
7944
    state_temp = None
7945

7946
    def analyse_expressions(self, env):
7947
        return self
7948 7949

    def generate_execution_code(self, code):
7950 7951 7952 7953 7954
        if self.state_temp:
            variable = self.state_temp.result()
        else:
            variable = None

7955
        if self.state == 'gil':
7956
            code.put_release_ensured_gil(variable)
7957
        else:
7958
            code.put_acquire_gil(variable)
7959 7960


7961 7962 7963 7964 7965 7966 7967
class EnsureGILNode(GILExitNode):
    """
    Ensure the GIL in nogil functions for cleanup before returning.
    """

    def generate_execution_code(self, code):
        code.put_ensure_gil(declare_gilstate=False)
7968

7969

7970 7971 7972 7973 7974
def cython_view_utility_code():
    from . import MemoryView
    return MemoryView.view_utility_code


7975 7976 7977
utility_code_for_cimports = {
    # utility code (or inlining c) in a pxd (or pyx) file.
    # TODO: Consider a generic user-level mechanism for importing
7978 7979 7980
    'cpython.array'         : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cpython.array.array'   : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cython.view'           : cython_view_utility_code,
7981
}
7982

7983 7984 7985
utility_code_for_imports = {
    # utility code used when special modules are imported.
    # TODO: Consider a generic user-level mechanism for importing
7986 7987
    'asyncio': ("__Pyx_patch_asyncio", "PatchAsyncIO", "Coroutine.c"),
    'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"),
7988 7989 7990
}


William Stein's avatar
William Stein committed
7991 7992 7993 7994 7995
class CImportStatNode(StatNode):
    #  cimport statement
    #
    #  module_name   string           Qualified name of module being imported
    #  as_name       string or None   Name specified in "as" clause, if any
7996
    #  is_absolute   bool             True for absolute imports, False otherwise
7997 7998

    child_attrs = []
7999
    is_absolute = False
8000

William Stein's avatar
William Stein committed
8001
    def analyse_declarations(self, env):
8002 8003 8004
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
8005 8006
        module_scope = env.find_module(
            self.module_name, self.pos, relative_level=0 if self.is_absolute else -1)
William Stein's avatar
William Stein committed
8007
        if "." in self.module_name:
8008
            names = [EncodedString(name) for name in self.module_name.split(".")]
William Stein's avatar
William Stein committed
8009 8010 8011 8012 8013 8014 8015 8016 8017 8018
            top_name = names[0]
            top_module_scope = env.context.find_submodule(top_name)
            module_scope = top_module_scope
            for name in names[1:]:
                submodule_scope = module_scope.find_submodule(name)
                module_scope.declare_module(name, submodule_scope, self.pos)
                module_scope = submodule_scope
            if self.as_name:
                env.declare_module(self.as_name, module_scope, self.pos)
            else:
8019
                env.add_imported_module(module_scope)
William Stein's avatar
William Stein committed
8020 8021 8022 8023
                env.declare_module(top_name, top_module_scope, self.pos)
        else:
            name = self.as_name or self.module_name
            env.declare_module(name, module_scope, self.pos)
8024
        if self.module_name in utility_code_for_cimports:
8025
            env.use_utility_code(utility_code_for_cimports[self.module_name]())
William Stein's avatar
William Stein committed
8026 8027

    def analyse_expressions(self, env):
8028
        return self
8029

William Stein's avatar
William Stein committed
8030 8031
    def generate_execution_code(self, code):
        pass
8032

William Stein's avatar
William Stein committed
8033 8034 8035 8036

class FromCImportStatNode(StatNode):
    #  from ... cimport statement
    #
8037
    #  module_name     string                        Qualified name of module
8038
    #  relative_level  int or None                   Relative import: number of dots before module_name
8039
    #  imported_names  [(pos, name, as_name, kind)]  Names to be imported
8040

8041
    child_attrs = []
8042 8043 8044
    module_name = None
    relative_level = None
    imported_names = None
8045

William Stein's avatar
William Stein committed
8046
    def analyse_declarations(self, env):
8047 8048 8049
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
8050 8051
        if self.relative_level and self.relative_level > env.qualified_name.count('.'):
            error(self.pos, "relative cimport beyond main package is not allowed")
8052
            return
8053 8054
        module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level)
        module_name = module_scope.qualified_name
William Stein's avatar
William Stein committed
8055
        env.add_imported_module(module_scope)
8056
        for pos, name, as_name, kind in self.imported_names:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8057
            if name == "*":
8058
                for local_name, entry in list(module_scope.entries.items()):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8059 8060
                    env.add_imported_entry(local_name, entry, pos)
            else:
8061 8062 8063 8064
                entry = module_scope.lookup(name)
                if entry:
                    if kind and not self.declaration_matches(entry, kind):
                        entry.redeclared(pos)
8065
                    entry.used = 1
8066 8067
                else:
                    if kind == 'struct' or kind == 'union':
8068 8069
                        entry = module_scope.declare_struct_or_union(
                            name, kind=kind, scope=None, typedef_flag=0, pos=pos)
8070
                    elif kind == 'class':
8071
                        entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name)
8072
                    else:
8073 8074
                        submodule_scope = env.context.find_module(
                            name, relative_to=module_scope, pos=self.pos, absolute_fallback=False)
8075 8076 8077
                        if submodule_scope.parent_module is module_scope:
                            env.declare_module(as_name or name, submodule_scope, self.pos)
                        else:
8078
                            error(pos, "Name '%s' not declared in module '%s'" % (name, module_name))
8079

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8080 8081 8082
                if entry:
                    local_name = as_name or name
                    env.add_imported_entry(local_name, entry, pos)
8083

8084
        if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now
8085
            if module_name in utility_code_for_cimports:
8086
                env.use_utility_code(utility_code_for_cimports[module_name]())
8087
            for _, name, _, _ in self.imported_names:
8088
                fqname = '%s.%s' % (module_name, name)
8089
                if fqname in utility_code_for_cimports:
8090
                    env.use_utility_code(utility_code_for_cimports[fqname]())
8091

8092
    def declaration_matches(self, entry, kind):
8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104
        if not entry.is_type:
            return 0
        type = entry.type
        if kind == 'class':
            if not type.is_extension_type:
                return 0
        else:
            if not type.is_struct_or_union:
                return 0
            if kind != type.kind:
                return 0
        return 1
William Stein's avatar
William Stein committed
8105 8106

    def analyse_expressions(self, env):
8107
        return self
8108

William Stein's avatar
William Stein committed
8109 8110 8111 8112 8113 8114 8115 8116 8117
    def generate_execution_code(self, code):
        pass


class FromImportStatNode(StatNode):
    #  from ... import statement
    #
    #  module           ImportNode
    #  items            [(string, NameNode)]
8118
    #  interned_items   [(string, NameNode, ExprNode)]
William Stein's avatar
William Stein committed
8119
    #  item             PyTempNode            used internally
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8120
    #  import_star      boolean               used internally
8121 8122

    child_attrs = ["module"]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8123
    import_star = 0
8124

William Stein's avatar
William Stein committed
8125
    def analyse_declarations(self, env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8126 8127 8128 8129 8130 8131 8132 8133 8134
        for name, target in self.items:
            if name == "*":
                if not env.is_module_scope:
                    error(self.pos, "import * only allowed at module level")
                    return
                env.has_import_star = 1
                self.import_star = 1
            else:
                target.analyse_target_declaration(env)
8135

William Stein's avatar
William Stein committed
8136
    def analyse_expressions(self, env):
8137
        from . import ExprNodes
8138
        self.module = self.module.analyse_expressions(env)
8139
        self.item = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
William Stein's avatar
William Stein committed
8140 8141
        self.interned_items = []
        for name, target in self.items:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8142 8143 8144
            if name == '*':
                for _, entry in env.entries.items():
                    if not entry.is_type and entry.type.is_extension_type:
8145
                        env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8146 8147
                        break
            else:
8148
                entry = env.lookup(target.name)
8149 8150
                # check whether or not entry is already cimported
                if (entry.is_type and entry.type.name == name
Stefan Behnel's avatar
Stefan Behnel committed
8151
                        and hasattr(entry.type, 'module_name')):
8152 8153 8154 8155 8156
                    if entry.type.module_name == self.module.module_name.value:
                        # cimported with absolute name
                        continue
                    try:
                        # cimported with relative name
8157 8158
                        module = env.find_module(self.module.module_name.value, pos=self.pos,
                                                 relative_level=self.module.level)
8159 8160 8161
                        if entry.type.module_name == module.qualified_name:
                            continue
                    except AttributeError:
8162
                        pass
8163
                target = target.analyse_target_expression(env, None)  # FIXME?
8164 8165 8166 8167
                if target.type is py_object_type:
                    coerced_item = None
                else:
                    coerced_item = self.item.coerce_to(target.type, env)
8168
                self.interned_items.append((name, target, coerced_item))
8169
        return self
8170

William Stein's avatar
William Stein committed
8171
    def generate_execution_code(self, code):
8172
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
8173
        self.module.generate_evaluation_code(code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
8174 8175 8176 8177 8178 8179
        if self.import_star:
            code.putln(
                'if (%s(%s) < 0) %s;' % (
                    Naming.import_star,
                    self.module.py_result(),
                    code.error_goto(self.pos)))
8180 8181
        item_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.item.set_cname(item_temp)
8182 8183 8184
        if self.interned_items:
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ImportFrom", "ImportExport.c"))
8185
        for name, target, coerced_item in self.interned_items:
8186
            code.putln(
8187
                '%s = __Pyx_ImportFrom(%s, %s); %s' % (
8188
                    item_temp,
8189
                    self.module.py_result(),
8190 8191
                    code.intern_identifier(name),
                    code.error_goto_if_null(item_temp, self.pos)))
8192
            code.put_gotref(item_temp)
8193 8194 8195 8196 8197 8198
            if coerced_item is None:
                target.generate_assignment_code(self.item, code)
            else:
                coerced_item.allocate_temp_result(code)
                coerced_item.generate_result_code(code)
                target.generate_assignment_code(coerced_item, code)
8199 8200
            code.put_decref_clear(item_temp, py_object_type)
        code.funcstate.release_temp(item_temp)
William Stein's avatar
William Stein committed
8201
        self.module.generate_disposal_code(code)
8202
        self.module.free_temps(code)
William Stein's avatar
William Stein committed
8203

8204

Mark Florisson's avatar
Mark Florisson committed
8205 8206 8207 8208 8209 8210 8211 8212 8213 8214
class ParallelNode(Node):
    """
    Base class for cython.parallel constructs.
    """

    nogil_check = None


class ParallelStatNode(StatNode, ParallelNode):
    """
8215
    Base class for 'with cython.parallel.parallel():' and 'for i in prange():'.
Mark Florisson's avatar
Mark Florisson committed
8216 8217 8218 8219 8220

    assignments     { Entry(var) : (var.pos, inplace_operator_or_None) }
                    assignments to variables in this parallel section

    parent          parent ParallelStatNode or None
8221 8222 8223
    is_parallel     indicates whether this node is OpenMP parallel
                    (true for #pragma omp parallel for and
                              #pragma omp parallel)
Mark Florisson's avatar
Mark Florisson committed
8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234

    is_parallel is true for:

        #pragma omp parallel
        #pragma omp parallel for

    sections, but NOT for

        #pragma omp for

    We need this to determine the sharing attributes.
8235 8236 8237

    privatization_insertion_point   a code insertion point used to make temps
                                    private (esp. the "nsteps" temp)
8238 8239 8240 8241

    args         tuple          the arguments passed to the parallel construct
    kwargs       DictNode       the keyword arguments passed to the parallel
                                construct (replaced by its compile time value)
Mark Florisson's avatar
Mark Florisson committed
8242 8243
    """

8244
    child_attrs = ['body', 'num_threads']
Mark Florisson's avatar
Mark Florisson committed
8245 8246 8247 8248

    body = None

    is_prange = False
8249
    is_nested_prange = False
Mark Florisson's avatar
Mark Florisson committed
8250

8251
    error_label_used = False
8252

8253
    num_threads = None
8254
    chunksize = None
8255

8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272
    parallel_exc = (
        Naming.parallel_exc_type,
        Naming.parallel_exc_value,
        Naming.parallel_exc_tb,
    )

    parallel_pos_info = (
        Naming.parallel_filename,
        Naming.parallel_lineno,
        Naming.parallel_clineno,
    )

    pos_info = (
        Naming.filename_cname,
        Naming.lineno_cname,
        Naming.clineno_cname,
    )
8273

8274 8275
    critical_section_counter = 0

Mark Florisson's avatar
Mark Florisson committed
8276 8277
    def __init__(self, pos, **kwargs):
        super(ParallelStatNode, self).__init__(pos, **kwargs)
8278 8279

        # All assignments in this scope
Mark Florisson's avatar
Mark Florisson committed
8280 8281
        self.assignments = kwargs.get('assignments') or {}

8282 8283 8284 8285
        # All seen closure cnames and their temporary cnames
        self.seen_closure_vars = set()

        # Dict of variables that should be declared (first|last|)private or
8286 8287
        # reduction { Entry: (op, lastprivate) }.
        # If op is not None, it's a reduction.
8288
        self.privates = {}
Mark Florisson's avatar
Mark Florisson committed
8289

Mark Florisson's avatar
Mark Florisson committed
8290 8291 8292
        # [NameNode]
        self.assigned_nodes = []

Mark Florisson's avatar
Mark Florisson committed
8293 8294 8295
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)

8296 8297
        self.num_threads = None

8298
        if self.kwargs:
8299 8300
            # Try to find num_threads and chunksize keyword arguments
            pairs = []
8301
            seen = set()
8302
            for dictitem in self.kwargs.key_value_pairs:
8303 8304 8305
                if dictitem.key.value in seen:
                    error(self.pos, "Duplicate keyword argument found: %s" % dictitem.key.value)
                seen.add(dictitem.key.value)
8306
                if dictitem.key.value == 'num_threads':
8307 8308
                    if not dictitem.value.is_none:
                       self.num_threads = dictitem.value
8309
                elif self.is_prange and dictitem.key.value == 'chunksize':
8310 8311
                    if not dictitem.value.is_none:
                        self.chunksize = dictitem.value
8312 8313 8314 8315
                else:
                    pairs.append(dictitem)

            self.kwargs.key_value_pairs = pairs
8316

8317 8318
            try:
                self.kwargs = self.kwargs.compile_time_value(env)
8319
            except Exception as e:
8320 8321
                error(self.kwargs.pos, "Only compile-time values may be "
                                       "supplied as keyword arguments")
8322 8323 8324
        else:
            self.kwargs = {}

8325
        for kw, val in self.kwargs.items():
8326 8327 8328 8329 8330
            if kw not in self.valid_keyword_arguments:
                error(self.pos, "Invalid keyword argument: %s" % kw)
            else:
                setattr(self, kw, val)

8331
    def analyse_expressions(self, env):
8332
        if self.num_threads:
8333
            self.num_threads = self.num_threads.analyse_expressions(env)
8334 8335

        if self.chunksize:
8336
            self.chunksize = self.chunksize.analyse_expressions(env)
8337

8338
        self.body = self.body.analyse_expressions(env)
8339
        self.analyse_sharing_attributes(env)
8340

8341
        if self.num_threads is not None:
8342 8343
            if self.parent and self.parent.num_threads is not None and not self.parent.is_prange:
                error(self.pos, "num_threads already declared in outer section")
8344
            elif self.parent and not self.parent.is_prange:
8345
                error(self.pos, "num_threads must be declared in the parent parallel section")
8346
            elif (self.num_threads.type.is_int and
8347 8348 8349
                    self.num_threads.is_literal and
                    self.num_threads.compile_time_value(env) <= 0):
                error(self.pos, "argument to num_threads must be greater than 0")
8350

8351
            if not self.num_threads.is_simple() or self.num_threads.type.is_pyobject:
Mark Florisson's avatar
Mark Florisson committed
8352
                self.num_threads = self.num_threads.coerce_to(
8353 8354
                    PyrexTypes.c_int_type, env).coerce_to_temp(env)
        return self
8355 8356

    def analyse_sharing_attributes(self, env):
Mark Florisson's avatar
Mark Florisson committed
8357
        """
8358 8359 8360
        Analyse the privates for this block and set them in self.privates.
        This should be called in a post-order fashion during the
        analyse_expressions phase
Mark Florisson's avatar
Mark Florisson committed
8361
        """
8362
        for entry, (pos, op) in self.assignments.items():
Mark Florisson's avatar
Mark Florisson committed
8363

8364 8365 8366 8367 8368
            if self.is_prange and not self.is_parallel:
                # closely nested prange in a with parallel block, disallow
                # assigning to privates in the with parallel block (we
                # consider it too implicit and magicky for users)
                if entry in self.parent.assignments:
8369
                    error(pos, "Cannot assign to private of outer parallel block")
8370 8371 8372 8373 8374 8375 8376
                    continue

            if not self.is_prange and op:
                # Again possible, but considered to magicky
                error(pos, "Reductions not allowed for parallel blocks")
                continue

Mark Florisson's avatar
Mark Florisson committed
8377 8378 8379
            # By default all variables should have the same values as if
            # executed sequentially
            lastprivate = True
8380
            self.propagate_var_privatization(entry, pos, op, lastprivate)
8381

8382
    def propagate_var_privatization(self, entry, pos, op, lastprivate):
8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410
        """
        Propagate the sharing attributes of a variable. If the privatization is
        determined by a parent scope, done propagate further.

        If we are a prange, we propagate our sharing attributes outwards to
        other pranges. If we are a prange in parallel block and the parallel
        block does not determine the variable private, we propagate to the
        parent of the parent. Recursion stops at parallel blocks, as they have
        no concept of lastprivate or reduction.

        So the following cases propagate:

            sum is a reduction for all loops:

                for i in prange(n):
                    for j in prange(n):
                        for k in prange(n):
                            sum += i * j * k

            sum is a reduction for both loops, local_var is private to the
            parallel with block:

                for i in prange(n):
                    with parallel:
                        local_var = ... # private to the parallel
                        for j in prange(n):
                            sum += i * j

Mark Florisson's avatar
Mark Florisson committed
8411 8412
        Nested with parallel blocks are disallowed, because they wouldn't
        allow you to propagate lastprivates or reductions:
8413 8414 8415 8416

            #pragma omp parallel for lastprivate(i)
            for i in prange(n):

Mark Florisson's avatar
Mark Florisson committed
8417 8418
                sum = 0

8419 8420 8421 8422 8423 8424 8425 8426 8427 8428 8429 8430 8431 8432 8433 8434
                #pragma omp parallel private(j, sum)
                with parallel:

                    #pragma omp parallel
                    with parallel:

                        #pragma omp for lastprivate(j) reduction(+:sum)
                        for j in prange(n):
                            sum += i

                    # sum and j are well-defined here

                # sum and j are undefined here

            # sum and j are undefined here
        """
8435
        self.privates[entry] = (op, lastprivate)
8436 8437 8438 8439 8440

        if entry.type.is_memoryviewslice:
            error(pos, "Memoryview slices can only be shared in parallel sections")
            return

8441 8442 8443 8444 8445 8446
        if self.is_prange:
            if not self.is_parallel and entry not in self.parent.assignments:
                # Parent is a parallel with block
                parent = self.parent.parent
            else:
                parent = self.parent
Mark Florisson's avatar
Mark Florisson committed
8447

8448 8449 8450
            # We don't need to propagate privates, only reductions and
            # lastprivates
            if parent and (op or lastprivate):
8451
                parent.propagate_var_privatization(entry, pos, op, lastprivate)
Mark Florisson's avatar
Mark Florisson committed
8452 8453 8454 8455 8456 8457 8458 8459 8460

    def _allocate_closure_temp(self, code, entry):
        """
        Helper function that allocate a temporary for a closure variable that
        is assigned to.
        """
        if self.parent:
            return self.parent._allocate_closure_temp(code, entry)

8461 8462 8463
        if entry.cname in self.seen_closure_vars:
            return entry.cname

8464
        cname = code.funcstate.allocate_temp(entry.type, True)
8465 8466 8467 8468 8469 8470

        # Add both the actual cname and the temp cname, as the actual cname
        # will be replaced with the temp cname on the entry
        self.seen_closure_vars.add(entry.cname)
        self.seen_closure_vars.add(cname)

Mark Florisson's avatar
Mark Florisson committed
8471 8472 8473 8474
        self.modified_entries.append((entry, entry.cname))
        code.putln("%s = %s;" % (cname, entry.cname))
        entry.cname = cname

8475
    def initialize_privates_to_nan(self, code, exclude=None):
8476
        first = True
8477

8478
        for entry, (op, lastprivate) in sorted(self.privates.items()):
8479
            if not op and (not exclude or entry != exclude):
8480
                invalid_value = entry.type.invalid_value()
8481

8482
                if invalid_value:
8483 8484 8485 8486
                    if first:
                        code.putln("/* Initialize private variables to "
                                   "invalid values */")
                        first = False
8487
                    code.putln("%s = %s;" % (entry.cname,
8488
                                             entry.type.cast_code(invalid_value)))
8489

8490
    def evaluate_before_block(self, code, expr):
Mark Florisson's avatar
Mark Florisson committed
8491
        c = self.begin_of_parallel_control_block_point_after_decls
8492 8493 8494 8495 8496 8497 8498 8499 8500 8501
        # we need to set the owner to ourselves temporarily, as
        # allocate_temp may generate a comment in the middle of our pragma
        # otherwise when DebugFlags.debug_temp_code_comments is in effect
        owner = c.funcstate.owner
        c.funcstate.owner = c
        expr.generate_evaluation_code(c)
        c.funcstate.owner = owner

        return expr.result()

8502 8503 8504 8505 8506
    def put_num_threads(self, code):
        """
        Write self.num_threads if set as the num_threads OpenMP directive
        """
        if self.num_threads is not None:
8507
            code.put(" num_threads(%s)" % self.evaluate_before_block(code, self.num_threads))
8508

8509

Mark Florisson's avatar
Mark Florisson committed
8510 8511 8512 8513 8514 8515 8516 8517 8518
    def declare_closure_privates(self, code):
        """
        If a variable is in a scope object, we need to allocate a temp and
        assign the value from the temp to the variable in the scope object
        after the parallel section. This kind of copying should be done only
        in the outermost parallel section.
        """
        self.modified_entries = []

8519
        for entry in sorted(self.assignments):
Mark Florisson's avatar
Mark Florisson committed
8520
            if entry.from_closure or entry.in_closure:
8521
                self._allocate_closure_temp(code, entry)
Mark Florisson's avatar
Mark Florisson committed
8522 8523

    def release_closure_privates(self, code):
8524 8525 8526
        """
        Release any temps used for variables in scope objects. As this is the
        outermost parallel block, we don't need to delete the cnames from
8527
        self.seen_closure_vars.
8528
        """
Mark Florisson's avatar
Mark Florisson committed
8529 8530 8531 8532 8533
        for entry, original_cname in self.modified_entries:
            code.putln("%s = %s;" % (original_cname, entry.cname))
            code.funcstate.release_temp(entry.cname)
            entry.cname = original_cname

8534 8535 8536 8537 8538
    def privatize_temps(self, code, exclude_temps=()):
        """
        Make any used temporaries private. Before the relevant code block
        code.start_collecting_temps() should have been called.
        """
8539 8540
        c = self.privatization_insertion_point
        self.privatization_insertion_point = None
8541

8542
        if self.is_parallel:
8543
            self.temps = temps = code.funcstate.stop_collecting_temps()
8544
            privates, firstprivates = [], []
8545
            for temp, type in sorted(temps):
8546
                if type.is_pyobject or type.is_memoryviewslice:
8547 8548 8549
                    firstprivates.append(temp)
                else:
                    privates.append(temp)
8550

8551 8552 8553 8554
            if privates:
                c.put(" private(%s)" % ", ".join(privates))
            if firstprivates:
                c.put(" firstprivate(%s)" % ", ".join(firstprivates))
8555 8556 8557 8558 8559 8560 8561 8562

            if self.breaking_label_used:
                shared_vars = [Naming.parallel_why]
                if self.error_label_used:
                    shared_vars.extend(self.parallel_exc)
                    c.put(" private(%s, %s, %s)" % self.pos_info)

                c.put(" shared(%s)" % ', '.join(shared_vars))
8563

8564 8565 8566 8567
    def cleanup_temps(self, code):
        # Now clean up any memoryview slice and object temporaries
        if self.is_parallel and not self.is_nested_prange:
            code.putln("/* Clean up any temporaries */")
8568
            for temp, type in sorted(self.temps):
8569 8570 8571 8572 8573
                if type.is_memoryviewslice:
                    code.put_xdecref_memoryviewslice(temp, have_gil=False)
                elif type.is_pyobject:
                    code.put_xdecref(temp, type)
                    code.putln("%s = NULL;" % temp)
8574

8575
    def setup_parallel_control_flow_block(self, code):
8576
        """
8577 8578
        Sets up a block that surrounds the parallel block to determine
        how the parallel section was exited. Any kind of return is
8579 8580 8581
        trapped (break, continue, return, exceptions). This is the idea:

        {
8582
            int why = 0;
8583 8584 8585 8586 8587 8588 8589

            #pragma omp parallel
            {
                return # -> goto new_return_label;
                goto end_parallel;

            new_return_label:
8590
                why = 3;
8591
                goto end_parallel;
8592

8593
            end_parallel:;
8594
                #pragma omp flush(why) # we need to flush for every iteration
8595 8596
            }

8597
            if (why == 3)
8598 8599 8600 8601 8602 8603 8604 8605
                goto old_return_label;
        }
        """
        self.old_loop_labels = code.new_loop_labels()
        self.old_error_label = code.new_error_label()
        self.old_return_label = code.return_label
        code.return_label = code.new_label(name="return")

8606 8607
        code.begin_block() # parallel control flow block
        self.begin_of_parallel_control_block_point = code.insertion_point()
Mark Florisson's avatar
Mark Florisson committed
8608
        self.begin_of_parallel_control_block_point_after_decls = code.insertion_point()
8609

8610 8611
        self.undef_builtin_expect_apple_gcc_bug(code)

8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623
    def begin_parallel_block(self, code):
        """
        Each OpenMP thread in a parallel section that contains a with gil block
        must have the thread-state initialized. The call to
        PyGILState_Release() then deallocates our threadstate. If we wouldn't
        do this, each with gil block would allocate and deallocate one, thereby
        losing exception information before it can be saved before leaving the
        parallel section.
        """
        self.begin_of_parallel_block = code.insertion_point()

    def end_parallel_block(self, code):
8624 8625 8626 8627 8628 8629 8630 8631 8632
        """
        To ensure all OpenMP threads have thread states, we ensure the GIL
        in each thread (which creates a thread state if it doesn't exist),
        after which we release the GIL.
        On exit, reacquire the GIL and release the thread state.

        If compiled without OpenMP support (at the C level), then we still have
        to acquire the GIL to decref any object temporaries.
        """
8633 8634 8635
        begin_code = self.begin_of_parallel_block
        self.begin_of_parallel_block = None

8636 8637 8638
        if self.error_label_used:
            end_code = code

8639
            begin_code.putln("#ifdef _OPENMP")
8640 8641
            begin_code.put_ensure_gil(declare_gilstate=True)
            begin_code.putln("Py_BEGIN_ALLOW_THREADS")
8642
            begin_code.putln("#endif /* _OPENMP */")
8643

8644
            end_code.putln("#ifdef _OPENMP")
8645
            end_code.putln("Py_END_ALLOW_THREADS")
8646 8647 8648 8649 8650
            end_code.putln("#else")
            end_code.put_safe("{\n")
            end_code.put_ensure_gil()
            end_code.putln("#endif /* _OPENMP */")
            self.cleanup_temps(end_code)
8651
            end_code.put_release_ensured_gil()
8652 8653
            end_code.putln("#ifndef _OPENMP")
            end_code.put_safe("}\n")
8654
            end_code.putln("#endif /* _OPENMP */")
8655

8656 8657 8658 8659
    def trap_parallel_exit(self, code, should_flush=False):
        """
        Trap any kind of return inside a parallel construct. 'should_flush'
        indicates whether the variable should be flushed, which is needed by
8660 8661 8662 8663 8664 8665 8666 8667 8668
        prange to skip the loop. It also indicates whether we need to register
        a continue (we need this for parallel blocks, but not for prange
        loops, as it is a direct jump there).

        It uses the same mechanism as try/finally:
            1 continue
            2 break
            3 return
            4 error
8669
        """
8670
        save_lastprivates_label = code.new_label()
8671
        dont_return_label = code.new_label()
8672 8673 8674

        self.any_label_used = False
        self.breaking_label_used = False
8675
        self.error_label_used = False
8676

8677 8678 8679 8680 8681 8682
        self.parallel_private_temps = []

        all_labels = code.get_all_labels()

        # Figure this out before starting to generate any code
        for label in all_labels:
8683 8684
            if code.label_used(label):
                self.breaking_label_used = (self.breaking_label_used or
8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695
                                            label != code.continue_label)
                self.any_label_used = True

        if self.any_label_used:
            code.put_goto(dont_return_label)

        for i, label in enumerate(all_labels):
            if not code.label_used(label):
                continue

            is_continue_label = label == code.continue_label
8696

8697
            code.put_label(label)
8698

8699 8700 8701 8702
            if not (should_flush and is_continue_label):
                if label == code.error_label:
                    self.error_label_used = True
                    self.fetch_parallel_exception(code)
8703

8704
                code.putln("%s = %d;" % (Naming.parallel_why, i + 1))
8705

8706 8707 8708 8709
            if (self.breaking_label_used and self.is_prange and not
                    is_continue_label):
                code.put_goto(save_lastprivates_label)
            else:
8710 8711
                code.put_goto(dont_return_label)

8712
        if self.any_label_used:
8713 8714 8715 8716 8717
            if self.is_prange and self.breaking_label_used:
                # Don't rely on lastprivate, save our lastprivates
                code.put_label(save_lastprivates_label)
                self.save_parallel_vars(code)

8718 8719 8720
            code.put_label(dont_return_label)

            if should_flush and self.breaking_label_used:
8721 8722
                code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_why)

8723 8724 8725 8726 8727 8728 8729 8730 8731
    def save_parallel_vars(self, code):
        """
        The following shenanigans are instated when we break, return or
        propagate errors from a prange. In this case we cannot rely on
        lastprivate() to do its job, as no iterations may have executed yet
        in the last thread, leaving the values undefined. It is most likely
        that the breaking thread has well-defined values of the lastprivate
        variables, so we keep those values.
        """
8732
        section_name = "__pyx_parallel_lastprivates%d" % self.critical_section_counter
8733 8734 8735 8736 8737 8738 8739 8740
        code.putln_openmp("#pragma omp critical(%s)" % section_name)
        ParallelStatNode.critical_section_counter += 1

        code.begin_block() # begin critical section

        c = self.begin_of_parallel_control_block_point

        temp_count = 0
8741
        for entry, (op, lastprivate) in sorted(self.privates.items()):
8742 8743 8744
            if not lastprivate or entry.type.is_pyobject:
                continue

8745
            type_decl = entry.type.empty_declaration_code()
8746 8747 8748 8749 8750
            temp_cname = "__pyx_parallel_temp%d" % temp_count
            private_cname = entry.cname

            temp_count += 1

8751 8752
            invalid_value = entry.type.invalid_value()
            if invalid_value:
8753
                init = ' = ' + entry.type.cast_code(invalid_value)
8754 8755
            else:
                init = ''
8756
            # Declare the parallel private in the outer block
8757
            c.putln("%s %s%s;" % (type_decl, temp_cname, init))
8758 8759 8760 8761 8762 8763 8764 8765

            # Initialize before escaping
            code.putln("%s = %s;" % (temp_cname, private_cname))

            self.parallel_private_temps.append((temp_cname, private_cname))

        code.end_block() # end critical section

8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799
    def fetch_parallel_exception(self, code):
        """
        As each OpenMP thread may raise an exception, we need to fetch that
        exception from the threadstate and save it for after the parallel
        section where it can be re-raised in the master thread.

        Although it would seem that __pyx_filename, __pyx_lineno and
        __pyx_clineno are only assigned to under exception conditions (i.e.,
        when we have the GIL), and thus should be allowed to be shared without
        any race condition, they are in fact subject to the same race
        conditions that they were previously when they were global variables
        and functions were allowed to release the GIL:

            thread A                thread B
                acquire
                set lineno
                release
                                        acquire
                                        set lineno
                                        release
                acquire
                fetch exception
                release
                                        skip the fetch

                deallocate threadstate  deallocate threadstate
        """
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

        code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_exc_type)
        code.putln(
            "if (!%s) {" % Naming.parallel_exc_type)

8800
        code.putln("__Pyx_ErrFetchWithState(&%s, &%s, &%s);" % self.parallel_exc)
8801
        pos_info = chain(*zip(self.parallel_pos_info, self.pos_info))
8802
        code.funcstate.uses_error_indicator = True
8803
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
8804
        code.put_gotref(Naming.parallel_exc_type)
8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816

        code.putln(
            "}")

        code.put_release_ensured_gil()
        code.end_block()

    def restore_parallel_exception(self, code):
        "Re-raise a parallel exception"
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

8817
        code.put_giveref(Naming.parallel_exc_type)
8818
        code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc)
8819 8820 8821 8822 8823
        pos_info = chain(*zip(self.pos_info, self.parallel_pos_info))
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))

        code.put_release_ensured_gil()
        code.end_block()
8824 8825

    def restore_labels(self, code):
8826 8827 8828 8829 8830 8831
        """
        Restore all old labels. Call this before the 'else' clause to for
        loops and always before ending the parallel control flow block.
        """
        code.set_all_labels(self.old_loop_labels + (self.old_return_label,
                                                    self.old_error_label))
8832

8833 8834
    def end_parallel_control_flow_block(
            self, code, break_=False, continue_=False, return_=False):
8835 8836 8837 8838 8839
        """
        This ends the parallel control flow block and based on how the parallel
        section was exited, takes the corresponding action. The break_ and
        continue_ parameters indicate whether these should be propagated
        outwards:
8840

8841 8842 8843 8844 8845 8846 8847
            for i in prange(...):
                with cython.parallel.parallel():
                    continue

        Here break should be trapped in the parallel block, and propagated to
        the for loop.
        """
8848
        c = self.begin_of_parallel_control_block_point
8849 8850
        self.begin_of_parallel_control_block_point = None
        self.begin_of_parallel_control_block_point_after_decls = None
8851 8852 8853

        # Firstly, always prefer errors over returning, continue or break
        if self.error_label_used:
8854 8855
            c.putln("const char *%s = NULL; int %s = 0, %s = 0;" % self.parallel_pos_info)
            c.putln("PyObject *%s = NULL, *%s = NULL, *%s = NULL;" % self.parallel_exc)
8856 8857 8858 8859 8860 8861 8862 8863 8864

            code.putln(
                "if (%s) {" % Naming.parallel_exc_type)
            code.putln("/* This may have been overridden by a continue, "
                       "break or return in another thread. Prefer the error. */")
            code.putln("%s = 4;" % Naming.parallel_why)
            code.putln(
                "}")

8865 8866 8867 8868 8869 8870 8871
        if continue_:
            any_label_used = self.any_label_used
        else:
            any_label_used = self.breaking_label_used

        if any_label_used:
            # __pyx_parallel_why is used, declare and initialize
8872 8873
            c.putln("int %s;" % Naming.parallel_why)
            c.putln("%s = 0;" % Naming.parallel_why)
8874

8875 8876 8877 8878 8879 8880
            code.putln(
                "if (%s) {" % Naming.parallel_why)

            for temp_cname, private_cname in self.parallel_private_temps:
                code.putln("%s = %s;" % (private_cname, temp_cname))

8881
            code.putln("switch (%s) {" % Naming.parallel_why)
8882 8883 8884 8885 8886 8887 8888 8889
            if continue_:
                code.put("    case 1: ")
                code.put_goto(code.continue_label)

            if break_:
                code.put("    case 2: ")
                code.put_goto(code.break_label)

8890 8891 8892
            if return_:
                code.put("    case 3: ")
                code.put_goto(code.return_label)
8893 8894

            if self.error_label_used:
8895
                code.globalstate.use_utility_code(restore_exception_utility_code)
8896 8897 8898
                code.putln("    case 4:")
                self.restore_parallel_exception(code)
                code.put_goto(code.error_label)
8899

8900 8901 8902
            code.putln("}") # end switch
            code.putln(
                "}") # end if
8903 8904

        code.end_block() # end parallel control flow block
8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916
        self.redef_builtin_expect_apple_gcc_bug(code)

    # FIXME: improve with version number for OS X Lion
    buggy_platform_macro_condition = "(defined(__APPLE__) || defined(__OSX__))"
    have_expect_condition = "(defined(__GNUC__) && " \
                             "(__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))))"
    redef_condition = "(%s && %s)" % (buggy_platform_macro_condition, have_expect_condition)

    def undef_builtin_expect_apple_gcc_bug(self, code):
        """
        A bug on OS X Lion disallows __builtin_expect macros. This code avoids them
        """
8917
        if not self.parent:
8918
            code.undef_builtin_expect(self.redef_condition)
8919 8920

    def redef_builtin_expect_apple_gcc_bug(self, code):
8921
        if not self.parent:
8922
            code.redef_builtin_expect(self.redef_condition)
8923

Mark Florisson's avatar
Mark Florisson committed
8924 8925 8926

class ParallelWithBlockNode(ParallelStatNode):
    """
8927
    This node represents a 'with cython.parallel.parallel():' block
Mark Florisson's avatar
Mark Florisson committed
8928 8929
    """

8930 8931 8932 8933 8934 8935 8936 8937 8938
    valid_keyword_arguments = ['num_threads']

    num_threads = None

    def analyse_declarations(self, env):
        super(ParallelWithBlockNode, self).analyse_declarations(env)
        if self.args:
            error(self.pos, "cython.parallel.parallel() does not take "
                            "positional arguments")
8939

Mark Florisson's avatar
Mark Florisson committed
8940 8941
    def generate_execution_code(self, code):
        self.declare_closure_privates(code)
8942
        self.setup_parallel_control_flow_block(code)
Mark Florisson's avatar
Mark Florisson committed
8943 8944 8945

        code.putln("#ifdef _OPENMP")
        code.put("#pragma omp parallel ")
8946 8947

        if self.privates:
8948
            privates = [e.cname for e in self.privates
8949
                        if not e.type.is_pyobject]
8950
            code.put('private(%s)' % ', '.join(sorted(privates)))
8951

8952
        self.privatization_insertion_point = code.insertion_point()
8953
        self.put_num_threads(code)
8954
        code.putln("")
8955

8956 8957
        code.putln("#endif /* _OPENMP */")

8958
        code.begin_block()  # parallel block
8959
        self.begin_parallel_block(code)
8960
        self.initialize_privates_to_nan(code)
8961
        code.funcstate.start_collecting_temps()
Mark Florisson's avatar
Mark Florisson committed
8962
        self.body.generate_execution_code(code)
8963
        self.trap_parallel_exit(code)
8964
        self.privatize_temps(code)
8965
        self.end_parallel_block(code)
8966
        code.end_block()  # end parallel block
8967

8968 8969
        continue_ = code.label_used(code.continue_label)
        break_ = code.label_used(code.break_label)
8970
        return_ = code.label_used(code.return_label)
Mark Florisson's avatar
Mark Florisson committed
8971

8972 8973
        self.restore_labels(code)
        self.end_parallel_control_flow_block(code, break_=break_,
8974 8975
                                             continue_=continue_,
                                             return_=return_)
Mark Florisson's avatar
Mark Florisson committed
8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986
        self.release_closure_privates(code)


class ParallelRangeNode(ParallelStatNode):
    """
    This node represents a 'for i in cython.parallel.prange():' construct.

    target       NameNode       the target iteration variable
    else_clause  Node or None   the else clause of this loop
    """

8987 8988
    child_attrs = ['body', 'target', 'else_clause', 'args', 'num_threads',
                   'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8989 8990 8991 8992 8993 8994

    body = target = else_clause = args = None

    start = stop = step = None

    is_prange = True
8995

8996
    nogil = None
8997 8998
    schedule = None

8999
    valid_keyword_arguments = ['schedule', 'nogil', 'num_threads', 'chunksize']
Mark Florisson's avatar
Mark Florisson committed
9000

9001 9002 9003 9004 9005
    def __init__(self, pos, **kwds):
        super(ParallelRangeNode, self).__init__(pos, **kwds)
        # Pretend to be a ForInStatNode for control flow analysis
        self.iterator = PassStatNode(pos)

Mark Florisson's avatar
Mark Florisson committed
9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022
    def analyse_declarations(self, env):
        super(ParallelRangeNode, self).analyse_declarations(env)
        self.target.analyse_target_declaration(env)
        if self.else_clause is not None:
            self.else_clause.analyse_declarations(env)

        if not self.args or len(self.args) > 3:
            error(self.pos, "Invalid number of positional arguments to prange")
            return

        if len(self.args) == 1:
            self.stop, = self.args
        elif len(self.args) == 2:
            self.start, self.stop = self.args
        else:
            self.start, self.stop, self.step = self.args

Mark Florisson's avatar
Mark Florisson committed
9023 9024 9025
        if hasattr(self.schedule, 'decode'):
            self.schedule = self.schedule.decode('ascii')

9026 9027
        if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'):
            error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,))
Mark Florisson's avatar
Mark Florisson committed
9028 9029

    def analyse_expressions(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
9030
        was_nogil = env.nogil
9031 9032 9033
        if self.nogil:
            env.nogil = True

9034 9035
        if self.target is None:
            error(self.pos, "prange() can only be used as part of a for loop")
9036
            return self
Mark Florisson's avatar
Mark Florisson committed
9037

9038
        self.target = self.target.analyse_target_types(env)
Mark Florisson's avatar
Mark Florisson committed
9039

9040 9041 9042 9043 9044 9045 9046 9047
        if not self.target.type.is_numeric:
            # Not a valid type, assume one for now anyway

            if not self.target.type.is_pyobject:
                # nogil_check will catch the is_pyobject case
                error(self.target.pos,
                      "Must be of numeric type, not %s" % self.target.type)

9048
            self.index_type = PyrexTypes.c_py_ssize_t_type
9049 9050
        else:
            self.index_type = self.target.type
9051 9052 9053 9054
            if not self.index_type.signed:
                warning(self.target.pos,
                        "Unsigned index type not allowed before OpenMP 3.0",
                        level=2)
Mark Florisson's avatar
Mark Florisson committed
9055 9056 9057 9058 9059 9060 9061 9062 9063

        # Setup start, stop and step, allocating temps if needed
        self.names = 'start', 'stop', 'step'
        start_stop_step = self.start, self.stop, self.step

        for node, name in zip(start_stop_step, self.names):
            if node is not None:
                node.analyse_types(env)
                if not node.type.is_numeric:
9064 9065
                    error(node.pos, "%s argument must be numeric" % name)
                    continue
Mark Florisson's avatar
Mark Florisson committed
9066 9067 9068 9069 9070 9071 9072 9073

                if not node.is_literal:
                    node = node.coerce_to_temp(env)
                    setattr(self, name, node)

                # As we range from 0 to nsteps, computing the index along the
                # way, we need a fitting type for 'i' and 'nsteps'
                self.index_type = PyrexTypes.widest_numeric_type(
9074
                    self.index_type, node.type)
Mark Florisson's avatar
Mark Florisson committed
9075 9076

        if self.else_clause is not None:
9077
            self.else_clause = self.else_clause.analyse_expressions(env)
Mark Florisson's avatar
Mark Florisson committed
9078

9079 9080 9081 9082 9083 9084 9085 9086
        # Although not actually an assignment in this scope, it should be
        # treated as such to ensure it is unpacked if a closure temp, and to
        # ensure lastprivate behaviour and propagation. If the target index is
        # not a NameNode, it won't have an entry, and an error was issued by
        # ParallelRangeTransform
        if hasattr(self.target, 'entry'):
            self.assignments[self.target.entry] = self.target.pos, None

9087
        node = super(ParallelRangeNode, self).analyse_expressions(env)
9088

9089 9090 9091
        if node.chunksize:
            if not node.schedule:
                error(node.chunksize.pos,
9092
                      "Must provide schedule with chunksize")
9093 9094
            elif node.schedule == 'runtime':
                error(node.chunksize.pos,
9095
                      "Chunksize not valid for the schedule runtime")
9096 9097 9098 9099
            elif (node.chunksize.type.is_int and
                  node.chunksize.is_literal and
                  node.chunksize.compile_time_value(env) <= 0):
                error(node.chunksize.pos, "Chunksize must not be negative")
9100

9101 9102
            node.chunksize = node.chunksize.coerce_to(
                PyrexTypes.c_int_type, env).coerce_to_temp(env)
9103

9104
        if node.nogil:
9105 9106
            env.nogil = was_nogil

9107 9108 9109
        node.is_nested_prange = node.parent and node.parent.is_prange
        if node.is_nested_prange:
            parent = node
9110 9111 9112
            while parent.parent and parent.parent.is_prange:
                parent = parent.parent

9113 9114 9115 9116
            parent.assignments.update(node.assignments)
            parent.privates.update(node.privates)
            parent.assigned_nodes.extend(node.assigned_nodes)
        return node
9117

Mark Florisson's avatar
Mark Florisson committed
9118 9119
    def nogil_check(self, env):
        names = 'start', 'stop', 'step', 'target'
9120
        nodes = self.start, self.stop, self.step, self.target
Mark Florisson's avatar
Mark Florisson committed
9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165
        for name, node in zip(names, nodes):
            if node is not None and node.type.is_pyobject:
                error(node.pos, "%s may not be a Python object "
                                "as we don't have the GIL" % name)

    def generate_execution_code(self, code):
        """
        Generate code in the following steps

            1)  copy any closure variables determined thread-private
                into temporaries

            2)  allocate temps for start, stop and step

            3)  generate a loop that calculates the total number of steps,
                which then computes the target iteration variable for every step:

                    for i in prange(start, stop, step):
                        ...

                becomes

                    nsteps = (stop - start) / step;
                    i = start;

                    #pragma omp parallel for lastprivate(i)
                    for (temp = 0; temp < nsteps; temp++) {
                        i = start + step * temp;
                        ...
                    }

                Note that accumulation of 'i' would have a data dependency
                between iterations.

                Also, you can't do this

                    for (i = start; i < stop; i += step)
                        ...

                as the '<' operator should become '>' for descending loops.
                'for i from x < i < y:' does not suffer from this problem
                as the relational operator is known at compile time!

            4) release our temps and write back any private closure variables
        """
9166
        self.declare_closure_privates(code)
Mark Florisson's avatar
Mark Florisson committed
9167 9168 9169 9170 9171 9172 9173 9174

        # This can only be a NameNode
        target_index_cname = self.target.entry.cname

        # This will be used as the dict to format our code strings, holding
        # the start, stop , step, temps and target cnames
        fmt_dict = {
            'target': target_index_cname,
9175
            'target_type': self.target.type.empty_declaration_code()
Mark Florisson's avatar
Mark Florisson committed
9176 9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198
        }

        # Setup start, stop and step, allocating temps if needed
        start_stop_step = self.start, self.stop, self.step
        defaults = '0', '0', '1'
        for node, name, default in zip(start_stop_step, self.names, defaults):
            if node is None:
                result = default
            elif node.is_literal:
                result = node.get_constant_c_result_code()
            else:
                node.generate_evaluation_code(code)
                result = node.result()

            fmt_dict[name] = result

        fmt_dict['i'] = code.funcstate.allocate_temp(self.index_type, False)
        fmt_dict['nsteps'] = code.funcstate.allocate_temp(self.index_type, False)

        # TODO: check if the step is 0 and if so, raise an exception in a
        # 'with gil' block. For now, just abort
        code.putln("if (%(step)s == 0) abort();" % fmt_dict)

9199
        self.setup_parallel_control_flow_block(code) # parallel control flow block
9200

9201
        # Note: nsteps is private in an outer scope if present
9202
        code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict)
Mark Florisson's avatar
Mark Florisson committed
9203

9204 9205 9206 9207 9208 9209 9210
        # The target iteration variable might not be initialized, do it only if
        # we are executing at least 1 iteration, otherwise we should leave the
        # target unaffected. The target iteration variable is firstprivate to
        # shut up compiler warnings caused by lastprivate, as the compiler
        # erroneously believes that nsteps may be <= 0, leaving the private
        # target index uninitialized
        code.putln("if (%(nsteps)s > 0)" % fmt_dict)
9211
        code.begin_block() # if block
Mark Florisson's avatar
Mark Florisson committed
9212
        self.generate_loop(code, fmt_dict)
9213
        code.end_block() # end if block
Mark Florisson's avatar
Mark Florisson committed
9214

9215 9216 9217
        self.restore_labels(code)

        if self.else_clause:
9218
            if self.breaking_label_used:
9219
                code.put("if (%s < 2)" % Naming.parallel_why)
9220 9221 9222 9223 9224 9225 9226

            code.begin_block() # else block
            code.putln("/* else */")
            self.else_clause.generate_execution_code(code)
            code.end_block() # end else block

        # ------ cleanup ------
9227
        self.end_parallel_control_flow_block(code) # end parallel control flow block
9228

Mark Florisson's avatar
Mark Florisson committed
9229 9230
        # And finally, release our privates and write back any closure
        # variables
9231
        for temp in start_stop_step + (self.chunksize, self.num_threads):
Mark Florisson's avatar
Mark Florisson committed
9232 9233 9234 9235 9236 9237 9238 9239 9240 9241
            if temp is not None:
                temp.generate_disposal_code(code)
                temp.free_temps(code)

        code.funcstate.release_temp(fmt_dict['i'])
        code.funcstate.release_temp(fmt_dict['nsteps'])

        self.release_closure_privates(code)

    def generate_loop(self, code, fmt_dict):
9242 9243 9244 9245
        if self.is_nested_prange:
            code.putln("#if 0")
        else:
            code.putln("#ifdef _OPENMP")
Mark Florisson's avatar
Mark Florisson committed
9246 9247 9248

        if not self.is_parallel:
            code.put("#pragma omp for")
9249
            self.privatization_insertion_point = code.insertion_point()
9250
            reduction_codepoint = self.parent.privatization_insertion_point
Mark Florisson's avatar
Mark Florisson committed
9251
        else:
9252 9253
            code.put("#pragma omp parallel")
            self.privatization_insertion_point = code.insertion_point()
9254
            reduction_codepoint = self.privatization_insertion_point
9255 9256 9257 9258 9259 9260 9261 9262
            code.putln("")
            code.putln("#endif /* _OPENMP */")

            code.begin_block() # pragma omp parallel begin block

            # Initialize the GIL if needed for this thread
            self.begin_parallel_block(code)

9263 9264 9265 9266
            if self.is_nested_prange:
                code.putln("#if 0")
            else:
                code.putln("#ifdef _OPENMP")
9267
            code.put("#pragma omp for")
Mark Florisson's avatar
Mark Florisson committed
9268

9269
        for entry, (op, lastprivate) in sorted(self.privates.items()):
Mark Florisson's avatar
Mark Florisson committed
9270
            # Don't declare the index variable as a reduction
9271
            if op and op in "+*-&^|" and entry != self.target.entry:
9272 9273 9274
                if entry.type.is_pyobject:
                    error(self.pos, "Python objects cannot be reductions")
                else:
9275 9276 9277 9278
                    #code.put(" reduction(%s:%s)" % (op, entry.cname))
                    # This is the only way reductions + nesting works in gcc4.5
                    reduction_codepoint.put(
                                " reduction(%s:%s)" % (op, entry.cname))
9279
            else:
9280 9281
                if entry == self.target.entry:
                    code.put(" firstprivate(%s)" % entry.cname)
9282 9283
                    code.put(" lastprivate(%s)" % entry.cname)
                    continue
Mark Florisson's avatar
Mark Florisson committed
9284

9285
                if not entry.type.is_pyobject:
9286 9287 9288 9289
                    if lastprivate:
                        private = 'lastprivate'
                    else:
                        private = 'private'
Mark Florisson's avatar
Mark Florisson committed
9290

9291
                    code.put(" %s(%s)" % (private, entry.cname))
9292

Mark Florisson's avatar
Mark Florisson committed
9293
        if self.schedule:
9294
            if self.chunksize:
9295
                chunksize = ", %s" % self.evaluate_before_block(code, self.chunksize)
9296 9297 9298 9299
            else:
                chunksize = ""

            code.put(" schedule(%s%s)" % (self.schedule, chunksize))
9300

9301
        self.put_num_threads(reduction_codepoint)
9302

9303 9304
        code.putln("")
        code.putln("#endif /* _OPENMP */")
Mark Florisson's avatar
Mark Florisson committed
9305 9306

        code.put("for (%(i)s = 0; %(i)s < %(nsteps)s; %(i)s++)" % fmt_dict)
9307
        code.begin_block()  # for loop block
9308

9309
        guard_around_body_codepoint = code.insertion_point()
9310

9311 9312
        # Start if guard block around the body. This may be unnecessary, but
        # at least it doesn't spoil indentation
Mark Florisson's avatar
Mark Florisson committed
9313
        code.begin_block()
9314

9315
        code.putln("%(target)s = (%(target_type)s)(%(start)s + %(step)s * %(i)s);" % fmt_dict)
9316 9317
        self.initialize_privates_to_nan(code, exclude=self.target.entry)

9318 9319 9320
        if self.is_parallel:
            code.funcstate.start_collecting_temps()

Mark Florisson's avatar
Mark Florisson committed
9321
        self.body.generate_execution_code(code)
9322
        self.trap_parallel_exit(code, should_flush=True)
9323 9324
        self.privatize_temps(code)

9325 9326 9327
        if self.breaking_label_used:
            # Put a guard around the loop body in case return, break or
            # exceptions might be used
9328
            guard_around_body_codepoint.putln("if (%s < 2)" % Naming.parallel_why)
9329

9330 9331
        code.end_block()  # end guard around loop body
        code.end_block()  # end for loop block
Mark Florisson's avatar
Mark Florisson committed
9332

9333 9334 9335
        if self.is_parallel:
            # Release the GIL and deallocate the thread state
            self.end_parallel_block(code)
9336
            code.end_block()  # pragma omp parallel end block
9337

9338

9339 9340 9341 9342 9343 9344 9345 9346
class CnameDecoratorNode(StatNode):
    """
    This node is for the cname decorator in CythonUtilityCode:

        @cname('the_cname')
        cdef func(...):
            ...

9347 9348
    In case of a cdef class the cname specifies the objstruct_cname.

9349 9350 9351 9352 9353 9354 9355 9356
    node        the node to which the cname decorator is applied
    cname       the cname the node should get
    """

    child_attrs = ['node']

    def analyse_declarations(self, env):
        self.node.analyse_declarations(env)
9357

9358 9359 9360 9361 9362
        node = self.node
        if isinstance(node, CompilerDirectivesNode):
            node = node.body.stats[0]

        self.is_function = isinstance(node, FuncDefNode)
9363
        is_struct_or_enum = isinstance(node, (CStructOrUnionDefNode, CEnumDefNode))
9364
        e = node.entry
9365 9366 9367 9368

        if self.is_function:
            e.cname = self.cname
            e.func_cname = self.cname
9369
            e.used = True
9370 9371
            if e.pyfunc_cname and '.' in e.pyfunc_cname:
                e.pyfunc_cname = self.mangle(e.pyfunc_cname)
9372 9373
        elif is_struct_or_enum:
            e.cname = e.type.cname = self.cname
9374
        else:
9375
            scope = node.scope
9376 9377

            e.cname = self.cname
9378
            e.type.objstruct_cname = self.cname + '_obj'
9379
            e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
9380
            e.type.typeptr_cname = self.cname + '_type'
9381
            e.type.scope.namespace_cname = e.type.typeptr_cname
9382

9383
            e.as_variable.cname = e.type.typeptr_cname
9384 9385 9386

            scope.scope_prefix = self.cname + "_"

9387
            for name, entry in scope.entries.items():
9388
                if entry.func_cname:
9389 9390 9391 9392 9393 9394 9395 9396 9397
                    entry.func_cname = self.mangle(entry.cname)
                if entry.pyfunc_cname:
                    entry.pyfunc_cname = self.mangle(entry.pyfunc_cname)

    def mangle(self, cname):
        if '.' in cname:
            # remove __pyx_base from func_cname
            cname = cname.split('.')[-1]
        return '%s_%s' % (self.cname, cname)
9398 9399

    def analyse_expressions(self, env):
9400 9401
        self.node = self.node.analyse_expressions(env)
        return self
9402 9403

    def generate_function_definitions(self, env, code):
9404
        "Ensure a prototype for every @cname method in the right place"
9405 9406 9407 9408 9409 9410
        if self.is_function and env.is_c_class_scope:
            # method in cdef class, generate a prototype in the header
            h_code = code.globalstate['utility_code_proto']

            if isinstance(self.node, DefNode):
                self.node.generate_function_header(
9411
                    h_code, with_pymethdef=False, proto_only=True)
9412
            else:
9413
                from . import ModuleNode
9414 9415 9416 9417 9418
                entry = self.node.entry
                cname = entry.cname
                entry.cname = entry.func_cname

                ModuleNode.generate_cfunction_declaration(
9419 9420 9421 9422
                    entry,
                    env.global_scope(),
                    h_code,
                    definition=True)
9423 9424 9425 9426 9427 9428 9429

                entry.cname = cname

        self.node.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
        self.node.generate_execution_code(code)
Mark Florisson's avatar
Mark Florisson committed
9430

9431

William Stein's avatar
William Stein committed
9432 9433 9434 9435 9436 9437
#------------------------------------------------------------------------------------
#
#  Runtime support code
#
#------------------------------------------------------------------------------------

Robert Bradshaw's avatar
Robert Bradshaw committed
9438
if Options.gcc_branch_hints:
Stefan Behnel's avatar
Stefan Behnel committed
9439
    branch_prediction_macros = """
9440 9441 9442 9443 9444 9445
/* Test for GCC > 2.95 */
#if defined(__GNUC__) \
    && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
  #define likely(x)   __builtin_expect(!!(x), 1)
  #define unlikely(x) __builtin_expect(!!(x), 0)
#else /* !__GNUC__ or GCC < 2.95 */
9446 9447
  #define likely(x)   (x)
  #define unlikely(x) (x)
9448
#endif /* __GNUC__ */
Stefan Behnel's avatar
Stefan Behnel committed
9449
"""
Robert Bradshaw's avatar
Robert Bradshaw committed
9450
else:
Stefan Behnel's avatar
Stefan Behnel committed
9451
    branch_prediction_macros = """
Robert Bradshaw's avatar
Robert Bradshaw committed
9452 9453
#define likely(x)   (x)
#define unlikely(x) (x)
Stefan Behnel's avatar
Stefan Behnel committed
9454
"""
William Stein's avatar
William Stein committed
9455 9456 9457

#------------------------------------------------------------------------------------

9458 9459
printing_utility_code = UtilityCode.load_cached("Print", "Printing.c")
printing_one_utility_code = UtilityCode.load_cached("PrintOne", "Printing.c")
9460

William Stein's avatar
William Stein committed
9461 9462
#------------------------------------------------------------------------------------

9463 9464 9465 9466 9467 9468 9469
# Exception raising code
#
# Exceptions are raised by __Pyx_Raise() and stored as plain
# type/value/tb in PyThreadState->curexc_*.  When being caught by an
# 'except' statement, curexc_* is moved over to exc_* by
# __Pyx_GetException()

9470 9471 9472 9473 9474
restore_exception_utility_code = UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")
raise_utility_code = UtilityCode.load_cached("RaiseException", "Exceptions.c")
get_exception_utility_code = UtilityCode.load_cached("GetException", "Exceptions.c")
swap_exception_utility_code = UtilityCode.load_cached("SwapException", "Exceptions.c")
reset_exception_utility_code = UtilityCode.load_cached("SaveResetException", "Exceptions.c")
9475
traceback_utility_code = UtilityCode.load_cached("AddTraceback", "Exceptions.c")
9476 9477 9478

#------------------------------------------------------------------------------------

9479 9480
get_exception_tuple_utility_code = UtilityCode(
    proto="""
9481
static PyObject *__Pyx_GetExceptionTuple(PyThreadState *__pyx_tstate); /*proto*/
9482
""",
9483 9484 9485 9486 9487
    # I doubt that calling __Pyx_GetException() here is correct as it moves
    # the exception from tstate->curexc_* to tstate->exc_*, which prevents
    # exception handlers later on from receiving it.
    # NOTE: "__pyx_tstate" may be used by __Pyx_GetException() macro
    impl = """
9488
static PyObject *__Pyx_GetExceptionTuple(CYTHON_UNUSED PyThreadState *__pyx_tstate) {
9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504
    PyObject *type = NULL, *value = NULL, *tb = NULL;
    if (__Pyx_GetException(&type, &value, &tb) == 0) {
        PyObject* exc_info = PyTuple_New(3);
        if (exc_info) {
            Py_INCREF(type);
            Py_INCREF(value);
            Py_INCREF(tb);
            PyTuple_SET_ITEM(exc_info, 0, type);
            PyTuple_SET_ITEM(exc_info, 1, value);
            PyTuple_SET_ITEM(exc_info, 2, tb);
            return exc_info;
        }
    }
    return NULL;
}
""",
9505
    requires=[get_exception_utility_code])